\n ![](\"https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/refs/heads/main/docs/whats_new/v1.3.0/amlb2025_fig3a.png\")
\n ![](\"https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/refs/heads/main/docs/whats_new/v1.3.0/amlb2025_fig10d.png\")
\n
![](\"https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/refs/heads/main/docs/whats_new/v1.3.0/amlb2025_fig1.png\")
\n\n## Papers using AutoGluon\n\nBelow is a sampling of some interesting papers that have cited AutoGluon.\n\n* (2023/04/28) [Benchmarking Automated Machine Learning Methods for Price Forecasting Applications](https://arxiv.org/abs/2304.14735)\n * This paper compares various traditional and AutoML methods for price forecasting problems, with AutoGluon achieving the strongest results.\n"
},
{
"path": "CI/batch/cancel-job.py",
"content": "import argparse\nimport boto3\nimport re\n\nparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\nparser.add_argument('--profile', help='profile name of aws account.', type=str,\n default=None)\nparser.add_argument('--region', help='Default region when creating new connections', type=str,\n default='us-east-1')\nparser.add_argument('--name', help='name of the job to be cancelled.', type=str,\n default='')\nparser.add_argument('--job-type', help='name of the job to be cancelled.', type=str,\n default='CI-CPU')\nparser.add_argument('--reason', help='reason to cancel the job.', type=str,\n default='Canelling because new commits related to same PR is pushed')\nargs = parser.parse_args()\n\nprofile = args.profile\nregion = args.region\njob_name = re.sub('[^A-Za-z0-9_\\-]', '', args.name)[:128] # Enforce AWS Batch jobName rules\njob_type = args.job_type\nreason = args.reason\n\nsession = boto3.Session(profile_name=profile, region_name=region)\nbatch = session.client(service_name='batch')\n\n\ndef main():\n # Find all jobs with job_name that are running or about to run and terminate them all.\n list_args = {\n \"jobQueue\": job_type,\n \"filters\": [{\"name\": \"JOB_NAME\", \"values\": [job_name]}],\n }\n try:\n response = batch.list_jobs(**list_args)\n for job in response[\"jobSummaryList\"]:\n if job[\"status\"] in [\"SUBMITTED\", \"PENDING\", \"RUNNABLE\", \"STARTING\", \"RUNNING\"]:\n print(f'Terminate previous job {job[\"jobId\"]}')\n batch.terminate_job(jobId=job[\"jobId\"], reason=\"New job submitted\")\n except Exception as e:\n print(f'Failed to terminate the job because of exception: {e}')\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "CI/batch/docker/Dockerfile.cpu",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.8.0-cpu-py312-ubuntu22.04-ec2\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n pandoc \\\n python3.11-venv \\\n graphviz \\\n graphviz-dev \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN adduser --disabled-password --disabled-login ci\nWORKDIR /home/ci\n\n# add autogluon_job script\nADD autogluon_job.sh .\nRUN chmod +x autogluon_job.sh; chown ci autogluon_job.sh\n\nUSER ci\n\nENV VIRTUAL_ENV=/home/ci/opt/venv\nRUN python3 -m venv $VIRTUAL_ENV\nENV PATH=\"$VIRTUAL_ENV/bin:$PATH\"\n\nCMD [\"/bin/bash\"]\n"
},
{
"path": "CI/batch/docker/Dockerfile.gpu",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.8.0-gpu-py312-cu129-ubuntu22.04-ec2\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n pandoc \\\n python3.11-venv \\\n graphviz \\\n graphviz-dev \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN apt install tesseract-ocr -y\n\nRUN adduser --disabled-password --disabled-login ci\nWORKDIR /home/ci\n\n# add autogluon_job script\nADD autogluon_job.sh .\nRUN chmod +x autogluon_job.sh; chown ci autogluon_job.sh\n\nUSER ci\n\nENV VIRTUAL_ENV=/home/ci/opt/venv\nRUN python3 -m venv $VIRTUAL_ENV\nENV PATH=\"$VIRTUAL_ENV/bin:$PATH\"\n\nCMD [\"/bin/bash\"]\n"
},
{
"path": "CI/batch/docker/Dockerfile.pyodide",
"content": "FROM pyodide/pyodide-env:20221102-chrome107-firefox106\n\nWORKDIR /src\n# compile pyodide\nRUN git clone --depth 1 --branch 0.22.0 https://github.com/pyodide/pyodide.git \\\n && cd pyodide \\\n && pip install -r requirements.txt \\\n && make \\\n && PYODIDE_PACKAGES='*,!bcrypt,!sparseqr,!cryptography,!libmpfr,!pyerfa' make \\\n && cd pyodide-build && pip install -e '.[test]' && cd .. \\\n && pip install 'playwright<1.23.0' && python3 -m playwright install \\\n && cd ..\n\nWORKDIR /src/pyodide\n# add autogluon_job script\nADD autogluon_job.sh .\nRUN chmod +x autogluon_job.sh\n\nRUN python3 -m pip install numpy pandas scikit-learn\n\nCMD [\"/bin/bash\"]\n\n"
},
{
"path": "CI/batch/docker/README.md",
"content": "# Updating the Docker Image for AWS Batch\nThis is for AutoGluon Devs to update the CI docker environment.\n\n**IMPORTANT**:\nPlease push the changes to our github repository if you updated the docker file and pushed the new docker image to our ECR.\nThe new docker image will take effect even you didn't push the changes to our github repository.\nThis helps to make sure everyone sees the changes you made and build on top.\n\nTo update the docker:\n\n- Update the Dockerfile\n- Log into the AWS account that holds the ECR repo on your dev machine.\n- Export the AWS account credentials as environment variables\n- CD to the same folder as the Dockerfile and execute the following:\n\n```shell\n# First export your ecr repo address as a environment variable\nexport AWS_ECR_REPO=${your_repo}\n\n# Following script will build, tag, and push the image\n# For cpu\n./docker_deploy.sh cpu\n# For gpu\n./docker_deploy.sh gpu\n\n```\n"
},
{
"path": "CI/batch/docker/autogluon_job.sh",
"content": "#!/bin/bash\n\ndate\necho \"Args: $@\"\nenv\necho \"jobId: $AWS_BATCH_JOB_ID\"\necho \"jobQueue: $AWS_BATCH_JQ_NAME\"\necho \"computeEnvironment: $AWS_BATCH_CE_NAME\"\n\nSOURCE_REF=$1\nWORK_DIR=$2\nCOMMAND=$3\nSAVED_OUTPUT=$4\nSAVE_PATH=$5\nREMOTE=$6\nSAFE_TO_USE_SCRIPT=$7\n\n# Copy the workflow from master branch\ngit clone https://github.com/autogluon/autogluon.git\nWORKFLOW_SCRIPTS=autogluon/.github/workflow_scripts\nif [ -d \"$WORKFLOW_SCRIPTS\" ]; then\n cp -R autogluon/.github/workflow_scripts .\nfi\n\ncd autogluon\n\nif [ ! -z $REMOTE ]; then\n git remote set-url origin $REMOTE\nfi\n\ngit fetch origin $SOURCE_REF:working\ngit checkout working\n\n# If not safe to use script, we overwrite with the script from master branch\nTRUE=true\nif [[ ${SAFE_TO_USE_SCRIPT,,} != ${TRUE,,} ]]; then\n if [ -d ../workflow_scripts ]; then\n rm -rf .github/workflow_scripts\n mv ../workflow_scripts .github/\n else\n echo Not safe to use user provided script, and could not find script from master branches\n exit 1\n fi\nfi\n\ncd $WORK_DIR\n/bin/bash -o pipefail -c \"eval $COMMAND\"\nCOMMAND_EXIT_CODE=$?\n\nexit $COMMAND_EXIT_CODE\n"
},
{
"path": "CI/batch/docker/docker_deploy.sh",
"content": "#!/bin/bash\n\nTYPE=$1\n\n# This executes a command that logs into ECR for both our CI repo and the AutoGluon DLC container repo.\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 369469875935.dkr.ecr.us-east-1.amazonaws.com\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com\n\nif [ -z $TYPE ]; then\n\techo \"No type detected. Choices: cpu, gpu\"\n\texit 1\nfi;\n\nif [ $TYPE == cpu ] || [ $TYPE == CPU ]; then\n\tdocker build -f Dockerfile.cpu -t autogluon-ci:cpu-latest .\n\tdocker tag autogluon-ci:cpu-latest $AWS_ECR_REPO:cpu-latest\n\tdocker push $AWS_ECR_REPO:cpu-latest\nelif [ $TYPE == gpu ] || [ $TYPE == GPU ]; then\n\tdocker build -f Dockerfile.gpu -t autogluon-ci:gpu-latest .\n\tdocker tag autogluon-ci:gpu-latest $AWS_ECR_REPO:gpu-latest\n\tdocker push $AWS_ECR_REPO:gpu-latest\nelse\n\techo \"Invalid type detected. Choices: cpu, gpu\"\n\texit 1\nfi;\n"
},
{
"path": "CI/batch/submit-job.py",
"content": "import argparse\nimport random\nimport re\nimport sys\nimport time\nfrom datetime import datetime, timezone\n\nimport boto3\nfrom botocore.compat import total_seconds\nfrom botocore.config import Config\n\n\njob_type_info = {\n 'CI-CPU': {\n 'job_definition': 'autogluon-ci-cpu:3',\n 'job_queue': 'CI-CPU'\n },\n 'CI-GPU': {\n 'job_definition': 'autogluon-ci-gpu:3',\n 'job_queue': 'CI-GPU'\n },\n 'CI-WASM': {\n 'job_definition': 'autogluon-ci-wasm:1',\n 'job_queue': 'CI-CPU'\n },\n 'CI-MULTI-GPU': {\n 'job_definition': 'autogluon-ci-multi-gpu:6',\n 'job_queue': 'CI-MULTI-GPU'\n },\n 'CI-CPU-PUSH': {\n 'job_definition': 'autogluon-ci-cpu-push:3',\n 'job_queue': 'CI-CPU'\n },\n 'CI-GPU-PUSH': {\n 'job_definition': 'autogluon-ci-gpu-push:4',\n 'job_queue': 'CI-GPU'\n },\n 'CI-WASM-PUSH': {\n 'job_definition': 'autogluon-ci-wasm-push:2',\n 'job_queue': 'CI-CPU'\n },\n 'CI-MULTI-GPU-PUSH': {\n 'job_definition': 'autogluon-ci-multi-gpu-push:2',\n 'job_queue': 'CI-MULTI-GPU'\n },\n}\n\nparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\nparser.add_argument('--profile', help='profile name of aws account.', type=str,\n default=None)\nparser.add_argument('--region', help='Default region when creating new connections', type=str,\n default='us-east-1')\nparser.add_argument('--name', help='name of the job', type=str, default='dummy')\nparser.add_argument('--job-type', help='type of job to submit.', type=str,\n choices=job_type_info.keys(), default='CI-CPU')\nparser.add_argument('--source-ref',\n help='ref in AutoGluon main github. e.g. master, refs/pull/500/head',\n type=str, default='master')\nparser.add_argument('--work-dir',\n help='working directory inside the repo. e.g. scripts/preprocess',\n type=str, default='scripts/preprocess')\nparser.add_argument('--saved-output',\n help='output to be saved, relative to working directory. '\n 'it can be either a single file or a directory',\n type=str, default='None')\nparser.add_argument('--save-path',\n help='s3 path where files are saved.',\n type=str, default='batch/temp/{}'.format(datetime.now().isoformat()))\nparser.add_argument('--command', help='command to run', type=str,\n default='git rev-parse HEAD | tee stdout.log')\nparser.add_argument('--remote',\n help='git repo address. https://github.com/autogluon/autogluon',\n type=str, default=\"https://github.com/autogluon/autogluon\")\nparser.add_argument('--safe-to-use-script',\n help='whether the script changes from the actor is safe. We assume it is safe if the actor has write permission to our repo',\n action='store_true')\nparser.add_argument('--wait', help='block wait until the job completes. '\n 'Non-zero exit code if job fails.', action='store_true')\nparser.add_argument('--timeout', help='job timeout in seconds', default=None, type=int)\n\n\nargs = parser.parse_args()\n\nsession = boto3.Session(profile_name=args.profile, region_name=args.region)\nconfig = Config(\n retries = dict(\n max_attempts = 20\n )\n)\nbatch, cloudwatch = [session.client(service_name=sn, config=config) for sn in ['batch', 'logs']]\n\n\ndef printLogs(logGroupName, logStreamName, startTime):\n kwargs = {'logGroupName': logGroupName,\n 'logStreamName': logStreamName,\n 'startTime': startTime,\n 'startFromHead': True}\n\n lastTimestamp = startTime - 1\n while True:\n logEvents = cloudwatch.get_log_events(**kwargs)\n\n for event in logEvents['events']:\n lastTimestamp = event['timestamp']\n timestamp = datetime.fromtimestamp(lastTimestamp / 1000.0, timezone.utc).isoformat()\n print('[{}] {}'.format((timestamp + '.000')[:23] + 'Z', event['message']))\n\n nextToken = logEvents['nextForwardToken']\n if nextToken and kwargs.get('nextToken') != nextToken:\n kwargs['nextToken'] = nextToken\n else:\n break\n return lastTimestamp\n\n\ndef nowInMillis():\n endTime = int(total_seconds(datetime.now(timezone.utc).replace(tzinfo=None) - datetime(1970, 1, 1))) * 1000\n return endTime\n\n\ndef main():\n spin = ['-', '/', '|', '\\\\', '-', '/', '|', '\\\\']\n logGroupName = '/aws/batch/job'\n\n jobName = re.sub('[^A-Za-z0-9_\\-]', '', args.name)[:128] # Enforce AWS Batch jobName rules\n jobType = args.job_type\n jobQueue = job_type_info[jobType]['job_queue']\n jobDefinition = job_type_info[jobType]['job_definition']\n wait = args.wait\n\n safe_to_use_script = 'False'\n if args.safe_to_use_script:\n safe_to_use_script = 'True'\n\n parameters = {\n 'SOURCE_REF': args.source_ref,\n 'WORK_DIR': args.work_dir,\n 'SAVED_OUTPUT': args.saved_output,\n 'SAVE_PATH': args.save_path,\n 'COMMAND': f\"\\\"{args.command}\\\"\", # wrap command with double quotation mark, so that batch can treat it as a single command\n 'REMOTE': args.remote,\n 'SAFE_TO_USE_SCRIPT': safe_to_use_script,\n }\n kwargs = dict(\n jobName=jobName,\n jobQueue=jobQueue,\n jobDefinition=jobDefinition,\n parameters=parameters,\n )\n if args.timeout is not None:\n kwargs['timeout'] = {'attemptDurationSeconds': args.timeout}\n submitJobResponse = batch.submit_job(**kwargs)\n\n jobId = submitJobResponse['jobId']\n print('Submitted job [{} - {}] to the job queue [{}]'.format(jobName, jobId, jobQueue))\n\n spinner = 0\n running = False\n status_set = set()\n startTime = 0\n logStreamName = None\n while wait:\n time.sleep(random.randint(5, 10))\n describeJobsResponse = batch.describe_jobs(jobs=[jobId])\n status = describeJobsResponse['jobs'][0]['status']\n if status == 'SUCCEEDED' or status == 'FAILED':\n if logStreamName:\n startTime = printLogs(logGroupName, logStreamName, startTime) + 1\n print('=' * 80)\n print('Job [{} - {}] {}'.format(jobName, jobId, status))\n sys.exit(status == 'FAILED')\n\n elif status == 'RUNNING':\n logStreamName = describeJobsResponse['jobs'][0]['container']['logStreamName']\n if not running:\n running = True\n print('\\rJob [{}, {}] is RUNNING.'.format(jobName, jobId))\n if logStreamName:\n print('Output [{}]:\\n {}'.format(logStreamName, '=' * 80))\n if logStreamName:\n startTime = printLogs(logGroupName, logStreamName, startTime) + 1\n elif status not in status_set:\n status_set.add(status)\n print('\\rJob [%s - %s] is %-9s... %s' % (jobName, jobId, status, spin[spinner % len(spin)]),)\n sys.stdout.flush()\n spinner += 1\n\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "CI/bench/evaluate.py",
"content": "import argparse\nimport os\nimport subprocess\nfrom datetime import datetime\n\nimport pandas as pd\nimport yaml\n\ndef process_results(eval_flag: bool):\n try:\n paths = []\n frameworks = []\n for file in os.listdir(\"./results\"):\n if file.endswith(\".csv\") and not file.endswith(\"_min.csv\"):\n file = os.path.join(\"./results\", file)\n df = pd.read_csv(file)\n paths.append(os.path.basename(file))\n frameworks += list(df[\"framework\"].unique())\n\n modified_list_paths = []\n modified_list_frameworks = []\n\n for path in paths:\n modified_list_paths.append('--paths')\n modified_list_paths.append(path)\n\n for framework in frameworks:\n modified_list_frameworks.append('--frameworks-run')\n modified_list_frameworks.append(framework)\n \n paths = modified_list_paths\n frameworks = modified_list_frameworks\n subprocess.run(\n [\n \"agbench\",\n \"evaluate-amlb-results\",\n *frameworks,\n \"--results-dir-input\",\n \"./results/\",\n *paths,\n f\"--results-dir-output\",\n f\"./evaluate\",\n \"--no-clean-data\",\n ],\n check=True\n )\n\n unique_framework = {}\n # Renaming the frameworks for dashboard formatting\n for file in os.listdir(\"./evaluate\"):\n if file.endswith(\"dataset_valid.csv\"):\n file_path = os.path.join(\"./evaluate\", file)\n df = pd.read_csv(file_path)\n for index, row in df.iterrows():\n if (row['framework'].split('_')[-1] not in unique_framework) and (\"AutoGluon\" in row['framework']):\n unique_framework[row['framework']] = row['framework'].split('_')[-1]\n \n if len(unique_framework) > 1:\n unique_framework = dict(sorted(unique_framework.items(), key=lambda item: item[1]))\n earliest_timestamp = next(iter(unique_framework))\n for index, (key, value) in enumerate(unique_framework.items()):\n if eval_flag:\n if index > 0:\n unique_framework[key] = f'AutoGluon_master'\n else:\n unique_framework[key] = f'AutoGluon_v1.0'\n else:\n if index > 0:\n unique_framework[key] = f'AutoGluon_PR_{index}'\n else:\n unique_framework[key] = f'AutoGluon_master_branch'\n\n df['framework'] = df['framework'].map(unique_framework)\n df.to_csv(file_path, index=False)\n\n for file in os.listdir(\"./evaluate/pairwise/\"):\n if file.endswith(\".csv\"):\n file_path = os.path.join(\"./evaluate/pairwise/\", file)\n df = pd.read_csv(file_path)\n cols_to_drop = [\"% Loss Reduction (median)\", \"Avg Fit Speed Diff\"]\n df = df.drop(columns=cols_to_drop, errors='ignore')\n df = df.rename(columns={'% Loss Reduction':'% Less Avg. Errors'})\n\n df['framework'] = df['framework'].map(unique_framework)\n df.to_csv(file_path, index=False)\n return df\n except Exception as e:\n raise Exception(f\"Failed to process results: {e}\") from e\n\n\ndef main():\n\n parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\n parser.add_argument(\n \"--config_path\", help=\"path to generated config path to fetch benchmark name\", type=str, required=True\n )\n parser.add_argument(\"--module_name\", help=\"module on which we run benchmark\", type=str, required=True)\n parser.add_argument(\"--time_limit\", help=\"time limit of the benchmark run\", type=str, required=True)\n parser.add_argument(\"--branch_name\", help=\"if it happens to be master then just push the cleaned result, do not evaluate\", type=str, required=True)\n parser.add_argument(\"--benchmark_type\", help=\"type of benchmark to run, tabular, timeseries, automm-text etc.\", type=str, required=True)\n\n\n args = parser.parse_args()\n\n config_path = args.config_path\n module_name = args.module_name\n time_limit = args.time_limit\n branch_name = args.branch_name\n benchmark_type = args.benchmark_type\n df1 = pd.DataFrame()\n\n try:\n for root, dirs, files in os.walk(config_path):\n for file in files:\n if file == f\"{module_name}_cloud_configs.yaml\":\n config_file = os.path.join(root, file)\n break\n\n with open(config_file, \"r\") as f:\n config = yaml.safe_load(f)\n benchmark_name = config[\"benchmark_name\"]\n\n subprocess.run(\n [\n \"agbench\",\n \"aggregate-amlb-results\",\n \"autogluon-ci-benchmark\",\n module_name,\n benchmark_name,\n \"--constraint\",\n time_limit,\n ],\n check=True,\n )\n\n subprocess.run(\n [\n \"agbench\",\n \"clean-amlb-results\",\n benchmark_name,\n f\"--results-dir-input\",\n f\"s3://autogluon-ci-benchmark/aggregated/{module_name}/{benchmark_name}/\",\n \"--file-prefix\",\n f\"results_automlbenchmark_{time_limit}\",\n \"--benchmark-name-in-input-path\",\n \"--results-dir-output\",\n \"./results\",\n ],\n check=True,\n )\n\n subprocess.run(\n [\n \"sh\",\n \"-c\",\n \"rm -f ./results/*_min.csv ./results/*_min.parquet ./results/*.parquet\",\n ],\n check=True,\n )\n\n # If branch is master Copy v1.0 results from S3\n if branch_name == \"master\":\n if benchmark_type.startswith(\"tabular\") or benchmark_type.startswith(\"timeseries\"):\n subprocess.run(\n [\n \"aws\",\n \"s3\",\n \"cp\",\n \"--recursive\",\n f\"s3://autogluon-ci-benchmark/version_1.0/cleaned/{module_name}/\",\n \"./results\",\n ],\n check=True,\n ) \n else:\n subprocess.run(\n [\n \"aws\",\n \"s3\",\n \"cp\",\n \"--recursive\",\n f\"s3://autogluon-ci-benchmark/version_1.0/cleaned/{module_name}/{benchmark_type}/\",\n \"./results\",\n ],\n check=True,\n )\n\n df = process_results(eval_flag=True)\n\n for root, dirs, files in os.walk(\"./evaluate\"):\n for file in files:\n if file.startswith(\"AutoGluon\") and file.endswith(\".csv\") and \"pairwise\" in root:\n file_path = os.path.join(root, file)\n df1 = pd.read_csv(file_path, usecols=[\"framework\", \"Winrate\", \"time_train_s\", \"time_infer_s\",\"rank\"])\n\n df1.to_csv(\"./report_results.csv\", index=False, mode='w')\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n\n if benchmark_type.startswith(\"tabular\") or benchmark_type.startswith(\"timeseries\"):\n subprocess.run(\n [\n \"aws\",\n \"s3\",\n \"cp\",\n \"--recursive\",\n \"./evaluate\",\n f\"s3://autogluon-ci-benchmark/version_1.0/evaluated/{module_name}/{timestamp}/\",\n ],\n check=True,\n )\n else:\n subprocess.run(\n [\n \"aws\",\n \"s3\",\n \"cp\",\n \"--recursive\",\n \"./evaluate\",\n f\"s3://autogluon-ci-benchmark/version_1.0/evaluated/{module_name}/{benchmark_type}/{timestamp}/\",\n ],\n check=True,\n )\n # If it is not master then it is a PR, perform the evaluation w.r.t cleaned master bench results\n else:\n df = process_results(eval_flag=False)\n\n # Compare aggregated results with Master branch and return comment\n master_win_rate = 0\n for _, row in df.iterrows():\n if \"master\" in row['framework']:\n master_win_rate = row['Winrate']\n\n pr_comment = f\"\\nBenchmark Test Result - Pass\\nEvaluation Results Path: s3://autogluon-ci-benchmark/evaluation/{module_name}/{branch_name}\\n\"\n for _, row in df.iterrows():\n if (\"master\" not in row['framework']) and (master_win_rate >= row['Winrate']):\n pr_comment = \"\"\n pr_comment = f\"\\nBenchmark Test Result - Fail\\nEvaluation Results Path: s3://autogluon-ci-benchmark/evaluation/{module_name}/{branch_name}\\n\"\n\n with open(\"final_eval.txt\", \"w\") as file:\n file.write(pr_comment)\n except Exception as e:\n print(f\"An exception occurred: {e}\")\n\nif __name__ == \"__main__\":\n main()"
},
{
"path": "CI/bench/generate_amlb_user_dir.sh",
"content": "#!/usr/bin/env bash\n\nMODULE=$1\nREPOSITORY=$2\nBRANCH=$3\nSHORT_SHA=$4\nPR_NUMBER=$5\nFOLDS=$6\n\n# generate custom amlb configs\nif [ -z \"$FOLDS\" ] || [ $MODULE == 'multimodal' ] ; then\n python $(dirname \"$0\")/generate_framework.py --module $MODULE --repository https://github.com/$REPOSITORY.git --branch $BRANCH --folds_to_run -1\nelse\n python $(dirname \"$0\")/generate_framework.py --module $MODULE --repository https://github.com/$REPOSITORY.git --branch $BRANCH --folds_to_run $FOLDS\nfi\n\nif [ -n \"$PR_NUMBER\" ]\nthen\n CONFIG_PATH=$MODULE/$PR_NUMBER\nelse\n CONFIG_PATH=$MODULE/$BRANCH\nfi\n\nUSER_DIR=\"amlb_user_dir\"\nif [ $MODULE == 'multimodal' ]; then\n USER_DIR=\"custom_user_dir\"\n aws s3 cp --recursive s3://autogluon-ci-benchmark/configs/custom-dataloaders/ $(dirname \"$0\")/$MODULE/$USER_DIR/dataloaders/\n aws s3 cp --recursive s3://autogluon-ci-benchmark/configs/custom-metrics/ $(dirname \"$0\")/$MODULE/custom_metrics/\nfi\n\n# keep commit sha for future reference\naws s3 cp --recursive $(dirname \"$0\")/$MODULE/$USER_DIR/ s3://autogluon-ci-benchmark/configs/$CONFIG_PATH/$SHORT_SHA/\naws s3 rm --recursive s3://autogluon-ci-benchmark/configs/$CONFIG_PATH/latest/\naws s3 cp --recursive $(dirname \"$0\")/$MODULE/$USER_DIR/ s3://autogluon-ci-benchmark/configs/$CONFIG_PATH/latest/\n"
},
{
"path": "CI/bench/generate_bench_config.sh",
"content": "#!/usr/bin/env bash\n\nMODULE=$1\nPRESET=$2\nBENCHMARK=$3\nTIME_LIMIT=$4\nUSER_DIR_S3_PREFIX=$5 # where to find the pre-generated config. This will either be a branch name or a PR number\n\nCDK_DEPLOY_ACCOUNT=369469875935\nCDK_DEPLOY_REGION=us-east-1\nMETRICS_BUCKET=autogluon-ci-benchmark\nMAX_MACHINE_NUM=1040\n\n# Function to convert time format to seconds\nconvert_time_to_seconds() {\n local time_str=$1\n if [[ $time_str =~ ^([0-9]+)h$ ]]; then\n echo $(( ${BASH_REMATCH[1]} * 3600 ))\n elif [[ $time_str =~ ^([0-9]+)m$ ]]; then\n echo $(( ${BASH_REMATCH[1]} * 60 ))\n elif [[ $time_str =~ ^([0-9]+)s$ ]]; then\n echo ${BASH_REMATCH[1]}\n elif [[ $time_str =~ ^[0-9]+$ ]]; then\n echo $time_str # Already in seconds\n else\n echo \"3600\" # Default to 1 hour if format is unrecognized\n fi\n}\n\n# Convert TIME_LIMIT to seconds for AWS infrastructure timeout\nTIME_LIMIT_SECONDS=$(convert_time_to_seconds \"$TIME_LIMIT\")\n\nif [ $MODULE == \"tabular\" ] || [ $MODULE == \"timeseries\" ]; then\n FRAMEWORK=AutoGluon_$PRESET:benchmark\n INSTANCE_TYPE=m5.2xlarge\n aws s3 cp --recursive s3://autogluon-ci-benchmark/configs/$MODULE/$USER_DIR_S3_PREFIX/latest/ $(dirname \"$0\")/amlb_user_dir/\n agbench generate-cloud-config \\\n --prefix ag-bench-${INSTANCE_TYPE//./} \\\n --module $MODULE \\\n --cdk-deploy-account $CDK_DEPLOY_ACCOUNT \\\n --cdk-deploy-region $CDK_DEPLOY_REGION \\\n --metrics-bucket $METRICS_BUCKET \\\n --max-machine-num $MAX_MACHINE_NUM \\\n --instance $INSTANCE_TYPE \\\n --framework $FRAMEWORK \\\n --amlb-benchmark $BENCHMARK \\\n --amlb-constraint $TIME_LIMIT \\\n --time-limit $TIME_LIMIT_SECONDS \\\n --amlb-user-dir $(dirname \"$0\")/amlb_user_dir \\\n --git-uri-branch https://github.com/Innixma/automlbenchmark.git#autogluon_switch_to_uv\nelse\n FRAMEWORK=AutoGluon_$PRESET\n aws s3 cp --recursive s3://autogluon-ci-benchmark/configs/$MODULE/$USER_DIR_S3_PREFIX/latest/ $(dirname \"$0\")/custom_user_dir/\n dataloader_file=\"\"\n class_name=\"\"\n dataset_file=\"\"\n custom_dataloader_value=\"\"\n custom_metrics_path=\"\"\n custom_function_name=\"\"\n optimum=0\n if [ $BENCHMARK == \"automm-image\" ]; then\n dataloader_file=\"vision_dataloader.py\"\n class_name=\"VisionDataLoader\"\n dataset_file=\"automm_cv_datasets.yaml\"\n custom_dataloader_value=\"dataloader_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataloader_file;class_name:$class_name;dataset_config_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataset_file\"\n elif [ $BENCHMARK == \"automm-text-tabular\" ]; then\n dataloader_file=\"text_tabular_dataloader.py\"\n class_name=\"TextTabularDataLoader\"\n dataset_file=\"text_tabular_datasets.yaml\"\n custom_dataloader_value=\"dataloader_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataloader_file;class_name:$class_name;dataset_config_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataset_file\"\n elif [ $BENCHMARK == \"automm-text\" ]; then\n dataloader_file=\"text_dataloader.py\"\n class_name=\"TextDataLoader\"\n dataset_file=\"text_datasets.yaml\"\n custom_dataloader_value=\"dataloader_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataloader_file;class_name:$class_name;dataset_config_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataset_file\"\n elif [ $BENCHMARK == \"automm-text-tabular-image\" ]; then\n dataloader_file=\"text_tabular_image_dataloader.py\"\n class_name=\"TextTabularImageDataLoader\"\n dataset_file=\"text_tabular_image_datasets.yaml\"\n custom_dataloader_value=\"dataloader_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataloader_file;class_name:$class_name;dataset_config_file:$(dirname \"$0\")/custom_user_dir/dataloaders/$dataset_file\"\n custom_metrics_path=\"$(dirname \"$0\")/custom_metrics/cpp_coverage.py\"\n custom_function_name=\"coverage\"\n optimum=1\n else\n echo \"Error: Unsupported benchmark '$BENCHMARK'\"\n exit 1\n fi\n\n DATASET_YAML_PATH=\"$(dirname \"$0\")/custom_user_dir/dataloaders/$dataset_file\"\n dataset_names=\"\"\n # Use yq to extract the dataset names and concatenate them with commas\n for name in $(yq -r '. | keys[]' \"$DATASET_YAML_PATH\"); do\n if [ \"$name\" != \"base\" ]; then\n if [ -n \"$dataset_names\" ]; then\n dataset_names=\"$dataset_names,$name\"\n else\n dataset_names=\"$name\"\n fi\n fi\n done\n\n gen_bench_command=\"agbench generate-cloud-config \\\n --prefix ag-bench \\\n --module $MODULE \\\n --cdk-deploy-account $CDK_DEPLOY_ACCOUNT \\\n --cdk-deploy-region $CDK_DEPLOY_REGION \\\n --metrics-bucket $METRICS_BUCKET \\\n --max-machine-num $MAX_MACHINE_NUM \\\n --data-bucket automl-mm-bench \\\n --framework $FRAMEWORK \\\n --constraint $TIME_LIMIT \\\n --time-limit $TIME_LIMIT_SECONDS \\\n --custom-resource-dir $(dirname \"$0\")/custom_user_dir \\\n --dataset-names \"$dataset_names\" \\\n --custom-dataloader '$custom_dataloader_value'\"\n\n if [ $BENCHMARK == \"automm-text\" ]; then\n gen_bench_command=\"$gen_bench_command\"\n elif [ $BENCHMARK == \"automm-text-tabular-image\" ]; then\n gen_bench_command=\"$gen_bench_command --custom-metrics --metrics-path $custom_metrics_path --function-name $custom_function_name --optimum $optimum --greater-is-better\"\n fi\n eval \"$gen_bench_command\"\nfi\n"
},
{
"path": "CI/bench/generate_framework.py",
"content": "import argparse\nimport os\nimport yaml\n\nparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\nparser.add_argument(\"--module\", help=\"module to run ag-bench on\", type=str, required=True)\nparser.add_argument(\"--repository\", help=\"git repository to run autogluon on\", type=str, required=True)\nparser.add_argument(\"--branch\", help=\"git branch to run autogluon on\", type=str, required=True)\nparser.add_argument(\"--folds_to_run\", help=\"number of folds to run, has to be greater than 0\", type=int, required=False)\n\nargs = parser.parse_args()\n\nmodule = args.module\nrepository = args.repository\nbranch = args.branch\nfolds_to_run = args.folds_to_run\ncurrent_dir = os.path.dirname(__file__)\n\nif module == 'multimodal':\n framework_template_file = os.path.join(current_dir, f\"{module}/custom_user_dir\", \"multimodal_frameworks_template.yaml\")\n framework_benchmark_file = os.path.join(os.path.dirname(framework_template_file), \"multimodal_frameworks.yaml\")\n constraints_file = os.path.join(current_dir, f\"{module}/custom_user_dir\", \"multimodal_constraints.yaml\")\nelse:\n framework_template_file = os.path.join(current_dir, f\"{module}/amlb_user_dir\", \"frameworks_template.yaml\")\n framework_benchmark_file = os.path.join(os.path.dirname(framework_template_file), \"frameworks_benchmark.yaml\")\n constraints_file = os.path.join(current_dir, f\"{module}/amlb_user_dir\", \"constraints.yaml\")\n\nif folds_to_run > 0 and module != 'multimodal':\n constraints = {}\n with open(constraints_file, \"r\") as f:\n constraints = yaml.safe_load(f)\n\n for constraint in constraints.values():\n constraint[\"folds\"] = folds_to_run\n\n with open(constraints_file, \"w\") as f:\n yaml.safe_dump(constraints, f)\n\nframeworks = {}\nwith open(framework_template_file, \"r\") as f:\n frameworks = yaml.safe_load(f)\n\nfor framework in frameworks.values():\n framework[\"repo\"] = repository\n framework[\"version\"] = branch\n\nwith open(framework_benchmark_file, \"w\") as f:\n yaml.safe_dump(frameworks, f)\n"
},
{
"path": "CI/bench/multimodal/custom_user_dir/multimodal_constraints.yaml",
"content": "test:\n TIME_LIMIT: 1800\n INSTANCE: g4dn.2xlarge\n\n10m4x:\n TIME_LIMIT: 600\n INSTANCE: g4dn.4xlarge\n\ng4_12x:\n INSTANCE: g4dn.12xlarge\n MAX_MACHINE_NUM: 200 \n BLOCK_DEVICE_VOLUME: 300 \n RESERVED_MEMORY_SIZE: 8000\n TIME_LIMIT: 3600\n"
},
{
"path": "CI/bench/multimodal/custom_user_dir/multimodal_frameworks_template.yaml",
"content": "AutoGluon_multimodal_best:\n repo: https://github.com/autogluon/autogluon.git\n version: master\n params: # MultimodalPredictor.fit(params) # can add the actual job time limit here\n presets: best_quality\n hyperparameters:\n optim.max_epochs: 10 # 10 is default\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/benchmarks/tabular_full.yaml",
"content": "---\n# Combines openml/s/269 and openml/s/271\n# Locally stored instead of fetched from openml to avoid server errors\n\n- name: APSFailure\n openml_task_id: 168868\n\n- name: Airlines_DepDelay_10M\n openml_task_id: 359929\n\n- name: Allstate_Claims_Severity\n openml_task_id: 233212\n\n- name: Amazon_employee_access\n openml_task_id: 359979\n\n- name: Australian\n openml_task_id: 146818\n\n- name: Bioresponse\n openml_task_id: 359967\n\n- name: Brazilian_houses\n openml_task_id: 359938\n\n- name: Buzzinsocialmedia_Twitter\n openml_task_id: 233213\n\n- name: Click_prediction_small\n openml_task_id: 359992\n\n- name: Diabetes130US\n openml_task_id: 211986\n\n- name: Fashion-MNIST\n openml_task_id: 359976\n\n- name: GesturePhaseSegmentationProcessed\n openml_task_id: 359970\n\n- name: Higgs\n openml_task_id: 360114\n\n- name: Internet-Advertisements\n openml_task_id: 359966\n\n- name: KDDCup09-Upselling\n openml_task_id: 360975\n\n- name: KDDCup09_appetency\n openml_task_id: 3945\n\n- name: KDDCup99\n openml_task_id: 360112\n\n- name: MIP-2016-regression\n openml_task_id: 360945\n\n- name: Mercedes_Benz_Greener_Manufacturing\n openml_task_id: 233215\n\n- name: MiniBooNE\n openml_task_id: 359990\n\n- name: Moneyball\n openml_task_id: 167210\n\n- name: OnlineNewsPopularity\n openml_task_id: 359941\n\n- name: PhishingWebsites\n openml_task_id: 359971\n\n- name: QSAR-TID-10980\n openml_task_id: 360933\n\n- name: QSAR-TID-11\n openml_task_id: 360932\n\n- name: SAT11-HAND-runtime-regression\n openml_task_id: 359948\n\n- name: Santander_transaction_value\n openml_task_id: 233214\n\n- name: Satellite\n openml_task_id: 359975\n\n- name: Yolanda\n openml_task_id: 317614\n\n- name: abalone\n openml_task_id: 359944\n\n- name: ada\n openml_task_id: 190411\n\n- name: adult\n openml_task_id: 359983\n\n- name: airlines\n openml_task_id: 189354\n\n- name: albert\n openml_task_id: 189356\n\n- name: amazon-commerce-reviews\n openml_task_id: 10090\n\n- name: arcene\n openml_task_id: 190412\n\n- name: bank-marketing\n openml_task_id: 359982\n\n- name: black_friday\n openml_task_id: 359937\n\n- name: blood-transfusion-service-center\n openml_task_id: 359955\n\n- name: boston\n openml_task_id: 359950\n\n- name: car\n openml_task_id: 359960\n\n- name: christine\n openml_task_id: 359973\n\n- name: churn\n openml_task_id: 359968\n\n- name: cmc\n openml_task_id: 359959\n\n- name: cnae-9\n openml_task_id: 359957\n\n- name: colleges\n openml_task_id: 359942\n\n- name: connect-4\n openml_task_id: 359977\n\n- name: covertype\n openml_task_id: 7593\n\n- name: credit-g\n openml_task_id: 168757\n\n- name: diamonds\n openml_task_id: 233211\n\n- name: dilbert\n openml_task_id: 168909\n\n- name: dionis\n openml_task_id: 189355\n\n- name: dna\n openml_task_id: 359964\n\n- name: elevators\n openml_task_id: 359936\n\n- name: eucalyptus\n openml_task_id: 359954\n\n- name: fabert\n openml_task_id: 168910\n\n- name: first-order-theorem-proving\n openml_task_id: 359969\n\n- name: gina\n openml_task_id: 189922\n\n- name: guillermo\n openml_task_id: 359988\n\n- name: helena\n openml_task_id: 359984\n\n- name: house_16H\n openml_task_id: 359952\n\n- name: house_prices_nominal\n openml_task_id: 359951\n\n- name: house_sales\n openml_task_id: 359949\n\n- name: jannis\n openml_task_id: 211979\n\n- name: jasmine\n openml_task_id: 168911\n\n- name: jungle_chess_2pcs_raw_endgame_complete\n openml_task_id: 359981\n\n- name: kc1\n openml_task_id: 359962\n\n- name: kick\n openml_task_id: 359991\n\n- name: kr-vs-kp\n openml_task_id: 359965\n\n- name: madeline\n openml_task_id: 190392\n\n- name: mfeat-factors\n openml_task_id: 359961\n\n- name: micro-mass\n openml_task_id: 359953\n\n- name: nomao\n openml_task_id: 359980\n\n- name: numerai28.6\n openml_task_id: 167120\n\n- name: nyc-taxi-green-dec-2016\n openml_task_id: 359943\n\n- name: okcupid-stem\n openml_task_id: 359993\n\n- name: ozone-level-8hr\n openml_task_id: 190137\n\n- name: pc4\n openml_task_id: 359958\n\n- name: philippine\n openml_task_id: 190410\n\n- name: phoneme\n openml_task_id: 168350\n\n- name: pol\n openml_task_id: 359946\n\n- name: porto-seguro\n openml_task_id: 360113\n\n- name: qsar-biodeg\n openml_task_id: 359956\n\n- name: quake\n openml_task_id: 359930\n\n- name: riccardo\n openml_task_id: 359989\n\n- name: robert\n openml_task_id: 359986\n\n- name: segment\n openml_task_id: 359963\n\n- name: sensory\n openml_task_id: 359931\n\n- name: sf-police-incidents\n openml_task_id: 359994\n\n- name: shuttle\n openml_task_id: 359987\n\n- name: socmob\n openml_task_id: 359932\n\n- name: space_ga\n openml_task_id: 359933\n\n- name: steel-plates-fault\n openml_task_id: 168784\n\n- name: sylvine\n openml_task_id: 359972\n\n- name: tecator\n openml_task_id: 359934\n\n- name: topo_2_1\n openml_task_id: 359939\n\n- name: us_crime\n openml_task_id: 359945\n\n- name: vehicle\n openml_task_id: 190146\n\n- name: volkert\n openml_task_id: 359985\n\n- name: wilt\n openml_task_id: 146820\n\n- name: wine-quality-white\n openml_task_id: 359974\n\n- name: wine_quality\n openml_task_id: 359935\n\n- name: yeast\n openml_task_id: 2073\n\n- name: yprop_4_1\n openml_task_id: 359940\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/benchmarks/tabular_small.yaml",
"content": "---\n# Locally stored instead of fetched from openml to avoid server errors\n# Small datasets to test benchmarking - extracted from autogluon-bench/data/metadata/task_metadata.csv\n\n# binary\n- name: Australian\n openml_task_id: 146818\n\n# multiclass\n- name: vehicle\n openml_task_id: 190146\n\n# regression\n- name: sensory\n openml_task_id: 359931"
},
{
"path": "CI/bench/tabular/amlb_user_dir/benchmarks/tabular_test.yaml",
"content": "---\n# Combines openml/s/269 and openml/s/271\n# Locally stored instead of fetched from openml to avoid server errors\n\n- name: adult\n openml_task_id: 359983\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/config.yaml",
"content": "frameworks: # configuration namespace for the frameworks definitions.\n definition_file: # list of yaml files describing the frameworks base definitions.\n - '{user}/frameworks.yaml'\n - '{root}/resources/frameworks.yaml'\n allow_duplicates: true # if true, the last definition is used.\n tags: ['stable', 'latest', 'benchmark'] # the list of supported tags when looking up frameworks:\n # for example frmwk:latest will look for framework frmwk in a frameworks_latest.yaml file if present.\n\nbenchmarks: # configuration namespace for the benchmarks definitions.\n definition_dir: # list of directories containing the benchmarks yaml definitions.\n - '{user}/benchmarks'\n - '{root}/resources/benchmarks'\n constraints_file: # list of yaml files describing the benchmarks runtime constraints.\n - '{user}/constraints.yaml'\n - '{root}/resources/constraints.yaml' \n overhead_time_seconds: 72000 # amount of additional time allowed for the job to complete before sending an interruption signal\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/constraints.yaml",
"content": "---\n\n1h:\n folds: 10\n max_runtime_seconds: 3600\n cores: 8\n min_vol_size_mb: 100000\n\n4h:\n folds: 10\n max_runtime_seconds: 14400\n cores: 8\n min_vol_size_mb: 100000\n\n8h:\n folds: 10\n max_runtime_seconds: 28800\n cores: 8\n min_vol_size_mb: 100000\n\n16h:\n folds: 10\n max_runtime_seconds: 57600\n cores: 8\n min_vol_size_mb: 100000\n\n24h:\n folds: 10\n max_runtime_seconds: 86400\n cores: 8\n min_vol_size_mb: 100000\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/frameworks_template.yaml",
"content": "---\n\n#########################\n### AutoML frameworks ###\n#########################\n\n######### Do Not Remove #########\nAutoGluon:\n version: \"latest\"\n setup_cmd: python3 {user}/setup_hf_cache.py\n######### Do Not Remove #########\n\n\nAutoGluon_tabular_best:\n extends: AutoGluon\n repo: https://github.com/autogluon/autogluon.git\n version: master # branch name\n params: # TabularPredictor.fit(params)\n presets: best_quality\n\nAutoGluon_tabular_high:\n extends: AutoGluon\n repo: https://github.com/autogluon/autogluon.git\n version: master # branch name\n params: # TabularPredictor.fit(params)\n presets: high_quality\n\nAutoGluon_tabular_good:\n extends: AutoGluon\n repo: https://github.com/autogluon/autogluon.git\n version: master # branch name\n params: # TabularPredictor.fit(params)\n presets: good_quality\n\nAutoGluon_tabular_medium:\n extends: AutoGluon\n repo: https://github.com/autogluon/autogluon.git\n version: master # branch name\n params: # TabularPredictor.fit(params)\n presets: medium_quality\n"
},
{
"path": "CI/bench/tabular/amlb_user_dir/setup_hf_cache.py",
"content": "#!/usr/bin/env python3\n\"\"\"\nScript to cache HuggingFace models from S3 before AutoGluon training starts.\nThis prevents rate limiting issues when multiple parallel benchmark runs try to download TabPFNv2 models.\n\"\"\"\n\nimport subprocess\nfrom pathlib import Path\n\n\ndef setup_hf_cache():\n \"\"\"Download tabular foundational models from S3 to HuggingFace cache directory.\"\"\"\n\n # HuggingFace cache directory\n cache_dir = Path.home() / \".cache\" / \"huggingface\" / \"hub\"\n cache_dir.mkdir(parents=True, exist_ok=True)\n\n # S3 bucket with model mirror\n bucket = \"s3://autogluon-hf-model-mirror\"\n\n # tabular foundational models to cache\n models = [\n \"Prior-Labs/TabPFN-v2-clf\",\n \"Prior-Labs/TabPFN-v2-reg\"\n ]\n\n print(\"Setting up HuggingFace model cache for tabular foundational models...\")\n\n for model in models:\n # Convert model name to S3 path format\n model_name = \"--\".join(model.split('/'))\n s3_model_name = f\"models--{model_name}\"\n model_path = cache_dir / s3_model_name\n s3_path = f\"{bucket}/{s3_model_name}\"\n\n print(f\"Caching {model} from {s3_path}...\")\n\n try:\n # Create local directory\n model_path.mkdir(parents=True, exist_ok=True)\n\n # Download from S3 to local cache\n cmd = [\"aws\", \"s3\", \"cp\", s3_path, str(model_path), \"--recursive\", \"--quiet\"]\n result = subprocess.run(cmd, capture_output=True, text=True)\n\n if result.returncode == 0:\n print(f\"Successfully cached {model}\")\n else:\n print(f\"Warning: Failed to cache {model} from S3: {result.stderr}\")\n print(\"Tabular foundational models will be downloaded from HuggingFace Hub during training\")\n\n except Exception as e:\n print(f\"Warning: Exception while caching {model}: {e}\")\n print(\"Tabular foundational models will be downloaded from HuggingFace Hub during training\")\n\n print(\"HuggingFace model cache setup completed.\")\n\n\nif __name__ == \"__main__\":\n setup_hf_cache()\n"
},
{
"path": "CI/bench/timeseries/amlb_user_dir/benchmarks/timeseries_small.yaml",
"content": "# Last number after underscore is the unique identifier for the benchmark\n- name: m4_hourly_2\n dataset:\n path: https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_hourly/test.csv\n type: timeseries\n freq: H\n forecast_horizon_in_steps: 48\n seasonality: 24\n target: target\n id_column: item_id\n timestamp_column: timestamp\n metric: [mase, smape, mape, rmse, mql, wql, sql]\n quantile_levels: [0.05, 0.5, 0.95]\n folds: 2\n"
},
{
"path": "CI/bench/timeseries/amlb_user_dir/config.yaml",
"content": "frameworks: # configuration namespace for the frameworks definitions.\n definition_file: # list of yaml files describing the frameworks base definitions.\n - '{user}/frameworks.yaml'\n - '{root}/resources/frameworks.yaml'\n allow_duplicates: true # if true, the last definition is used.\n tags: ['stable', 'latest', '2020Q2', '2021Q3', '2023Q2', 'example', 'benchmark'] # the list of supported tags when looking up frameworks:\n # for example frmwk:latest will look for framework frmwk in a frameworks_latest.yaml file if present.\nbenchmarks: # configuration namespace for the benchmarks definitions.\n definition_dir: \n - '{user}/benchmarks'\n - '{root}/resources/benchmarks'\n constraints_file: # list of yaml files describing the benchmarks runtime constraints.\n - '{user}/constraints.yaml'\n - '{root}/resources/constraints.yaml'\n"
},
{
"path": "CI/bench/timeseries/amlb_user_dir/constraints.yaml",
"content": "---\n\n10m4c:\n folds: 3\n max_runtime_seconds: 600\n cores: 4\n min_vol_size_mb: 100000\n"
},
{
"path": "CI/bench/timeseries/amlb_user_dir/frameworks_template.yaml",
"content": "---\n\n#########################\n### AutoML frameworks ###\n#########################\n\n######### Do Not Remove #########\nAutoGluon:\n version: \"latest\"\n######### Do Not Remove #########\n\n\nAutoGluon_timeseries_best:\n extends: AutoGluon\n repo: https://github.com/autogluon/autogluon.git\n version: stable_GA4_update # branch name\n params: # TabularPredictor.fit(params)\n presets: best_quality\n"
},
{
"path": "CI/docker/Dockerfile.cpu-inference",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:2.5.1-cpu-py311-ubuntu22.04-sagemaker\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN pip3 install -U pip\nRUN pip3 install -U setuptools wheel\nRUN pip3 install -U numpy==1.24.4 # to match training container numpy version\n\nRUN git clone https://github.com/autogluon/autogluon.git\nCOPY full_install_image.sh autogluon/\nRUN cd autogluon && chmod +x full_install_image.sh && ./full_install_image.sh\n"
},
{
"path": "CI/docker/Dockerfile.cpu-training",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.5.1-cpu-py311-ubuntu22.04-sagemaker\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n # Install rsync to support ray distributed training\n && apt-get install rsync -y \\\n # Install poppler-utils for pdf2image and tesseract-ocr for pytesseract\n && apt-get install -y poppler-utils tesseract-ocr libtesseract-dev \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN pip3 install -U pip\nRUN pip3 install -U setuptools wheel\n\nRUN git clone https://github.com/autogluon/autogluon.git\nCOPY full_install_image.sh autogluon/\nRUN cd autogluon && chmod +x full_install_image.sh && ./full_install_image.sh\n"
},
{
"path": "CI/docker/Dockerfile.gpu-inference",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:2.5.1-gpu-py311-cu124-ubuntu22.04-sagemaker\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN pip3 install -U pip\nRUN pip3 install -U setuptools wheel\nRUN pip3 install -U numpy==1.24.4 # to match training container numpy version\n\nRUN git clone https://github.com/autogluon/autogluon.git\nCOPY full_install_image.sh autogluon/\nRUN cd autogluon && chmod +x full_install_image.sh && ./full_install_image.sh\n"
},
{
"path": "CI/docker/Dockerfile.gpu-training",
"content": "FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.5.1-gpu-py311-cu124-ubuntu22.04-sagemaker\n\nRUN apt-get update \\\n && apt-get -y upgrade \\\n && apt-get install -y --no-install-recommends \\\n # Install rsync to support ray distributed training\n && apt-get install rsync -y \\\n # Install poppler-utils for pdf2image and tesseract-ocr for pytesseract\n && apt-get install -y poppler-utils tesseract-ocr libtesseract-dev \\\n && apt-get autoremove -y \\\n && apt-get clean\n\nRUN pip3 install -U pip\nRUN pip3 install -U setuptools wheel\n\nRUN git clone https://github.com/autogluon/autogluon.git\nCOPY full_install_image.sh autogluon/\nRUN cd autogluon && chmod +x full_install_image.sh && ./full_install_image.sh\n"
},
{
"path": "CI/docker/full_install_image.sh",
"content": "#!/usr/bin/env bash\nset -euo pipefail\n\npython3 -m pip install common/[tests]\npython3 -m pip install features/\npython3 -m pip install core/[all,tests]\npython3 -m pip install tabular/[all,tests]\npython3 -m pip install multimodal/[tests]\npython3 -m pip install timeseries/[all,tests]\npython3 -m pip install autogluon/\n\nmim install \"mmcv==2.1.0\" --timeout 60\npython3 -m pip install --upgrade \"mmdet==3.2.0\"\n"
},
{
"path": "CI/docker/login_ecr.sh",
"content": "#!/bin/bash\n\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 369469875935.dkr.ecr.us-east-1.amazonaws.com\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com\n"
},
{
"path": "CI/hf_mirror/Dockerfile",
"content": "FROM python:3.8\n\nRUN python3 -m pip --no-cache-dir install --upgrade wheel setuptools \\\n && python3 -m pip --no-cache-dir install --upgrade huggingface-hub awscli\n\nCOPY download_hf_models.py ./\nCOPY download_hf_models.sh ./\nRUN chmod +x download_hf_models.sh\n\nCMD [\"/bin/bash\"]\n"
},
{
"path": "CI/hf_mirror/deploy.sh",
"content": "#!/bin/bash\nECR_REPO=369469875935.dkr.ecr.us-east-1.amazonaws.com\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ECR_REPO\ndocker build -t autogluon-hf-mirror .\ndocker tag autogluon-hf-mirror:latest $ECR_REPO/autogluon-hf-mirror:latest\ndocker push $ECR_REPO/autogluon-hf-mirror:latest"
},
{
"path": "CI/hf_mirror/download_hf_models.py",
"content": "import argparse\nimport os\nimport yaml\n\nfrom huggingface_hub import snapshot_download\nfrom huggingface_hub.utils import RepositoryNotFoundError\n\n\nparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n\nparser.add_argument('--model_list_file', help='file containing list of models to download.', type=str, required=True)\nparser.add_argument('--dataset_list_file', help='file containing list of datasets to download.', type=str, default=None)\n\nargs = parser.parse_args()\n\nmodel_list_file = args.model_list_file\ndataset_list_file = args.dataset_list_file\n\nwith open(model_list_file, 'r') as fp:\n model_list = yaml.safe_load(fp) # a dict containing models for different sub_folders\n model_list = list(model_list.values())\n model_list = set(sum(model_list, [])) # concatenate inner model lists\n\nfor model in model_list:\n print(f\"Downloading {model}\")\n try:\n snapshot_download(repo_id=model, cache_dir=\"/mnt/efs/\")\n print(f\"Finished downloading {model}\")\n except RepositoryNotFoundError:\n print(f\"Invalid repo_id: {model}. Failed to download\")\n\n# Download HuggingFace datasets if dataset list is provided\nif dataset_list_file and os.path.exists(dataset_list_file):\n from datasets import load_dataset\n\n with open(dataset_list_file, 'r') as fp:\n dataset_list = yaml.safe_load(fp)\n dataset_list = list(dataset_list.values())\n dataset_list = set(sum(dataset_list, [])) # concatenate inner dataset lists\n\n datasets_cache_dir = \"/mnt/efs/datasets\"\n os.makedirs(datasets_cache_dir, exist_ok=True)\n\n for dataset_spec in dataset_list:\n print(f\"Downloading dataset: {dataset_spec}\")\n try:\n # Parse \"dataset_name:config\" format\n if \":\" in dataset_spec:\n dataset_name, config = dataset_spec.split(\":\", 1)\n load_dataset(dataset_name, config, cache_dir=datasets_cache_dir)\n else:\n load_dataset(dataset_spec, cache_dir=datasets_cache_dir)\n print(f\"Finished downloading dataset: {dataset_spec}\")\n except Exception as e:\n print(f\"Failed to download dataset {dataset_spec}: {e}\")\n"
},
{
"path": "CI/hf_mirror/download_hf_models.sh",
"content": "#!/bin/bash\n\nif grep -qs '/mnt/efs ' /proc/mounts;\nthen\n echo EFS attached\nelse\n echo EFS failed to attach\n exit 1\nfi\ngit clone https://github.com/autogluon/autogluon.git\npython3 download_hf_models.py \\\n --model_list_file autogluon/multimodal/tests/hf_model_list.yaml \\\n --dataset_list_file autogluon/multimodal/tests/hf_dataset_list.yaml\naws s3 sync --delete /mnt/efs s3://autogluon-hf-model-mirror\n"
},
{
"path": "CITING.md",
"content": "The AutoGluon developers and community are committed to open source, and that extends to our research.\nBelow you can find a list of our published work relating to AutoGluon along with guidelines for citing our work.\n\n## Scientific Publications\n- [AutoGluon-Tabular: Robust and Accurate AutoML for Structured Data](https://arxiv.org/pdf/2003.06505.pdf) (*Arxiv*, 2020) ([BibTeX](#general-usage--autogluontabular))\n- [Fast, Accurate, and Simple Models for Tabular Data via Augmented Distillation](https://proceedings.neurips.cc/paper/2020/hash/62d75fb2e3075506e8837d8f55021ab1-Abstract.html) (*NeurIPS*, 2020) ([BibTeX](#tabular-distillation))\n- [Benchmarking Multimodal AutoML for Tabular Data with Text Fields](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/9bf31c7ff062936a96d3c8bd1f8f2ff3-Paper-round2.pdf) (*NeurIPS*, 2021) ([BibTeX](#autogluonmultimodal))\n- [XTab: Cross-table Pretraining for Tabular Transformers](https://proceedings.mlr.press/v202/zhu23k/zhu23k.pdf) (*ICML*, 2023)\n- [AutoGluon-TimeSeries: AutoML for Probabilistic Time Series Forecasting](https://arxiv.org/abs/2308.05566) (*AutoML Conf*, 2023) ([BibTeX](#autogluontimeseries))\n- [Multi-layer Stack Ensembles for Time Series Forecasting](https://arxiv.org/abs/2511.15350) (*AutoML Conf*, 2025) ([BibTeX](#autogluontimeseries))\n- [Chronos-2: From Univariate to Universal Forecasting](https://arxiv.org/abs/2510.15821) (*Arxiv*, 2025) ([BibTeX](#autogluontimeseries))\n- [TabRepo: A Large Scale Repository of Tabular Model Evaluations and its AutoML Applications](https://arxiv.org/pdf/2311.02971.pdf) (*AutoML Conf*, 2024)\n- [AutoGluon-Multimodal (AutoMM): Supercharging Multimodal AutoML with Foundation Models](https://arxiv.org/pdf/2404.16233) (*AutoML Conf*, 2024) ([BibTeX](#autogluonmultimodal))\n\n## Citing AutoGluon\n\nPlease cite the core AutoGluon paper (AutoGluon-Tabular) as well as any module specific paper that is relevant.\n\n### General Usage & autogluon.tabular\n\nIf you use AutoGluon in a scientific publication, please cite the following paper (regardless of which module is being used):\n- Erickson, Nick, et al. [\"AutoGluon-Tabular: Robust and Accurate AutoML for Structured Data.\"](https://arxiv.org/abs/2003.06505) arXiv preprint arXiv:2003.06505 (2020).\n\nBibTeX entry:\n\n```bibtex\n@article{agtabular,\n title={AutoGluon-Tabular: Robust and Accurate AutoML for Structured Data},\n author={Erickson, Nick and Mueller, Jonas and Shirkov, Alexander and Zhang, Hang and Larroy, Pedro and Li, Mu and Smola, Alexander},\n journal={arXiv preprint arXiv:2003.06505},\n year={2020}\n}\n```\n\n### autogluon.timeseries\n\nIf you use AutoGluon's time series forecasting functionality in a scientific publication, please cite the following paper:\n```bibtex\n@inproceedings{agtimeseries,\n title={{AutoGluon-TimeSeries}: {AutoML} for Probabilistic Time Series Forecasting},\n author={Shchur, Oleksandr and Turkmen, Caner and Erickson, Nick and Shen, Huibin and Shirkov, Alexander and Hu, Tony and Wang, Yuyang},\n booktitle={International Conference on Automated Machine Learning},\n year={2023}\n}\n```\n\nIf you use the Chronos pretrained model, please cite:\n```bibtex\n@article{ansari2024chronos,\n title={Chronos: Learning the Language of Time Series},\n author={Ansari, Abdul Fatir and Stella, Lorenzo and Turkmen, Caner and Zhang, Xiyuan and Mercado, Pedro and Shen, Huibin and Shchur, Oleksandr and Rangapuram, Syama Sundar and Pineda Arango, Sebastian and Kapoor, Shubham and Zschiegner, Jasper and Maddix, Danielle C. and Mahoney, Michael W. and Torkkola, Kari and Gordon Wilson, Andrew and Bohlke-Schneider, Michael and Wang, Yuyang},\n journal={Transactions on Machine Learning Research},\n issn={2835-8856},\n year={2024},\n url={https://openreview.net/forum?id=gerNCVqqtR}\n}\n```\n\nIf you use the Chronos-2 pretrained model, please cite:\n```bibtex\n@article{ansari2025chronos2,\n title={Chronos-2: From Univariate to Universal Forecasting},\n author={Abdul Fatir Ansari and Oleksandr Shchur and Jaris K\\\"uken and Andreas Auer and Boran Han and Pedro Mercado and Syama Sundar Rangapuram and Huibin Shen and Lorenzo Stella and Xiyuan Zhang and Mononito Goswami and Shubham Kapoor and Danielle C. Maddix and Pablo Guerron and Tony Hu and Junming Yin and Nick Erickson and Prateek Mutalik Desai and Hao Wang and Huzefa Rangwala and George Karypis and Yuyang Wang and Michael Bohlke-Schneider},\n journal={arXiv preprint arXiv:2510.15821},\n year={2025},\n url={https://arxiv.org/abs/2510.15821}\n}\n```\n\nIf you use multi-layer stack ensembles for time series forecasting, please cite:\n```bibtex\n@inproceedings{bosch2025multi,\n title={Multi-layer Stack Ensembles for Time Series Forecasting},\n author={Bosch, Nathanael and Shchur, Oleksandr and Erickson, Nick and Bohlke-Schneider, Michael and Turkmen, Ali Caner},\n booktitle={AutoML 2025 Methods Track},\n year={2025}\n}\n```\n\n\n### autogluon.multimodal\n\nIf you use AutoGluon's multimodal functionality in a scientific publication, please cite the following paper:\n\nZhiqiang, Tang, et al. [\"AutoGluon-Multimodal (AutoMM): Supercharging Multimodal AutoML with Foundation Models\"](https://arxiv.org/pdf/2404.16233), The International Conference on Automated Machine Learning (AutoML), 2024.\n\nBibTeX entry:\n\n```bibtex\n@article{tang2024autogluon,\n title={AutoGluon-Multimodal (AutoMM): Supercharging Multimodal AutoML with Foundation Models},\n author={Tang, Zhiqiang and Fang, Haoyang and Zhou, Su and Yang, Taojiannan and Zhong, Zihan and Hu, Tony and Kirchhoff, Katrin and Karypis, George},\n journal={arXiv preprint arXiv:2404.16233},\n year={2024}\n}\n```\n\nIf you use AutoGluon's TextPredictor, please cite the following paper:\n\nShi, Xingjian, et al. [\"Benchmarking Multimodal AutoML for Tabular Data with Text Fields.\"](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/9bf31c7ff062936a96d3c8bd1f8f2ff3-Paper-round2.pdf) Advances in Neural Information Processing Systems 35, Datasets and Benchmarks Track (2021).\n\n```bibtex\n@inproceedings{agmultimodaltext,\n title={Benchmarking Multimodal AutoML for Tabular Data with Text Fields},\n author={Shi, Xingjian and Mueller, Jonas and Erickson, Nick and Li, Mu and Smola, Alexander J},\n journal={Advances in Neural Information Processing Systems Datasets and Benchmarks Track},\n volume={35},\n year={2021}\n}\n```\n\n### Tabular Distillation\n\nIf you are using AutoGluon Tabular's model distillation functionality, please cite the following paper:\n\nFakoor, Rasool, et al. [\"Fast, Accurate, and Simple Models for Tabular Data via Augmented Distillation.\"](https://proceedings.neurips.cc/paper/2020/hash/62d75fb2e3075506e8837d8f55021ab1-Abstract.html) Advances in Neural Information Processing Systems 33 (2020).\n\nBibTeX entry:\n\n```bibtex\n@article{agtabulardistill,\n title={Fast, Accurate, and Simple Models for Tabular Data via Augmented Distillation},\n author={Fakoor, Rasool and Mueller, Jonas W and Erickson, Nick and Chaudhari, Pratik and Smola, Alexander J},\n journal={Advances in Neural Information Processing Systems},\n volume={33},\n year={2020}\n}\n```\n"
},
{
"path": "CODE_OF_CONDUCT.md",
"content": "## Code of Conduct\nThis project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).\nFor more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact\nopensource-codeofconduct@amazon.com with any additional questions or comments.\n"
},
{
"path": "CONTRIBUTING.md",
"content": "# Contributing Guidelines\n\nThank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional\ndocumentation, we greatly value feedback and contributions from our community.\n\nPlease read through this document before submitting any issues or pull requests to ensure we have all the necessary\ninformation to effectively respond to your bug report or contribution.\n\n\n## Reporting Bugs/Feature Requests\n\nWe welcome you to use the GitHub issue tracker to report bugs or suggest features.\n\nWhen filing an issue, please check [existing open](https://github.com/autogluon/autogluon/issues), or [recently closed](https://github.com/autogluon/autogluon/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already\nreported the issue. Please try to include as much information as you can. Details like these are incredibly useful:\n\n* A reproducible test case or series of steps\n* The version of AutoGluon being used, the version of pytorch\n* Any modifications you've made relevant to the bug\n* Anything unusual about your environment or deployment\n\nIdeally, you can install AutoGluon and its dependencies in a fresh virtualenv to reproduce the bug.\n\n## Contributing via Pull Requests\nCode contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:\n\n1. You are working against the latest source on the *master* branch.\n2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.\n3. You open an issue to discuss any significant work before implementing it. Major new features, submodules, and model contributions need to fit into our overall design philosophy and our users' interests. We also need to evaluate if the contribution is worth the maintenance overhead.\n\nTo send us a pull request, please:\n\n1. [Fork the repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) and [clone the source code to your local machine](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository).\n2. Modify the source (see details below); please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.\n3. Commit to your fork using clear commit messages.\n4. [Create a pull request](https://github.com/autogluon/autogluon/pulls) ([GitHub Docs](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)), answering any default questions in the pull request interface.\n5. Pay attention to any automated continuous integration (CI) failures reported in the pull request, and stay involved in the conversation.\n\nGitHub provides additional documentation on [forking a repository](https://help.github.com/articles/fork-a-repo/) and\n[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).\n\n\n## Tips for Modifying the Source Code\n\n- Using a fresh virtualenv, install the package via [these instructions](https://auto.gluon.ai/dev/install.html).\nBe sure to select the *Source* option from the installation preferences.\n\n- We recommend developing on Linux as this is the primary OS we develop on and is the primary OS used by our users. We also support Windows and MacOS. Try to avoid introducing changes that will only work on a particular OS. Changes to existing code that improve cross-platform compatibility are most welcome!\n\n- Use Python 3.10, 3.11, 3.12 or 3.13 for development, as these are the only versions where AutoGluon is fully functional.\n\n- Please try to avoid introducing additional dependencies / 3rd party packages (except for model contributions). We are currently working to reduce the number of external dependencies of our package. For now, we recommend [lazy-import](https://github.com/autogluon/autogluon/blob/master/common/src/autogluon/common/utils/try_import.py) of external packages if you are adding functionality that you believe will only be used by a small fraction of users.\n\n- All code should adhere to the [PEP8 style](https://www.python.org/dev/peps/pep-0008/).\n\n- (Optional) After you have edited the code, ensure your changes pass the unit tests via the below commands. Note that in practice we don't do this and instead submit the pull request so that our continuous integration on GitHub automatically runs the tests. This is because our unit tests require multiple hours of compute to complete, and thus it isn't practical to run all the tests on a local machine.\n ```\n # optional, not recommended to run all tests on local machine\n pytest common/tests\n pytest features/tests\n pytest core/tests\n pytest tabular/tests\n pytest multimodal/tests\n pytest timeseries/tests\n ```\n\n- Set up pre-commit hooks for automatic code formatting:\n ```bash\n pre-commit install\n ```\n\n The pre-commit hooks will automatically run formatting checks before each commit. You can also run them manually:\n ```bash\n # Check all files\n pre-commit run\n\n # Or run formatting manually if the previous check fails\n ruff format multimodal/ timeseries/ common/ core/ features/ tabular/\n ruff check --fix --select I common/ core/ features/ multimodal/ tabular/ timeseries/\n ```\n\n- After linting, make sure to commit the linting changes, so it appears in your pull request.\n\n- Use NumPy-style docstrings for public APIs. AutoGluon documentation is built with `numpydoc`, and NumPy docstrings are the expected style for new or updated docstrings:\n - Format reference: https://numpydoc.readthedocs.io/en/latest/format.html\n - VSCode: install `autoDocstring` and set `autoDocstring.docstringFormat` to `numpy`\n - PyCharm: configure docstring generation style to NumPy in the IDE settings\n\n- We encourage you to add your own unit tests, but please ensure they run quickly (unit tests should train models on small data-subsample with the lowest values of training iterations and time-limits that suffice to evaluate the intended functionality). You can run a specific unit test within a specific file like this:\n ```\n python3 -m pytest path_to_file::test_mytest\n ```\n Or remove the ::test_mytest suffix to run all tests in the file:\n ```\n python3 -m pytest path_to_file\n ```\n\n- If using PyCharm, we highly recommend `pip install pytest-pycharm` to [improve ease of debugging inside unit tests](https://stackoverflow.com/questions/14086067/debugging-pytest-post-mortem-exceptions-in-pycharm-pydev).\n\n- To otherwise test your code changes, we recommend running AutoGluon on multiple datasets and verifying the code runs correctly and the resulting accuracy of the trained models is not harmed by your change. One easy way to test is to simply modify the scripts in [`examples/`](https://github.com/autogluon/autogluon/tree/master/examples), or the [tutorial notebooks](https://github.com/autogluon/autogluon/tree/master/docs/tutorials), which already provide datasets.\n\n- Remember to update all existing examples/tutorials/documentation affected by your code changes.\n\n- We also encourage you to contribute new tutorials using AutoGluon for applications you think other users will be interested in. Please see [`docs/tutorials/`](https://github.com/autogluon/autogluon/tree/master/docs/tutorials). All tutorials should be Jupyter notebooks (.ipynb) files.\n\n- After you open your pull request, our CI system will run to check your code and report found errors. This may take a few hours. Please check back and fix any errors encountered at this stage (you can retrigger a new CI check by pushing updated code to the same PR in a new commit).\n\n## Finding Contributions to Work On\nLooking at the existing issues is a great way to find something to contribute on. As our project uses the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/autogluon/autogluon/labels/help%20wanted) issues is a great place to start.\n\n## Model Contributions (Tabular)\n\nIf you are interested in contributing a new model to AutoGluon Tabular, refer to our [custom model tutorial](https://auto.gluon.ai/stable/tutorials/tabular/advanced/tabular-custom-model.html) which provides a solid foundation to base your contribution.\nPlease be aware that it is very possible for a model to never be merged and for the PR to be closed for any number of reasons.\nNew model contributions have a **very** high bar for acceptance, and will often take months before being merged, if it ever becomes merged.\nThe value add for the model has to be substantial, as supporting a new model type is a large ongoing maintenance burden.\nIn order to evaluate the value a model provides, our developer team will run extensive benchmarking tests. These are currently manual, time-consuming, and require nuanced interpretation of the results.\n\nWe are actively working on ways to automate the evaluation of new model contributions, and hope to have this new logic ready by the end of 2025.\n\n## Code of Conduct\nThis project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).\nFor more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact\nopensource-codeofconduct@amazon.com with any additional questions or comments.\n\n\n## Security Issue Notifications\nIf you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue for a security vulnerability.\n\n\n## Instructions for Project Maintainers\n\nThe following instructions are for project maintainers with write access to the AutoGluon repository (those with permissions to merge PRs)\n\n### Platform Tests\n\nTo spin up the platform tests, which test AutoGluon on Linux, MacOS, and Windows separately for each supported Python version, do the following:\n\n1. Ensure that the PR branch is originating from the `autogluon/autogluon` repository. It cannot originate from your own personal repository or else the platform tests will not trigger. This is for security reasons.\n2. Comment on the PR with `/platform_tests` ([Example](https://github.com/autogluon/autogluon/pull/4714#issuecomment-2522270778)) (requires write permissions in AutoGluon repo). It is recommended to run the platform tests only after you have passed the default CI and only for changes that are likely to cause platform specific issues.\n\n### Benchmarking\n\nTo spin up automated benchmarking, do the following:\n\n1. Comment on the PR with `/benchmark module=tabular preset=tabular_best benchmark=tabular_full time_limit=1h` to benchmark the tabular module ([Example](https://github.com/autogluon/autogluon/pull/4714#issuecomment-2524696887)).\n2. Comment on the PR with `/benchmark module=multimodal preset=multimodal_best benchmark=automm-image time_limit=g4_12x` to benchmark the multimodal module ([Example](https://github.com/autogluon/autogluon/pull/4714#issuecomment-2524199294))\n\nAutomated benchmarking should only be performed for changes that are likely to impact performance, as it is computationally intensive.\n\n## Licensing\n\nThis project uses the Apache 2.0 license. See the [LICENSE](https://github.com/autogluon/autogluon/blob/master/LICENSE) file for details. We will ask you to confirm the licensing of your contribution.\n\nWe may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.\n"
},
{
"path": "LICENSE",
"content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n"
},
{
"path": "NOTICE",
"content": "AutoML for Text, Image, and Tabular Data\nCopyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. \n"
},
{
"path": "README.md",
"content": "\n\n\n![](\"https://user-images.githubusercontent.com/16392542/77208906-224aa500-6aba-11ea-96bd-e81806074030.png\")
\n\n## Fast and Accurate ML in 3 Lines of Code\n\n[](https://github.com/autogluon/autogluon/releases)\n[](https://anaconda.org/conda-forge/autogluon)\n[](https://pypi.org/project/autogluon/)\n[](https://pepy.tech/project/autogluon)\n[](./LICENSE)\n[](https://discord.gg/wjUmjqAc2N)\n[](https://twitter.com/autogluon)\n[](https://github.com/autogluon/autogluon/actions/workflows/continuous_integration.yml)\n[](https://github.com/autogluon/autogluon/actions/workflows/platform_tests-command.yml)\n\n[Installation](https://auto.gluon.ai/stable/install.html) | [Documentation](https://auto.gluon.ai/stable/index.html) | [Release Notes](https://auto.gluon.ai/stable/whats_new/index.html)\n\n
\n\nAutoGluon, developed by AWS AI, automates machine learning tasks enabling you to easily achieve strong predictive performance in your applications. With just a few lines of code, you can train and deploy high-accuracy machine learning and deep learning models on image, text, time series, and tabular data.\n\n\n## đž Installation\n\nAutoGluon is supported on Python 3.10 - 3.13 and is available on Linux, MacOS, and Windows.\n\nYou can install AutoGluon with:\n\n```python\npip install autogluon\n```\n\nVisit our [Installation Guide](https://auto.gluon.ai/stable/install.html) for detailed instructions, including GPU support, Conda installs, and optional dependencies.\n\n## :zap: Quickstart\n\nBuild accurate end-to-end ML models in just 3 lines of code!\n\n```python\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(label=\"class\").fit(\"train.csv\", presets=\"best\")\npredictions = predictor.predict(\"test.csv\")\n```\n\n| AutoGluon Task | Quickstart | API |\n|:--------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------:|\n| TabularPredictor | [](https://auto.gluon.ai/stable/tutorials/tabular/tabular-quick-start.html) | [](https://auto.gluon.ai/stable/api/autogluon.tabular.TabularPredictor.html) |\n| TimeSeriesPredictor | [](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-quick-start.html) | [](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.html) |\n| MultiModalPredictor | [](https://auto.gluon.ai/stable/tutorials/multimodal/multimodal_prediction/multimodal-quick-start.html) | [](https://auto.gluon.ai/stable/api/autogluon.multimodal.MultiModalPredictor.html) |\n\n## :mag: Resources\n\n### Hands-on Tutorials / Talks\n\nBelow is a curated list of recent tutorials and talks on AutoGluon. A comprehensive list is available [here](AWESOME.md#videos--tutorials).\n\n| Title | Format | Location | Date |\n|--------------------------------------------------------------------------------------------------------------------------|----------|----------------------------------------------------------------------------------|------------|\n| :tv: [Structured Foundation Models Meets AutoML](https://icml.cc/virtual/2025/46786) | Expo Talk | [ICML 2025](https://icml.cc/Conferences/2025) | 2025/07/13 |\n| :tv: [AutoGluon 1.2: Advancing AutoML with Foundational Models and LLM Agents](https://neurips.cc/virtual/2024/expo-workshop/100328) | Expo Workshop | [NeurIPS 2024](https://neurips.cc/Conferences/2024) | 2024/12/10 |\n| :tv: [AutoGluon: Towards No-Code Automated Machine Learning](https://www.youtube.com/watch?v=SwPq9qjaN2Q) | Tutorial | [AutoML 2024](https://2024.automl.cc/) | 2024/09/09 |\n| :tv: [AutoGluon 1.0: Shattering the AutoML Ceiling with Zero Lines of Code](https://www.youtube.com/watch?v=5tvp_Ihgnuk) | Tutorial | [AutoML 2023](https://2023.automl.cc/) | 2023/09/12 |\n| :sound: [AutoGluon: The Story](https://automlpodcast.com/episode/autogluon-the-story) | Podcast | [The AutoML Podcast](https://automlpodcast.com/) | 2023/09/05 |\n| :tv: [AutoGluon: AutoML for Tabular, Multimodal, and Time Series Data](https://youtu.be/Lwu15m5mmbs?si=jSaFJDqkTU27C0fa) | Tutorial | PyData Berlin | 2023/06/20 |\n| :tv: [Solving Complex ML Problems in a few Lines of Code with AutoGluon](https://www.youtube.com/watch?v=J1UQUCPB88I) | Tutorial | PyData Seattle | 2023/06/20 |\n| :tv: [The AutoML Revolution](https://www.youtube.com/watch?v=VAAITEds-28) | Tutorial | [Fall AutoML School 2022](https://sites.google.com/view/automl-fall-school-2022) | 2022/10/18 |\n\n### Scientific Publications\n- [AutoGluon-Tabular: Robust and Accurate AutoML for Structured Data](https://arxiv.org/pdf/2003.06505.pdf) (*Arxiv*, 2020) ([BibTeX](CITING.md#general-usage--autogluontabular))\n- [Fast, Accurate, and Simple Models for Tabular Data via Augmented Distillation](https://proceedings.neurips.cc/paper/2020/hash/62d75fb2e3075506e8837d8f55021ab1-Abstract.html) (*NeurIPS*, 2020) ([BibTeX](CITING.md#tabular-distillation))\n- [Benchmarking Multimodal AutoML for Tabular Data with Text Fields](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/9bf31c7ff062936a96d3c8bd1f8f2ff3-Paper-round2.pdf) (*NeurIPS*, 2021) ([BibTeX](CITING.md#autogluonmultimodal))\n- [XTab: Cross-table Pretraining for Tabular Transformers](https://proceedings.mlr.press/v202/zhu23k/zhu23k.pdf) (*ICML*, 2023)\n- [AutoGluon-TimeSeries: AutoML for Probabilistic Time Series Forecasting](https://arxiv.org/abs/2308.05566) (*AutoML Conf*, 2023) ([BibTeX](CITING.md#autogluontimeseries))\n- [TabRepo: A Large Scale Repository of Tabular Model Evaluations and its AutoML Applications](https://arxiv.org/pdf/2311.02971.pdf) (*AutoML Conf*, 2024)\n- [AutoGluon-Multimodal (AutoMM): Supercharging Multimodal AutoML with Foundation Models](https://arxiv.org/pdf/2404.16233) (*AutoML Conf*, 2024) ([BibTeX](CITING.md#autogluonmultimodal))\n- [Chronos: Learning the Language of Time Series](https://arxiv.org/abs/2403.07815) (*TMLR*, 2024)\n- [Multi-layer Stack Ensembles for Time Series Forecasting](https://arxiv.org/abs/2511.15350) (*AutoML Conf*, 2025) ([BibTeX](CITING.md#autogluontimeseries))\n- [Chronos-2: From Univariate to Universal Forecasting](https://arxiv.org/abs/2510.15821) (*Arxiv*, 2025) ([BibTeX](CITING.md#autogluontimeseries))\n- [TabArena: A Living Benchmark for Machine Learning on Tabular Data](https://arxiv.org/abs/2506.16791) (*NeurIPS Spotlight*, 2025)\n- [Mitra: Mixed Synthetic Priors for Enhancing Tabular Foundation Models](https://arxiv.org/abs/2510.21204) (*NeurIPS*, 2025)\n- [MLZero: A Multi-Agent System for End-to-end Machine Learning Automation](https://arxiv.org/abs/2505.13941) (*NeurIPS*, 2025)\n- [fev-bench: A Realistic Benchmark for Time Series Forecasting](https://arxiv.org/abs/2509.26468) (*Arxiv*, 2025)\n\n### Articles\n- [AutoGluon-TimeSeries: Every Time Series Forecasting Model In One Library](https://towardsdatascience.com/autogluon-timeseries-every-time-series-forecasting-model-in-one-library-29a3bf6879db) (*Towards Data Science*, Jan 2024)\n- [AutoGluon for tabular data: 3 lines of code to achieve top 1% in Kaggle competitions](https://aws.amazon.com/blogs/opensource/machine-learning-with-autogluon-an-open-source-automl-library/) (*AWS Open Source Blog*, Mar 2020)\n- [AutoGluon overview & example applications](https://towardsdatascience.com/autogluon-deep-learning-automl-5cdb4e2388ec?source=friends_link&sk=e3d17d06880ac714e47f07f39178fdf2) (*Towards Data Science*, Dec 2019)\n\n### Train/Deploy AutoGluon in the Cloud\n- [AutoGluon Cloud](https://auto.gluon.ai/cloud/stable/index.html) (Recommended)\n- [AutoGluon on Amazon SageMaker](https://auto.gluon.ai/stable/tutorials/cloud_fit_deploy/cloud-aws-sagemaker-train-deploy.html)\n- [AutoGluon Deep Learning Containers](https://github.com/aws/deep-learning-containers/blob/master/available_images.md#autogluon-training-containers) (Security certified & maintained by the AutoGluon developers)\n- [AutoGluon Official Docker Container](https://hub.docker.com/r/autogluon/autogluon)\n\n#### Outdated / Unsupported Cloud Options\n- [AutoGluon on SageMaker AutoPilot](https://auto.gluon.ai/stable/tutorials/cloud_fit_deploy/autopilot-autogluon.html) (Uses an old AutoGluon 0.4 release)\n- [AutoGluon-Tabular on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-n4zf5pmjt7ism) (Outdated and not maintained by us)\n\n## :pencil: Citing AutoGluon\n\nIf you use AutoGluon in a scientific publication, please refer to our [citation guide](CITING.md).\n\n## :wave: How to get involved\n\nWe are actively accepting code contributions to the AutoGluon project. If you are interested in contributing to AutoGluon, please read the [Contributing Guide](https://github.com/autogluon/autogluon/blob/master/CONTRIBUTING.md) to get started.\n\n## :classical_building: License\n\nThis library is licensed under the Apache 2.0 License.\n"
},
{
"path": "SECURITY.md",
"content": "# Security Policy\n\n## Supported Versions\n\n| Version | Supported |\n|---------| ------------------ |\n| 1.4.0 | :white_check_mark: |\n| < 1.4.0 | :x: |\n\n## How we do security\n\n\nAs much as possible, AutoGluon relies on automated tools to do security scanning. In particular, we support:\n\n1. Dependency Analysis: Using Dependabot\n2. Docker Scanning: Using Snyk\n3. Code Analysis: Using CodeGuru\n\n\n## Reporting a Vulnerability\n\nReport any security vulnerabilities to `autogluon-security@amazon.com`. This email directly links to the Autogluon security maintenance team. Once the security vulnerability is confirmed, we will work privately on a patch, aiming to produce a dedicated bugfix release as swiftly as complexity allows.\n"
},
{
"path": "VERSION",
"content": "1.5.1\n"
},
{
"path": "autogluon/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport os\nfrom setuptools import setup\nimport importlib.util\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, '..', 'core', 'src', 'autogluon', 'core', '_setup_utils.py')\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec)\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag)\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = None # None since this module is special (it isn't a real submodule)\ninstall_requires = [\n f'autogluon.core[all]=={version}',\n f'autogluon.features=={version}',\n f'autogluon.tabular[all]=={version}',\n f'autogluon.multimodal=={version}',\n f'autogluon.timeseries[all]=={version}',\n] if not ag.LITE_MODE else [\n f'{ag.PACKAGE_NAME}.core=={version}',\n f'{ag.PACKAGE_NAME}.features=={version}',\n f'{ag.PACKAGE_NAME}.tabular=={version}',\n]\n\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\n\nextras_require = {\n \"tabarena\": [f'autogluon.tabular[tabarena]=={version}']\n}\nif __name__ == '__main__':\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "autogluon/src/autogluon/_internal_/__init__.py",
"content": "# Placeholder to resolve empty package"
},
{
"path": "common/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport importlib.util\nimport os\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec) # type: ignore\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag) # type: ignore\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version, use_file_if_exists=False, create_file=True)\n\nsubmodule = \"common\"\ninstall_requires = (\n [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"pyarrow\", # version range defined in `core/_setup_utils.py`\n \"boto3\", # version range defined in `core/_setup_utils.py`\n \"psutil\", # version range defined in `core/_setup_utils.py`\n \"tqdm\", # version range defined in `core/_setup_utils.py`\n \"requests\",\n \"joblib\", # version range defined in `core/_setup_utils.py`\n \"pyyaml\", # version range defined in `core/_setup_utils.py`\n # s3fs is removed due to doubling install time due to version range resolution\n # \"s3fs\", # version range defined in `core/_setup_utils.py`\n ]\n if not ag.LITE_MODE\n else {\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"tqdm\", # version range defined in `core/_setup_utils.py`\n \"requests\",\n }\n)\n\nextras_require = dict()\n\ntest_requirements = [\n \"pytest\",\n \"types-requests\",\n \"types-setuptools\",\n \"pytest-mypy\",\n]\n\ntest_requirements = list(set(test_requirements))\nextras_require[\"tests\"] = test_requirements\n\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\nfor key in extras_require:\n extras_require[key] = ag.get_dependency_version_ranges(extras_require[key])\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "common/src/autogluon/common/__init__.py",
"content": "from .dataset import TabularDataset\nfrom .features.feature_metadata import FeatureMetadata\nfrom .utils.log_utils import _add_stream_handler\nfrom .utils.log_utils import fix_logging_if_kaggle as __fix_logging_if_kaggle\nfrom .version import __version__\n\n# Fixes logger in Kaggle to show logs in notebook.\n__fix_logging_if_kaggle()\n\n_add_stream_handler()\n"
},
{
"path": "common/src/autogluon/common/dataset.py",
"content": "from __future__ import annotations\n\nfrom pathlib import Path\n\nimport pandas as pd\n\nfrom .loaders import load_pd\nfrom .savers import save_pd\n\n__all__ = [\"TabularDataset\"]\n\n\nclass TabularDataset:\n \"\"\"\n A dataset in tabular format (with rows = samples, columns = features/variables).\n This class returns a :class:`pd.DataFrame` when initialized and all existing pandas methods can be applied to it.\n For full list of methods/attributes, see pandas Dataframe documentation: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html\n\n The purpose of this class is to provide an easy-to-use shorthand for loading a pandas DataFrame to use in AutoGluon.\n\n Parameters\n ----------\n data : str, :class:`pd.DataFrame`, :class:`np.ndarray`, Iterable, or dict\n If str, path to data file (CSV or Parquet format).\n If you already have your data in a :class:`pd.DataFrame`, you can specify it here. In this case, the same DataFrame will be returned with no changes.\n\n Examples\n --------\n >>> import pandas as pd\n >>> from autogluon.common import TabularDataset\n >>> train_data = TabularDataset(\"https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv\")\n >>> train_data_pd = pd.read_csv(\"https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv\")\n >>> assert isinstance(train_data, pd.DataFrame) # True\n >>> assert train_data.equals(train_data_pd) # True\n >>> assert type(train_data) == type(train_data_pd) # True\n \"\"\"\n\n def __new__(cls, data, **kwargs) -> pd.DataFrame:\n if isinstance(data, (str, Path)):\n data = cls.load(path=data)\n return pd.DataFrame(data, **kwargs)\n\n @classmethod\n def load(cls, path: str | Path, **kwargs) -> pd.DataFrame:\n return load_pd.load(path, **kwargs)\n\n @classmethod\n def save(cls, path: str | Path, df: pd.DataFrame, **kwargs):\n save_pd.save(path=path, df=df, **kwargs)\n"
},
{
"path": "common/src/autogluon/common/features/__init__.py",
"content": ""
},
{
"path": "common/src/autogluon/common/features/feature_metadata.py",
"content": "import copy\nimport logging\nfrom collections import defaultdict\nfrom typing import Any, Dict, List, Set, Tuple, Union\n\nimport pandas as pd\n\nfrom .infer_types import get_type_group_map_special, get_type_map_raw\n\nlogger = logging.getLogger(__name__)\n\n\nclass FeatureMetadata:\n \"\"\"\n Feature metadata contains information about features that are not directly apparent in the raw data itself.\n This enables feature generators to properly process features, and allows downstream models to properly handle features during training and inference.\n\n Parameters\n ----------\n type_map_raw : Dict[str, str]\n Dictionary of feature names to raw types.\n The values can be anything, but it is generally recommended they be one of:\n ['int', 'float', 'object', 'category', 'datetime']\n type_group_map_special : Dict[str, List[str]], optional\n Dictionary of special types to lists of feature names.\n The keys can be anything, but it is generally recommended they be one of:\n ['binned', 'datetime_as_int', 'datetime_as_object', 'text', 'text_as_category', 'text_special', 'text_ngram', 'image_path', 'stack']\n For descriptions of each special feature-type, see: `autogluon.common.features.types`\n Feature names that appear in the value lists must also be keys in type_map_raw.\n Feature names are not required to have special types.\n Only one of type_group_map_special and type_map_special can be specified.\n type_map_special : Dict[str, List[str]], optional\n Dictionary of feature names to lists of special types.\n This is an alternative representation of the special types.\n Only one of type_group_map_special and type_map_special can be specified.\n \"\"\"\n\n def __init__(\n self,\n type_map_raw: Dict[str, str],\n type_group_map_special: Dict[str, List[str]] = None,\n type_map_special: Dict[str, List[str]] = None,\n ):\n if type_group_map_special is None:\n if type_map_special is not None:\n type_group_map_special = self.get_type_group_map_special_from_type_map_special(type_map_special)\n else:\n type_group_map_special = defaultdict(list)\n elif type_map_special is not None:\n raise ValueError(\"Only one of type_group_map_special and type_map_special can be specified in init.\")\n if not isinstance(type_group_map_special, defaultdict):\n type_group_map_special = defaultdict(list, type_group_map_special)\n\n self.type_map_raw = type_map_raw\n self.type_group_map_special = type_group_map_special\n\n self._validate()\n\n def __eq__(self, other) -> bool:\n if set(self.type_map_raw.keys()) != set(other.type_map_raw.keys()):\n return False\n for k in self.type_map_raw.keys():\n if self.type_map_raw[k] != other.type_map_raw[k]:\n return False\n if set(self.type_group_map_special.keys()) != set(other.type_group_map_special.keys()):\n return False\n for k in self.type_group_map_special.keys():\n if set(self.type_group_map_special[k]) != set(other.type_group_map_special[k]):\n return False\n return True\n\n # Confirms if inputs are valid\n def _validate(self):\n type_group_map_special_expanded = []\n for key in self.type_group_map_special:\n type_group_map_special_expanded += self.type_group_map_special[key]\n\n features_invalid = []\n type_map_raw_keys = self.type_map_raw.keys()\n for feature in type_group_map_special_expanded:\n if feature not in type_map_raw_keys:\n features_invalid.append(feature)\n if features_invalid:\n raise AssertionError(\n f\"{len(features_invalid)} features are present in type_group_map_special but not in type_map_raw. Invalid features: {features_invalid}\"\n )\n\n # Note: This is not optimized for speed. Do not rely on this function during inference.\n # TODO: Add valid_names, invalid_names arguments which override all other arguments for the features listed?\n def get_features(\n self,\n valid_raw_types: list = None,\n valid_special_types: list = None,\n invalid_raw_types: list = None,\n invalid_special_types: list = None,\n required_special_types: list = None,\n required_raw_special_pairs: List[Tuple[str, Union[List[str], Set[str]]]] = None,\n required_exact=False,\n required_at_least_one_special=False,\n ) -> List[str]:\n \"\"\"\n Returns a list of features held within the feature metadata object after being pruned through the available parameters.\n\n Parameters\n ----------\n valid_raw_types : list, default None\n If a feature's raw type is not in this list, it is pruned.\n If None, then no features are pruned through this logic.\n valid_special_types : list, default None\n If a feature has a special type not in this list, it is pruned.\n Features without special types are never pruned through this logic.\n If None, then no features are pruned through this logic.\n invalid_raw_types : list, default None\n If a feature's raw type is in this list, it is pruned.\n If None, then no features are pruned through this logic.\n invalid_special_types : list, default None\n If a feature has a special type in this list, it is pruned.\n Features without special types are never pruned through this logic.\n If None, then no features are pruned through this logic.\n required_special_types : list, default None\n If a feature does not have all of the special types in this list, it is pruned.\n Features without special types are pruned through this logic.\n If None, then no features are pruned through this logic.\n required_raw_special_pairs : List[Tuple[str, List[str]]], default None\n If a feature does not satisfy the (raw_type, special_types) requirement of at least one of the elements in this list, it is pruned.\n Identical to getting the union of calling get_features(valid_raw_types=[raw_type], required_special_types=special_types) for every\n element of (raw_type, special_types) in required_raw_special_pairs\n If raw_type is None, then any feature will satisfy the raw type requirement.\n If special_types is None, then any feature will satisfy the special type requirement (including those with no special types).\n required_exact : bool, default False\n If True, then if a feature does not have the exact same special types (with no extra special types) as required_special_types, it is pruned.\n This is also applied to required_raw_special_pairs if specified.\n Has no effect if required_special_types and required_raw_special_pairs are None.\n required_at_least_one_special : bool, default False\n If True, then if a feature has zero special types, it is pruned.\n\n Returns\n -------\n features : list of feature names in feature metadata that satisfy all checks dictated by the parameters.\n\n \"\"\"\n features = list(self.type_map_raw.keys())\n\n if valid_raw_types is not None:\n features = [feature for feature in features if self.get_feature_type_raw(feature) in valid_raw_types]\n if valid_special_types is not None:\n valid_special_types_set = set(valid_special_types)\n features = [\n feature\n for feature in features\n if not valid_special_types_set.isdisjoint(self.get_feature_types_special(feature))\n or not self.get_feature_types_special(feature)\n ]\n if invalid_raw_types is not None:\n features = [feature for feature in features if self.get_feature_type_raw(feature) not in invalid_raw_types]\n if invalid_special_types is not None:\n invalid_special_types_set = set(invalid_special_types)\n features = [\n feature\n for feature in features\n if invalid_special_types_set.isdisjoint(self.get_feature_types_special(feature))\n ]\n if required_special_types is not None:\n required_special_types_set = set(required_special_types)\n if required_exact:\n features = [\n feature\n for feature in features\n if required_special_types_set == set(self.get_feature_types_special(feature))\n ]\n else:\n features = [\n feature\n for feature in features\n if required_special_types_set.issubset(self.get_feature_types_special(feature))\n ]\n if required_at_least_one_special:\n features = [feature for feature in features if self.get_feature_types_special(feature)]\n if required_raw_special_pairs is not None:\n features_og = copy.deepcopy(features)\n features_to_keep = []\n for valid_raw, valid_special in required_raw_special_pairs:\n if valid_special is not None:\n valid_special = set(valid_special)\n features_to_keep_inner = []\n for feature in features:\n feature_type_raw = self.get_feature_type_raw(feature)\n feature_types_special = set(self.get_feature_types_special(feature))\n if valid_raw is None or feature_type_raw == valid_raw:\n if valid_special is None:\n features_to_keep_inner.append(feature)\n elif required_exact:\n if valid_special == feature_types_special:\n features_to_keep_inner.append(feature)\n elif valid_special.issubset(feature_types_special):\n features_to_keep_inner.append(feature)\n features = [feature for feature in features if feature not in features_to_keep_inner]\n features_to_keep += features_to_keep_inner\n features = [feature for feature in features_og if feature in features_to_keep]\n\n return features\n\n def get_feature_type_raw(self, feature: str) -> str:\n return self.type_map_raw[feature]\n\n def get_feature_types_special(self, feature: str) -> list:\n if feature not in self.type_map_raw:\n raise KeyError(f\"{feature} does not exist in {self.__class__.__name__}.\")\n return self._get_feature_types(feature=feature, feature_types_dict=self.type_group_map_special)\n\n def get_type_map_special(self) -> dict:\n return {feature: self.get_feature_types_special(feature) for feature in self.get_features()}\n\n @staticmethod\n def get_type_group_map_special_from_type_map_special(type_map_special: Dict[str, List[str]]):\n type_group_map_special = defaultdict(list)\n for feature in type_map_special:\n for type_special in type_map_special[feature]:\n type_group_map_special[type_special].append(feature)\n return type_group_map_special\n\n def get_type_group_map_raw(self):\n type_group_map_raw = defaultdict(list)\n for feature, dtype in self.type_map_raw.items():\n type_group_map_raw[dtype].append(feature)\n return type_group_map_raw\n\n def remove_features(self, features: list, inplace=False):\n \"\"\"Removes all features from metadata that are in features\"\"\"\n if inplace:\n metadata = self\n else:\n metadata = copy.deepcopy(self)\n features_invalid = [feature for feature in features if feature not in self.get_features()]\n if features_invalid:\n raise KeyError(\n f\"remove_features was called with a feature that does not exist in feature metadata. Invalid Features: {features_invalid}\"\n )\n metadata._remove_features_from_type_map(d=metadata.type_map_raw, features=features)\n metadata._remove_features_from_type_group_map(d=metadata.type_group_map_special, features=features)\n return metadata\n\n def keep_features(self, features: list, inplace=False):\n \"\"\"Removes all features from metadata except for those in features\"\"\"\n features_invalid = [feature for feature in features if feature not in self.get_features()]\n if features_invalid:\n raise KeyError(\n f\"keep_features was called with a feature that does not exist in feature metadata. Invalid Features: {features_invalid}\"\n )\n features_to_remove = [feature for feature in self.get_features() if feature not in features]\n return self.remove_features(features=features_to_remove, inplace=inplace)\n\n def add_special_types(self, type_map_special: Dict[str, List[str]], inplace=False):\n \"\"\"\n Adds special types to features.\n\n Parameters\n ----------\n type_map_special : Dict[str, List[str]]\n Dictionary of feature -> list of special types to add.\n Features in dictionary must already exist in the FeatureMetadata object.\n inplace : bool, default False\n If True, updates self inplace and returns self.\n If False, updates a copy of self and returns copy.\n Returns\n -------\n :class:`FeatureMetadata` object.\n\n Examples\n --------\n >>> from autogluon.common.features.feature_metadata import FeatureMetadata\n >>> feature_metadata = FeatureMetadata({'FeatureA': 'int', 'FeatureB': 'object'})\n >>> feature_metadata = feature_metadata.add_special_types({'FeatureA': ['MySpecialType'], 'FeatureB': ['MySpecialType', 'text']})\n \"\"\"\n if inplace:\n metadata = self\n else:\n metadata = copy.deepcopy(self)\n valid_features = set(self.get_features())\n\n for feature, special_types in type_map_special.items():\n if feature not in valid_features:\n raise ValueError(\n f'\"{feature}\" does not exist in this FeatureMetadata object. Only existing features can be assigned special types.'\n )\n for special_type in special_types:\n metadata.type_group_map_special[special_type].append(feature)\n return metadata\n\n @staticmethod\n def _remove_features_from_type_group_map(d, features):\n for key, features_orig in d.items():\n d[key] = [feature for feature in features_orig if feature not in features]\n\n @staticmethod\n def _remove_features_from_type_map(d, features):\n for feature in features:\n if feature in d:\n d.pop(feature)\n\n def rename_features(self, rename_map: dict, inplace=False):\n \"\"\"Rename all features from metadata that are keys in rename_map to their values.\"\"\"\n if inplace:\n metadata = self\n else:\n metadata = copy.deepcopy(self)\n before_len = len(metadata.type_map_raw.keys())\n metadata.type_map_raw = {rename_map.get(key, key): val for key, val in metadata.type_map_raw.items()}\n after_len = len(metadata.type_map_raw.keys())\n if before_len != after_len:\n raise AssertionError(\n \"key names conflicted during renaming. Do not rename features to exist feature names.\"\n )\n for dtype in metadata.type_group_map_special:\n metadata.type_group_map_special[dtype] = [\n rename_map.get(feature, feature) for feature in metadata.type_group_map_special[dtype]\n ]\n return metadata\n\n # TODO: Add documentation on shared_raw_features usage\n def join_metadata(self, metadata, shared_raw_features=\"error\"):\n \"\"\"Join two FeatureMetadata objects together, returning a new FeatureMetadata object\"\"\"\n if shared_raw_features not in [\"error\", \"error_if_diff\", \"overwrite\"]:\n raise ValueError(\n f\"shared_raw_features must be one of {['error', 'error_if_diff', 'overwrite']}, but was: '{shared_raw_features}'\"\n )\n type_map_raw = copy.deepcopy(self.type_map_raw)\n shared_features = []\n shared_features_diff_types = []\n for key, features in metadata.type_map_raw.items():\n if key in type_map_raw:\n shared_features.append(key)\n if type_map_raw[key] != metadata.type_map_raw[key]:\n shared_features_diff_types.append(key)\n if shared_features:\n if shared_raw_features == \"error\":\n logger.error(\"ERROR: Conflicting metadata:\")\n logger.error(\"Metadata 1:\")\n self.print_feature_metadata_full(log_prefix=\"\\t\", log_level=40)\n logger.error(\"Metadata 2:\")\n metadata.print_feature_metadata_full(log_prefix=\"\\t\", log_level=40)\n raise AssertionError(\n f\"Metadata objects to join share raw features, but `shared_raw_features='error'`. Shared features: {shared_features}\"\n )\n if shared_features_diff_types:\n if shared_raw_features == \"overwrite\":\n logger.log(\n 20,\n f\"Overwriting type_map_raw during FeatureMetadata join. \"\n f\"Shared features with conflicting types: {shared_features_diff_types}\",\n )\n shared_features = []\n elif shared_raw_features == \"error_if_diff\":\n logger.error(\"ERROR: Conflicting metadata:\")\n logger.error(\"Metadata 1:\")\n self.print_feature_metadata_full(log_prefix=\"\\t\", log_level=40)\n logger.error(\"Metadata 2:\")\n metadata.print_feature_metadata_full(log_prefix=\"\\t\", log_level=40)\n raise AssertionError(\n f\"Metadata objects to join share raw features but do not agree on raw dtypes, \"\n f\"and `shared_raw_features='error_if_diff'`. Shared conflicting features: {shared_features_diff_types}\"\n )\n type_map_raw.update({key: val for key, val in metadata.type_map_raw.items() if key not in shared_features})\n\n type_group_map_special = self._add_type_group_map_special(\n [self.type_group_map_special, metadata.type_group_map_special]\n )\n\n return FeatureMetadata(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n\n @staticmethod\n def _add_type_group_map_special(type_group_map_special_lst: List[dict]) -> dict:\n if not type_group_map_special_lst:\n return defaultdict(list)\n type_group_map_special_combined = copy.deepcopy(type_group_map_special_lst[0])\n for type_group_map_special in type_group_map_special_lst[1:]:\n for key, features in type_group_map_special.items():\n if key in type_group_map_special_combined:\n features_to_add = [\n feature for feature in features if feature not in type_group_map_special_combined[key]\n ]\n type_group_map_special_combined[key] += features_to_add\n else:\n type_group_map_special_combined[key] = features\n return type_group_map_special_combined\n\n @staticmethod\n def _get_feature_types(feature: str, feature_types_dict: dict) -> list:\n feature_types = []\n for dtype_family in feature_types_dict:\n if feature in feature_types_dict[dtype_family]:\n feature_types.append(dtype_family)\n feature_types = sorted(feature_types)\n return feature_types\n\n # Joins a list of metadata objects together, returning a new metadata object\n @staticmethod\n def join_metadatas(metadata_list, shared_raw_features=\"error\"):\n metadata_new = copy.deepcopy(metadata_list[0])\n for metadata in metadata_list[1:]:\n metadata_new = metadata_new.join_metadata(metadata, shared_raw_features=shared_raw_features)\n return metadata_new\n\n def to_dict(self, inverse=False) -> dict:\n if not inverse:\n feature_metadata_dict: Dict[Union[str, Tuple[str, tuple]], Any] = dict()\n else:\n feature_metadata_dict = defaultdict(list)\n\n for feature in self.get_features():\n feature_type_raw = self.type_map_raw[feature]\n feature_types_special = tuple(self.get_feature_types_special(feature))\n if not inverse:\n feature_metadata_dict[feature] = (feature_type_raw, feature_types_special)\n else:\n feature_metadata_dict[(feature_type_raw, feature_types_special)].append(feature)\n\n if inverse:\n feature_metadata_dict = dict(feature_metadata_dict)\n\n return feature_metadata_dict\n\n def print_feature_metadata_full(\n self, log_prefix=\"\", print_only_one_special=False, log_level=20, max_list_len=5, return_str=False\n ):\n feature_metadata_dict = self.to_dict(inverse=True)\n if not feature_metadata_dict:\n if return_str:\n return \"\"\n else:\n return\n keys = list(feature_metadata_dict.keys())\n keys = sorted(keys)\n output = [((key[0], list(key[1])), feature_metadata_dict[key]) for key in keys]\n output_str = \"\"\n if print_only_one_special:\n for i, ((raw, special), features) in enumerate(output):\n if len(special) == 1:\n output[i] = ((raw, special[0]), features)\n elif len(special) > 1:\n output[i] = ((raw, special[0]), features)\n logger.warning(\n f\"Warning: print_only_one_special=True was set, but features with {len(special)} special types were found. \"\n f\"Invalid Types: {output[i]}\"\n )\n else:\n output[i] = ((raw, None), features)\n max_key_len = max([len(str(key)) for key, _ in output])\n max_val_len = max([len(str(len(val))) for _, val in output])\n for key, val in output:\n key_len = len(str(key))\n val_len = len(str(len(val)))\n max_key_minus_cur = max(max_key_len - key_len, 0)\n max_val_minus_cur = max(max_val_len - val_len, 0)\n if max_list_len is not None:\n features = str(val[:max_list_len])\n if len(val) > max_list_len:\n features = features[:-1] + \", ...]\"\n else:\n features = str(val)\n if val:\n message = (\n f\"{log_prefix}{key}{' ' * max_key_minus_cur} : {' ' * max_val_minus_cur}{len(val)} | {features}\"\n )\n if return_str:\n output_str += message + \"\\n\"\n else:\n logger.log(log_level, message)\n if return_str:\n if output_str[-1] == \"\\n\":\n output_str = output_str[:-1]\n return output_str\n\n @classmethod\n def from_df(cls, df: pd.DataFrame):\n \"\"\"\n Construct FeatureMetadata based on the inferred feature types of an input :class:`pd.DataFrame`.\n\n Parameters\n ----------\n df : :class:`pd.DataFrame`\n DataFrame used to infer FeatureMetadata.\n\n Returns\n -------\n :class:`FeatureMetadata` object.\n \"\"\"\n type_map_raw = get_type_map_raw(df)\n type_group_map_special = get_type_group_map_special(df)\n return cls(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n\n def verify_data(self, df: pd.DataFrame) -> bool:\n \"\"\"\n Returns True if the DataFrame's raw types match FeatureMetadata, else False\n \"\"\"\n type_map_raw = get_type_map_raw(df)\n features = set(type_map_raw.keys())\n if features != set(self.get_features()):\n return False\n return self._verify_data_type_raw(type_map_raw=type_map_raw)\n\n def verify_data_subset(self, df: pd.DataFrame) -> bool:\n \"\"\"\n Returns True if the DataFrame's features are a subset of FeatureMetadata and its raw types match FeatureMetadata, else False\n \"\"\"\n type_map_raw = get_type_map_raw(df)\n features = set(type_map_raw.keys())\n if not features.issubset(set(self.get_features())):\n return False\n return self._verify_data_type_raw(type_map_raw=type_map_raw)\n\n def verify_data_superset(self, df: pd.DataFrame) -> bool:\n \"\"\"\n Returns True if the DataFrame's features are a superset of FeatureMetadata and its raw types match FeatureMetadata, else False\n \"\"\"\n type_map_raw = get_type_map_raw(df)\n features = set(type_map_raw.keys())\n features_self = set(self.get_features())\n if not features.issuperset(features_self):\n return False\n type_map_raw = {k: v for k, v in type_map_raw.items() if k in features_self}\n return self._verify_data_type_raw(type_map_raw=type_map_raw)\n\n def _verify_data_type_raw(self, type_map_raw: dict) -> bool:\n features = set(type_map_raw.keys())\n for feature in features:\n if type_map_raw[feature] != self.type_map_raw[feature]:\n return False\n return True\n\n def __str__(self):\n return self.print_feature_metadata_full(return_str=True)\n"
},
{
"path": "common/src/autogluon/common/features/infer_types.py",
"content": "import logging\nfrom collections import defaultdict\nfrom typing import List\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_type_family_raw(dtype) -> str:\n \"\"\"From dtype, gets the dtype family.\"\"\"\n try:\n if isinstance(dtype, pd.SparseDtype):\n dtype = dtype.subtype\n if dtype.name == \"category\":\n return \"category\"\n if \"datetime\" in dtype.name:\n return \"datetime\"\n if \"string\" in dtype.name:\n return \"object\"\n elif np.issubdtype(dtype, np.integer):\n return \"int\"\n elif np.issubdtype(dtype, np.floating):\n return \"float\"\n except Exception as err:\n logger.error(\n f\"Warning: dtype {dtype} is not recognized as a valid dtype by numpy! \"\n f\"AutoGluon may incorrectly handle this feature...\"\n )\n logger.error(err)\n\n if dtype.name in [\"bool\", \"bool_\"]:\n return \"bool\"\n elif dtype.name in [\"str\", \"string\", \"object\"]:\n return \"object\"\n else:\n return dtype.name\n\n\n# Real dtypes\ndef get_type_map_real(df: DataFrame) -> dict:\n features_types = df.dtypes.to_dict()\n return {k: v.name for k, v in features_types.items()}\n\n\n# Raw dtypes (Real dtypes family)\ndef get_type_map_raw(df: DataFrame) -> dict:\n features_types = df.dtypes.to_dict()\n return {k: get_type_family_raw(v) for k, v in features_types.items()}\n\n\ndef get_type_map_special(X: DataFrame) -> dict:\n type_map_special = {}\n for column in X:\n types_special = get_types_special(X[column])\n if types_special:\n type_map_special[column] = types_special\n return type_map_special\n\n\ndef get_types_special(X: Series) -> List[str]:\n types_special = []\n if isinstance(X.dtype, pd.SparseDtype):\n types_special.append(\"sparse\")\n if check_if_datetime_as_object_feature(X):\n types_special.append(\"datetime_as_object\")\n elif check_if_nlp_feature(X):\n types_special.append(\"text\")\n return types_special\n\n\ndef get_type_group_map(type_map: dict) -> defaultdict:\n type_group_map = defaultdict(list)\n for key, val in type_map.items():\n if isinstance(val, list):\n for feature_type in val:\n type_group_map[feature_type].append(key)\n else:\n type_group_map[val].append(key)\n return type_group_map\n\n\ndef get_type_group_map_real(df: DataFrame) -> defaultdict:\n type_map_real = get_type_map_real(df)\n return get_type_group_map(type_map_real)\n\n\ndef get_type_group_map_raw(df: DataFrame) -> defaultdict:\n type_map_raw = get_type_map_raw(df)\n return get_type_group_map(type_map_raw)\n\n\ndef get_type_group_map_special(df: DataFrame) -> defaultdict:\n type_map_special = get_type_map_special(df)\n return get_type_group_map(type_map_special)\n\n\n# TODO: Expand to int64 -> date features (milli from epoch etc)\n# TODO: This takes a surprisingly long time to run, ~30 seconds a laptop for\n# 50,000 rows of datetime_as_object for a single column. Try to optimize.\ndef check_if_datetime_as_object_feature(X: Series) -> bool:\n type_family = get_type_family_raw(X.dtype)\n # TODO: Check if low numeric numbers, could be categorical encoding!\n # TODO: If low numeric, potentially it is just numeric instead of date\n if X.isnull().all():\n return False\n if type_family != \"object\": # TODO: seconds from epoch support\n return False\n try:\n # TODO: pd.Series(['20170204','20170205','20170206']) is incorrectly not detected as datetime_as_object\n # But we don't want pd.Series(['184','822828','20170206']) to be detected as datetime_as_object\n # Need some smart logic (check min/max values?, check last 2 values don't go >31?)\n pd.to_numeric(X)\n except (ValueError, TypeError):\n try:\n if len(X) > 500:\n # Sample to speed-up type inference\n X = X.sample(n=500, random_state=0)\n result = pd.to_datetime(X, errors=\"coerce\", format=\"mixed\")\n if result.isnull().mean() > 0.8: # If over 80% of the rows are NaN\n return False\n return True\n except (ValueError, TypeError):\n return False\n else:\n return False\n\n\ndef check_if_nlp_feature(X: Series) -> bool:\n type_family = get_type_family_raw(X.dtype)\n if type_family != \"object\":\n return False\n if len(X) > 5000:\n # Sample to speed-up type inference\n X = X.sample(n=5000, random_state=0)\n X_unique = X.unique()\n num_unique = len(X_unique)\n num_rows = len(X)\n unique_ratio = num_unique / num_rows\n if unique_ratio <= 0.01:\n return False\n try:\n avg_words = Series(X_unique.astype(str)).str.split().str.len().mean()\n except AttributeError:\n return False\n if avg_words < 3:\n return False\n\n return True\n\n\ndef get_bool_true_val(uniques):\n \"\"\"\n From a pandas series with `uniques = series.unique()`, get the replace_val to convert to boolean when calling:\n series_bool = series == replace_val\n\n Therefore, any value other than `replace_val` will be set to `False` when converting to boolean.\n\n series must have exactly 2 unique values\n\n We make the assumption that the value chosen as `True` between the two options is mostly arbitrary,\n with the exception that np.nan will not be considered `True`.\n When possible, we try to sort the values so that (0, 1) will choose 1 as True, however this decision\n should ideally not impact downstream models much.\n Any new unseen values (including nan) at inference time will be mapped to `False` automatically.\n\n In this code, 0 and 0.0 (int and float) are treated as the same value. Similarly with any other integer and float (such as 1 and 1.0).\n\n \"\"\"\n # This is a safety net in case the unique types are mixed (such as string and int). In this scenario, an exception is raised\n # and therefore we use the unsorted values.\n try:\n # Sort the values to avoid relying on row-order when determining which value is mapped to `True`.\n uniques = np.sort(uniques)\n except (ValueError, TypeError):\n pass\n replace_val = uniques[1]\n try:\n # This is to ensure that we don't map np.nan to `True` in the boolean.\n is_nan = np.isnan(replace_val)\n except (ValueError, TypeError):\n if replace_val is None:\n is_nan = True\n else:\n is_nan = False\n if is_nan:\n replace_val = uniques[0]\n return replace_val\n"
},
{
"path": "common/src/autogluon/common/features/types.py",
"content": "# Raw types: Raw data type information grouped into families.\n# For example: uint8, int8, int16, int32, and int64 features all map to 'int'\nR_INT = \"int\"\nR_FLOAT = \"float\"\nR_OBJECT = \"object\"\nR_CATEGORY = \"category\"\nR_DATETIME = \"datetime\"\nR_BOOL = \"bool\" # TODO: R_BOOL/R_BOOLEAN?\n# TODO: R_FLOAT_SPARSE/R_INT_SPARSE/R_CATEGORY_SPARSE?\n\n# Special types: Meta information about the special meaning of a feature that is not present in the raw data.\n# feature has been converted to int8 raw dtype with only 0 and 1 values (2 unique values, missing converted to 0)\n# this differs from R_BOOL as R_BOOL refers to a feature with the bool raw dtype (Values are False and True instead of 0 and 1)\nS_BOOL = \"bool\"\n\n# feature has been binned into discrete integer values from its original representation\nS_BINNED = \"binned\"\n\n# feature was originally a datetime type that was converted to numeric\nS_DATETIME_AS_INT = \"datetime_as_int\"\n\n# feature is a datetime in object form (string dates), which can be converted to datetime via pd.to_datetime\nS_DATETIME_AS_OBJECT = \"datetime_as_object\"\n\n# feature is in sparse form, such as pd.SparseDtype ex: 'Sparse[uint8, 0]'\n# Convert features in this form to a scipy coo matrix with `DataFrame.sparse.to_coo()`\nS_SPARSE = \"sparse\"\n\n# feature is an object type that contains text information that can be utilized in natural language processing\nS_TEXT = \"text\"\n\n# feature is a categorical that was originally text information. It may or may not still contain the raw text in its data.\nS_TEXT_AS_CATEGORY = \"text_as_category\"\n\n# feature is a generated feature based off of a text feature but is not an ngram. Examples include character count, word count, symbol count, etc.\nS_TEXT_SPECIAL = \"text_special\"\n\n# feature is a generated feature based off of a text feature that is an ngram.\nS_TEXT_NGRAM = \"text_ngram\"\n\n# feature is a generated feature based off of a text feature that is a text embedding.\nS_TEXT_EMBEDDING = \"text_embedding\"\n# feature is a collection of S_TEXT_EMBEDDING features after dimensionality reduction has been applied.\nS_TEXT_EMBEDDING_DR = \"text_embedding_dr\"\n\n# feature is an object type that contains a string path to an image that can be utilized in computer vision\nS_IMAGE_PATH = \"image_path\"\n\n# feature is an object type that contains bytearray of an image that can be utilized in computer vision\nS_IMAGE_BYTEARRAY = \"image_bytearray\"\n\n# feature is a generated feature based off of a ML model's prediction probabilities of the label column for the row.\n# Any model which takes a stack feature as input is a stack ensemble.\nS_STACK = \"stack\"\n\n# Feature that comes from dimensionality reduction\nS_DIMENSIONALITY_REDUCTION = \"dimensionality_reduction\"\n"
},
{
"path": "common/src/autogluon/common/loaders/__init__.py",
"content": ""
},
{
"path": "common/src/autogluon/common/loaders/_utils.py",
"content": "import functools\nimport hashlib\nimport logging\nimport os\nimport sys\nimport uuid\nimport warnings\nimport zipfile\nfrom typing import Optional\n\nimport boto3\nimport numpy as np\nimport requests\nimport tqdm\n\nS3_PREFIX = \"s3://\"\n\nINT_TYPES = (int, np.uint8, np.int8, np.int32, np.int64)\nFLOAT_TYPES = (float, np.float16, np.float32, np.float64)\nBOOL_TYPES = (bool, np.bool_)\n\n\nif not sys.platform.startswith(\"win32\"):\n # refer to https://github.com/untitaker/python-atomicwrites\n def replace_file(src, dst):\n \"\"\"Implement atomic os.replace with linux and OSX.\n Parameters\n ----------\n src : source file path\n dst : destination file path\n \"\"\"\n try:\n os.rename(src, dst)\n except OSError:\n try:\n os.remove(src)\n except OSError:\n pass\n finally:\n raise OSError(\n \"Moving downloaded temp file - {}, to {} failed. \\\n Please retry the download.\".format(src, dst)\n )\n\nelse:\n import ctypes\n\n _MOVEFILE_REPLACE_EXISTING = 0x1\n # Setting this value guarantees that a move performed as a copy\n # and delete operation is flushed to disk before the function returns.\n # The flush occurs at the end of the copy operation.\n _MOVEFILE_WRITE_THROUGH = 0x8\n _windows_default_flags = _MOVEFILE_WRITE_THROUGH\n\n def _str_to_unicode(x):\n \"\"\"Handle text decoding. Internal use only\"\"\"\n if not isinstance(x, str):\n return x.decode(sys.getfilesystemencoding())\n return x\n\n def _handle_errors(rv, src):\n \"\"\"Handle WinError. Internal use only\"\"\"\n if not rv:\n msg = ctypes.FormatError(ctypes.GetLastError())\n # if the MoveFileExW fails(e.g. fail to acquire file lock), removes the tempfile\n try:\n os.remove(src)\n except OSError:\n pass\n finally:\n raise OSError(msg)\n\n def replace_file(src, dst):\n \"\"\"Implement atomic os.replace with windows.\n refer to https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-movefileexw\n The function fails when one of the process(copy, flush, delete) fails.\n Parameters\n ----------\n src : source file path\n dst : destination file path\n \"\"\"\n _handle_errors(\n ctypes.windll.kernel32.MoveFileExW(\n _str_to_unicode(src), _str_to_unicode(dst), _windows_default_flags | _MOVEFILE_REPLACE_EXISTING\n ),\n src,\n )\n\n\ndef sha1sum(filename):\n \"\"\"Calculate the sha1sum of a file\n Parameters\n ----------\n filename\n Name of the file\n Returns\n -------\n ret\n The sha1sum\n \"\"\"\n with open(filename, mode=\"rb\") as f:\n d = hashlib.sha1()\n for buf in iter(functools.partial(f.read, 1024 * 100), b\"\"):\n d.update(buf)\n return d.hexdigest()\n\n\ndef download(\n url: str,\n path: Optional[str] = None,\n overwrite: Optional[bool] = False,\n sha1_hash: Optional[str] = None,\n retries: int = 5,\n verify_ssl: Optional[bool] = True,\n) -> str:\n \"\"\"Download a given URL\n\n Parameters\n ----------\n url\n URL to download\n path\n Destination path to store downloaded file. By default stores to the\n current directory with same name as in url.\n overwrite\n Whether to overwrite destination file if already exists.\n sha1_hash\n Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified\n but doesn't match.\n retries\n The number of times to attempt the download in case of failure or non 200 return codes\n verify_ssl\n Verify SSL certificates.\n Returns\n -------\n fname\n The file path of the downloaded file.\n \"\"\"\n is_s3 = url.startswith(S3_PREFIX)\n if is_s3:\n s3 = boto3.resource(\"s3\")\n if boto3.session.Session().get_credentials() is None:\n from botocore.handlers import disable_signing\n\n s3.meta.client.meta.events.register(\"choose-signer.s3.*\", disable_signing)\n components = url[len(S3_PREFIX) :].split(\"/\")\n if len(components) < 2:\n raise ValueError(\"Invalid S3 url. Received url={}\".format(url))\n s3_bucket_name = components[0]\n s3_key = \"/\".join(components[1:])\n if path is None:\n fname = url.split(\"/\")[-1]\n # Empty filenames are invalid\n assert fname, \"Can't construct file-name from this URL. Please set the `path` option manually.\"\n else:\n path = os.path.expanduser(path)\n if os.path.isdir(path):\n fname = os.path.join(path, url.split(\"/\")[-1])\n else:\n fname = path\n assert retries >= 0, \"Number of retries should be at least 0, currently it's {}\".format(retries)\n\n if not verify_ssl:\n warnings.warn(\n \"Unverified HTTPS request is being made (verify_ssl=False). \"\n \"Adding certificate verification is strongly advised.\"\n )\n\n if overwrite or not os.path.exists(fname) or (sha1_hash and not sha1sum(fname) == sha1_hash):\n dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname)))\n if not os.path.exists(dirname):\n os.makedirs(dirname, exist_ok=True)\n while retries + 1 > 0:\n # Disable pyling too broad Exception\n # pylint: disable=W0703\n try:\n logging.info(\"Downloading {} from {}...\".format(fname, url))\n if is_s3:\n response = s3.meta.client.head_object(Bucket=s3_bucket_name, Key=s3_key)\n total_size = int(response.get(\"ContentLength\", 0))\n random_uuid = str(uuid.uuid4())\n tmp_path = \"{}.{}\".format(fname, random_uuid)\n if tqdm is not None:\n\n def hook(t_obj):\n def inner(bytes_amount):\n t_obj.update(bytes_amount)\n\n return inner\n\n with tqdm.tqdm(total=total_size, unit=\"iB\", unit_scale=True) as t:\n s3.meta.client.download_file(s3_bucket_name, s3_key, tmp_path, Callback=hook(t))\n else:\n s3.meta.client.download_file(s3_bucket_name, s3_key, tmp_path)\n else:\n r = requests.get(url, stream=True, verify=verify_ssl)\n if r.status_code != 200:\n raise RuntimeError(\"Failed downloading url {}\".format(url))\n # create uuid for temporary files\n random_uuid = str(uuid.uuid4())\n total_size = int(r.headers.get(\"content-length\", 0))\n chunk_size = 1024\n if tqdm is not None:\n t = tqdm.tqdm(total=total_size, unit=\"iB\", unit_scale=True)\n with open(\"{}.{}\".format(fname, random_uuid), \"wb\") as f:\n for chunk in r.iter_content(chunk_size=chunk_size):\n if chunk: # filter out keep-alive new chunks\n if tqdm is not None:\n t.update(len(chunk))\n f.write(chunk)\n if tqdm is not None:\n t.close()\n # if the target file exists(created by other processes)\n # and have the same hash with target file\n # delete the temporary file\n if not os.path.exists(fname) or (sha1_hash and not sha1sum(fname) == sha1_hash):\n # atomic operation in the same file system\n replace_file(\"{}.{}\".format(fname, random_uuid), fname)\n else:\n try:\n os.remove(\"{}.{}\".format(fname, random_uuid))\n except OSError:\n pass\n finally:\n warnings.warn(\"File {} exists in file system so the downloaded file is deleted\".format(fname))\n if sha1_hash and not sha1sum(fname) == sha1_hash:\n raise UserWarning(\n \"File {} is downloaded but the content hash does not match.\"\n \" The repo may be outdated or download may be incomplete. \"\n 'If the \"repo_url\" is overridden, consider switching to '\n \"the default repo.\".format(fname)\n )\n break\n except Exception as e:\n retries -= 1\n if retries <= 0:\n raise e\n\n logging.warning(\n \"download failed due to {}, retrying, {} attempt{} left\".format(\n repr(e), retries, \"s\" if retries > 1 else \"\"\n )\n )\n\n return fname\n\n\ndef path_expander(path, base_folder):\n path_l = path.split(\";\")\n return \";\".join([os.path.join(base_folder, path) for path in path_l])\n\n\ndef protected_zip_extraction(zipfile_path, sha1_hash, folder):\n \"\"\"Extract zip file to the folder.\n\n A signature file named \".SHA1HASH.sig\" will be created if the extraction has been finished.\n\n Returns\n -------\n folder\n The directory to extract the zipfile\n \"\"\"\n os.makedirs(folder, exist_ok=True)\n\n if sha1_hash:\n sha1_hash = sha1_hash[:6]\n signature = \".{}.sig\".format(sha1_hash)\n\n if os.path.exists(os.path.join(folder, signature)):\n # We have found the signature file. Thus, we will not extract again.\n return folder\n else:\n signature = None\n\n # Extract the file\n logging.info(\"Extract files...\")\n with zipfile.ZipFile(zipfile_path, \"r\") as zip_ref:\n zip_ref.extractall(folder)\n\n if signature:\n # Create the signature\n with open(os.path.join(folder, signature), \"w\"):\n pass\n\n return folder\n"
},
{
"path": "common/src/autogluon/common/loaders/load_json.py",
"content": "from __future__ import annotations\n\nimport json\nimport logging\n\nfrom ..utils import s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\ndef load(path: str, *, verbose=True) -> dict | list:\n if verbose:\n logger.log(15, \"Loading: %s\" % path)\n is_s3_path = s3_utils.is_s3_url(path)\n if is_s3_path:\n import boto3\n\n bucket, key = s3_utils.s3_path_to_bucket_prefix(path)\n s3_client = boto3.client(\"s3\")\n result = s3_client.get_object(Bucket=bucket, Key=key)\n out = json.loads(result[\"Body\"].read())\n else:\n with open(path, \"r\") as f:\n out = json.load(f)\n return out\n"
},
{
"path": "common/src/autogluon/common/loaders/load_pd.py",
"content": "from __future__ import annotations\n\nimport logging\nimport multiprocessing\nfrom os import listdir\nfrom os.path import isfile, join\nfrom pathlib import Path\n\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom ..utils import multiprocessing_utils, s3_utils\nfrom .load_s3 import list_bucket_prefix_suffix_contains_s3\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: v1.0 consider renaming function so it isn't 'load'. Consider instead 'load_pd', or something more descriptive.\n# TODO: Add full docstring\ndef load(\n path: str | Path | list[str | Path],\n delimiter=None,\n encoding=\"utf-8\",\n columns_to_keep=None,\n dtype=None,\n header=0,\n names=None,\n format=None,\n nrows=None,\n skiprows=None,\n usecols=None,\n low_memory=False,\n converters=None,\n worker_count=None,\n multiprocessing_method=\"forkserver\",\n) -> DataFrame:\n if isinstance(path, Path):\n path = str(path)\n if isinstance(path, list):\n path = [str(p) if isinstance(p, Path) else p for p in path]\n return _load_multipart(\n paths=path,\n delimiter=delimiter,\n encoding=encoding,\n columns_to_keep=columns_to_keep,\n dtype=dtype,\n header=header,\n names=names,\n format=format,\n nrows=nrows,\n skiprows=skiprows,\n usecols=usecols,\n low_memory=low_memory,\n converters=converters,\n worker_count=worker_count,\n multiprocessing_method=multiprocessing_method,\n )\n if format is not None:\n pass\n elif path[-1] == \"/\" and s3_utils.is_s3_url(path): # and path[:2] == 's3'\n format = \"multipart_s3\"\n elif path[-1] == \"/\" and not s3_utils.is_s3_url(path): # and path[:2] != 's3'\n format = \"multipart_local\"\n elif \".parquet\" in path or \".pq\" in path or path[-1] == \"/\":\n format = \"parquet\"\n else:\n format = \"csv\"\n if delimiter is None:\n if path.endswith(\".tsv\"):\n delimiter = \"\\t\"\n logger.debug(\n f\"File delimiter for {path} inferred as '\\\\t' (tab). If this is incorrect, please manually load the data as a pandas DataFrame.\"\n )\n else:\n delimiter = \",\"\n logger.debug(\n f\"File delimiter for {path} inferred as ',' (comma). If this is incorrect, please manually load the data as a pandas DataFrame.\"\n )\n\n if format == \"multipart_s3\":\n bucket, prefix = s3_utils.s3_path_to_bucket_prefix(path)\n return _load_multipart_s3(\n bucket=bucket,\n prefix=prefix,\n columns_to_keep=columns_to_keep,\n dtype=dtype,\n worker_count=worker_count,\n multiprocessing_method=multiprocessing_method,\n ) # TODO: Add arguments!\n elif format == \"multipart_local\":\n paths = [join(path, f) for f in listdir(path) if (isfile(join(path, f))) & (f.startswith(\"part-\"))]\n return _load_multipart(\n paths=paths,\n delimiter=delimiter,\n encoding=encoding,\n columns_to_keep=columns_to_keep,\n dtype=dtype,\n header=header,\n names=names,\n format=None,\n nrows=nrows,\n skiprows=skiprows,\n usecols=usecols,\n low_memory=low_memory,\n converters=converters,\n worker_count=worker_count,\n multiprocessing_method=multiprocessing_method,\n )\n elif format == \"parquet\":\n df = pd.read_parquet(path, columns=columns_to_keep)\n column_count_full = len(df.columns)\n elif format == \"csv\":\n df = pd.read_csv(\n path,\n converters=converters,\n delimiter=delimiter,\n encoding=encoding,\n header=header,\n names=names,\n dtype=dtype,\n low_memory=low_memory,\n nrows=nrows,\n skiprows=skiprows,\n usecols=usecols,\n )\n column_count_full = len(list(df.columns.values))\n if columns_to_keep is not None:\n df = df[columns_to_keep]\n else:\n raise Exception(\"file format \" + format + \" not supported!\")\n\n row_count = df.shape[0]\n\n column_count_trimmed = len(list(df.columns.values))\n\n logger.log(\n 20,\n \"Loaded data from: \"\n + str(path)\n + \" | Columns = \"\n + str(column_count_trimmed)\n + \" / \"\n + str(column_count_full)\n + \" | Rows = \"\n + str(row_count)\n + \" -> \"\n + str(len(df)),\n )\n return df\n\n\ndef _load_multipart_child(chunk):\n (\n path,\n delimiter,\n encoding,\n columns_to_keep,\n dtype,\n header,\n names,\n format,\n nrows,\n skiprows,\n usecols,\n low_memory,\n converters,\n ) = chunk\n df = load(\n path=path,\n delimiter=delimiter,\n encoding=encoding,\n columns_to_keep=columns_to_keep,\n dtype=dtype,\n header=header,\n names=names,\n format=format,\n nrows=nrows,\n skiprows=skiprows,\n usecols=usecols,\n low_memory=low_memory,\n converters=converters,\n )\n return df\n\n\ndef _load_multipart(\n paths,\n delimiter=\",\",\n encoding=\"utf-8\",\n columns_to_keep=None,\n dtype=None,\n header=0,\n names=None,\n format=None,\n nrows=None,\n skiprows=None,\n usecols=None,\n low_memory=False,\n converters=None,\n worker_count=None,\n multiprocessing_method=\"forkserver\",\n):\n cpu_count = multiprocessing.cpu_count()\n workers = int(round(cpu_count))\n if worker_count is not None:\n if worker_count <= workers:\n workers = worker_count\n\n logger.log(15, \"Load multipart running pool with \" + str(workers) + \" workers...\")\n\n full_chunks = [\n [\n path,\n delimiter,\n encoding,\n columns_to_keep,\n dtype,\n header,\n names,\n format,\n nrows,\n skiprows,\n usecols,\n low_memory,\n converters,\n ]\n for path in paths\n ]\n\n df_list = multiprocessing_utils.execute_multiprocessing(\n workers_count=workers,\n transformer=_load_multipart_child,\n chunks=full_chunks,\n multiprocessing_method=multiprocessing_method,\n )\n\n df_combined = pd.concat(df_list, axis=0, ignore_index=True)\n\n column_count = len(list(df_combined.columns.values))\n row_count = df_combined.shape[0]\n\n logger.log(20, \"Loaded data from multipart file | Columns = \" + str(column_count) + \" | Rows = \" + str(row_count))\n return df_combined\n\n\ndef _load_multipart_s3(\n bucket,\n prefix,\n columns_to_keep=None,\n dtype=None,\n worker_count=None,\n multiprocessing_method=\"forkserver\",\n):\n if prefix[-1] == \"/\":\n prefix = prefix[:-1]\n prefix_multipart = prefix + \"/part-\"\n # exclude_contains=['/'] to disallow loading files like 's3://bucket/prefix/part-directory/some_file.abc'\n files = list_bucket_prefix_suffix_contains_s3(bucket=bucket, prefix=prefix_multipart, exclude_contains=[\"/\"])\n paths_full = [s3_utils.s3_bucket_prefix_to_path(bucket=bucket, prefix=file, version=\"s3\") for file in files]\n\n df = load(\n path=paths_full,\n columns_to_keep=columns_to_keep,\n dtype=dtype,\n worker_count=worker_count,\n multiprocessing_method=multiprocessing_method,\n )\n return df\n"
},
{
"path": "common/src/autogluon/common/loaders/load_pkl.py",
"content": "from __future__ import annotations\n\nimport io\nimport logging\nimport pickle\nfrom typing import Any\nfrom urllib.parse import urlparse\n\nimport requests\n\nfrom ..utils import compression_utils, s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\ndef load(path: str, format: str | None = None, verbose: bool = True, **kwargs) -> Any:\n \"\"\"\n\n Parameters\n ----------\n path: str\n The path to the pickle file.\n Can be either local path, a web url, or a private s3 path.\n Local Path Example:\n \"local/path/to/file.pkl\"\n Web Url Example:\n \"https://path/to/file.pkl\"\n S3 Path Example:\n \"s3://bucket/prefix/file.pkl\"\n\n format: str, optional\n Legacy argument, unused.\n verbose: bool, default True\n kwargs\n\n Returns\n -------\n object\n The contents of the pickle file\n\n \"\"\"\n compression_fn = kwargs.get(\"compression_fn\", None)\n compression_fn_kwargs = kwargs.get(\"compression_fn_kwargs\", None)\n\n if s3_utils.is_s3_url(path):\n format = \"s3\"\n elif _is_web_url(path=path):\n format = \"url\"\n\n if verbose:\n logger.log(15, f\"Loading: {path}\")\n\n if format == \"s3\":\n import boto3\n\n s3_bucket, s3_prefix = s3_utils.s3_path_to_bucket_prefix(s3_path=path)\n s3 = boto3.resource(\"s3\")\n return pickle.loads(s3.Bucket(s3_bucket).Object(s3_prefix).get()[\"Body\"].read())\n elif format == \"url\":\n return _load_pickle_from_url(url=path)\n\n compression_fn_map = compression_utils.get_compression_map()\n validated_path = compression_utils.get_validated_path(path, compression_fn=compression_fn)\n\n if compression_fn_kwargs is None:\n compression_fn_kwargs = {}\n\n if compression_fn in compression_fn_map:\n with compression_fn_map[compression_fn][\"open\"](validated_path, \"rb\", **compression_fn_kwargs) as fin:\n object = pickle.load(fin)\n else:\n raise ValueError(\n f\"compression_fn={compression_fn} or compression_fn_kwargs={compression_fn_kwargs} are not valid.\"\n f\" Valid function values: {compression_fn_map.keys()}\"\n )\n\n return object\n\n\ndef _is_web_url(path: str) -> bool:\n try:\n result = urlparse(path)\n return result.scheme in (\"http\", \"https\") and bool(result.netloc)\n except ValueError:\n return False\n\n\ndef _load_pickle_from_url(url: str):\n response = requests.get(url)\n response.raise_for_status() # Raise an error for bad status codes\n return pickle.loads(response.content)\n\n\ndef load_with_fn(path, pickle_fn, format=None, verbose=True):\n if s3_utils.is_s3_url(path):\n format = \"s3\"\n if format == \"s3\":\n import boto3\n\n if verbose:\n logger.log(15, \"Loading: %s\" % path)\n s3_bucket, s3_prefix = s3_utils.s3_path_to_bucket_prefix(s3_path=path)\n s3 = boto3.resource(\"s3\")\n # Has to be wrapped in IO buffer since s3 stream does not implement seek()\n buff = io.BytesIO(s3.Bucket(s3_bucket).Object(s3_prefix).get()[\"Body\"].read())\n return pickle_fn(buff)\n\n if verbose:\n logger.log(15, \"Loading: %s\" % path)\n with open(path, \"rb\") as fin:\n object = pickle_fn(fin)\n return object\n"
},
{
"path": "common/src/autogluon/common/loaders/load_s3.py",
"content": "import logging\nimport os\nimport pathlib\nfrom typing import List, Optional, Union\n\nlogger = logging.getLogger(__name__)\n\n\ndef list_bucket_s3(bucket):\n import boto3\n\n logger.log(15, \"Listing s3 bucket: \" + str(bucket))\n\n s3bucket = boto3.resource(\"s3\")\n my_bucket = s3bucket.Bucket(bucket)\n files = []\n for object in my_bucket.objects.all():\n files.append(object.key)\n logger.log(15, str(object.key))\n return files\n\n\ndef download(input_bucket, input_prefix, local_path):\n import boto3\n\n directory = os.path.dirname(local_path)\n pathlib.Path(directory).mkdir(parents=True, exist_ok=True)\n\n s3 = boto3.resource(\"s3\")\n s3.Bucket(input_bucket).download_file(input_prefix, local_path)\n\n\n# TODO: Expand to support arbitrary list of candidate file paths\n# TODO: add local file path support as a more general function\n# TODO: Consider renaming and deprecating old name\n# TODO: consider using a single parameter supporting wildcards or regex - this will solve all possible use cases for filtering\n# list_bucket_prefix_suffix_contains_s3(..., exclude=['**/*.bak', '**/data/*_excl.csv'])\n# TODO: Add unit tests for non-boto3 logic\ndef list_bucket_prefix_suffix_contains_s3(\n bucket: str,\n prefix: str,\n suffix: Optional[Union[str, List[str]]] = None,\n exclude_suffix: Optional[Union[str, List[str]]] = None,\n contains: Optional[Union[str, List[str]]] = None,\n exclude_contains: Optional[Union[str, List[str]]] = None,\n) -> List[str]:\n \"\"\"\n Returns a list of file paths within an S3 bucket that satisfies the constraints.\n\n Parameters\n ----------\n bucket : str\n The S3 bucket to list files from.\n You must have read permissions to the S3 bucket and its files for this function to work.\n prefix : str\n The string prefix to search for files within the S3 bucket. Any file outside of this prefix will not be considered.\n For example, if `bucket='autogluon'` and `prefix='datasets/'`,\n only files starting under `s3://autogluon/datasets/` will be considered.\n To check all files in the bucket, specify `prefix=''` (empty string)\n suffix : str or List[str], default = None\n If specified, filters files to ensure their paths end with the specified suffix (if str)\n or at least one element of `suffix` (if list) in the post-prefix string path.\n exclude_suffix : str or List[str], default = None\n If specified, filters files to ensure their paths do not end with any element in `exclude_suffix`.\n contains : str or List[str], default = None\n If specified, will filter any result that doesn't contain `contains` (if str)\n or at least one element of `contains` (if list) in the post-prefix string path.\n exclude_contains : str or List[str], default = None\n If specified, filters files to ensure their paths do not contain any element in `exclude_contains`.\n\n Returns\n -------\n Returns a list of file paths within an S3 bucket that satisfies the constraints.\n\n \"\"\"\n import boto3\n\n if exclude_suffix is None:\n exclude_suffix = []\n if exclude_contains is None:\n exclude_contains = []\n if suffix is not None and not isinstance(suffix, list):\n suffix = [suffix]\n if exclude_suffix is not None and not isinstance(exclude_suffix, list):\n exclude_suffix = [exclude_suffix]\n if contains is not None and not isinstance(contains, list):\n contains = [contains]\n if exclude_contains is not None and not isinstance(exclude_contains, list):\n exclude_contains = [exclude_contains]\n\n s3 = boto3.resource(\"s3\")\n my_bucket = s3.Bucket(bucket)\n\n files = []\n for object_summary in my_bucket.objects.filter(Prefix=prefix):\n suffix_full = object_summary.key.split(prefix, 1)[1] if len(prefix) > 0 else object_summary.key\n is_banned = False\n for banned_s in exclude_suffix:\n if suffix_full.endswith(banned_s):\n is_banned = True\n break\n if is_banned:\n continue\n for banned_c in exclude_contains:\n if banned_c in suffix_full:\n is_banned = True\n break\n if is_banned:\n continue\n if suffix is not None:\n has_valid_suffix = False\n for s in suffix:\n if suffix_full.endswith(s):\n has_valid_suffix = True\n break\n if not has_valid_suffix:\n continue\n if contains is None:\n files.append(object_summary.key)\n else:\n for c in contains:\n if c in suffix_full:\n files.append(object_summary.key)\n break\n return files\n"
},
{
"path": "common/src/autogluon/common/loaders/load_str.py",
"content": "import logging\n\nfrom ..utils import s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\ndef load(path: str) -> str:\n \"\"\"\n Loads the `data` value from a file saved via `savers.save_str.save(path=path, data=data)`.\n This function is compatible with local and s3 files.\n\n Parameters\n ----------\n path : str\n Path to the file to load the data from.\n Can be local or s3 path.\n\n Returns\n -------\n data : str\n The string object that is contained in the loaded file.\n\n Examples\n --------\n >>> from autogluon.core.utils.loaders import load_str\n >>> from autogluon.core.utils.savers import save_str\n >>> data = 'the string value i want to save and load'\n >>> path = 'path/to/a/new/file'\n >>> save_str.save(path=path, data=data)\n >>> data_loaded = load_str.load(path=path)\n >>> assert data == data_loaded\n \"\"\"\n\n is_s3_path = s3_utils.is_s3_url(path)\n if is_s3_path:\n import boto3\n\n bucket, key = s3_utils.s3_path_to_bucket_prefix(path)\n s3_client = boto3.client(\"s3\")\n s3_object = s3_client.get_object(Bucket=bucket, Key=key)\n data = s3_object[\"Body\"].read().decode(\"utf-8\")\n else:\n with open(path, \"r\") as f:\n data = f.read()\n return data\n"
},
{
"path": "common/src/autogluon/common/loaders/load_zip.py",
"content": "import logging\nimport os\n\nlogger = logging.getLogger(__name__)\n\n\ndef unzip(path, sha1sum=None, unzip_dir=None):\n \"\"\"Unzip a .zip file from path to unzip_dir.\"\"\"\n from ._utils import download, protected_zip_extraction\n\n local_file = unzip_dir + os.sep + \"file.zip\"\n download(path, path=local_file, sha1_hash=sha1sum)\n\n logger.log(20, f\"Unzipping {local_file} to {unzip_dir}\")\n\n protected_zip_extraction(local_file, sha1_hash=sha1sum, folder=unzip_dir)\n\n return\n"
},
{
"path": "common/src/autogluon/common/model_filter/__init__.py",
"content": "from ._model_filter import ModelFilter\n"
},
{
"path": "common/src/autogluon/common/model_filter/_model_filter.py",
"content": "import logging\nfrom typing import Any, Dict, List, Optional, Union\n\nlogger = logging.getLogger(__name__)\n\n\nclass ModelFilter:\n \"\"\"Class to filter models given user requirements\"\"\"\n\n @staticmethod\n def include_models(\n models: Union[Dict[str, Any], List[str]], included_model_types: List[str]\n ) -> Union[Dict[str, Any], List[str]]:\n \"\"\"\n Only include models specified in `included_model_types`, other models will be removed\n If model specified in `included_model_types` doesn't present in `models`, will warn users and ignore\n\n Parameters\n ----------\n models: Union[Dict[str, Any], List[str]]\n A dictionary containing models and their hyperparameters\n included_model_types: List[str]\n List of model types to be included\n\n Return\n ------\n Union[Dict[str, Any], List[str]]\n Updated dictionary or list with correct models\n \"\"\"\n if isinstance(models, dict):\n included_models = {model: val for model, val in models.items() if model in included_model_types}\n missing_models = set(included_model_types) - set(included_models.keys())\n elif isinstance(models, list):\n included_models = [model for model in models if model in included_model_types]\n missing_models = set(included_model_types) - set(included_models)\n if included_model_types is not None:\n logger.log(\n 20,\n f\"Included models: {list(included_model_types)} (Specified by `included_model_types`, all other model types will be skipped)\",\n )\n if len(missing_models) > 0:\n logger.warning(\n f\"\\tThe models types {list(missing_models)} are not present in the model list specified by the user and will be ignored:\"\n )\n return included_models\n\n @staticmethod\n def exclude_models(\n models: Union[Dict[str, Any], List[str]], excluded_model_types: List[str]\n ) -> Union[Dict[str, Any], List[str]]:\n \"\"\"\n Exclude models from the current dictionary.\n All models specified in `excluded_model_types` will be removed\n\n Parameters\n ----------\n models: Union[Dict[str, Any], List[str]]\n A dictionary containing models and their hyperparameters\n excluded_model_types: List[str]\n List of model types to be excluded\n\n Return\n ------\n Union[Dict[str, Any], List[str]]\n Updated dictionary or list with correct models\n \"\"\"\n excluded_models = None\n if isinstance(models, dict):\n models_after_exclusion = {model: val for model, val in models.items() if model not in excluded_model_types}\n excluded_models = set(models.keys()) - set(models_after_exclusion.keys())\n elif isinstance(models, list):\n models_after_exclusion = [model for model in models if model not in excluded_model_types]\n excluded_models = set(models) - set(models_after_exclusion)\n if excluded_models is not None:\n logger.log(20, f\"Excluded models: {list(excluded_models)} (Specified by `excluded_model_types`)\")\n return models_after_exclusion\n\n @staticmethod\n def filter_models(\n models: Union[Dict[str, Any], List[str]],\n included_model_types: Optional[List[str]] = None,\n excluded_model_types: Optional[List[str]] = None,\n ) -> Union[Dict[str, Any], List[str]]:\n \"\"\"\n Filter models given `included_model_types` or `excluded_model_types`\n If both are provided, will only respect `included_model_types`\n\n Parameters\n ----------\n models: Union[Dict[str, Any], List[str]]\n A dictionary containing models and their hyperparameters\n included_model_types: List[str]\n List of model types to be included\n excluded_model_types: List[str]\n List of model types to be excluded\n\n Return\n ------\n Union[Dict[str, Any], List[str]]\n Updated dictionary or list with correct models\n \"\"\"\n if included_model_types is not None and excluded_model_types is not None:\n logger.warning(\n \"Both `included_model_types` and `excluded_model_types` are specified. Will use `included_model_types` only\"\n )\n excluded_model_types = None\n if included_model_types is not None:\n return ModelFilter.include_models(models=models, included_model_types=included_model_types)\n if excluded_model_types is not None:\n return ModelFilter.exclude_models(models=models, excluded_model_types=excluded_model_types)\n return models\n"
},
{
"path": "common/src/autogluon/common/savers/__init__.py",
"content": ""
},
{
"path": "common/src/autogluon/common/savers/save_json.py",
"content": "# TODO: Standardize / unify this code with ag.save()\nimport json\nimport logging\nimport os\nimport tempfile\n\nfrom ..utils import s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\ndef save(path, obj, sanitize=True):\n if sanitize:\n obj = sanitize_object_to_primitives(obj=obj)\n is_s3_path = s3_utils.is_s3_url(path)\n if is_s3_path:\n import boto3\n\n data = json.dumps(obj).encode(\"UTF-8\")\n\n bucket, key = s3_utils.s3_path_to_bucket_prefix(path)\n s3_client = boto3.client(\"s3\")\n s3_client.put_object(Body=data, Bucket=bucket, Key=key)\n else:\n dirname = os.path.dirname(path)\n if dirname:\n os.makedirs(dirname, exist_ok=True)\n with open(path, \"w\") as fp:\n json.dump(obj, fp, indent=2)\n\n\ndef sanitize_object_to_primitives(obj):\n if isinstance(obj, dict):\n obj_sanitized = dict()\n for key, val in obj.items():\n obj_sanitized[key] = sanitize_object_to_primitives(val)\n else:\n try:\n json.dumps(obj)\n obj_sanitized = obj\n except (TypeError, OverflowError):\n json.dumps(type(obj).__name__)\n obj_sanitized = type(obj).__name__\n return obj_sanitized\n"
},
{
"path": "common/src/autogluon/common/savers/save_pd.py",
"content": "from __future__ import annotations\n\nimport logging\nimport multiprocessing\nimport os\nfrom io import StringIO\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom ..utils import multiprocessing_utils, s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Update so verbose prints at level 20, and adjust calls to save accordingly\n# gzip compression produces random deflate issues on linux machines - use with caution\n# TODO: v1.0 deprecate 'df', replace with 'data', or align with Pandas for parameter names\n# TODO: v1.0 consider renaming function so it isn't 'save'. Consider instead 'save_pd', or something more descriptive.\n# TODO: Add full docstring\n# TODO: Add `allow_overwrite=True` so that users can instead force\n# an error by setting to False if saving would overwrite an existing file.\ndef save(\n path: str | Path,\n df: pd.DataFrame,\n index: bool = False,\n verbose: bool = True,\n type: str | None = None,\n sep: str = \",\",\n compression: str = \"gzip\",\n header: bool = True,\n):\n \"\"\"\n Save pandas DataFrame to the file path.\n If local path, directories will be created automatically if necessary to save the file.\n If local, will be relative to working directory unless specified as absolute.\n If S3 path, you must have permissions to save to the S3 location available via boto3 in the current session.\n\n By default will save the header and index.\n If saving to CSV, column dtypes may not be maintained upon loading the file.\n To ensure identical column dtypes when loading, save in Parquet format.\n\n For large DataFrames (>1 GB), it is highly recommended to save in Parquet format.\n For massive DataFrames (>10 GB), it is highly recommended to save in multipart Parquet format.\n\n When saving to multipart Parquet on S3, files under the S3 path that already exist will be deleted\n to avoid corrupting the multipart save.\n Please ensure the S3 path is empty or you are ok with the files being deleted.\n\n Example paths:\n CSV (local) : \"local/path/out.csv\"\n Parquet (local) : \"local/path/out.parquet\"\n Multipart Parquet (local) : \"local/path/\"\n CSV (absolute local) : \"/home/ubuntu/path/out.csv\"\n Parquet (absolute local) : \"/home/ubuntu/path/out.parquet\"\n Multipart Parquet (abs loc) : \"/home/ubuntu/path/\"\n CSV (S3) : \"s3://bucket/pre/fix/out.csv\"\n Parquet (S3) : \"s3://bucket/pre/fix/out.parquet\"\n Multipart Parquet (S3) : \"s3://bucket/pre/fix/\"\n\n Note: Once saved via this function, the same path can be used to\n load the file via `autogluon.common.loaders.load_pd.load(path=path)`.\n\n \"\"\"\n if isinstance(path, Path):\n path = str(path)\n if type is None:\n if path[-1] == \"/\" and s3_utils.is_s3_url(path): # and path[:2] == 's3'\n type = \"multipart_s3\"\n elif path[-1] == \"/\" and not s3_utils.is_s3_url(path): # and path[:2] != 's3'\n type = \"multipart_local\"\n elif \".csv\" in path:\n type = \"csv\"\n elif \".parquet\" in path:\n type = \"parquet\"\n else:\n type = \"csv\"\n if \"s3\" not in path[:2]:\n is_local = True\n else:\n is_local = False\n column_count = len(list(df.columns.values))\n row_count = df.shape[0]\n if is_local:\n path_abs = os.path.abspath(path)\n path_abs_dirname = os.path.dirname(path_abs)\n if path_abs_dirname:\n os.makedirs(path_abs_dirname, exist_ok=True)\n if type == \"csv\":\n if is_local:\n df.to_csv(path, index=index, sep=sep, header=header)\n else:\n import boto3\n\n buffer = StringIO()\n df.to_csv(buffer, index=index, sep=sep, header=header)\n bucket, prefix = s3_utils.s3_path_to_bucket_prefix(s3_path=path)\n s3_resource = boto3.resource(\"s3\")\n s3_resource.Object(bucket, prefix).put(Body=buffer.getvalue(), ACL=\"bucket-owner-full-control\")\n if verbose:\n logger.log(15, \"Saved \" + str(path) + \" | Columns = \" + str(column_count) + \" | Rows = \" + str(row_count))\n elif type == \"parquet\":\n df.to_parquet(path, compression=compression)\n if verbose:\n logger.log(15, \"Saved \" + str(path) + \" | Columns = \" + str(column_count) + \" | Rows = \" + str(row_count))\n elif type == \"multipart_s3\":\n bucket, prefix = s3_utils.s3_path_to_bucket_prefix(s3_path=path)\n # We delete the prior files because imagine we are saving a 10-part multipart parquet now,\n # but we saved a 20 part multipart parquet prior.\n # In this situation, the output would be corrupted, since the first 10 parts would be the new output,\n # while parts 11-20 would be the old output.\n # Multipart Parquet loading would see 20 parts and try to load all of them, resulting in at best an exception,\n # and at worst the unintended and silent concatenation of two different DataFrames.\n s3_utils.delete_s3_prefix(bucket=bucket, prefix=prefix) # TODO: Might only delete the first 1000!\n _save_multipart(\n path=path,\n df=df,\n index=index,\n verbose=verbose,\n type=\"parquet\",\n sep=sep,\n compression=compression,\n header=header,\n )\n elif type == \"multipart_local\":\n # TODO: v1.0 : Ensure the same file deletion process best practice occurs during multipart local saving.\n if os.path.isdir(path):\n for file in os.listdir(path):\n file_path = os.path.join(path, file)\n try:\n if os.path.isfile(file_path):\n os.unlink(file_path)\n except Exception as e:\n logger.exception(e)\n _save_multipart(\n path=path,\n df=df,\n index=index,\n verbose=verbose,\n type=\"parquet\",\n sep=sep,\n compression=compression,\n header=header,\n )\n else:\n raise Exception(\"Unknown save type: \" + type)\n\n\ndef _save_multipart_child(chunk):\n path, df, index, verbose, type, sep, compression, header = chunk\n save(\n path=path,\n df=df,\n index=index,\n verbose=verbose,\n type=type,\n sep=sep,\n compression=compression,\n header=header,\n )\n\n\ndef _save_multipart(path, df, index=False, verbose=True, type=None, sep=\",\", compression=\"snappy\", header=True):\n cpu_count = multiprocessing.cpu_count()\n workers_count = int(round(cpu_count))\n parts = workers_count\n\n logger.log(15, \"Save_multipart running pool with \" + str(workers_count) + \" workers\")\n\n paths = [path + \"part-\" + \"0\" * (5 - min(5, len(str(i)))) + str(i) + \".parquet\" for i in range(parts)]\n df_parts = np.array_split(df, parts)\n\n full_chunks = [\n [\n path,\n df_part,\n index,\n verbose,\n type,\n sep,\n compression,\n header,\n ]\n for path, df_part in zip(paths, df_parts)\n ]\n\n multiprocessing_utils.execute_multiprocessing(\n workers_count=workers_count, transformer=_save_multipart_child, chunks=full_chunks\n )\n\n logger.log(15, \"Saved multipart file to \" + str(path))\n"
},
{
"path": "common/src/autogluon/common/savers/save_pkl.py",
"content": "# TODO: Standardize / unify this code with ag.save()\nimport logging\nimport os\nimport pickle\nimport tempfile\n\nfrom ..utils import compression_utils, s3_utils\n\nlogger = logging.getLogger(__name__)\n\ncompression_fn_map = compression_utils.get_compression_map()\n\n\n# TODO: object -> obj?\ndef save(path, object, format=None, verbose=True, **kwargs):\n compression_fn = kwargs.get(\"compression_fn\", None)\n compression_fn_kwargs = kwargs.get(\"compression_fn_kwargs\", None)\n\n if compression_fn in compression_fn_map:\n validated_path = compression_utils.get_validated_path(path, compression_fn)\n else:\n raise ValueError(\n f\"compression_fn={compression_fn} is not a valid compression_fn. Valid values: {compression_fn_map.keys()}\"\n )\n\n def pickle_fn(o, buffer):\n return pickle.dump(o, buffer, protocol=4)\n\n save_with_fn(\n validated_path,\n object,\n pickle_fn,\n format=format,\n verbose=verbose,\n compression_fn=compression_fn,\n compression_fn_kwargs=compression_fn_kwargs,\n )\n\n\ndef save_with_fn(path, object, pickle_fn, format=None, verbose=True, compression_fn=None, compression_fn_kwargs=None):\n if verbose:\n logger.log(15, \"Saving \" + str(path))\n if s3_utils.is_s3_url(path):\n format = \"s3\"\n if format == \"s3\":\n save_s3(path, object, pickle_fn, verbose=verbose)\n else:\n path_parent = os.path.dirname(path)\n if path_parent == \"\":\n path_parent = \".\" # Allows saving to working directory root without crashing\n os.makedirs(path_parent, exist_ok=True)\n\n if compression_fn_kwargs is None:\n compression_fn_kwargs = {}\n\n with compression_fn_map[compression_fn][\"open\"](path, \"wb\", **compression_fn_kwargs) as fout:\n pickle_fn(object, fout)\n\n\ndef save_s3(path: str, obj, pickle_fn, verbose=True):\n import boto3\n\n if verbose:\n logger.info(f\"save object to {path}\")\n with tempfile.TemporaryFile() as f:\n pickle_fn(obj, f)\n f.flush()\n f.seek(0)\n\n bucket, key = s3_utils.s3_path_to_bucket_prefix(path)\n s3_client = boto3.client(\"s3\")\n try:\n config = boto3.s3.transfer.TransferConfig() # enable multipart uploading for files larger than 8MB\n s3_client.upload_fileobj(f, bucket, key, Config=config)\n except Exception:\n logger.error(\"Failed to save object to s3\")\n raise\n"
},
{
"path": "common/src/autogluon/common/savers/save_str.py",
"content": "import logging\nimport os\n\nfrom ..utils import s3_utils\n\nlogger = logging.getLogger(__name__)\n\n\ndef save(path, data: str, verbose=True):\n \"\"\"\n Saves the `data` value to a file.\n This function is compatible with local and s3 files.\n\n Parameters\n ----------\n path : str\n Path to the file to load the data from.\n Can be local or s3 path.\n data : str\n The string object to be saved.\n verbose : bool, default = True\n Whether to log that the file was saved.\n\n Examples\n --------\n >>> from autogluon.core.utils.loaders import load_str\n >>> from autogluon.core.utils.savers import save_str\n >>> data = 'the string value i want to save and load'\n >>> path = 'path/to/a/new/file'\n >>> save_str.save(path=path, data=data)\n >>> data_loaded = load_str.load(path=path)\n >>> assert data == data_loaded\n \"\"\"\n\n is_s3_path = s3_utils.is_s3_url(path)\n if is_s3_path:\n import boto3\n\n bucket, key = s3_utils.s3_path_to_bucket_prefix(path)\n s3_client = boto3.client(\"s3\")\n s3_client.put_object(Body=data, Bucket=bucket, Key=key)\n else:\n dirname = os.path.dirname(path)\n if dirname:\n os.makedirs(os.path.dirname(path), exist_ok=True)\n with open(path, \"w\") as f:\n f.write(data)\n\n if verbose:\n logger.log(15, f'Saving {path} with contents \"{data}\"')\n"
},
{
"path": "common/src/autogluon/common/space.py",
"content": "__all__ = [\"Space\", \"Categorical\", \"Real\", \"Int\", \"Bool\"]\n\n\nclass Space(object):\n \"\"\"Basic search space describing set of possible candidate values for hyperparameter.\"\"\"\n\n @property\n def default(self):\n \"\"\"Return default value of hyperparameter corresponding to this search space. This value is tried first during hyperparameter optimization.\"\"\"\n raise NotImplementedError\n\n\nclass SimpleSpace(Space):\n def __init__(self, default):\n self._default = default\n\n \"\"\"Non-nested search space (i.e. corresponds to a single simple hyperparameter).\"\"\"\n\n def __repr__(self):\n reprstr = self.__class__.__name__\n if hasattr(self, \"lower\") and hasattr(self, \"upper\"):\n reprstr += \": lower={}, upper={}\".format(self.lower, self.upper)\n if hasattr(self, \"value\"):\n reprstr += \": value={}\".format(self.value)\n return reprstr\n\n @property\n def default(self):\n \"\"\"Return default value of hyperparameter corresponding to this search space. This value is tried first during hyperparameter optimization.\"\"\"\n return self._default\n\n @default.setter\n def default(self, value):\n \"\"\"Set default value for hyperparameter corresponding to this search space. The default value is always tried in the first trial of HPO.\"\"\"\n self._default = value\n\n\nclass DiscreteSpace(SimpleSpace):\n \"\"\"\n Search space with the requirement of having a discrete number of options, such that it is possible to exhaust the search space.\n \"\"\"\n\n def __len__(self) -> int:\n \"\"\"Returns the number of unique spaces within the discrete search space.\"\"\"\n raise NotImplementedError\n\n\nclass Categorical(DiscreteSpace):\n \"\"\"Nested search space for hyperparameters which are categorical. Such a hyperparameter takes one value out of the discrete set of provided options.\n The first value in the list of options will be the default value that gets tried first during HPO.\n\n Parameters\n ----------\n data : Space or python built-in objects\n the choice candidates\n\n Examples\n --------\n >>> a = Categorical('a', 'b', 'c', 'd') # 'a' will be default value tried first during HPO\n \"\"\"\n\n def __init__(self, *data):\n self.data = [*data]\n super().__init__(self.data[0])\n\n def __iter__(self):\n for elem in self.data:\n yield elem\n\n def __getitem__(self, index):\n return self.data[index]\n\n def __setitem__(self, index, data):\n self.data[index] = data\n\n def __len__(self):\n return len(self.data)\n\n def convert_to_sklearn(self):\n return self.data\n\n def __repr__(self):\n reprstr = self.__class__.__name__ + str(self.data)\n return reprstr\n\n\nclass Real(SimpleSpace):\n \"\"\"Search space for numeric hyperparameter that takes continuous values.\n\n Parameters\n ----------\n lower : float\n The lower bound of the search space (minimum possible value of hyperparameter)\n upper : float\n The upper bound of the search space (maximum possible value of hyperparameter)\n default : float (optional)\n Default value tried first during hyperparameter optimization\n log : (True/False)\n Whether to search the values on a logarithmic rather than linear scale.\n This is useful for numeric hyperparameters (such as learning rates) whose search space spans many orders of magnitude.\n\n Examples\n --------\n >>> learning_rate = Real(0.01, 0.1, log=True)\n \"\"\"\n\n def __init__(self, lower, upper, default=None, log=False):\n if log and lower <= 0:\n raise AssertionError(f\"lower must be greater than 0 when `log=True`. lower: {lower}\")\n if lower >= upper:\n raise AssertionError(f\"lower must be less than upper. lower: {lower}, upper: {upper}\")\n if default is None:\n default = lower\n super().__init__(default=default)\n self.lower = lower\n self.upper = upper\n self.log = log\n\n def convert_to_sklearn(self):\n from scipy.stats import loguniform, uniform\n\n if self.log:\n sampler = loguniform(self.lower, self.upper)\n else:\n sampler = uniform(self.lower, self.upper - self.lower)\n return sampler\n\n\nclass Int(DiscreteSpace):\n \"\"\"Search space for numeric hyperparameter that takes integer values.\n\n Parameters\n ----------\n lower : int\n The lower bound of the search space (minimum possible value of hyperparameter)\n upper : int\n The upper bound of the search space (maximum possible value of hyperparameter)\n default : int (optional)\n Default value tried first during hyperparameter optimization\n\n\n Examples\n --------\n >>> range = Int(0, 100)\n \"\"\"\n\n def __init__(self, lower, upper, default=None):\n if default is None:\n default = lower\n super().__init__(default=default)\n self.lower = lower\n self.upper = upper\n\n def convert_to_sklearn(self):\n from scipy.stats import randint\n\n return randint(self.lower, self.upper + 1)\n\n def __len__(self):\n return self.upper - self.lower + 1\n\n\nclass Bool(Int):\n \"\"\"Search space for hyperparameter that is either True or False.\n `Bool()` serves as shorthand for: `Categorical(True, False)`\n\n Examples\n --------\n >>> pretrained = Bool()\n \"\"\"\n\n def __init__(self):\n super(Bool, self).__init__(0, 1)\n"
},
{
"path": "common/src/autogluon/common/utils/__init__.py",
"content": "from .deprecated_utils import Deprecated\n"
},
{
"path": "common/src/autogluon/common/utils/cache_presets_to_yaml.py",
"content": "from __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import Any, Mapping\nfrom urllib.parse import urlparse\n\n\ndef _is_s3_uri(path: str) -> bool:\n return urlparse(path).scheme == \"s3\"\n\n\ndef _parse_s3_uri(uri: str) -> tuple[str, str]:\n p = urlparse(uri)\n if p.scheme != \"s3\":\n raise ValueError(f\"Not an s3 uri: {uri!r}\")\n bucket = p.netloc\n key = p.path.lstrip(\"/\")\n if not bucket or not key:\n raise ValueError(f\"Invalid s3 uri (expected s3://bucket/key): {uri!r}\")\n return bucket, key\n\n\ndef _s3_put_text(\n uri: str,\n text: str,\n *,\n content_type: str = \"text/yaml; charset=utf-8\",\n upload_as_public: bool = False,\n) -> None:\n import boto3\n\n bucket, key = _parse_s3_uri(uri)\n\n put_kwargs = dict(\n Bucket=bucket,\n Key=key,\n Body=text.encode(\"utf-8\"),\n ContentType=content_type,\n )\n\n if upload_as_public:\n put_kwargs[\"ACL\"] = \"public-read\"\n\n boto3.client(\"s3\").put_object(**put_kwargs)\n\n\ndef presets_to_yaml_files(\n presets: Mapping[str, Mapping[str, Any]] | Mapping[str, Any],\n output: str | Path,\n *,\n multi: bool | None = None,\n per_preset_files: bool = False,\n overwrite: bool = False,\n sort_keys: bool = False,\n upload_as_public: bool = False, # NEW\n) -> list[str]:\n \"\"\"\n Export AutoGluon-style presets to YAML.\n\n Supports:\n - Local file output (single file) or local directory output (per preset files)\n - S3 output:\n * single file: s3://bucket/path/presets.yaml\n * per preset: s3://bucket/path/presets/ (prefix; each becomes \n\n## Fast and Accurate ML in 3 Lines of Code\n\n[](https://github.com/autogluon/autogluon/releases)\n[](https://anaconda.org/conda-forge/autogluon)\n[](https://pypi.org/project/autogluon/)\n[](https://pepy.tech/project/autogluon)\n[](./LICENSE)\n[](https://discord.gg/wjUmjqAc2N)\n[](https://twitter.com/autogluon)\n[](https://github.com/autogluon/autogluon/actions/workflows/continuous_integration.yml)\n[](https://github.com/autogluon/autogluon/actions/workflows/platform_tests-command.yml)\n\n[Installation](https://auto.gluon.ai/stable/install.html) | [Documentation](https://auto.gluon.ai/stable/index.html) | [Release Notes](https://auto.gluon.ai/stable/whats_new/index.html)\n\nbetween key-value pairs.\"\"\"\n s = \"\"\n for key in d:\n new_s = str(key) + \": \" + str(d[key]) + \"
\"\n s += new_s\n return s[:-4]\n\n\ndef mousover_plot(\n datadict,\n attr_x,\n attr_y,\n attr_color=None,\n attr_size=None,\n save_file=None,\n plot_title=\"\",\n point_transparency=0.5,\n point_size=20,\n default_color=\"#2222aa\",\n hidden_keys=[],\n show_plot=False,\n):\n \"\"\"Produces dynamic scatter plot that can be interacted with by mousing over each point to see its label\n Args:\n datadict (dict): keys contain attributes, values of lists of data from each attribute to plot (each list index corresponds to datapoint).\n The values of all extra keys in this dict are considered (string) labels to assign to datapoints when they are moused over.\n Apply _formatDict() to any entries in datadict which are themselves dicts.\n attr_x (str): name of column in dataframe whose values are shown on x-axis (eg. 'latency'). Can be categorical or numeric values\n attr_y (str): name of column in dataframe whose values are shown on y-axis (eg. 'validation performance'). Must be numeric values.\n attr_size (str): name of column in dataframe whose values determine size of dots (eg. 'memory consumption'). Must be numeric values.\n attr_color (str): name of column in dataframe whose values determine color of dots (eg. one of the hyperparameters). Can be categorical or numeric values\n point_labels (list): list of strings describing the label for each dot (must be in same order as rows of dataframe)\n save_file (str): where to save plot to (html) file (if None, plot is not saved)\n plot_title (str): Title of plot and html file\n point_transparency (float): alpha value of points, lower = more transparent\n point_size (int): size of points, higher = larger\n hidden keys (list[str]): which keys of datadict NOT to show labels for.\n show_plot (bool): whether to show plot\n \"\"\"\n try:\n with warning_filter():\n import bokeh\n from bokeh.models import CategoricalColorMapper, ColorBar, HoverTool, Legend, LegendItem, LinearColorMapper\n from bokeh.palettes import Category20\n from bokeh.plotting import ColumnDataSource, figure, output_file, save, show\n except ImportError:\n warnings.warn(\n 'AutoGluon summary plots cannot be created because bokeh is not installed. To see plots, please do: \"pip install bokeh==2.0.1 Jinja2==3.0.3\"'\n )\n return None\n\n n = len(datadict[attr_x])\n for key in datadict.keys(): # Check lengths are all the same\n if len(datadict[key]) != n:\n raise ValueError(\"Key %s in datadict has different length than %s\" % (key, attr_x))\n\n attr_x_is_string = any([isinstance(val, str) for val in datadict[attr_x]])\n if attr_x_is_string:\n attr_x_levels = list(set(datadict[attr_x])) # use this to translate between int-indices and x-values\n og_x_vals = datadict[attr_x][:]\n attr_x2 = attr_x + \"___\" # this key must not already be in datadict.\n hidden_keys.append(attr_x2)\n datadict[attr_x2] = [attr_x_levels.index(category) for category in og_x_vals] # convert to ints\n\n legend = None\n if attr_color is not None:\n attr_color_is_string = any([isinstance(val, str) for val in datadict[attr_color]])\n color_datavals = datadict[attr_color]\n if attr_color_is_string:\n attr_color_levels = list(set(color_datavals))\n colorpalette = Category20[20]\n color_mapper = CategoricalColorMapper(\n factors=attr_color_levels,\n palette=[colorpalette[2 * i % len(colorpalette)] for i in range(len(attr_color_levels))],\n )\n legend = attr_color\n else:\n color_mapper = LinearColorMapper(\n palette=\"Magma256\", low=min(datadict[attr_color]), high=max(datadict[attr_color]) * 1.25\n )\n default_color = {\"field\": attr_color, \"transform\": color_mapper}\n\n if attr_size is not None: # different size for each point, ensure mean-size == point_size\n attr_size2 = attr_size + \"____\"\n hidden_keys.append(attr_size2)\n og_sizevals = np.array(datadict[attr_size])\n sizevals = point_size + (og_sizevals - np.mean(og_sizevals)) / np.std(og_sizevals) * (point_size / 2)\n if np.min(sizevals) < 0:\n sizevals = -np.min(sizevals) + sizevals + 1.0\n datadict[attr_size2] = list(sizevals)\n point_size = attr_size2\n\n if save_file is not None:\n output_file(save_file, title=plot_title)\n print(\"Plot summary of models saved to file: %s\" % save_file)\n\n source = ColumnDataSource(datadict)\n TOOLS = \"crosshair,pan,wheel_zoom,box_zoom,reset,hover,save\"\n p = figure(title=plot_title, tools=TOOLS)\n if attr_x_is_string:\n circ = p.circle(\n attr_x2,\n attr_y,\n line_color=default_color,\n line_alpha=point_transparency,\n fill_color=default_color,\n fill_alpha=point_transparency,\n size=point_size,\n source=source,\n )\n else:\n circ = p.circle(\n attr_x,\n attr_y,\n line_color=default_color,\n line_alpha=point_transparency,\n fill_color=default_color,\n fill_alpha=point_transparency,\n size=point_size,\n source=source,\n )\n hover = p.select(dict(type=HoverTool))\n hover.tooltips = OrderedDict([(key, \"@\" + key + \"{safe}\") for key in datadict.keys() if key not in hidden_keys])\n # Format axes:\n p.xaxis.axis_label = attr_x\n p.yaxis.axis_label = attr_y\n if attr_x_is_string: # add x-ticks:\n p.xaxis.ticker = list(range(len(attr_x_levels)))\n p.xaxis.major_label_overrides = {i: attr_x_levels[i] for i in range(len(attr_x_levels))}\n\n # Legend additions:\n if attr_color is not None and attr_color_is_string:\n legend_it = []\n for i in range(len(attr_color_levels)):\n legend_it.append(\n LegendItem(\n label=attr_color_levels[i],\n renderers=[circ],\n index=datadict[attr_color].index(attr_color_levels[i]),\n )\n )\n legend = Legend(items=legend_it, location=(0, 0))\n p.add_layout(legend, \"right\")\n\n if attr_color is not None and not attr_color_is_string:\n color_bar = ColorBar(\n color_mapper=color_mapper, title=attr_color, label_standoff=12, border_line_color=None, location=(0, 0)\n )\n p.add_layout(color_bar, \"right\")\n\n if attr_size is not None:\n p.add_layout(Legend(items=[LegendItem(label='Size of points based on \"' + attr_size + '\"')]), \"below\")\n\n if show_plot:\n show(p)\n elif save_file is not None:\n save(p)\n" }, { "path": "core/src/autogluon/core/utils/savers/__init__.py", "content": "from autogluon.common.savers import save_json, save_pd, save_pkl, save_str\n" }, { "path": "core/src/autogluon/core/utils/time.py", "content": "import logging\nimport time\nfrom typing import Optional\n\nfrom pandas import DataFrame\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Consider replacing with timeit, but need to ensure repeat functionality plays nicely with time limit logic.\ndef time_func(f, args: list = None, kwargs: dict = None, time_limit: float = 0.2, max_repeats: int = 10) -> float:\n \"\"\"\n Returns the average time taken by `f(*args, **kwargs)`.\n\n Parameters\n ----------\n f : callable\n Function to be called.\n Must not alter outer context or otherwise could have negative side effects.\n args : list, default = None\n args to f\n kwargs : dict, default = None\n kwargs to f\n time_limit : float, default = 0.2\n Time limit in seconds to spend repeatedly calling `f`.\n Once time limit is exceeded, return current estimate.\n max_repeats : int, default = 10\n Maximum repeats of calling `f` to obtain a better average time estimate.\n\n Returns\n -------\n Average time taken in seconds by `f(*args, **kwargs)`.\n \"\"\"\n if args is None:\n args = list()\n if kwargs is None:\n kwargs = dict()\n time_start = time.time()\n f(*args, **kwargs)\n time_end = time.time()\n avg_time = time_end - time_start\n if max_repeats == 1:\n return avg_time\n if time_limit is None or avg_time < (time_limit / max_repeats):\n time_start_loop = time.time()\n for i in range(max_repeats - 1):\n f(*args, **kwargs)\n time_end_loop = time.time()\n total_time_loop = time_end_loop - time_start_loop\n avg_time = (avg_time + total_time_loop) / max_repeats\n return avg_time\n\n\ndef sample_df_for_time_func(df: DataFrame, sample_size: int, max_sample_size: Optional[int] = 10000) -> DataFrame:\n \"\"\"\n Samples df for the purposes of passing as an argument to `time_func`.\n A common use-case is for batch-inference speed calculations of batch = sample_size.\n The rows of data returned should not be important beyond that each row is a valid pre-existing row in df (which could appear multiple times).\n\n Parameters\n ----------\n df : DataFrame\n The DataFrame to sample from.\n sample_size : int\n The sample_size to try to return such that len(X_out) == sample_size.\n max_sample_size : int, default = 10000\n The sample_size to limit the result to if sample_size>max_sample_size and max_sample_size>len(df)\n This avoids df_out taking up large amounts of memory when it shouldn't be necessary for the purposes of timing.\n For example, avoids crash if user specifies sample_size=99999999.\n If len(df)>max_sample_size and sample_size>len(df), then df is returned unaltered.\n If None, then max_sample_size == sample_size.\n\n Returns\n -------\n df_out : DataFrame\n Sampled DataFrame. User can get the effective sample_size via len(df_out).\n \"\"\"\n len_df = len(df)\n if max_sample_size is None:\n max_sample_size = sample_size\n assert isinstance(sample_size, int), \"sample_size must be of type int\"\n assert isinstance(max_sample_size, int), \"max_sample_size must be of type int\"\n if sample_size < 1:\n raise AssertionError(f\"sample_size must be >=1, but was {sample_size}\")\n if max_sample_size < 1:\n raise AssertionError(f\"max_sample_size must be >=1, but was {max_sample_size}\")\n\n if len_df > sample_size:\n df_out = df.head(sample_size)\n elif len_df < sample_size and len_df < max_sample_size:\n sample_size = min(sample_size, max_sample_size) # No need to sample more than 10,000 generally\n df_out = df.sample(sample_size, replace=True, random_state=0, ignore_index=True)\n else:\n # Not enough rows of data to satisfy batch size, instead use all available\n df_out = df\n return df_out\n" }, { "path": "core/src/autogluon/core/utils/utils.py", "content": "from __future__ import annotations\n\nimport logging\nimport math\nimport pickle\nimport random\nimport sys\nimport time\nfrom typing import Callable, List, Tuple, Union\n\nimport numpy as np\nimport pandas as pd\nimport scipy.stats\nfrom numpy.typing import ArrayLike\nfrom pandas import DataFrame, Series\nfrom sklearn.model_selection import train_test_split\n\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom ..constants import (\n BINARY,\n LARGE_DATA_THRESHOLD,\n MULTICLASS,\n MULTICLASS_UPPER_LIMIT,\n QUANTILE,\n REGRESS_THRESHOLD_LARGE_DATA,\n REGRESS_THRESHOLD_SMALL_DATA,\n REGRESSION,\n SOFTCLASS,\n)\nfrom ..metrics import Scorer, accuracy, pinball_loss, root_mean_squared_error\n\nlogger = logging.getLogger(__name__)\n\n\ndef setup_compute(nthreads_per_trial, ngpus_per_trial):\n if nthreads_per_trial is None or nthreads_per_trial == \"all\":\n nthreads_per_trial = ResourceManager.get_cpu_count() # Use all of processing power / trial by default. To use just half: # int(np.floor(multiprocessing.cpu_count()/2))\n if ngpus_per_trial is None:\n ngpus_per_trial = 0 # do not use GPU by default\n elif ngpus_per_trial == \"all\":\n ngpus_per_trial = ResourceManager.get_gpu_count()\n if not isinstance(nthreads_per_trial, int) and nthreads_per_trial != \"auto\":\n raise ValueError(f'nthreads_per_trial must be an integer or \"auto\": nthreads_per_trial = {nthreads_per_trial}')\n if not isinstance(ngpus_per_trial, int) and ngpus_per_trial != \"auto\":\n raise ValueError(f'ngpus_per_trial must be an integer or \"auto\": ngpus_per_trial = {ngpus_per_trial}')\n return nthreads_per_trial, ngpus_per_trial\n\n\ndef setup_trial_limits(time_limit, num_trials, hyperparameters):\n \"\"\"Adjust default time limits / num_trials\"\"\"\n if num_trials is None:\n if time_limit is None:\n time_limit = 10 * 60 # run for 10min by default\n time_limit /= float(len(hyperparameters)) # each model type gets half the available time\n num_trials = 1000 # run up to 1000 trials (or as you can within the given time_limit)\n elif time_limit is None:\n time_limit = int(1e6) # user only specified num_trials, so run all of them regardless of time_limit\n else:\n time_limit /= float(len(hyperparameters)) # each model type gets half the available time\n\n if time_limit <= 10: # threshold = 10sec, ie. too little time to run >1 trial.\n num_trials = 1\n time_limit *= 0.9 # reduce slightly to account for extra time overhead\n return time_limit, num_trials\n\n\ndef get_leaderboard_pareto_frontier(\n leaderboard: DataFrame, score_col=\"score_val\", inference_time_col=\"pred_time_val_full\"\n) -> DataFrame:\n \"\"\"\n Given a set of models, returns in ranked order from best score to worst score models which satisfy the criteria:\n 1. No other model in the set has both a lower inference time and a better or equal score.\n\n :param leaderboard: Leaderboard DataFrame of model info containing score_col and inference_time_col\n :param score_col: Column name in leaderboard of model score values\n :param inference_time_col: Column name in leaderboard of model inference times\n :return: Subset of the original leaderboard DataFrame containing only models that are a valid optimal choice at different valuations of score and inference time.\n \"\"\"\n leaderboard = leaderboard.sort_values(by=[score_col, inference_time_col], ascending=[False, True]).reset_index(\n drop=True\n )\n leaderboard_unique = leaderboard.drop_duplicates(subset=[score_col])\n\n pareto_frontier = []\n inference_time_min = None\n for index, row in leaderboard_unique.iterrows():\n if row[inference_time_col] is None or row[score_col] is None:\n pass\n elif (inference_time_min is None) or (row[inference_time_col] < inference_time_min):\n inference_time_min = row[inference_time_col]\n pareto_frontier.append(index)\n leaderboard_pareto_frontier = leaderboard_unique.loc[pareto_frontier].reset_index(drop=True)\n return leaderboard_pareto_frontier\n\n\ndef shuffle_df_rows(X: DataFrame, seed=0, reset_index=True):\n \"\"\"Returns DataFrame with rows shuffled based on seed value.\"\"\"\n row_count = X.shape[0]\n np.random.seed(seed)\n rand_shuffle = np.random.randint(0, row_count, size=row_count)\n X_shuffled = X.iloc[rand_shuffle]\n if reset_index:\n X_shuffled.reset_index(inplace=True, drop=True)\n return X_shuffled\n\n\ndef normalize_binary_probas(y_predprob, eps):\n \"\"\"Remaps the predicted probabilities to open interval (0,1) while maintaining rank order\"\"\"\n (pmin, pmax) = (eps, 1 - eps) # predicted probs outside this range will be remapped into (0,1)\n which_toobig = y_predprob > pmax\n if np.sum(which_toobig) > 0: # remap overly large probs\n y_predprob = np.logical_not(which_toobig) * y_predprob + which_toobig * (\n 1 - (eps * np.exp(-(y_predprob - pmax)))\n )\n which_toosmall = y_predprob < pmin\n if np.sum(which_toosmall) > 0: # remap overly small probs\n y_predprob = np.logical_not(which_toosmall) * y_predprob + which_toosmall * eps * np.exp(-(pmin - y_predprob))\n return y_predprob\n\n\ndef normalize_multi_probas(y_predprob, eps):\n \"\"\"Remaps the predicted probabilities to lie in (0,1) where eps controls how far from 0 smallest class-probability lies\"\"\"\n min_predprob = np.min(y_predprob)\n if min_predprob < 0: # ensure nonnegative rows\n most_negative_rowvals = np.clip(np.min(y_predprob, axis=1), a_min=None, a_max=0)\n y_predprob = y_predprob - most_negative_rowvals[:, None]\n if min_predprob < eps:\n y_predprob = np.clip(y_predprob, a_min=eps, a_max=None) # ensure no entries < eps\n y_predprob = y_predprob / y_predprob.sum(axis=1, keepdims=1) # renormalize\n return y_predprob\n\n\ndef default_holdout_frac(num_train_rows, hyperparameter_tune: bool = False):\n \"\"\"Returns default holdout_frac used in fit().\n Between row count 5,000 and 25,000 keep 0.1 holdout_frac, as we want to grow validation set to a stable 2500 examples.\n \"\"\"\n if num_train_rows < 5000:\n holdout_frac = max(0.1, min(0.2, 500.0 / num_train_rows))\n else:\n holdout_frac = max(0.01, min(0.1, 2500.0 / num_train_rows))\n\n if hyperparameter_tune:\n holdout_frac = min(\n 0.2, holdout_frac * 2\n ) # We want to allocate more validation data for HPO to avoid overfitting\n\n return holdout_frac\n\n\ndef augment_rare_classes(X, label, threshold):\n \"\"\"Use this method when using certain eval_metrics like log_loss, for which no classes may be filtered out.\n This method will augment dataset with additional examples of rare classes.\n \"\"\"\n class_counts = X[label].value_counts()\n class_counts_invalid = class_counts[class_counts < threshold]\n if len(class_counts_invalid) == 0:\n logger.debug(\"augment_rare_classes did not need to duplicate any data from rare classes\")\n return X\n\n missing_classes = []\n for clss, n_clss in class_counts_invalid.items():\n if n_clss == 0:\n missing_classes.append(clss)\n if missing_classes:\n logger.warning(\n f\"WARNING: {len(missing_classes)} classes were found that have 0 training examples, \"\n f\"and may lead to downstream issues. \"\n f\"Consider either providing data for these classes or removing them from the class categories. \"\n f\"These classes will be ignored: {missing_classes}\"\n )\n class_counts_invalid = class_counts_invalid[~class_counts_invalid.index.isin(set(missing_classes))]\n\n if len(class_counts_invalid) == 0:\n # This avoids crash when the only invalid classes were those that appeared 0 times\n return X\n\n dfs_to_add = []\n for clss, n_clss in class_counts_invalid.items():\n n_toadd = threshold - n_clss\n clss_df = X.loc[X[label] == clss]\n duplicate_times = int(np.floor(n_toadd / n_clss))\n remainder = n_toadd % n_clss\n\n if duplicate_times > 0:\n logger.debug(f\"Duplicating data from rare class: {clss}\")\n dfs_to_add.extend([clss_df] * duplicate_times)\n if remainder > 0:\n dfs_to_add.append(clss_df.iloc[:remainder])\n\n if not dfs_to_add:\n return X\n\n aug_df = pd.concat(dfs_to_add, axis=0)\n\n # Ensure new samples generated via augmentation have unique indices\n aug_df = aug_df.reset_index(drop=True)\n aug_df_len = len(aug_df)\n X_index_aug_start = X.index.max() + 1\n aug_index = [X_index_aug_start + i for i in range(aug_df_len)]\n aug_df.index = aug_index\n\n logger.log(\n 20,\n f\"Duplicated {len(aug_df)} samples from {len(class_counts_invalid)} rare classes in training set because eval_metric requires all classes have at least {threshold} samples.\",\n )\n\n X = pd.concat([X, aug_df], axis=0)\n class_counts = X[label].value_counts()\n class_counts_invalid = class_counts[class_counts < threshold]\n class_counts_invalid = class_counts_invalid[~class_counts_invalid.index.isin(set(missing_classes))]\n if len(class_counts_invalid) > 0:\n raise RuntimeError(\"augment_rare_classes failed to produce enough data from rare classes\")\n return X\n\n\ndef get_pred_from_proba_df(\n y_pred_proba: pd.DataFrame, problem_type: str = BINARY, decision_threshold: float = None\n) -> pd.Series:\n \"\"\"\n From input DataFrame of pred_proba, return Series of pred.\n The input DataFrame's columns must be the names of the target classes.\n \"\"\"\n if problem_type == REGRESSION:\n y_pred = y_pred_proba\n elif problem_type == QUANTILE:\n y_pred = y_pred_proba\n elif problem_type == BINARY and decision_threshold is not None:\n negative_class, positive_class = y_pred_proba.columns\n y_pred = get_pred_from_proba(\n y_pred_proba=y_pred_proba.values, problem_type=problem_type, decision_threshold=decision_threshold\n )\n y_pred = [positive_class if pred == 1 else negative_class for pred in y_pred]\n y_pred = pd.Series(data=y_pred, index=y_pred_proba.index)\n else:\n y_pred = y_pred_proba.idxmax(axis=1)\n return y_pred\n\n\ndef get_pred_from_proba(\n y_pred_proba: np.ndarray, problem_type: str = BINARY, decision_threshold: float = None\n) -> np.ndarray:\n if problem_type == BINARY:\n if decision_threshold is None:\n decision_threshold = 0.5\n # Using > instead of >= to align with Pandas `.idxmax` logic which picks the left-most column during ties.\n # If this is not done, then predictions can be inconsistent when converting in binary classification from multiclass-form pred_proba and\n # binary-form pred_proba when the pred_proba is 0.5 for positive and negative classes.\n if len(y_pred_proba.shape) == 2:\n assert y_pred_proba.shape[1] == 2\n # Assume positive class is in 2nd position\n y_pred = (y_pred_proba[:, 1] > decision_threshold).astype(int)\n else:\n y_pred = (y_pred_proba > decision_threshold).astype(int)\n elif problem_type == REGRESSION:\n y_pred = y_pred_proba\n elif problem_type == QUANTILE:\n y_pred = y_pred_proba\n else:\n y_pred = []\n if not len(y_pred_proba) == 0:\n y_pred = np.argmax(y_pred_proba, axis=1)\n return y_pred\n\n\ndef convert_pred_probas_to_df(\n pred_proba_list: List[ArrayLike], columns: List[str], problem_type: str, index: pd.Index = None\n) -> DataFrame:\n \"\"\"\n Converts a list of pred_proba model outputs to a DataFrame\n\n Parameters\n ----------\n pred_proba_list : List[ArrayLike]\n A list of prediction probabilities.\n Each element is a numpy array or ndarray of prediction probabilities for a given model.\n columns : List[str]\n The column names for the output DataFrame\n problem_type : str\n The problem type. This informs the way in which the DataFrame is constructed.\n index : pd.Index, default = None\n If specified, sets the output DataFrame's index.\n\n Returns\n -------\n DataFrame\n \"\"\"\n if problem_type in [MULTICLASS, SOFTCLASS, QUANTILE]:\n pred_proba_list = np.concatenate(pred_proba_list, axis=1)\n pred_proba_df = pd.DataFrame(data=pred_proba_list, columns=columns)\n else:\n pred_proba_df = pd.DataFrame(data=np.asarray(pred_proba_list).T, columns=columns)\n if index is not None:\n pred_proba_df.set_index(index, inplace=True)\n return pred_proba_df\n\n\ndef extract_label(data: DataFrame, label: str) -> (DataFrame, Series):\n \"\"\"\n Extract the label column from a dataset and return X, y.\n\n Parameters\n ----------\n data : DataFrame\n The data containing features and the label column.\n label : str\n The label column name.\n\n Returns\n -------\n X, y : (DataFrame, Series)\n X is the data with the label column dropped.\n y is the label column as a pd.Series.\n \"\"\"\n if label not in list(data.columns):\n raise ValueError(f\"Provided DataFrame does not contain label column: {label}\")\n y = data[label].copy()\n X = data.drop(label, axis=1)\n return X, y\n\n\ndef generate_train_test_split_combined(\n data: DataFrame,\n label: str,\n problem_type: str = None,\n test_size: int | float = None,\n train_size: int | float = None,\n random_state: int = 0,\n stratify: bool | Series = None,\n min_cls_count_train: int = 1,\n) -> Tuple[DataFrame, DataFrame]:\n \"\"\"\n Generate a train test split from a DataFrame that contains the label column.\n\n Parameters\n ----------\n data : DataFrame\n DataFrame containing the features plus the label column to split into train and test sets.\n label : str\n The label column name.\n Used for stratification and to ensure all classes in multiclass classification are preserved in train data.\n problem_type : str, default = None\n The problem_type the label is used for. Determines if stratification is used.\n If \"binary\" or \"multiclass\", enables the `min_cls_count_train` logic.\n Options: [\"binary\", \"multiclass\", \"regression\", \"softclass\", \"quantile\"]\n stratify : bool | Series, default = None\n The stratification strategy.\n If True, will stratify using `y`.\n If False, will not use stratification.\n If None and problem_type is specified, will be set to True or False depending on the problem_type.\n True if problem_type in [\"binary\", \"multiclass\"], else False.\n If None and problem_type is None, defaults to False.\n test_size : int | float, default = None\n If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the test split.\n If int, represents the absolute number of test samples.\n If test_size is None and train_size is None, test_size defaults to 0.1\n train_size : int | float, default = None\n If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the train split.\n If int, represents the absolute number of train samples.\n If test_size is None, represents the complement of `test_size`.\n For example, `train_size=0.7, test_size=None` is equivalent to `train_size=None, test_size=0.3`.\n Note: It is possible for exceptions to be raised if both train_size and test_size are specified, stratification is enabled, and rare classes exist.\n random_state : int, default = 0\n Random seed to use during the split.\n min_cls_count_train : int, default = 1\n The minimum number of instances of each class that must occur in the training set (for classification).\n If not satisfied by the original split, instances of unsatisfied classes are\n taken from test and put into train until satisfied.\n Raises an exception if impossible to satisfy.\n\n Returns\n -------\n train_data, test_data : (DataFrame, DataFrame)\n The train_data and test_data after performing the split. Includes the label column.\n \"\"\"\n X, y = extract_label(data=data, label=label)\n train_data, test_data, y_train, y_test = generate_train_test_split(\n X=X,\n y=y,\n problem_type=problem_type,\n test_size=test_size,\n train_size=train_size,\n random_state=random_state,\n stratify=stratify,\n min_cls_count_train=min_cls_count_train,\n )\n train_data[label] = y_train\n test_data[label] = y_test\n return train_data, test_data\n\n\ndef generate_train_test_split(\n X: DataFrame,\n y: Series,\n problem_type: str = None,\n test_size: int | float = None,\n train_size: int | float = None,\n random_state: int = 0,\n stratify: bool | Series = None,\n min_cls_count_train: int = 1,\n) -> Tuple[DataFrame, DataFrame, Series, Series]:\n \"\"\"\n Generate a train test split from input X, y.\n If you have a combined X, y DataFrame, refer to `generate_train_test_split_combined` instead.\n\n Parameters\n ----------\n X : DataFrame\n pd.DataFrame containing the features minus the label column to split into train and test sets.\n y : Series\n pd.Series containing the label with matching indices to X.\n Used for stratification and to ensure all classes in multiclass classification are preserved in train data.\n problem_type : str, default = None\n The problem_type the label is used for. Determines if stratification is used.\n If \"binary\" or \"multiclass\", enables the `min_cls_count_train` logic.\n Options: [\"binary\", \"multiclass\", \"regression\", \"softclass\", \"quantile\"]\n stratify : bool | Series, default = None\n The stratification strategy.\n If True, will stratify using `y`.\n If False, will not use stratification.\n If None and problem_type is specified, will be set to True or False depending on the problem_type.\n True if problem_type in [\"binary\", \"multiclass\"], else False.\n If None and problem_type is None, defaults to False.\n test_size : int | float, default = None\n If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the test split.\n If int, represents the absolute number of test samples.\n If test_size is None and train_size is None, test_size defaults to 0.1\n train_size : int | float, default = None\n If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to include in the train split.\n If int, represents the absolute number of train samples.\n If test_size is None, represents the complement of `test_size`.\n For example, `train_size=0.7, test_size=None` is equivalent to `train_size=None, test_size=0.3`.\n Note: It is possible for exceptions to be raised if both train_size and test_size are specified, stratification is enabled, and rare classes exist.\n random_state : int, default = 0\n Random seed to use during the split.\n min_cls_count_train : int, default = 1\n The minimum number of instances of each class that must occur in the training set (for classification).\n If not satisfied by the original split, instances of unsatisfied classes are\n taken from test and put into train until satisfied.\n Raises an exception if impossible to satisfy.\n\n Returns\n -------\n X_train, X_test, y_train, y_test : (DataFrame, DataFrame, Series, Series)\n The train_data and test_data after performing the split, separated into X and y.\n\n \"\"\"\n if len(X) == 1:\n raise ValueError(f\"Cannot split data into train/val as it contains only one sample.\")\n if test_size is None and train_size is None:\n test_size = 0.1\n if train_size is not None:\n if isinstance(train_size, float):\n if (train_size <= 0.0) or (train_size >= 1.0):\n raise ValueError(f\"train_size fraction must be specified between 0 and 1. Value: {train_size}\")\n elif isinstance(train_size, int):\n assert train_size > 0\n else:\n raise TypeError(\n f\"train_size was specified, but is not of type int or float! Type: {type(train_size)}, Value: {train_size}\"\n )\n if train_size is not None and test_size is None:\n # Set train_size to None to avoid edge-case exceptions with rare classes during stratification\n if isinstance(train_size, float):\n test_size = (\n 1.0 - train_size - 1e-10\n ) # -1e-10 to fix numerical imprecision issues, ensuring `test_size=0.1` gets same result as `train_size=0.9`\n else:\n test_size = len(X) - train_size\n train_size = None\n if test_size is not None:\n if isinstance(test_size, float):\n if (test_size <= 0.0) or (test_size >= 1.0):\n raise ValueError(f\"test_size fraction must be specified between 0 and 1. Value: {test_size}\")\n elif isinstance(test_size, int):\n assert test_size > 0\n else:\n raise TypeError(\n f\"test_size was specified, but is not of type int or float! Type: {type(test_size)}, Value: {test_size}\"\n )\n if problem_type is not None:\n valid_problem_types = [BINARY, MULTICLASS, REGRESSION, SOFTCLASS, QUANTILE]\n assert problem_type in valid_problem_types, (\n f'Unknown problem type \"{problem_type}\" | Valid problem types: {valid_problem_types}'\n )\n\n X_split = X\n y_split = y\n if stratify is None:\n if problem_type is not None and problem_type in [BINARY, MULTICLASS]:\n stratify = y\n elif isinstance(stratify, bool):\n stratify = y if stratify else None\n\n # This code block is necessary to avoid crashing when performing a stratified split and only 1 sample exists for a class.\n # This code will ensure that the sample will be part of the train split, meaning the test split will have 0 samples of the rare class.\n rare_indices = None\n if stratify is not None:\n cls_counts = y.value_counts()\n cls_counts = cls_counts[cls_counts > 0] # > 0 to avoid error if missing class when dtype is categorical.\n cls_counts_invalid = cls_counts[cls_counts < min_cls_count_train]\n\n if len(cls_counts_invalid) > 0:\n logger.error(\n f\"ERROR: Classes have too few samples to split the data! At least {min_cls_count_train} are required.\"\n )\n logger.error(cls_counts_invalid)\n raise AssertionError(\"Not enough data to split data into train and val without dropping classes!\")\n elif min_cls_count_train < 2:\n cls_counts_rare = cls_counts[cls_counts < 2]\n if len(cls_counts_rare) > 0:\n cls_counts_rare_val = set(cls_counts_rare.index)\n y_rare = y[y.isin(cls_counts_rare_val)]\n rare_indices = list(y_rare.index)\n X_split = X.drop(index=rare_indices)\n y_split = y.drop(index=rare_indices)\n stratify = y_split\n\n try:\n X_train, X_test, y_train, y_test = train_test_split(\n X_split,\n y_split.values,\n test_size=test_size,\n train_size=train_size,\n shuffle=True,\n random_state=random_state,\n stratify=stratify,\n )\n except ValueError:\n # This logic is necessary to avoid an edge-case limitation of scikit-learn's train_test_split function that leads to the following error:\n # ValueError: The test_size = FOO should be greater or equal to the number of classes = BAR\n # When the number of classes is greater than the resulting number of test rows, and stratification is enabled, it will raise an exception.\n # Logically the code should still work, but for some reason scikit-learn doesn't allow this scenario.\n # To handle it without erroring, we disable stratification in this case. This isn't ideal, but proper solutions involve patching scikit-learn.\n if stratify is None:\n raise\n X_train, X_test, y_train, y_test = train_test_split(\n X_split,\n y_split.values,\n test_size=test_size,\n train_size=train_size,\n shuffle=True,\n random_state=random_state,\n stratify=None,\n )\n if len(y_test) >= len(y_split.unique()):\n # This should never occur, otherwise the original exception is not an expected one\n raise\n cls_y = type(y)\n y_train = cls_y(y_train, index=X_train.index)\n y_test = cls_y(y_test, index=X_test.index)\n\n if rare_indices:\n X_train = pd.concat([X_train, X.loc[rare_indices]])\n y_train = pd.concat([y_train, y.loc[rare_indices]])\n\n if problem_type is not None and problem_type in [BINARY, MULTICLASS]:\n class_counts_dict_orig = y.value_counts()\n class_counts_dict_orig = class_counts_dict_orig[class_counts_dict_orig > 0]\n class_counts_dict_orig = class_counts_dict_orig.to_dict()\n class_counts_dict = y_train.value_counts().to_dict()\n class_counts_dict_test = y_test.value_counts().to_dict()\n\n indices_to_move = []\n random_state_init = random.getstate()\n random.seed(random_state)\n for cls in class_counts_dict_orig.keys():\n count = class_counts_dict.get(cls, 0)\n if count >= min_cls_count_train:\n continue\n count_test = class_counts_dict_test.get(cls, 0)\n if count + count_test < min_cls_count_train:\n raise AssertionError(\"Not enough data to split data into train and val without dropping classes!\")\n count_to_move = min_cls_count_train - count\n indices_of_cls_test = list(y_test[y_test == cls].index)\n indices_to_move_cls = random.sample(indices_of_cls_test, count_to_move)\n indices_to_move += indices_to_move_cls\n random.setstate(random_state_init)\n if indices_to_move:\n y_test_moved = y_test.loc[indices_to_move].copy()\n X_test_moved = X_test.loc[indices_to_move].copy()\n y_train = pd.concat([y_train, y_test_moved])\n X_train = pd.concat([X_train, X_test_moved])\n y_test = y_test.drop(index=indices_to_move)\n X_test = X_test.drop(index=indices_to_move)\n y_train.name = y_split.name\n y_test.name = y_split.name\n return X_train, X_test, y_train, y_test\n\n\ndef normalize_pred_probas(y_predprob, problem_type, eps=1e-7):\n \"\"\"Remaps the predicted probabilities to ensure there are no zeros (needed for certain metrics like log-loss)\n and that no predicted probability exceeds [0,1] (eg. in distillation when classification is treated as regression).\n Args:\n y_predprob: 1D (for binary classification) or 2D (for multiclass) numpy array of predicted probabilities\n problem_type: We only consider normalization if the problem_type is one of: [BINARY, MULTICLASS, SOFTCLASS]\n eps: controls around how far from 0 remapped predicted probabilities should be (larger `eps` means predicted probabilities will lie further from 0).\n \"\"\"\n if (problem_type == REGRESSION) and (len(y_predprob.shape) > 1) and (y_predprob.shape[1] > 1):\n problem_type = SOFTCLASS # this was MULTICLASS problem converted to REGRESSION (as done in distillation)\n\n if problem_type in [BINARY, REGRESSION]:\n if len(y_predprob.shape) > 1 and min(y_predprob.shape) > 1:\n raise ValueError(\n f\"cannot call normalize_pred_probas with problem_type={problem_type} and y_predprob.shape=={y_predprob.shape}\"\n )\n return normalize_binary_probas(y_predprob, eps)\n elif problem_type in [MULTICLASS, SOFTCLASS]: # clip all probs below at eps and then renormalize\n if len(y_predprob.shape) == 1:\n return normalize_binary_probas(y_predprob, eps)\n else:\n return normalize_multi_probas(y_predprob, eps)\n else:\n raise ValueError(f\"Invalid problem_type\")\n\n\ndef infer_problem_type(y: Series, silent=False) -> str:\n \"\"\"Identifies which type of prediction problem we are interested in (if user has not specified).\n Ie. binary classification, multi-class classification, or regression.\n \"\"\"\n # treat None, NaN, INF, NINF as NA\n y = y.replace([np.inf, -np.inf], np.nan, inplace=False)\n y = y.dropna()\n num_rows = len(y)\n\n if num_rows == 0:\n raise ValueError(\"Label column cannot have 0 valid values\")\n\n unique_values = y.unique()\n\n if num_rows > LARGE_DATA_THRESHOLD:\n regression_threshold = REGRESS_THRESHOLD_LARGE_DATA # if the unique-ratio is less than this, we assume multiclass classification, even when labels are integers\n else:\n regression_threshold = REGRESS_THRESHOLD_SMALL_DATA\n\n unique_count = len(unique_values)\n if unique_count == 2:\n problem_type = BINARY\n reason = \"only two unique label-values observed\"\n elif y.dtype.name in [\"object\", \"category\", \"string\"]:\n problem_type = MULTICLASS\n reason = f\"dtype of label-column == {y.dtype.name}\"\n elif np.issubdtype(y.dtype, np.floating):\n unique_ratio = unique_count / float(num_rows)\n if (unique_ratio <= regression_threshold) and (unique_count <= MULTICLASS_UPPER_LIMIT):\n try:\n can_convert_to_int = np.array_equal(y, y.astype(int))\n if can_convert_to_int:\n problem_type = MULTICLASS\n reason = \"dtype of label-column == float, but few unique label-values observed and label-values can be converted to int\"\n else:\n problem_type = REGRESSION\n reason = \"dtype of label-column == float and label-values can't be converted to int\"\n except:\n problem_type = REGRESSION\n reason = \"dtype of label-column == float and label-values can't be converted to int\"\n else:\n problem_type = REGRESSION\n reason = \"dtype of label-column == float and many unique label-values observed\"\n elif np.issubdtype(y.dtype, np.integer):\n unique_ratio = unique_count / float(num_rows)\n if (unique_ratio <= regression_threshold) and (unique_count <= MULTICLASS_UPPER_LIMIT):\n problem_type = MULTICLASS # TODO: Check if integers are from 0 to n-1 for n unique values, if they have a wide spread, it could still be regression\n reason = \"dtype of label-column == int, but few unique label-values observed\"\n else:\n problem_type = REGRESSION\n reason = \"dtype of label-column == int and many unique label-values observed\"\n else:\n raise NotImplementedError(f\"label dtype {y.dtype} not supported!\")\n if not silent:\n logger.log(25, f\"AutoGluon infers your prediction problem is: '{problem_type}' (because {reason}).\")\n\n # TODO: Move this outside of this function so it is visible even if problem type was not inferred.\n if problem_type in [BINARY, MULTICLASS]:\n if unique_count > 10:\n logger.log(20, f\"\\tFirst 10 (of {unique_count}) unique label values: {list(unique_values[:10])}\")\n else:\n logger.log(20, f\"\\t{unique_count} unique label values: {list(unique_values)}\")\n elif problem_type == REGRESSION:\n y_max = y.max()\n y_min = y.min()\n y_mean = y.mean()\n y_stddev = y.std()\n logger.log(\n 20,\n f\"\\tLabel info (max, min, mean, stddev): ({y_max}, {y_min}, {round(y_mean, 5)}, {round(y_stddev, 5)})\",\n )\n\n logger.log(\n 25,\n f\"\\tIf '{problem_type}' is not the correct problem_type, please manually specify the problem_type parameter during Predictor init \"\n f\"(You may specify problem_type as one of: {[BINARY, MULTICLASS, REGRESSION, QUANTILE]})\",\n )\n return problem_type\n\n\ndef infer_eval_metric(problem_type: str) -> Scorer:\n \"\"\"Infers appropriate default eval metric based on problem_type. Useful when no eval_metric was provided.\"\"\"\n if problem_type == BINARY:\n return accuracy\n elif problem_type == MULTICLASS:\n return accuracy\n elif problem_type == QUANTILE:\n return pinball_loss\n else:\n return root_mean_squared_error\n\n\ndef extract_column(X, col_name):\n \"\"\"Extract specified column from dataframe.\"\"\"\n if col_name is None or col_name not in list(X.columns):\n return X, None\n w = X[col_name].copy()\n X = X.drop(col_name, axis=1)\n return X, w\n\n\n# TODO: v1.4: Remove this\ndef compute_weighted_metric(\n y: np.ndarray, y_pred: np.ndarray, metric: Scorer, weights: np.ndarray, weight_evaluation: bool = None, **kwargs\n) -> float:\n \"\"\"Report weighted metric if: weights is not None, weight_evaluation=True, and the given metric supports sample weights.\n If weight_evaluation=None, it will be set to False if weights=None, True otherwise.\n \"\"\"\n logger.log(\n 30,\n f\"WARNING: `compute_weighted_metric` is deprecated as of AutoGluon 1.2 and will be removed in AutoGluon 1.4. \"\n f\"Please use `autogluon.core.metrics.compute_metric` instead.\",\n )\n if not metric.needs_quantile:\n kwargs.pop(\"quantile_levels\", None)\n if weight_evaluation is None:\n weight_evaluation = not (weights is None)\n if weight_evaluation and weights is None:\n raise ValueError(\"Sample weights cannot be None when weight_evaluation=True.\")\n if not weight_evaluation:\n return metric(y, y_pred, **kwargs)\n try:\n weighted_metric = metric(y, y_pred, sample_weight=weights, **kwargs)\n except (ValueError, TypeError, KeyError):\n if hasattr(metric, \"name\"):\n metric_name = metric.name\n else:\n metric_name = metric\n logger.log(\n 30,\n f\"WARNING: eval_metric='{metric_name}' does not support sample weights so they will be ignored in reported metric.\",\n )\n weighted_metric = metric(y, y_pred, **kwargs)\n return weighted_metric\n\n\n# Note: Do not send training data as input or the importances will be overfit.\n# TODO: Improve time estimate (Currently pessimistic)\ndef compute_permutation_feature_importance(\n X: pd.DataFrame,\n y: pd.Series,\n predict_func: Callable[..., np.ndarray],\n eval_metric: Scorer,\n features: list = None,\n subsample_size=None,\n num_shuffle_sets: int = None,\n predict_func_kwargs: dict = None,\n transform_func: Callable[..., pd.DataFrame] = None,\n transform_func_kwargs: dict = None,\n time_limit: float = None,\n silent=False,\n log_prefix=\"\",\n importance_as_list=False,\n random_state=0,\n **kwargs,\n) -> pd.DataFrame:\n \"\"\"\n Computes a trained model's feature importance via permutation shuffling (https://explained.ai/rf-importance/).\n A feature's importance score represents the performance drop that results when the model makes predictions on a perturbed copy of the dataset where this feature's values have been randomly shuffled across rows.\n A feature score of 0.01 would indicate that the predictive performance dropped by 0.01 when the feature was randomly shuffled.\n The higher the score a feature has, the more important it is to the model's performance.\n If a feature has a negative score, this means that the feature is likely harmful to the final model, and a model trained with the feature removed would be expected to achieve a better predictive performance.\n Note that calculating feature importance can be a very computationally expensive process, particularly if the model uses hundreds or thousands of features. In many cases, this can take longer than the original model training.\n\n Note: For highly accurate stddev and z_score estimates, it is recommend to set `subsample_size` to at least 5,000 if possible and `num_shuffle_sets` to at least 10.\n\n Parameters\n ----------\n X : pd.DataFrame\n Validation data to permute when calculating feature importances.\n Do not use training data as it will result in overfit feature importances.\n y : pd.Series\n Label values of X. The index of X and y must align.\n predict_func : Callable[..., np.ndarray]\n Function that computes model predictions or prediction probabilities on input data.\n Output must be in the form of a numpy ndarray or pandas Series or DataFrame.\n Output `y_pred` must be in a form acceptable as input to `eval_metric(y, y_pred)`.\n If using a fit model object, this is typically `model.predict` or `model.predict_proba`, depending on the `eval_metric` being used.\n If `eval_metric.needs_pred==True`, use `model.predict`, otherwise use `model.predict_proba`.\n eval_metric : Scorer\n Object that computes a score given ground truth labels and predictions or prediction probabilities (depending on the type of metric).\n If using a fit model object, this is typically `model.eval_metric`.\n Feature importances will be based on the delta permutation shuffling has on the score produced by `eval_metric`.\n features : list, default None\n List of features to calculate importances for.\n If None, all features' importances will be calculated.\n Can contain tuples as elements of (feature_name, feature_list) form.\n feature_name can be any string so long as it is unique with all other feature names / features in the list.\n feature_list can be any list of valid features in the data.\n This will compute importance of the combination of features in feature_list, naming the set of features in the returned DataFrame feature_name.\n This importance will differ from adding the individual importances of each feature in feature_list, and will be more accurate to the overall group importance.\n Example: ['featA', 'featB', 'featC', ('featBC', ['featB', 'featC'])]\n In this example, the importance of 'featBC' will be calculated by jointly permuting 'featB' and 'featC' together as if they were a single two-dimensional feature.\n subsample_size : int, default None\n The amount of data rows to sample when computing importances.\n Higher values will improve the quality of feature importance estimates, but linearly increase the runtime.\n If None, all provided data will be used.\n num_shuffle_sets : int, default None\n The number of different permutation shuffles of the data that are evaluated.\n Shuffle sets are generated with different random seeds and importances are averaged across all shuffle sets to get the final feature importance values.\n Higher values will improve the quality of feature importance estimates, but linearly increase the runtime.\n `subsample_size` should be increased before `num_shuffle_sets` if runtime is a concern.\n Defaults to 1 if `time_limit` is None or 10 if `time_limit` is specified.\n When `num_shuffle_sets` is greater than 1, feature importance standard deviation and z-score will additionally be computed by using the results of each shuffle set as samples.\n predict_func_kwargs : dict, default {}\n Keyword arguments to be appended to calls to `predict_func(X, **kwargs)`.\n transform_func : Callable[..., pd.DataFrame], default None\n Transformation function that takes the raw input and transforms it row-wise to the input expected by `predict_func`.\n Common examples include `model.preprocess` and `feature_generator.transform`.\n If None, then no transformation is done on the data prior to calling `predict_func`.\n This is necessary to compute importance of original data features in `X` prior to their transformation assuming `predict_func` does not perform the transformation already.\n Example: `transform_func` is necessary to compute the importance of a text feature prior to being transformed into ngrams by `transform_func` when `predict_func` expects ngram features as input.\n transform_func_kwargs : dict, default {}\n Keyword arguments to be appended to calls to `transform_func(X, **kwargs)`.\n time_limit : float, default None\n Time in seconds to limit the calculation of feature importance.\n If None, feature importance will calculate without early stopping.\n A minimum of 1 full shuffle set will always be evaluated. If a shuffle set evaluation takes longer than `time_limit`, the method will take the length of a shuffle set evaluation to return regardless of the `time_limit`.\n If `num_shuffle_sets==1`, `time_limit` will be ignored.\n silent : bool, default False\n Whether to suppress logging output.\n log_prefix : str, default ''\n Prefix to add to logging statements.\n importance_as_list : bool, default False\n Whether to return the 'importance' column values as a list of the importance from each shuffle (True) or a single averaged value (False).\n random_state : int, default 0\n Acts as a seed for data subsampling and permuting feature values.\n\n Returns\n -------\n Pandas `pandas.DataFrame` of feature importance scores with 4 columns:\n index: The feature name.\n 'importance': The estimated feature importance score.\n 'stddev': The standard deviation of the feature importance score. If NaN, then not enough num_shuffle_sets were used to calculate a variance.\n 'p_value': P-value for a statistical t-test of the null hypothesis: importance = 0, vs the (one-sided) alternative: importance > 0.\n Features with low p-value appear confidently useful to the predictor, while the other features may be useless to the predictor (or even harmful to include in its training data).\n A p-value of 0.01 indicates that there is a 1% chance that the feature is useless or harmful, and a 99% chance that the feature is useful.\n A p-value of 0.99 indicates that there is a 99% chance that the feature is useless or harmful, and a 1% chance that the feature is useful.\n 'n': The number of shuffles performed to estimate importance score (corresponds to sample-size used to determine confidence interval for true score).\n \"\"\"\n if not eval_metric.needs_quantile:\n kwargs.pop(\"quantile_levels\", None)\n\n if num_shuffle_sets is None:\n num_shuffle_sets = 1 if time_limit is None else 10\n\n time_start = time.time()\n if predict_func_kwargs is None:\n predict_func_kwargs = dict()\n if transform_func_kwargs is None:\n transform_func_kwargs = dict()\n if features is None:\n features = list(X.columns)\n\n _validate_features(features=features, valid_features=list(X.columns))\n\n num_features = len(features)\n\n if subsample_size is not None:\n num_rows = min(len(X), subsample_size)\n else:\n num_rows = len(X)\n subsample = num_rows < len(X)\n\n if not silent:\n logging_message = f\"{log_prefix}Computing feature importance via permutation shuffling for {num_features} features using {num_rows} rows with {num_shuffle_sets} shuffle sets...\"\n if time_limit is not None:\n logging_message = f\"{logging_message} Time limit: {time_limit}s...\"\n logger.log(20, logging_message)\n\n time_permutation_start = time.time()\n fi_dict_list = []\n shuffle_repeats_completed = 0\n log_final_suffix = \"\"\n\n X_orig = X\n y_orig = y\n feature_batch_count = None\n X_raw = None\n score_baseline = None\n initial_random_state = random_state\n # TODO: Can speedup shuffle_repeats by incorporating into X_raw (do multiple repeats in a single predict call)\n for shuffle_repeat in range(num_shuffle_sets):\n fi = dict()\n random_state = initial_random_state + shuffle_repeat\n\n if subsample:\n # TODO: Stratify? We currently don't know in this function the problem_type (could pass as additional arg).\n X = X_orig.sample(subsample_size, random_state=random_state)\n y = y_orig.loc[X.index]\n\n if subsample or shuffle_repeat == 0:\n time_start_score = time.time()\n X_transformed = X if transform_func is None else transform_func(X, **transform_func_kwargs)\n y_pred = predict_func(X_transformed, **predict_func_kwargs)\n score_baseline = eval_metric(y, y_pred, **kwargs)\n if shuffle_repeat == 0:\n if not silent:\n time_score = time.time() - time_start_score\n time_estimated = (\n ((num_features + 1) * time_score) * num_shuffle_sets + time_start_score - time_start\n )\n time_estimated_per_set = time_estimated / num_shuffle_sets\n logger.log(\n 20,\n f\"{log_prefix}\\t{round(time_estimated, 2)}s\\t= Expected runtime ({round(time_estimated_per_set, 2)}s per shuffle set)\",\n )\n\n if transform_func is None:\n feature_batch_count = _get_safe_fi_batch_count(X=X, num_features=num_features)\n else:\n feature_batch_count = _get_safe_fi_batch_count(\n X=X, num_features=num_features, X_transformed=X_transformed\n )\n\n # creating copy of original data N=feature_batch_count times for parallel processing\n X_raw = pd.concat(\n [X.copy() for _ in range(feature_batch_count)], ignore_index=True, sort=False\n ).reset_index(drop=True)\n\n row_count = len(X)\n\n X_shuffled = shuffle_df_rows(X=X, seed=random_state)\n\n for i in range(0, num_features, feature_batch_count):\n parallel_computed_features = features[i : i + feature_batch_count]\n\n # if final iteration, leaving only necessary part of X_raw\n num_features_processing = len(parallel_computed_features)\n final_iteration = i + num_features_processing == num_features\n\n row_index = 0\n for feature in parallel_computed_features:\n if isinstance(feature, tuple):\n feature = feature[1]\n row_index_end = row_index + row_count\n X_raw.loc[row_index : row_index_end - 1, feature] = X_shuffled[feature].values\n row_index = row_index_end\n\n if (num_features_processing < feature_batch_count) and final_iteration:\n X_raw_transformed = X_raw.loc[: row_count * num_features_processing - 1]\n X_raw_transformed = (\n X_raw_transformed\n if transform_func is None\n else transform_func(X_raw_transformed, **transform_func_kwargs)\n )\n else:\n X_raw_transformed = X_raw if transform_func is None else transform_func(X_raw, **transform_func_kwargs)\n y_pred = predict_func(X_raw_transformed, **predict_func_kwargs)\n\n row_index = 0\n for feature in parallel_computed_features:\n if isinstance(feature, tuple):\n feature_name = feature[0]\n feature_list = feature[1]\n else:\n feature_name = feature\n feature_list = feature\n # calculating importance score for given feature\n row_index_end = row_index + row_count\n y_pred_cur = y_pred[row_index:row_index_end]\n score = eval_metric(y, y_pred_cur, **kwargs)\n fi[feature_name] = score_baseline - score\n\n # resetting to original values for processed feature\n X_raw.loc[row_index : row_index_end - 1, feature_list] = X[feature_list].values\n\n row_index = row_index_end\n fi_dict_list.append(fi)\n shuffle_repeats_completed = shuffle_repeat + 1\n if time_limit is not None and shuffle_repeat != (num_shuffle_sets - 1):\n time_now = time.time()\n time_left = time_limit - (time_now - time_start)\n time_permutation_average = (time_now - time_permutation_start) / (shuffle_repeat + 1)\n if time_left < (time_permutation_average * 1.1):\n log_final_suffix = \" (Early stopping due to lack of time...)\"\n break\n\n fi_list_dict = dict()\n for val in fi_dict_list:\n for key in val:\n if key not in fi_list_dict:\n fi_list_dict[key] = []\n fi_list_dict[key].append(val[key])\n fi_df = _compute_fi_with_stddev(fi_list_dict, importance_as_list=importance_as_list)\n\n if not silent:\n logger.log(\n 20,\n f\"{log_prefix}\\t{round(time.time() - time_start, 2)}s\\t= Actual runtime (Completed {shuffle_repeats_completed} of {num_shuffle_sets} shuffle sets){log_final_suffix}\",\n )\n\n return fi_df\n\n\ndef _validate_features(features: list, valid_features: list):\n \"\"\"Raises exception if features list contains invalid features or duplicate features\"\"\"\n valid_features = set(valid_features)\n used_features = set()\n # validate features\n for feature in features:\n if isinstance(feature, tuple):\n feature_name = feature[0]\n feature_list = feature[1]\n feature_list_set = set(feature_list)\n if len(feature_list_set) != len(feature_list):\n raise ValueError(f\"Feature list contains duplicate features:\\n{feature_list}\")\n for feature_in_list in feature_list:\n if feature_in_list not in valid_features:\n raise ValueError(\n f\"Feature does not exist in data: {feature_in_list}\\n\"\n f\"This feature came from the following feature set:\\n\"\n f\"{feature}\\n\"\n f\"Valid Features:\\n\"\n f\"{valid_features}\"\n )\n else:\n feature_name = feature\n if feature_name not in valid_features:\n raise ValueError(f\"Feature does not exist in data: {feature_name}\\nValid Features:\\n{valid_features}\")\n if feature_name in used_features:\n raise ValueError(f\"Feature is present multiple times in feature list: {feature_name}\")\n used_features.add(feature_name)\n\n\ndef _compute_fi_with_stddev(fi_list_dict: dict, importance_as_list=False) -> DataFrame:\n features = list(fi_list_dict.keys())\n fi = dict()\n fi_stddev = dict()\n fi_p_value = dict()\n fi_n = dict()\n for feature in features:\n fi[feature], fi_stddev[feature], fi_p_value[feature], fi_n[feature] = _compute_mean_stddev_and_p_value(\n fi_list_dict[feature]\n )\n if importance_as_list:\n fi[feature] = fi_list_dict[feature]\n\n fi = pd.Series(fi).sort_values(ascending=False)\n fi_stddev = pd.Series(fi_stddev)\n fi_p_value = pd.Series(fi_p_value)\n fi_n = pd.Series(fi_n, dtype=\"int64\")\n\n fi_df = fi.to_frame(name=\"importance\")\n fi_df[\"stddev\"] = fi_stddev\n fi_df[\"p_value\"] = fi_p_value\n fi_df[\"n\"] = fi_n\n return fi_df\n\n\ndef _compute_mean_stddev_and_p_value(values: list):\n mean = np.mean(values)\n n = len(values)\n p_value = np.nan\n stddev = np.std(values, ddof=1) if n > 1 else np.nan\n if not np.isnan(stddev) and stddev != 0:\n t_stat = mean / (stddev / math.sqrt(n))\n p_value = scipy.stats.t.sf(t_stat, n - 1)\n elif stddev == 0:\n if mean > 0:\n p_value = 0.0\n elif mean < 0:\n p_value = 1.0\n else:\n p_value = 0.5\n\n return mean, stddev, p_value, n\n\n\ndef _get_safe_fi_batch_count(X, num_features, X_transformed=None, max_memory_ratio=0.2, max_feature_batch_count=None):\n # calculating maximum number of features that are safe to process in parallel\n if max_feature_batch_count is None:\n # If None, use a heuristic: limit total rows*features processed in one batch to ~2,500,000.\n # This balances memory usage and vectorization speed.\n # For example, if X has 100 rows, batch size will be capped at 10,000 features (hard cap).\n # If X has 1,000 rows, batch size can include 2,500 features.\n # If X has 1,000,000 rows, batch size can be 2 features.\n max_rows_per_batch = 2500000\n num_rows = X.shape[0] if hasattr(X, \"shape\") else len(X)\n max_feature_batch_count = max(1, max_rows_per_batch // num_rows)\n max_feature_batch_count = min(max_feature_batch_count, 10000)\n\n if isinstance(X, pd.DataFrame):\n X_size_bytes = get_approximate_df_mem_usage(X).sum()\n else:\n X_size_bytes = sys.getsizeof(pickle.dumps(X, protocol=4))\n\n if X_transformed is not None:\n if isinstance(X_transformed, pd.DataFrame):\n X_size_bytes += get_approximate_df_mem_usage(X_transformed).sum()\n else:\n X_size_bytes += sys.getsizeof(pickle.dumps(X_transformed, protocol=4))\n available_mem = ResourceManager.get_available_virtual_mem()\n X_memory_ratio = X_size_bytes / available_mem\n\n feature_batch_count_safe = math.floor(max_memory_ratio / X_memory_ratio)\n feature_batch_count = max(1, min(max_feature_batch_count, feature_batch_count_safe))\n feature_batch_count = min(feature_batch_count, num_features)\n return feature_batch_count\n\n\ndef unevaluated_fi_df_template(features: List[str]) -> pd.DataFrame:\n importance_df = pd.DataFrame({\"importance\": None, \"stddev\": None, \"p_value\": None, \"n\": 0}, index=features)\n return importance_df\n" }, { "path": "core/src/autogluon/core/utils/version_utils.py", "content": "import pkgutil\nimport platform\nimport re\nimport sys\nfrom datetime import datetime\nfrom importlib.metadata import distribution, version\n\nimport autogluon\nfrom autogluon.common.utils.nvutil import cudaInit, cudaSystemGetNVMLVersion\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\n\ndef _get_autogluon_versions():\n \"\"\"Retrieve version of all autogluon subpackages and its dependencies\"\"\"\n versions = dict()\n for pkg in list(pkgutil.iter_modules(autogluon.__path__, autogluon.__name__ + \".\")):\n # The following packages will be recognized as a submodule by pkgutil -exclude them.\n if pkg.name in [\"autogluon.version\", \"autogluon.setup\", \"autogluon._internal_\"]:\n continue\n try:\n versions[pkg.name] = version(pkg.name)\n versions.update(_get_dependency_versions(pkg.name))\n except ImportError:\n versions[pkg.name] = None\n return versions\n\n\ndef _get_dependency_versions(package):\n \"\"\"Retrieve direct dependency of the given package\n\n Args:\n package (str): name of the package\n \"\"\"\n # Get all requires for the package\n dependencies = distribution(package).requires\n # Filter-out test dependencies\n dependencies = [req for req in dependencies if not bool(re.search(\"extra.*test\", req))]\n # keep only package name\n dependencies = [re.findall(\"[a-zA-Z0-9_\\\\-]+\", req)[0].strip() for req in dependencies]\n versions = dict()\n for dependency in dependencies:\n try:\n versions[dependency] = version(dependency)\n except ImportError:\n versions[dependency] = None\n return versions\n\n\ndef _get_sys_info():\n \"\"\"Retrieve system information\"\"\"\n uname = platform.uname()\n cuda_version = None\n if cudaInit():\n try:\n cuda_version = cudaSystemGetNVMLVersion()\n except:\n cuda_version = None\n\n return {\n \"date\": datetime.date(datetime.now()),\n \"time\": datetime.time(datetime.now()),\n \"python\": \".\".join(str(i) for i in sys.version_info),\n \"OS\": uname.system,\n \"OS-release\": uname.release,\n \"Version\": uname.version,\n \"machine\": uname.machine,\n \"processor\": uname.processor,\n \"num_cores\": ResourceManager.get_cpu_count(),\n \"cpu_ram_mb\": ResourceManager.get_memory_size(\"MB\"),\n \"cuda version\": cuda_version,\n \"num_gpus\": ResourceManager.get_gpu_count(),\n \"gpu_ram_mb\": ResourceManager.get_gpu_free_memory(),\n \"avail_disk_size_mb\": ResourceManager.get_available_disk_size(),\n }\n\n\ndef show_versions():\n \"\"\"\n Provide useful information, important for bug reports.\n It comprises info about hosting operation system, autogluon subpackage versions,\n and versions of other installed relative packages.\n \"\"\"\n sys_info = _get_sys_info()\n versions = _get_autogluon_versions()\n sorted_keys = sorted(versions.keys(), key=lambda x: x.lower())\n\n maxlen = 0 if len(versions) == 0 else max(len(x) for x in versions)\n print(\"\\nINSTALLED VERSIONS\")\n print(\"------------------\")\n for k, v in sys_info.items():\n print(f\"{k:<{maxlen}}: {v}\")\n print(\"\")\n for k in sorted_keys:\n print(f\"{k:<{maxlen}}: {versions[k]}\")\n" }, { "path": "core/tests/__init__.py", "content": "" }, { "path": "core/tests/conftest.py", "content": "import pytest\n\n\ndef pytest_addoption(parser):\n parser.addoption(\"--runslow\", action=\"store_true\", default=False, help=\"run slow tests\")\n parser.addoption(\"--runplatform\", action=\"store_true\", default=False, help=\"run all skipped platform tests\")\n\n\ndef pytest_configure(config):\n config.addinivalue_line(\"markers\", \"slow: mark test as slow to run\")\n config.addinivalue_line(\"markers\", \"platform: mark test as Ubuntu/Linux/Mac platform test\")\n plugin = config.pluginmanager.getplugin(\"mypy\")\n plugin.mypy_argv.append(\"--ignore-missing-imports\")\n\n\ndef pytest_collection_modifyitems(config, items):\n skip_slow = pytest.mark.skip(reason=\"need --runslow option to run\")\n skip_platform = pytest.mark.skip(reason=\"need --runplatform option to run\")\n custom_markers = dict(slow=skip_slow, platform=skip_platform)\n if config.getoption(\"--runslow\"):\n # --runslow given in cli: do not skip slow tests\n custom_markers.pop(\"slow\", None)\n if config.getoption(\"--runplatform\"):\n # --runplatform given in cli: do not skip platform tests\n custom_markers.pop(\"platform\", None)\n for item in items:\n for marker in custom_markers:\n if marker in item.keywords:\n item.add_marker(custom_markers[marker])\n" }, { "path": "core/tests/test_check_style.py", "content": "import logging\nimport warnings\nfrom subprocess import PIPE, Popen\n\n\ndef test_check_style():\n logging.getLogger().setLevel(logging.INFO)\n logging.info(\"PEP8 Style check\")\n flake8_proc = Popen([\"flake8\", \"--count\", \"--max-line-length\", \"300\"], stdout=PIPE)\n flake8_out = flake8_proc.communicate()[0]\n lines = flake8_out.splitlines()\n count = int(lines[-1].decode())\n if count > 0:\n warnings.warn(f\"{count} PEP8 warnings remaining\")\n assert count < 2500, \"Too many PEP8 warnings found, improve code quality to pass test.\"\n" }, { "path": "core/tests/unittests/__init__.py", "content": "" }, { "path": "core/tests/unittests/calibrate/__init__.py", "content": "" }, { "path": "core/tests/unittests/calibrate/test_decision_threshold.py", "content": "import numpy as np\nimport pytest\n\nfrom autogluon.core.calibrate import calibrate_decision_threshold\nfrom autogluon.core.metrics import balanced_accuracy, f1, roc_auc\n\n\ndef _get_sample_data():\n y = np.array(\n [\n 1,\n 0,\n 1,\n 1,\n 1,\n 0,\n ]\n )\n y_pred_proba = np.array(\n [\n 0.0,\n 0.24,\n 0.25,\n 0.25,\n 0.5,\n 1.0,\n ]\n )\n return y, y_pred_proba\n\n\ndef test_calibrate_decision_threshold():\n y, y_pred_proba = _get_sample_data()\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=f1,\n decision_thresholds=50,\n secondary_decision_thresholds=None,\n )\n assert decision_threshold == 0.24\n\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=f1,\n )\n assert decision_threshold == 0.249\n\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n )\n assert decision_threshold == 0.249\n\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=10,\n )\n assert decision_threshold == 1.0\n\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=[0.88],\n )\n assert decision_threshold == 0.88\n\n\ndef test_calibrate_decision_threshold_select_closer_to_0_5():\n \"\"\"Test that calibration will choose the threshold closer to 0.5 in the case of a tie\"\"\"\n y, y_pred_proba = _get_sample_data()\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=[0.5, 0.244, 0.247],\n secondary_decision_thresholds=None,\n )\n assert decision_threshold == 0.247\n\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=[0.5, 0.247, 0.244],\n secondary_decision_thresholds=None,\n )\n assert decision_threshold == 0.247\n\n\ndef test_calibrate_decision_threshold_proba_metric_0_5():\n \"\"\"Test that non-pred metrics will always return 0.5\"\"\"\n y, y_pred_proba = _get_sample_data()\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=roc_auc,\n )\n assert decision_threshold == 0.5\n decision_threshold = calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=roc_auc,\n decision_thresholds=[0.1],\n )\n assert decision_threshold == 0.5\n\n\ndef test_calibrate_decision_threshold_out_of_bounds():\n y, y_pred_proba = _get_sample_data()\n with pytest.raises(ValueError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=[1.0, 0.5, 2.0],\n )\n with pytest.raises(ValueError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=[-0.01, 0.5],\n )\n\n\ndef test_calibrate_decision_threshold_invalid_args():\n y, y_pred_proba = _get_sample_data()\n with pytest.raises(AssertionError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=\"invalid\",\n )\n with pytest.raises(AssertionError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=0.01,\n )\n with pytest.raises(AssertionError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n decision_thresholds=None,\n )\n with pytest.raises(AssertionError):\n calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=balanced_accuracy,\n secondary_decision_thresholds=[0.2, 0.4],\n )\n" }, { "path": "core/tests/unittests/core/__init__.py", "content": "" }, { "path": "core/tests/unittests/core/scheduler/__init__.py", "content": "" }, { "path": "core/tests/unittests/core/scheduler/test_scheduler_factory.py", "content": "import pytest\n\nfrom autogluon.core.scheduler.scheduler_factory import get_hyperparameter_tune_kwargs_preset, scheduler_factory\nfrom autogluon.core.scheduler.seq_scheduler import LocalSequentialScheduler\n\n\ndef test_scheduler_factory__can_construct_valid_config_with_str_scheduler():\n scheduler_cls, scheduler_params = scheduler_factory(\n hyperparameter_tune_kwargs={\"scheduler\": \"local\", \"searcher\": \"grid\", \"custom_option\": \"value\"}\n )\n\n assert scheduler_cls == LocalSequentialScheduler, \"scheduler_cls must be correct\"\n assert scheduler_params[\"resource\"][\"num_cpus\"] is not None, \"resources/num_cpus must be present\"\n assert scheduler_params[\"resource\"][\"num_gpus\"] is not None, \"resources/num_cpus must be present\"\n\n expected_values = {\n \"searcher\": \"grid\",\n \"search_options\": {},\n \"checkpoint\": None,\n \"resume\": False,\n \"num_trials\": None,\n \"reward_attr\": \"validation_performance\",\n \"time_attr\": \"epoch\",\n \"visualizer\": \"none\",\n \"dist_ip_addrs\": [],\n \"scheduler\": \"local\",\n \"custom_option\": \"value\",\n }\n for k, v in expected_values.items():\n assert scheduler_params[k] == v, f\"{k} must be {v}\"\n\n\ndef test_scheduler_factory__can_construct_valid_config_with_class_scheduler():\n scheduler_cls, scheduler_params = scheduler_factory(\n hyperparameter_tune_kwargs={\"scheduler\": LocalSequentialScheduler, \"searcher\": \"local_random\"}\n )\n assert scheduler_cls == LocalSequentialScheduler, \"scheduler_cls must be correct\"\n\n\ndef test_scheduler_factory__reaises_exception_on_missing_scheduler():\n with pytest.raises(ValueError, match=\"Required key 'scheduler' is not present in hyperparameter_tune_kwargs\"):\n scheduler_factory(hyperparameter_tune_kwargs={\"searcher\": \"local_random\"})\n\n\ndef test_scheduler_factory__reaises_exception_on_unknown_str_scheduler():\n with pytest.raises(\n ValueError, match=\"Required key 'scheduler' in hyperparameter_tune_kwargs must be one of the values dict_keys\"\n ):\n scheduler_factory(hyperparameter_tune_kwargs={\"scheduler\": \"_some_value_\", \"searcher\": \"local_random\"})\n\n\ndef test_scheduler_factory__reaises_exception_on_missing_searcher():\n with pytest.raises(ValueError, match=\"Required key 'searcher' is not present in hyperparameter_tune_kwargs\"):\n scheduler_factory(hyperparameter_tune_kwargs={\"scheduler\": \"local\"})\n\n\ndef test_get_hyperparameter_tune_kwargs_preset__preset_exists():\n assert get_hyperparameter_tune_kwargs_preset(preset=\"auto\") == {\"scheduler\": \"local\", \"searcher\": \"local_random\"}\n\n\ndef test_get_hyperparameter_tune_kwargs_preset__preset_missing():\n with pytest.raises(ValueError, match='Invalid hyperparameter_tune_kwargs preset value \"unknown\"'):\n get_hyperparameter_tune_kwargs_preset(preset=\"unknown\")\n" }, { "path": "core/tests/unittests/hpo/__init__.py", "content": "" }, { "path": "core/tests/unittests/hpo/test_ray_hpo.py", "content": "import sys\nimport tempfile\n\nimport pytest\n\nif sys.platform == \"win32\" and sys.version_info >= (3, 13):\n pytest.skip(\"No ray support on Windows with Python 3.13\", allow_module_level=True)\n\nfrom ray import tune\n\nfrom autogluon.core.hpo.ray_hpo import AutommRayTuneAdapter, RayTuneAdapter, TabularRayTuneAdapter, run\nfrom autogluon.core.hpo.ray_tune_constants import SCHEDULER_PRESETS, SEARCHER_PRESETS\n\n\nclass DummyAdapter(RayTuneAdapter):\n supported_searchers = list(SEARCHER_PRESETS.keys())\n supported_schedulers = list(SCHEDULER_PRESETS.keys())\n\n @property\n def adapter_type(self):\n return \"dummy\"\n\n def get_resource_calculator(self, **kwargs):\n pass\n\n def get_resources_per_trial(self, total_resources, num_samples, **kwargs):\n return {\"cpu\": 1}\n\n def trainable_args_update_method(self, trainable_args):\n return {}\n\n\nDUMMY_SEARCH_SPACE = {\"a\": tune.uniform(0, 1), \"b\": tune.uniform(0, 20)}\n\n\ndef _dummy_objective(x, a, b):\n return a * (x**0.5) + b\n\n\ndef _dummy_trainable(config):\n for x in range(20):\n score = _dummy_objective(x, config[\"a\"], config[\"b\"])\n\n tune.report({\"score\": score})\n\n\ndef test_invalid_searcher():\n hyperparameter_tune_kwargs = dict(\n searcher=\"abc\",\n scheduler=\"FIFO\",\n num_trials=1,\n )\n with tempfile.TemporaryDirectory() as root:\n with pytest.raises(Exception) as e_info:\n run(\n trainable=_dummy_trainable,\n trainable_args=dict(),\n search_space=DUMMY_SEARCH_SPACE,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=\"score\",\n mode=\"min\",\n save_dir=root,\n ray_tune_adapter=DummyAdapter(),\n )\n\n\ndef test_invalid_scheduler():\n hyperparameter_tune_kwargs = dict(\n searcher=\"random\",\n scheduler=\"abc\",\n num_trials=1,\n )\n with tempfile.TemporaryDirectory() as root:\n with pytest.raises(Exception) as e_info:\n run(\n trainable=_dummy_trainable,\n trainable_args=dict(),\n search_space=DUMMY_SEARCH_SPACE,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=\"score\",\n mode=\"min\",\n save_dir=root,\n ray_tune_adapter=DummyAdapter(),\n )\n\n\ndef test_invalid_preset():\n hyperparameter_tune_kwargs = \"abc\"\n with tempfile.TemporaryDirectory() as root:\n with pytest.raises(Exception) as e_info:\n run(\n trainable=_dummy_trainable,\n trainable_args=dict(),\n search_space=DUMMY_SEARCH_SPACE,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=\"score\",\n mode=\"min\",\n save_dir=root,\n ray_tune_adapter=DummyAdapter(),\n )\n\n\ndef test_empty_search_space():\n hyperparameter_tune_kwargs = dict(\n searcher=\"random\",\n scheduler=\"FIFO\",\n num_trials=1,\n )\n with tempfile.TemporaryDirectory() as root:\n with pytest.raises(Exception) as e_info:\n run(\n trainable=_dummy_trainable,\n trainable_args=dict(),\n search_space=dict(),\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=\"score\",\n mode=\"min\",\n save_dir=root,\n ray_tune_adapter=DummyAdapter(),\n )\n\n\n@pytest.mark.platform\n@pytest.mark.parametrize(\"searcher\", list(SEARCHER_PRESETS.keys()))\n@pytest.mark.parametrize(\"scheduler\", list(SCHEDULER_PRESETS.keys()))\ndef test_run(searcher, scheduler):\n hyperparameter_tune_kwargs = dict(\n searcher=searcher,\n scheduler=scheduler,\n num_trials=2,\n )\n with tempfile.TemporaryDirectory() as root:\n analysis = run(\n trainable=_dummy_trainable,\n trainable_args=dict(),\n search_space=DUMMY_SEARCH_SPACE,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=\"score\",\n mode=\"min\",\n save_dir=root,\n ray_tune_adapter=DummyAdapter(),\n )\n assert analysis is not None\n assert analysis.best_result is not None\n assert analysis.best_result[\"score\"] is not None\n" }, { "path": "core/tests/unittests/hpo/test_space_converter.py", "content": "import sys\n\nimport pytest\n\nif sys.platform == \"win32\" and sys.version_info >= (3, 13):\n pytest.skip(\"No ray support on Windows with Python 3.13\", allow_module_level=True)\n\nfrom ray import tune\n\nfrom autogluon.common import space\nfrom autogluon.core.hpo.space_converter import RaySpaceConverterFactory\n\n\n@pytest.mark.parametrize(\n \"space, expected_space\",\n [\n (space.Categorical([1, 2]), tune.choice([1, 2])),\n (space.Real(1, 2, log=True), tune.loguniform(1, 2)),\n (space.Real(1, 2, log=False), tune.uniform(1, 2)),\n (space.Int(1, 2), tune.randint(1, 3)),\n (space.Bool(), tune.randint(0, 2)),\n ],\n)\ndef test_space_converter(space, expected_space):\n ray_space = RaySpaceConverterFactory.get_space_converter(space.__class__.__name__).convert(space)\n assert type(ray_space) == type(expected_space)\n" }, { "path": "core/tests/unittests/metrics/__init__.py", "content": "" }, { "path": "core/tests/unittests/metrics/test_classification_metrics.py", "content": "import numpy as np\nimport pytest\nimport sklearn\n\nfrom autogluon.core.metrics import confusion_matrix, log_loss, quadratic_kappa, roc_auc\nfrom autogluon.core.metrics.softclass_metrics import soft_log_loss\n\n\ndef test_confusion_matrix_with_valid_inputs_without_labels_and_weights():\n # Given\n input_solution = [2, 0, 2, 2, 0, 1]\n input_prediction = [0, 0, 2, 2, 0, 2]\n expected_output = np.array([[2, 0, 0], [0, 0, 1], [1, 0, 2]])\n\n # When\n observed_output = confusion_matrix(input_solution, input_prediction)\n\n # Then\n assert np.array_equal(expected_output, observed_output)\n\n\ndef test_confusion_matrix_with_valid_inputs_with_labels_and_without_weights():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = [\"ant\", \"bird\", \"cat\"]\n expected_output = np.array([[2, 0, 0], [0, 0, 1], [1, 0, 2]])\n\n # When\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n # Then\n assert np.array_equal(expected_output, observed_output)\n\n\ndef test_confusion_matrix_with_valid_inputs_with_labels_and_with_weights():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = [\"ant\", \"bird\", \"cat\"]\n weights = [0.1, 0.3, 1.0, 0.8, 0.2, 2.0]\n expected_output = np.array([[0.5, 0.0, 0.0], [0.0, 0.0, 2.0], [0.1, 0.0, 1.8]])\n\n # When\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels, weights=weights)\n\n # Then\n assert np.array_equal(expected_output, observed_output)\n\n\ndef test_confusion_matrix_with_valid_inputs_with_lesser_number_of_labels_and_without_weights():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = [\"bird\", \"cat\"]\n expected_output = np.array([[0, 1], [0, 2]])\n\n # When\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n # Then\n assert np.array_equal(expected_output, observed_output)\n\n\ndef test_confusion_matrix_with_unequal_samples():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = confusion_matrix(input_solution, input_prediction)\n\n\ndef test_confusion_matrix_with_multioutput_samples():\n # Given\n input_solution = [[\"cat\", \"ant\", \"cat\"]]\n input_prediction = [[\"ant\", \"ant\", \"cat\"]]\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = confusion_matrix(input_solution, input_prediction)\n\n\ndef test_confusion_matrix_with_empty_labels():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = []\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n\ndef test_confusion_matrix_with_multiDimensional_labels():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = [[\"ant\", \"bird\"], \"cat\"]\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n\ndef test_confusion_matrix_with_invalid_weights():\n # Given\n input_solution = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n input_prediction = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n labels = [[1, 2], 0.1, [0.1], 3, 1]\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n\ndef test_confusion_matrix_with_empty_inputs():\n # Given\n input_solution = []\n input_prediction = []\n labels = [\"bird\", \"cat\"]\n expected_output = np.array([[0, 0], [0, 0]])\n\n # When\n observed_output = confusion_matrix(input_solution, input_prediction, labels=labels)\n\n # Then\n assert np.array_equal(expected_output, observed_output)\n\n\n@pytest.mark.parametrize(\n \"gt,probs\",\n [\n ([0, 2, 1, 0], [[0.1, 0.2, 0.7], [0.2, 0.1, 0.7], [0.3, 0.4, 0.3], [0.01, 0.9, 0.09]]),\n ([0, 2, 0, 0], [[0.1, 0.2, 0.7], [0.2, 0.1, 0.7], [0.3, 0.4, 0.3], [0.01, 0.9, 0.09]]),\n ],\n)\ndef test_log_loss(gt, probs):\n gt = np.array(gt, dtype=np.int64)\n probs = np.array(probs, dtype=np.float32)\n ag_loss = log_loss(gt, probs)\n expected = np.log(probs[np.arange(probs.shape[0]), gt]).mean()\n np.testing.assert_allclose(ag_loss, expected)\n\n\n@pytest.mark.parametrize(\n \"gt,probs\",\n [\n (\n [[0.2, 0.3, 0.5], [0.1, 0.6, 0.3], [0.9, 0.05, 0.05], [0.3, 0.5, 0.2]],\n [[0.1, 0.2, 0.7], [0.2, 0.1, 0.7], [0.3, 0.4, 0.3], [0.01, 0.9, 0.09]],\n )\n ],\n)\ndef test_soft_log_loss(gt, probs):\n gt = np.array(gt, dtype=np.float32)\n probs = np.array(probs, dtype=np.float32)\n ag_loss = soft_log_loss(gt, probs)\n expected = -1.4691482\n np.testing.assert_allclose(ag_loss, expected)\n\n\ndef test_log_loss_single_binary_class():\n gt = np.array([1, 1, 1])\n probs = np.array([0.1, 0.2, 0.3])\n np.testing.assert_allclose(log_loss(gt, probs), np.log(probs).mean())\n np.testing.assert_allclose(log_loss(1 - gt, probs), np.log(1 - probs).mean())\n\n\n@pytest.mark.parametrize(\n \"gt,probs\",\n [\n ([0, 2, 1, 1], [[0.1, 0.2, 0.7], [0.2, 0.1, 0.7], [0.3, 0.4, 0.3], [0.01, 0.9, 0.09]]),\n ([0, 1, 0, 1], [0.1, 0.2, 0.3, 0.4]),\n ],\n)\ndef test_log_loss_with_sklearn(gt, probs):\n gt = np.array(gt, dtype=np.int64)\n probs = np.array(probs, dtype=np.float32)\n ag_loss = log_loss(gt, probs)\n sklearn_log_loss = sklearn.metrics.log_loss(gt, probs)\n # In AutoGluon, the metrics will always return score that is higher the better.\n # Thus, the true value should be the negation of the real log_loss\n np.testing.assert_allclose(ag_loss, -sklearn_log_loss)\n\n ag_loss_as_sklearn = log_loss.convert_score_to_original(ag_loss)\n np.testing.assert_allclose(ag_loss_as_sklearn, sklearn_log_loss)\n\n\ndef test_roc_auc_score_with_sklearn():\n \"\"\"\n Ensure AutoGluon's custom fast roc_auc_score produces the same result as sklearn's roc_auc_score.\n \"\"\"\n y_true = np.array([0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1])\n y_score = np.array([0, 1, 0, 1, 0.1, 0.81, 0.76, 0.1, 0.31, 0.32, 0.34, 0.9])\n expected_score = sklearn.metrics.roc_auc_score(y_true, y_score)\n actual_score = roc_auc(y_true, y_score)\n\n assert np.isclose(actual_score, expected_score)\n\n\ndef test_roc_auc_score_with_sklearn_single_raise():\n y_true = np.array([1])\n y_score = np.array([0.9])\n\n # Check sklearn behavior: newer versions return NaN with a warning, older versions raise ValueError\n import warnings\n\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n try:\n result = sklearn.metrics.roc_auc_score(y_true, y_score)\n # Newer sklearn returns NaN\n assert np.isnan(result)\n except ValueError:\n # Older sklearn raises ValueError - this is expected\n pass\n\n # Our implementation should still raise ValueError for consistency\n with pytest.raises(ValueError):\n roc_auc(y_true, y_score)\n\n\ndef test_roc_auc_score_with_sklearn_zero_raise():\n y_true = np.array([])\n y_score = np.array([])\n with pytest.raises(ValueError):\n sklearn.metrics.roc_auc_score(y_true, y_score)\n with pytest.raises(ValueError):\n roc_auc(y_true, y_score)\n\n\ndef test_quadratic_kappa():\n actuals = np.array([4, 4, 3, 4, 4, 4, 1, 1, 2, 1])\n preds = np.array([0, 2, 1, 0, 0, 0, 1, 1, 2, 1])\n value = quadratic_kappa(actuals, preds)\n assert round(value, 3) == -0.139\n\n actuals = np.array([0, 1, 0, 1])\n preds = np.array([[0.8, 0.1, 0.1], [0.7, 0.1, 0.2], [0.1, 0.8, 0.1], [0.1, 0.1, 0.8]])\n value = quadratic_kappa(actuals, preds)\n assert value == 0.25\n" }, { "path": "core/tests/unittests/metrics/test_metric_kwargs.py", "content": "import numpy as np\nimport sklearn.metrics\n\nfrom autogluon.core.metrics import f1, make_scorer\n\n\ndef test_metric_kwargs():\n y_true = np.array([1, 1, 1, 0, 0, 0])\n y_pred = np.array([0, 1, 1, 0, 1, 1])\n score = f1(y_true, y_pred)\n\n score_pos_label_1 = f1(y_true, y_pred, pos_label=1)\n score_pos_label_0 = f1(y_true, y_pred, pos_label=0)\n\n assert score == score_pos_label_1\n assert score > score_pos_label_0\n assert score_pos_label_0 == 0.4\n\n error = f1.error(y_true, y_pred)\n error_pos_label_1 = f1.error(y_true, y_pred, pos_label=1)\n error_pos_label_0 = f1.error(y_true, y_pred, pos_label=0)\n\n assert f1.optimum == 1\n assert error == (f1.optimum - score)\n assert error_pos_label_1 == (f1.optimum - score_pos_label_1)\n assert error_pos_label_0 == (f1.optimum - score_pos_label_0)\n assert error_pos_label_0 == 0.6\n\n\ndef test_metric_kwargs_init():\n y_true = np.array([1, 1, 1, 0, 0, 0])\n y_pred = np.array([0, 1, 1, 0, 1, 1])\n f1_pos_label_0 = make_scorer(\"f1\", sklearn.metrics.f1_score, pos_label=0, needs_class=True)\n f1_pos_label_0_v2 = make_scorer(\"f1\", sklearn.metrics.f1_score, metric_kwargs=dict(pos_label=0), needs_class=True)\n f1_pos_label_0_test_override = make_scorer(\n \"f1\", sklearn.metrics.f1_score, metric_kwargs=dict(pos_label=0), pos_label=1, needs_class=True\n )\n\n score_og = f1(y_true, y_pred)\n score_pos_label_0_og = f1(y_true, y_pred, pos_label=0)\n score_pos_label_0 = f1_pos_label_0(y_true, y_pred)\n score_pos_label_0_v2 = f1_pos_label_0_v2(y_true, y_pred)\n score_pos_label_0_test_override = f1_pos_label_0_test_override(y_true, y_pred)\n\n assert score_og > score_pos_label_0_og\n assert score_pos_label_0_og == score_pos_label_0\n assert score_pos_label_0_og == score_pos_label_0_v2\n assert score_pos_label_0_og == score_pos_label_0_test_override\n\n # assert that kwargs passed during call override init kwargs\n assert score_og == f1_pos_label_0(y_true, y_pred, pos_label=1)\n" }, { "path": "core/tests/unittests/metrics/test_metrics.py", "content": "from math import isclose\n\nimport numpy as np\nimport pytest\nimport sklearn\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\nfrom autogluon.core.metrics import METRICS, Scorer, make_scorer, rmse_func\n\nMETRICS_NEEDS_CLASS = {}\nMETRICS_NEEDS_PROBA = {}\nMETRICS_NEEDS_THRESHOLD = {}\nNOT_METRICS_NEEDS_THRESHOLD = {}\nfor problem_type in METRICS:\n METRICS_NEEDS_CLASS[problem_type] = [k for k, v in METRICS[problem_type].items() if v.needs_class]\n METRICS_NEEDS_PROBA[problem_type] = [\n k for k, v in METRICS[problem_type].items() if v.needs_proba or v.needs_threshold\n ]\n METRICS_NEEDS_THRESHOLD[problem_type] = [k for k, v in METRICS[problem_type].items() if v.needs_threshold]\n NOT_METRICS_NEEDS_THRESHOLD[problem_type] = [k for k, v in METRICS[problem_type].items() if not v.needs_threshold]\n\nBINARY_METRICS = list(METRICS[BINARY].keys())\nMULTICLASS_METRICS = list(METRICS[MULTICLASS].keys())\nREGRESSION_METRICS = list(METRICS[REGRESSION].keys())\n\nBINARY_METRICS_NEEDS_POS_LABEL = [k for k, v in METRICS[BINARY].items() if v.needs_pos_label]\n\nEXPECTED_BINARY_METRICS = {\n \"acc\",\n \"accuracy\",\n \"average_precision\",\n \"balanced_accuracy\",\n \"f1\",\n \"f1_macro\",\n \"f1_micro\",\n \"f1_weighted\",\n \"log_loss\",\n \"mcc\",\n \"nll\",\n \"pac\",\n \"pac_score\",\n \"precision\",\n \"precision_macro\",\n \"precision_micro\",\n \"precision_weighted\",\n \"quadratic_kappa\",\n \"recall\",\n \"recall_macro\",\n \"recall_micro\",\n \"recall_weighted\",\n \"roc_auc\",\n}\n\nEXPECTED_MULTICLASS_METRICS = {\n \"acc\",\n \"accuracy\",\n \"balanced_accuracy\",\n \"f1_macro\",\n \"f1_micro\",\n \"f1_weighted\",\n \"log_loss\",\n \"mcc\",\n \"nll\",\n \"pac\",\n \"pac_score\",\n \"precision_macro\",\n \"precision_micro\",\n \"precision_weighted\",\n \"quadratic_kappa\",\n \"recall_macro\",\n \"recall_micro\",\n \"recall_weighted\",\n \"roc_auc_ovo\",\n \"roc_auc_ovo_macro\",\n \"roc_auc_ovo_weighted\",\n \"roc_auc_ovr\",\n \"roc_auc_ovr_macro\",\n \"roc_auc_ovr_micro\",\n \"roc_auc_ovr_weighted\",\n}\n\nEXPECTED_REGRESSION_METRICS = {\n \"mae\",\n \"mape\",\n \"mean_absolute_error\",\n \"mean_absolute_percentage_error\",\n \"mean_squared_error\",\n \"median_absolute_error\",\n \"mse\",\n \"pearsonr\",\n \"r2\",\n \"rmse\",\n \"root_mean_squared_error\",\n \"smape\",\n \"spearmanr\",\n \"symmetric_mean_absolute_percentage_error\",\n}\n\nEXPECTED_QUANTILE_METRICS = {\n \"pinball\",\n \"pinball_loss\",\n}\n\n\n@pytest.mark.parametrize(\n \"metrics,expected_metrics_and_aliases\",\n [\n [METRICS[BINARY], EXPECTED_BINARY_METRICS],\n [METRICS[MULTICLASS], EXPECTED_MULTICLASS_METRICS],\n [METRICS[REGRESSION], EXPECTED_REGRESSION_METRICS],\n [METRICS[QUANTILE], EXPECTED_QUANTILE_METRICS],\n ],\n ids=[\n BINARY,\n MULTICLASS,\n REGRESSION,\n QUANTILE,\n ],\n) # noqa\ndef test_metric_exists(metrics: dict, expected_metrics_and_aliases: set):\n \"\"\"\n Ensure all expected metrics are present and no unexpected metrics are present\n \"\"\"\n seen_metrics = set()\n\n for metric_name, metric_obj in metrics.items():\n assert (metric_name == metric_obj.name) or (metric_name in metric_obj.alias)\n assert metric_obj.greater_is_better is True\n if metric_name not in seen_metrics:\n seen_metrics.add(metric_name)\n diff_metrics = seen_metrics.symmetric_difference(expected_metrics_and_aliases)\n if len(diff_metrics) > 0:\n missing_metrics = expected_metrics_and_aliases.difference(seen_metrics)\n if len(missing_metrics) > 0:\n raise AssertionError(f\"Missing metrics: {list(missing_metrics)}\")\n else:\n unknown_metrics = seen_metrics.difference(expected_metrics_and_aliases)\n raise AssertionError(\n f\"Invalid metrics (If you have added a new metric, \"\n f\"please include it in the variable `expected_metrics_and_aliases`):\"\n f\"\\n\\t{list(unknown_metrics)}\"\n )\n\n\n@pytest.mark.parametrize(\"metric\", BINARY_METRICS, ids=BINARY_METRICS) # noqa\ndef test_metrics_perfect_binary(metric: str):\n _assert_valid_scorer_classifier(scorer=METRICS[BINARY][metric])\n _assert_perfect_score(scorer=METRICS[BINARY][metric])\n\n\n@pytest.mark.parametrize(\"metric\", MULTICLASS_METRICS, ids=MULTICLASS_METRICS) # noqa\ndef test_metrics_perfect_multiclass(metric: str):\n _assert_valid_scorer_classifier(scorer=METRICS[MULTICLASS][metric])\n _assert_perfect_score(scorer=METRICS[MULTICLASS][metric])\n\n\n@pytest.mark.skip(\n reason=\"average_precision doesn't raise an exception here when it should, so this test currently fails\"\n)\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_THRESHOLD[BINARY], ids=METRICS_NEEDS_THRESHOLD[BINARY]) # noqa\ndef test_metrics_perfect_raises_binary_single_sample(metric: str):\n with pytest.raises(Exception):\n # threshold metrics should not be able to predict on a single sample\n _assert_perfect_score_single_sample(scorer=METRICS[BINARY][metric])\n\n\n@pytest.mark.skip(reason=\"mcc, quadradic_kappa, and pac produce unexpected values here, so this test currently fails\")\n@pytest.mark.parametrize(\"metric\", NOT_METRICS_NEEDS_THRESHOLD[BINARY], ids=NOT_METRICS_NEEDS_THRESHOLD[BINARY]) # noqa\ndef test_metrics_perfect_binary_single_sample(metric: str):\n _assert_perfect_score_single_sample(scorer=METRICS[BINARY][metric])\n\n\n@pytest.mark.skip(reason=\"mcc and quadradic_kappa produce unexpected values here, so this test currently fails\")\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_CLASS[MULTICLASS], ids=METRICS_NEEDS_CLASS[MULTICLASS]) # noqa\ndef test_metrics_perfect_multiclass_single_sample(metric: str):\n _assert_perfect_score_single_sample(scorer=METRICS[MULTICLASS][metric])\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_CLASS[MULTICLASS], ids=METRICS_NEEDS_CLASS[MULTICLASS]) # noqa\ndef test_metrics_perfect_str_multiclass(metric: str):\n scorer = METRICS[MULTICLASS][metric]\n _assert_perfect_score_str_binary(scorer=scorer)\n _assert_perfect_score_str_multiclass(scorer=scorer)\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_CLASS[BINARY], ids=METRICS_NEEDS_CLASS[BINARY]) # noqa\ndef test_metrics_perfect_str_binary(metric: str):\n scorer = METRICS[BINARY][metric]\n if metric not in BINARY_METRICS_NEEDS_POS_LABEL:\n _assert_perfect_score_str_binary(scorer=scorer)\n else:\n with pytest.raises(ValueError):\n # pos_label should raise exception when passed string values\n _assert_perfect_score_str_binary(scorer=scorer)\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_PROBA[BINARY], ids=METRICS_NEEDS_PROBA[BINARY]) # noqa\ndef test_metrics_perfect_proba_raises_str_binary(metric: str):\n scorer = METRICS[BINARY][metric]\n with pytest.raises(Exception):\n # proba metrics should fail with string inputs\n scorer(np.array([\"a\", \"a\", \"b\"]), np.array([\"a\", \"a\", \"b\"]))\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_PROBA[MULTICLASS], ids=METRICS_NEEDS_PROBA[MULTICLASS]) # noqa\ndef test_metrics_perfect_proba_raises_str_multiclass(metric: str):\n scorer = METRICS[MULTICLASS][metric]\n with pytest.raises(Exception):\n # proba metrics should fail with string inputs\n scorer(np.array([\"a\", \"a\", \"b\"]), np.array([\"a\", \"a\", \"b\"]))\n\n\n@pytest.mark.parametrize(\"metric\", REGRESSION_METRICS, ids=REGRESSION_METRICS) # noqa\ndef test_metrics_perfect_regression(metric: str):\n _assert_valid_scorer_regressor(scorer=METRICS[REGRESSION][metric])\n _assert_perfect_score(scorer=METRICS[REGRESSION][metric])\n\n\n@pytest.mark.parametrize(\"metric\", BINARY_METRICS, ids=BINARY_METRICS) # noqa\ndef test_metrics_imperfect_binary(metric: str):\n _assert_imperfect_score(scorer=METRICS[BINARY][metric])\n\n\n@pytest.mark.parametrize(\"metric\", MULTICLASS_METRICS, ids=MULTICLASS_METRICS) # noqa\ndef test_metrics_imperfect_multiclass(metric: str):\n _assert_imperfect_score(scorer=METRICS[MULTICLASS][metric])\n\n\n@pytest.mark.parametrize(\"metric\", REGRESSION_METRICS, ids=REGRESSION_METRICS) # noqa\ndef test_metrics_imperfect_regression(metric: str):\n _assert_imperfect_score(scorer=METRICS[REGRESSION][metric])\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_CLASS[BINARY], ids=METRICS_NEEDS_CLASS[BINARY]) # noqa\ndef test_metrics_imperfect_str_binary(metric: str):\n if metric not in BINARY_METRICS_NEEDS_POS_LABEL:\n _assert_imperfect_score_str_binary(scorer=METRICS[BINARY][metric])\n else:\n with pytest.raises(ValueError):\n # pos_label should raise exception when passed string values\n _assert_imperfect_score_str_binary(scorer=METRICS[BINARY][metric])\n\n\n@pytest.mark.parametrize(\"metric\", METRICS_NEEDS_CLASS[MULTICLASS], ids=METRICS_NEEDS_CLASS[MULTICLASS]) # noqa\ndef test_metrics_imperfect_str_multiclass(metric: str):\n _assert_imperfect_score_str_multiclass(scorer=METRICS[MULTICLASS][metric])\n _assert_imperfect_score_str_binary(scorer=METRICS[MULTICLASS][metric])\n\n\ndef _assert_valid_scorer_classifier(scorer: Scorer):\n _assert_valid_scorer(scorer=scorer)\n num_true = sum([1 if needs else 0 for needs in [scorer.needs_proba, scorer.needs_threshold, scorer.needs_class]])\n if num_true != 1:\n raise AssertionError(\n f\"Classification scorer '{scorer.name}' (class={scorer.__class__.__name__}) has invalid needs (exactly 1 must be True): \"\n f\"(needs_proba={scorer.needs_proba}, needs_threshold={scorer.needs_threshold}, needs_class={scorer.needs_class})\"\n )\n\n\ndef _assert_valid_scorer_regressor(scorer: Scorer):\n _assert_valid_scorer(scorer=scorer)\n num_true = sum([1 if needs else 0 for needs in [scorer.needs_pred, scorer.needs_quantile]])\n if num_true != 1:\n raise AssertionError(\n f\"Regression scorer '{scorer.name}' (class={scorer.__class__.__name__}) has invalid needs (exactly 1 must be True): \"\n f\"(needs_pred={scorer.needs_pred}, needs_quantile={scorer.needs_quantile})\"\n )\n\n\ndef _assert_valid_scorer(scorer: Scorer):\n if scorer.needs_class and not scorer.needs_pred:\n raise AssertionError(\n f\"Invalid Scorer definition! If `needs_class=True`, then `needs_pred` must also be True. \"\n f\"(name={scorer.name}, needs_class={scorer.needs_class}, needs_pred={scorer.needs_pred})\"\n )\n\n\ndef _assert_perfect_score(scorer: Scorer, abs_tol=1e-5):\n \"\"\"\n Ensure a perfect prediction has an error of 0\n and a score equal to scorer's optimum for a given scorer.\n \"\"\"\n y_true = np.array([0, 0, 1])\n y_pred = np.array([0.0, 0.0, 1.0])\n _assert_perfect_score_generic(scorer=scorer, abs_tol=abs_tol, y_true=y_true, y_pred=y_pred)\n\n\ndef _assert_perfect_score_single_sample(scorer: Scorer, abs_tol=1e-5):\n \"\"\"\n Ensure a perfect prediction has an error of 0 when given only a single sample\n and a score equal to scorer's optimum for a given scorer.\n \"\"\"\n y_true = np.array([1])\n y_pred = np.array([1.0])\n _assert_perfect_score_generic(scorer=scorer, abs_tol=abs_tol, y_true=y_true, y_pred=y_pred)\n\n\ndef _assert_perfect_score_generic(scorer: Scorer, y_true, y_pred, abs_tol=1e-5):\n \"\"\"\n Ensure a perfect prediction has an error of 0\n and a score equal to scorer's optimum for a given scorer.\n \"\"\"\n score = scorer(y_true, y_pred)\n assert score == scorer.score(y_true, y_pred)\n error = scorer.error(y_true, y_pred)\n assert error == scorer.convert_score_to_error(score)\n assert isclose(score, scorer.convert_error_to_score(error), abs_tol=abs_tol)\n assert isclose(error, 0, abs_tol=abs_tol)\n assert isclose(score, scorer.optimum, abs_tol=abs_tol)\n\n\ndef _assert_perfect_score_str_binary(scorer: Scorer, abs_tol=1e-5):\n \"\"\"\n Ensure a perfect prediction has an error of 0\n and a score equal to scorer's optimum for a given scorer.\n \"\"\"\n y_true = np.array([\"a\", \"a\", \"b\"])\n y_pred = np.array([\"a\", \"a\", \"b\"])\n _assert_perfect_score_generic(scorer=scorer, abs_tol=abs_tol, y_true=y_true, y_pred=y_pred)\n\n\ndef _assert_perfect_score_str_multiclass(scorer: Scorer, abs_tol=1e-5):\n \"\"\"\n Ensure a perfect prediction has an error of 0\n and a score equal to scorer's optimum for a given scorer.\n \"\"\"\n y_true = np.array([\"b\", \"a\", \"b\", \"c\"])\n y_pred = np.array([\"b\", \"a\", \"b\", \"c\"])\n _assert_perfect_score_generic(scorer=scorer, abs_tol=abs_tol, y_true=y_true, y_pred=y_pred)\n\n\ndef _assert_imperfect_score(scorer: Scorer, abs_tol: float = 1e-5) -> float:\n \"\"\"\n Ensure an imperfect prediction has an error greater than 0\n and a score less than the scorer's optimum for a given scorer.\n \"\"\"\n y_true = np.array([0, 0, 1])\n y_pred = np.array([1.0, 1.0, 0.0])\n return _assert_imperfect_score_generic(scorer=scorer, y_true=y_true, y_pred=y_pred, abs_tol=abs_tol)\n\n\ndef _assert_imperfect_score_str_binary(scorer: Scorer, abs_tol: float = 1e-5):\n \"\"\"\n Ensure an imperfect prediction has an error greater than 0\n and a score less than the scorer's optimum for a given scorer.\n\n Also ensure that both numeric and string representations of the input get the same score without raising an exception.\n \"\"\"\n y_true = np.array([0, 0, 1])\n y_pred = np.array([0.0, 1.0, 0.0])\n score_numeric = _assert_imperfect_score_generic(scorer=scorer, y_true=y_true, y_pred=y_pred, abs_tol=abs_tol)\n\n y_true = np.array([\"b\", \"b\", \"a\"])\n y_pred = np.array([\"b\", \"a\", \"b\"])\n score_str = _assert_imperfect_score_generic(scorer=scorer, y_true=y_true, y_pred=y_pred, abs_tol=abs_tol)\n\n assert isclose(score_numeric, score_str, abs_tol=abs_tol)\n\n\ndef _assert_imperfect_score_str_multiclass(scorer: Scorer, abs_tol: float = 1e-5):\n \"\"\"\n Ensure an imperfect prediction has an error greater than 0\n and a score less than the scorer's optimum for a given scorer.\n\n Also ensure that both numeric and string representations of the input get the same score without raising an exception.\n \"\"\"\n y_true = np.array([1, 0, 1, 2, 2, 4])\n y_pred = np.array([1.0, 1.0, 0.0, 0.0, 3.0, 1.0])\n score_numeric = _assert_imperfect_score_generic(scorer=scorer, y_true=y_true, y_pred=y_pred, abs_tol=abs_tol)\n\n y_true = np.array([\"b\", \"a\", \"b\", \"c\", \"c\", \"e\"])\n y_pred = np.array([\"b\", \"b\", \"a\", \"a\", \"d\", \"b\"])\n score_str = _assert_imperfect_score_generic(scorer=scorer, y_true=y_true, y_pred=y_pred, abs_tol=abs_tol)\n\n assert isclose(score_numeric, score_str, abs_tol=abs_tol)\n\n\ndef _assert_imperfect_score_generic(scorer: Scorer, y_true, y_pred, abs_tol: float = 1e-5) -> float:\n \"\"\"\n Ensure an imperfect prediction has an error greater than 0\n and a score less than the scorer's optimum for a given scorer.\n \"\"\"\n score = scorer(y_true, y_pred)\n assert score == scorer.score(y_true, y_pred)\n error = scorer.error(y_true, y_pred)\n assert error == scorer.convert_score_to_error(score)\n assert isclose(score, scorer.convert_error_to_score(error), abs_tol=abs_tol)\n assert error > 0\n assert score < scorer.optimum\n assert not isclose(error, 0, abs_tol=abs_tol)\n assert not isclose(score, scorer.optimum, abs_tol=abs_tol)\n return score\n\n\n@pytest.mark.parametrize(\"sample_weight\", [None, np.array([0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.3])])\ndef test_rmse_with_sklearn(sample_weight):\n \"\"\"\n Ensure\n (1) Without sample_weight, AutoGluon's custom rmse produces the same result as sklearn's rmse\n (2) With sample_weight, computed wrmse is as expected\n \"\"\"\n y_true = np.array([0, 0, 1, 1, 1, 1, 0])\n y_pred = np.array([0.13, 0.09, 0.78, 0.43, 0.8, 0.91, 0.32])\n expected_rmse = sklearn.metrics.root_mean_squared_error(y_true, y_pred, sample_weight=sample_weight)\n\n kwargs = {\"y_true\": y_true, \"y_pred\": y_pred}\n if sample_weight is not None:\n kwargs[\"sample_weight\"] = sample_weight\n computed_rmse = rmse_func(**kwargs)\n\n assert np.isclose(computed_rmse, expected_rmse)\n\n\ndef test_invalid_scorer():\n \"\"\"\n Ensure various edge-cases are appropriately handled when Scorers are created incorrectly\n \"\"\"\n with pytest.raises(ValueError):\n # Invalid: Specifying multiple needs_*\n make_scorer(\"dummy\", score_func=sklearn.metrics.accuracy_score, needs_proba=True, needs_class=True)\n\n with pytest.raises(ValueError):\n # Invalid: Specifying False for all needs_*\n make_scorer(\"dummy\", score_func=sklearn.metrics.accuracy_score, needs_pred=False)\n\n with pytest.raises(ValueError):\n # Invalid: Specifying needs_pred=False when needs_class=True\n make_scorer(\"dummy\", score_func=sklearn.metrics.accuracy_score, needs_pred=False, needs_class=True)\n\n # Valid\n make_scorer(\"dummy\", score_func=sklearn.metrics.accuracy_score, needs_pred=True, needs_class=True)\n\n with pytest.raises(ValueError):\n # Invalid: Specifying needs_pred=True when needs_proba=True\n make_scorer(\"dummy\", score_func=sklearn.metrics.accuracy_score, needs_pred=True, needs_proba=True)\n" }, { "path": "core/tests/unittests/metrics/test_quantile_metrics.py", "content": "import numpy as np\nimport pytest\n\nfrom autogluon.core.metrics.quantile_metrics import pinball_loss\n\n\ndef test_invalid_quantile_values_shape_raises():\n # Given\n input_target_values = [1.0, 2.0, 3.0]\n input_quantile_values = [1.0, 2.0, 3.0]\n input_quantile_levels = [0.25, 0.5, 0.75, 0.9, 0.95]\n\n # When-Then\n with pytest.raises(ValueError):\n observed_output = pinball_loss(input_target_values, input_quantile_values, input_quantile_levels)\n\n\ndef test_mismatched_target_prediction_length_raises():\n # Given\n input_target_values = [1.0, 2.0]\n input_quantile_values = [[1.0], [2.0], [3.0]]\n input_quantile_levels = [0.5, 0.75]\n\n # When/Then\n with pytest.raises(ValueError):\n observed_output = pinball_loss(input_target_values, input_quantile_values, input_quantile_levels)\n\n\ndef test_mismatched_quantiles_raises():\n # Given\n input_target_values = [1.0, 2.0]\n input_quantile_values = [[1.0], [2.0]]\n input_quantile_levels = [0.5, 0.75]\n\n # When/Then\n with pytest.raises(ValueError):\n observed_output = pinball_loss(input_target_values, input_quantile_values, input_quantile_levels)\n\n\ndef test_single_prediction():\n # Given\n input_target_values = [100]\n input_quantile_values = [[90.0]]\n input_quantile_levels = [0.9]\n expected_output = 9.0\n\n # When\n observed_output = pinball_loss(input_target_values, input_quantile_values, input_quantile_levels)\n\n # Then\n assert np.isclose(expected_output, observed_output)\n\n\ndef test_multiple_predictions():\n # Given\n input_target_values = [1.0, 2.0, 3.0]\n input_quantile_values = [[1.0, 1.1], [2.0, 1.9], [2.0, 4.0]]\n input_quantile_levels = [0.5, 0.75]\n expected_output = 0.425 / 3\n\n # When\n observed_output = pinball_loss(input_target_values, input_quantile_values, input_quantile_levels)\n\n # Then\n assert np.isclose(expected_output, observed_output)\n\n\ndef test_multiple_predictions_with_weights():\n # Given\n input_target_values = [1.0, 2.0, 3.0]\n input_quantile_values = [[1.0, 1.1], [2.0, 1.9], [2.0, 4.0]]\n input_quantile_levels = [0.5, 0.75]\n input_sample_weights = [0.25, 0.5, 0.25]\n input_quantile_weights = [0.4, 0.6]\n expected_output = 0.11375\n\n # When\n observed_output = pinball_loss(\n input_target_values, input_quantile_values, input_quantile_levels, input_sample_weights, input_quantile_weights\n )\n\n # Then\n assert np.isclose(expected_output, observed_output)\n" }, { "path": "core/tests/unittests/models/__init__.py", "content": "" }, { "path": "core/tests/unittests/models/abstract_model/__init__.py", "content": "" }, { "path": "core/tests/unittests/models/abstract_model/test_init_user_params.py", "content": "import copy\nfrom typing import Any, Dict, Optional\n\nfrom autogluon.core.models import AbstractModel\n\n\ndef _assert_init_user_params(\n params_og: Optional[Dict[str, Any]], expected_params: Dict[str, Any], expected_params_aux: Dict[str, Any], **kwargs\n):\n \"\"\"\n Assert that `AbstractModel._init_user_params` works as intended\n and that `AbstractModel` calls `AbstractModel._init_user_params` in the expected way during init.\n \"\"\"\n expected_params_og = copy.deepcopy(params_og) if params_og is not None else params_og\n params, params_aux = AbstractModel._init_user_params(params=params_og, **kwargs)\n assert params_og == expected_params_og # Ensure no outer context update\n assert params == expected_params\n assert params_aux == expected_params_aux\n\n if kwargs is None or len(kwargs.keys()) == 0:\n abstract_model = AbstractModel(name=\"\", path=\"\", hyperparameters=params_og)\n assert params_og == expected_params_og\n assert abstract_model._user_params == expected_params\n assert abstract_model._user_params_aux == expected_params_aux\n\n\ndef test_init_user_params_none():\n params_og = None\n expected_params = {}\n expected_params_aux = {}\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_simple():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {}\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_ag_args_fit_none():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag_args_fit\": None,\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {}\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_with_prefix():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag.foo\": 3,\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {\"foo\": 3}\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_with_ag_args_fit():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag_args_fit\": {\"foo\": 3},\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {\"foo\": 3}\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_with_ag_args_fit_and_prefix():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag_args_fit\": {\"foo\": 3, \"ag.foo\": 4, \"ag.bar\": 5, \"ag.ag.bar\": 7},\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {\n \"foo\": 4,\n \"bar\": 5,\n \"ag.bar\": 7,\n }\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_with_all():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag.foo\": 12,\n \"ag_args_fit\": {\"foo\": 3, \"ag.foo\": 4, \"ag.bar\": 5, \"ag.ag.bar\": 7},\n }\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n }\n expected_params_aux = {\n \"foo\": 12,\n \"bar\": 5,\n \"ag.bar\": 7,\n }\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux\n )\n\n\ndef test_init_user_params_with_all_and_custom():\n params_og = {\n \"foo\": 1,\n \"bar\": 2,\n \"custom.\": \"hello\",\n \"ag.foo\": 12,\n \"ag_args_fit\": {\"foo\": 3, \"ag.foo\": 4, \"ag.bar\": 5, \"ag.ag.bar\": 7},\n \"hello\": {\"custom.5\": 22, \"ag.custom.5\": 33},\n }\n kwargs = {\"ag_args_fit\": \"hello\", \"ag_arg_prefix\": \"custom.\"}\n expected_params = {\n \"foo\": 1,\n \"bar\": 2,\n \"ag.foo\": 12,\n \"ag_args_fit\": {\"foo\": 3, \"ag.foo\": 4, \"ag.bar\": 5, \"ag.ag.bar\": 7},\n }\n expected_params_aux = {\n \"\": \"hello\",\n \"5\": 22,\n \"ag.custom.5\": 33,\n }\n _assert_init_user_params(\n params_og=params_og, expected_params=expected_params, expected_params_aux=expected_params_aux, **kwargs\n )\n" }, { "path": "core/tests/unittests/models/test_bagged_ensemble_model.py", "content": "import pandas as pd\n\nfrom autogluon.common.utils.cv_splitter import CVSplitter\nfrom autogluon.core.models import BaggedEnsembleModel\n\n\ndef test_generate_fold_configs():\n y = pd.Series([0, 0, 0, 1, 1, 1, 1, 1])\n X = pd.DataFrame([[0], [0], [0], [0], [0], [0], [0], [0]])\n\n k_fold_start = 2\n k_fold_end = 1\n k_fold = 3\n n_repeat_start = 2\n n_repeats = 5\n\n cv_splitter = CVSplitter(n_splits=k_fold, n_repeats=n_repeats, stratify=True, random_state=0)\n\n fold_fit_args_list, n_repeats_started, n_repeats_finished = BaggedEnsembleModel._generate_fold_configs(\n X=X,\n y=y,\n cv_splitter=cv_splitter,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeat_start=n_repeat_start,\n n_repeat_end=n_repeats,\n vary_seed_across_folds=True,\n random_seed_offset=0,\n )\n\n assert fold_fit_args_list[0][\"model_name_suffix\"] == \"S3F3\"\n assert fold_fit_args_list[-1][\"model_name_suffix\"] == \"S5F1\"\n assert len(fold_fit_args_list) == 5\n assert n_repeats_started == 2\n assert n_repeats_finished == 2\n\n assert fold_fit_args_list[0][\"is_last_fold\"] is False\n assert fold_fit_args_list[1][\"is_last_fold\"] is False\n assert fold_fit_args_list[2][\"is_last_fold\"] is False\n assert fold_fit_args_list[3][\"is_last_fold\"] is False\n assert fold_fit_args_list[4][\"is_last_fold\"] is True\n\n assert fold_fit_args_list[0][\"random_seed\"] == 8\n assert fold_fit_args_list[1][\"random_seed\"] == 9\n assert fold_fit_args_list[2][\"random_seed\"] == 10\n assert fold_fit_args_list[3][\"random_seed\"] == 11\n assert fold_fit_args_list[4][\"random_seed\"] == 12\n\n k_fold_start = 0\n k_fold_end = 3\n k_fold = 3\n n_repeat_start = 0\n n_repeats = 5\n cv_splitter = CVSplitter(n_splits=k_fold, n_repeats=n_repeats, stratify=True, random_state=0)\n\n fold_fit_args_list, n_repeats_started, n_repeats_finished = BaggedEnsembleModel._generate_fold_configs(\n X=X,\n y=y,\n cv_splitter=cv_splitter,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeat_start=n_repeat_start,\n n_repeat_end=n_repeats,\n vary_seed_across_folds=False,\n random_seed_offset=0,\n )\n\n assert fold_fit_args_list[0][\"random_seed\"] == 0\n assert fold_fit_args_list[1][\"random_seed\"] == 0\n assert fold_fit_args_list[2][\"random_seed\"] == 0\n assert fold_fit_args_list[3][\"random_seed\"] == 0\n assert fold_fit_args_list[4][\"random_seed\"] == 0\n\n fold_fit_args_list, n_repeats_started, n_repeats_finished = BaggedEnsembleModel._generate_fold_configs(\n X=X,\n y=y,\n cv_splitter=cv_splitter,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeat_start=n_repeat_start,\n n_repeat_end=n_repeats,\n vary_seed_across_folds=True,\n random_seed_offset=42,\n )\n\n assert fold_fit_args_list[0][\"random_seed\"] == 42\n assert fold_fit_args_list[1][\"random_seed\"] == 43\n assert fold_fit_args_list[2][\"random_seed\"] == 44\n assert fold_fit_args_list[3][\"random_seed\"] == 45\n assert fold_fit_args_list[4][\"random_seed\"] == 46\n" }, { "path": "core/tests/unittests/ray/__init__.py", "content": "" }, { "path": "core/tests/unittests/ray/test_resource_calculator.py", "content": "import pytest\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.ray.resources_calculator import (\n CpuResourceCalculator,\n GpuResourceCalculator,\n NonParallelGpuResourceCalculator,\n ResourceCalculatorFactory,\n)\n\n\ndef test_cpu_calculator_no_bottleneck():\n num_cpus = 32\n num_jobs = 20\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=\"cpu\")\n assert type(calculator) == CpuResourceCalculator\n\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n num_jobs=num_jobs,\n minimum_cpu_per_job=4, # allows 8 jobs to run in parallel\n )\n\n expected_resources_per_trial = dict(\n cpu=4,\n )\n expected_num_parallel_jobs = 8\n expected_batches = 3\n\n assert expected_resources_per_trial == resources_info[\"resources_per_job\"]\n assert expected_num_parallel_jobs == resources_info[\"num_parallel_jobs\"]\n assert expected_batches == resources_info[\"batches\"]\n\n\ndef test_cpu_calculator_mem_bottleneck():\n num_cpus = 32\n num_jobs = 20\n mem_available = ResourceManager.get_available_virtual_mem()\n model_estimate_memory_usage = mem_available // 2.5 # allows 2 jobs to run in parallel\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=\"cpu\")\n assert type(calculator) == CpuResourceCalculator\n\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n num_jobs=num_jobs,\n model_estimate_memory_usage=model_estimate_memory_usage,\n minimum_cpu_per_job=4, # allows 8 jobs to run in parallel\n )\n\n expected_num_parallel_jobs = (\n 2 # even user wants to run 20 jobs in prallel, cpu can run 8 jobs in parallel, memory only allows for 2 jobs\n )\n expected_resources_per_trial = dict(\n cpu=16,\n )\n expected_batches = 10\n\n assert expected_resources_per_trial == resources_info[\"resources_per_job\"]\n assert expected_num_parallel_jobs == resources_info[\"num_parallel_jobs\"]\n assert expected_batches == resources_info[\"batches\"]\n\n\ndef test_gpu_calculator_no_bottleneck():\n num_cpus = 32\n num_gpus = 4\n num_jobs = 20\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=\"gpu\")\n assert type(calculator) == GpuResourceCalculator\n\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n total_num_gpus=num_gpus,\n num_jobs=num_jobs,\n minimum_cpu_per_job=1, # allows 32 jobs to run in parallel\n minimum_gpu_per_job=0.5, # allows 8 jobs to run in parallel\n )\n\n expected_num_parallel_jobs = 8\n expected_resources_per_trial = dict(\n cpu=4,\n gpu=0.5,\n )\n expected_batches = 3\n\n assert expected_resources_per_trial == resources_info[\"resources_per_job\"]\n assert expected_num_parallel_jobs == resources_info[\"num_parallel_jobs\"]\n assert expected_batches == resources_info[\"batches\"]\n\n\ndef test_gpu_calculator_cpu_bottleneck():\n num_cpus = 4\n num_gpus = 4\n num_jobs = 20\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=\"gpu\")\n assert type(calculator) == GpuResourceCalculator\n\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n total_num_gpus=num_gpus,\n num_jobs=num_jobs,\n minimum_cpu_per_job=1, # allows 4 jobs to run in parallel\n minimum_gpu_per_job=0.5, # allows 8 jobs to run in parallel\n )\n\n expected_num_parallel_jobs = 4\n expected_resources_per_trial = dict(\n cpu=1,\n gpu=1,\n )\n expected_batches = 5\n\n assert expected_resources_per_trial == resources_info[\"resources_per_job\"]\n assert expected_num_parallel_jobs == resources_info[\"num_parallel_jobs\"]\n assert expected_batches == resources_info[\"batches\"]\n\n\ndef test_non_parallel_gpu_calculator():\n num_cpus = 32\n num_gpus = 4\n num_jobs = 2\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=\"non_parallel_gpu\")\n assert type(calculator) == NonParallelGpuResourceCalculator\n\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n total_num_gpus=num_gpus,\n num_jobs=num_jobs,\n minimum_cpu_per_job=1,\n minimum_gpu_per_job=1,\n )\n\n expected_num_parallel_jobs = 2\n expected_resources_per_trial = dict(\n cpu=16,\n gpu=1,\n )\n expected_batches = 1\n\n assert expected_resources_per_trial == resources_info[\"resources_per_job\"]\n assert expected_num_parallel_jobs == resources_info[\"num_parallel_jobs\"]\n assert expected_batches == resources_info[\"batches\"]\n\n\n@pytest.mark.parametrize(\"calculator_type\", [\"cpu\", \"gpu\", \"non_parallel_gpu\"])\ndef test_resource_not_enough(calculator_type):\n num_cpus = 0\n num_gpus = 0\n num_jobs = 20\n\n calculator = ResourceCalculatorFactory.get_resource_calculator(calculator_type=calculator_type)\n with pytest.raises(Exception, match=r\"Cannot train model with provided resources! .*\") as e_info:\n resources_info = calculator.get_resources_per_job(\n total_num_cpus=num_cpus,\n total_num_gpus=num_gpus,\n num_jobs=num_jobs,\n minimum_cpu_per_job=1,\n minimum_gpu_per_job=1,\n )\n" }, { "path": "core/tests/unittests/scheduler/__init__.py", "content": "" }, { "path": "core/tests/unittests/scheduler/test_scheduler.py", "content": "import pickle\nimport time\n\nimport numpy as np\n\nfrom autogluon.common import space\nfrom autogluon.core.scheduler import LocalSequentialScheduler\n\n\ndef test_local_sequential_scheduler():\n search_space = dict(\n lr=space.Real(1e-3, 1e-2, log=True),\n wd=space.Real(1e-3, 1e-2),\n epochs=10,\n )\n\n def train_fn(args, reporter):\n for e in range(args[\"epochs\"]):\n dummy_reward = 1 - np.power(1.8, -np.random.uniform(e, 2 * e))\n reporter(epoch=e + 1, reward=dummy_reward, lr=args.lr, wd=args.wd)\n\n scheduler = LocalSequentialScheduler(train_fn, search_space=search_space, num_trials=10)\n scheduler.run()\n scheduler.join_jobs()\n best_config = scheduler.get_best_config()\n best_task_id = scheduler.get_best_task_id()\n assert pickle.dumps(scheduler.config_history[best_task_id]) == pickle.dumps(best_config)\n\n\ndef test_timeout_scheduler():\n search_space = dict(\n lr=space.Real(1e-5, 1e-3),\n )\n\n def train_fn(args, reporter):\n start_tick = time.time()\n time.sleep(0.01)\n reporter(reward=time.time() - start_tick, time_attr=0)\n\n scheduler = LocalSequentialScheduler(\n train_fn, search_space=search_space, num_trials=7, time_attr=\"time_attr\", time_out=0.025\n )\n scheduler.run()\n scheduler.join_jobs()\n assert len(scheduler.config_history) <= 2\n" }, { "path": "core/tests/unittests/scheduler/test_seq_scheduler.py", "content": "import pytest\n\nfrom autogluon.common import space\nfrom autogluon.core.scheduler.seq_scheduler import LocalSequentialScheduler\n\ncls = LocalSequentialScheduler\n\n\ndef test_get_average_trial_time_():\n running_time = cls.get_average_trial_time_(0, avg_trial_run_time=None, trial_start_time=100, time_end=102)\n assert running_time == 2\n running_time = cls.get_average_trial_time_(1, avg_trial_run_time=running_time, trial_start_time=110, time_end=114)\n assert running_time == 3.0\n running_time = cls.get_average_trial_time_(2, avg_trial_run_time=running_time, trial_start_time=120, time_end=126)\n assert running_time == 4.0\n\n\ndef test_has_enough_time_for_trial__enough_time__no_avg_time():\n # Enough time - no average time\n assert cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=106, avg_trial_run_time=None\n )\n\n\ndef test_has_enough_time_for_trial__enough_time__avg_time_allows_trials():\n # Enough time - average time allows more trial\n assert cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=106, avg_trial_run_time=1\n )\n\n\ndef test_has_enough_time_for_trial__enough_time__avg_time_not_allows_trials():\n # Enough time - average time does not allow more trial\n assert not cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=106, avg_trial_run_time=5\n )\n\n\ndef test_has_enough_time_for_trial__time_exceeded_no_avg_time():\n # Time exceeded - no average time\n assert not cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=116, avg_trial_run_time=None\n )\n\n\ndef test_has_enough_time_for_trial__avg_time():\n # Time exceeded - no average time\n assert not cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=116, avg_trial_run_time=0\n )\n\n\ndef test_has_enough_time_for_trial__enough_time__avg_time_not_allows_trials_by_fill_factor():\n # Enough time - average time does not allow more trial\n assert not cls.has_enough_time_for_trial_(\n time_out=10, time_start=100, trial_start_time=105, trial_end_time=106, avg_trial_run_time=1, fill_factor=5\n )\n\n\ndef test_LocalSequentialScheduler_no_criteria():\n search_space = {\"lr\": space.Real(1e-2, 1e-1, log=True)}\n\n def _train_fn_():\n pass\n\n with pytest.raises(AssertionError, match=\"Need stopping criterion: Either num_trials or time_out\"):\n LocalSequentialScheduler(\n train_fn=_train_fn_, search_space=search_space, reward_attr=\"reward_attr\", resource={}\n )\n\n\ndef test_search_space():\n search_space = dict(\n a=space.Real(1e-3, 1e-2, log=True),\n b=space.Real(1e-3, 1e-2),\n c=space.Int(1, 10),\n d=space.Categorical(\"a\", \"b\", \"c\", \"d\"),\n e=space.Bool(),\n )\n\n def train_fn(args, reporter):\n a, b, c, d, e = args[\"a\"], args[\"b\"], args[\"c\"], args[\"d\"], args[\"e\"]\n\n assert a <= 1e-2 and a >= 1e-3\n assert b <= 1e-2 and b >= 1e-3\n assert c <= 10 and c >= 1\n assert d in [\"a\", \"b\", \"c\", \"d\"]\n assert e in [True, False]\n reporter(epoch=1, accuracy=0)\n\n scheduler = LocalSequentialScheduler(\n train_fn,\n search_space=search_space,\n resource={\"num_cpus\": \"all\", \"num_gpus\": 0},\n num_trials=10,\n reward_attr=\"accuracy\",\n time_attr=\"epoch\",\n checkpoint=None,\n )\n\n scheduler.run()\n\n\ndef test_scheduler_can_handle_failing_jobs():\n trails_outcomes = []\n best_result = [-1]\n\n search_space = dict(a=space.Real(0, 1))\n\n def train_fn(args, reporter):\n test_should_fail = args[\"a\"] > 0.7\n trails_outcomes.append(test_should_fail)\n if test_should_fail:\n raise Exception(\"Failed Trial\")\n elif args[\"a\"] > best_result[0]:\n best_result[0] = args[\"a\"]\n reporter(epoch=1, accuracy=args[\"a\"])\n\n scheduler = LocalSequentialScheduler(\n train_fn,\n search_space=search_space,\n resource={\"num_cpus\": \"all\", \"num_gpus\": 0},\n num_trials=10,\n reward_attr=\"accuracy\",\n time_attr=\"epoch\",\n checkpoint=None,\n )\n\n scheduler.run()\n\n actual_runs = []\n for trial in scheduler.training_history.values():\n is_failed = False\n for i in trial:\n if \"traceback\" in i:\n is_failed = True\n break\n actual_runs.append(is_failed)\n\n assert trails_outcomes == actual_runs\n assert scheduler.get_best_reward() == best_result[0]\n assert scheduler.get_best_config() == {\"a\": best_result[0]}\n" }, { "path": "core/tests/unittests/searcher/__init__.py", "content": "" }, { "path": "core/tests/unittests/searcher/test_local_grid_searcher.py", "content": "from autogluon.common import space\nfrom autogluon.core.searcher import LocalGridSearcher\n\n\ndef test_local_grid_searcher_categorical():\n search_space = dict(\n a=space.Categorical(\"a\", 7, [\"hello\", 2]),\n b=space.Categorical(12, 15),\n )\n\n searcher = LocalGridSearcher(search_space=search_space)\n\n expected_config_1 = {\"b\": 12, \"a\": \"a\"}\n expected_config_2 = {\"b\": 15, \"a\": \"a\"}\n expected_config_3 = {\"b\": 12, \"a\": 7}\n expected_config_4 = {\"b\": 15, \"a\": 7}\n expected_config_5 = {\"b\": 12, \"a\": [\"hello\", 2]}\n expected_config_6 = {\"b\": 15, \"a\": [\"hello\", 2]}\n\n config1 = searcher.get_config()\n searcher.update(config1, reward=0.2)\n assert searcher.get_reward(config1) == 0.2\n assert searcher.get_best_reward() == 0.2\n assert searcher.get_best_config() == config1\n\n config2 = searcher.get_config()\n\n config3 = searcher.get_config()\n\n config4 = searcher.get_config()\n searcher.update(config4, reward=0.1)\n assert searcher.get_reward(config4) == 0.1\n assert searcher.get_best_reward() == 0.2\n assert searcher.get_best_config() == config1\n searcher.update(config4, reward=0.5)\n assert searcher.get_reward(config4) == 0.5\n assert searcher.get_best_reward() == 0.5\n assert searcher.get_best_config() == config4\n\n config5 = searcher.get_config()\n\n config6 = searcher.get_config()\n\n assert expected_config_1 == config1\n assert expected_config_2 == config2\n assert expected_config_3 == config3\n assert expected_config_4 == config4\n assert expected_config_5 == config5\n assert expected_config_6 == config6\n\n assert len(searcher._results) == 2\n\n try:\n searcher.get_config()\n except AssertionError:\n pass\n else:\n raise AssertionError(\"GridSearcher should error due to being out of configs\")\n\n\ndef test_local_grid_searcher_numeric():\n search_spaces = [\n [dict(a=space.Bool()), [{\"a\": 0}, {\"a\": 1}]],\n [dict(a=space.Int(12, 15)), [{\"a\": 12}, {\"a\": 13}, {\"a\": 14}, {\"a\": 15}]],\n [dict(a=space.Real(12, 16)), [{\"a\": 12.0}, {\"a\": 13.333333333333334}, {\"a\": 14.666666666666666}, {\"a\": 16.0}]],\n [\n dict(a=space.Real(12, 16, log=True)),\n [{\"a\": 12.0}, {\"a\": 13.207708995578509}, {\"a\": 14.536964742657117}, {\"a\": 16.0}],\n ],\n ]\n for search_space, expected_values in search_spaces:\n searcher = LocalGridSearcher(search_space=search_space)\n actual_values = []\n while True:\n try:\n cfg = searcher.get_config()\n actual_values.append(cfg)\n searcher.update(cfg, reward=0.1)\n except AssertionError as e:\n assert expected_values == actual_values\n break\n\n\ndef test_local_grid_searcher_numeric_grid_settings():\n search_spaces = [\n [dict(a=space.Int(12, 15)), [{\"a\": 12}, {\"a\": 15}]],\n [dict(b=space.Int(12, 15)), [{\"b\": 12}, {\"b\": 13}, {\"b\": 15}]],\n [dict(c=space.Int(12, 15)), [{\"c\": 12}, {\"c\": 13}, {\"c\": 14}, {\"c\": 15}]],\n ]\n\n grid_num_sample_settings = {\n \"b\": 3,\n \"c\": 4,\n }\n\n for search_space, expected_values in search_spaces:\n searcher = LocalGridSearcher(\n search_space=search_space,\n grid_numeric_spaces_points_number=2,\n grid_num_sample_settings=grid_num_sample_settings,\n )\n actual_values = []\n while True:\n try:\n cfg = searcher.get_config()\n actual_values.append(cfg)\n searcher.update(cfg, reward=0.1)\n except AssertionError as e:\n assert expected_values == actual_values\n break\n" }, { "path": "core/tests/unittests/searcher/test_local_random_searcher.py", "content": "from math import isclose\nfrom numbers import Number\n\nfrom autogluon.common import space\nfrom autogluon.core.searcher import LocalRandomSearcher\n\n\ndef dictsAlmostEqual(dict1, dict2, rel_tol=1e-8):\n if len(dict1) != len(dict2):\n return False\n for key, item in dict1.items():\n if isinstance(item, dict):\n if not dictsAlmostEqual(dict1[key], dict2[key], rel_tol=rel_tol):\n return False\n else:\n if isinstance(item, Number):\n if not isclose(dict1[key], dict2[key], rel_tol=rel_tol):\n return False\n else:\n if not (dict1[key] == dict2[key]):\n return False\n return True\n\n\ndef test_local_random_searcher():\n search_space = dict(\n a=space.Real(0, 1, default=0.2),\n b=space.Real(0.05, 1, default=0.4, log=True),\n c=space.Int(5, 15),\n d=space.Int(7, 23, default=16),\n e=space.Categorical(\"a\", 7, [\"hello\", 2]),\n )\n\n searcher = LocalRandomSearcher(search_space=search_space)\n\n expected_config_1 = {\"a\": 0.2, \"b\": 0.4, \"c\": 5, \"d\": 16, \"e\": \"a\"}\n expected_config_2 = {\"a\": 0.5488135039273248, \"b\": 0.4260424000595025, \"c\": 8, \"d\": 10, \"e\": 7}\n expected_config_3 = {\"a\": 0.6235636967859723, \"b\": 0.15814742875130683, \"c\": 12, \"d\": 13, \"e\": \"a\"}\n expected_config_4 = {\"a\": 0.9636627605010293, \"b\": 0.1577026248478398, \"c\": 11, \"d\": 14, \"e\": [\"hello\", 2]}\n expected_config_5 = {\"a\": 0.5680445610939323, \"b\": 0.800200824711684, \"c\": 13, \"d\": 16, \"e\": \"a\"}\n\n assert searcher.get_best_reward() == float(\"-inf\")\n config1 = searcher.get_config()\n assert searcher.get_reward(config1) == float(\"-inf\")\n assert searcher.get_best_reward() == float(\"-inf\")\n searcher.update(config1, reward=0.2)\n assert searcher.get_reward(config1) == 0.2\n assert searcher.get_best_reward() == 0.2\n assert searcher.get_best_config() == config1\n\n config2 = searcher.get_config()\n\n config3 = searcher.get_config()\n\n config4 = searcher.get_config()\n searcher.update(config4, reward=0.1)\n assert searcher.get_reward(config4) == 0.1\n assert searcher.get_best_reward() == 0.2\n assert searcher.get_best_config() == config1\n searcher.update(config4, reward=0.5)\n assert searcher.get_reward(config4) == 0.5\n assert searcher.get_best_reward() == 0.5\n assert searcher.get_best_config() == config4\n\n config5 = searcher.get_config()\n\n assert dictsAlmostEqual(expected_config_1, config1)\n assert dictsAlmostEqual(expected_config_2, config2)\n assert dictsAlmostEqual(expected_config_3, config3)\n assert dictsAlmostEqual(expected_config_4, config4)\n assert dictsAlmostEqual(expected_config_5, config5)\n\n assert len(searcher._results) == 5\n" }, { "path": "core/tests/unittests/searcher/test_local_searcher.py", "content": "import unittest\n\nfrom autogluon.common import space\nfrom autogluon.core.searcher.local_searcher import LocalSearcher\n\n\nclass TestLocalSearcher(unittest.TestCase):\n def test_local_searcher(self):\n search_space = {\"param1\": space.Categorical(\"hello\", \"world\"), 7: 42}\n searcher = LocalSearcher(search_space=search_space)\n\n config1 = {\"param1\": \"hello\", 7: 42}\n config2 = {\"param1\": \"world\", 7: 42}\n config_edge_case_1 = {\"param1\": \"world\", 7: 0}\n config_invalid_2 = {\"param1\": \"invalid\", 7: 42}\n config_invalid_3 = {\"param1\": \"hello\"}\n config_invalid_4 = {7: 42}\n config_invalid_5 = {}\n config_invalid_6 = {\"param1\": \"hello\", 7: 42, \"unknown_param\": 7}\n config_invalid_7 = 0\n\n assert searcher.get_best_reward() == float(\"-inf\")\n searcher.update(config1, reward=0.2)\n assert searcher.get_best_reward() == 0.2\n assert searcher.get_best_config() == config1\n\n assert searcher.get_results() == [({\"param1\": \"hello\", 7: 42}, 0.2)]\n\n searcher.update(config1, reward=0.1)\n assert searcher.get_best_reward() == 0.1\n assert searcher.get_best_config() == config1\n\n assert searcher.get_results() == [({\"param1\": \"hello\", 7: 42}, 0.1)]\n\n searcher.update(config2, reward=0.7)\n assert searcher.get_best_reward() == 0.7\n assert searcher.get_best_config() == config2\n\n assert searcher.get_results() == [({\"param1\": \"world\", 7: 42}, 0.7), ({\"param1\": \"hello\", 7: 42}, 0.1)]\n assert len(searcher._results) == 2\n # This config is technically invalid, but for performance reasons is allowed to avoid having to pickle compare static parameters.\n # Since the static parameter should be fixed, this config is treated as being equivalent to config2\n searcher.update(config_edge_case_1, reward=0)\n assert searcher.get_best_reward() == 0.1\n assert len(searcher._results) == 2\n self.assertRaises(AssertionError, searcher.update, config_invalid_2, reward=0)\n self.assertRaises(AssertionError, searcher.update, config_invalid_3, reward=0)\n self.assertRaises(AssertionError, searcher.update, config_invalid_4, reward=0)\n self.assertRaises(AssertionError, searcher.update, config_invalid_5, reward=0)\n self.assertRaises(AssertionError, searcher.update, config_invalid_6, reward=0)\n self.assertRaises(AssertionError, searcher.update, config_invalid_7, reward=0)\n self.assertRaises(AssertionError, searcher.update, config1, reward=\"invalid_reward\")\n assert len(searcher._results) == 2\n assert searcher.get_results() == [({\"param1\": \"hello\", 7: 42}, 0.1), ({\"param1\": \"world\", 7: 42}, 0)]\n\n def test_local_searcher_pickle(self):\n search_space = {1: space.Categorical(1, 2), 2: space.Categorical(1, 2)}\n searcher = LocalSearcher(search_space=search_space)\n\n # Identical configs should have same pkl key, different configs should have different pkl keys\n config_1 = {1: 1, 2: 2}\n config_2 = {2: 2, 1: 1}\n config_diff = {1: 2, 2: 1}\n config_pkl_1 = searcher._pickle_config(config_1)\n config_pkl_2 = searcher._pickle_config(config_2)\n config_pkl_diff = searcher._pickle_config(config_diff)\n\n assert config_pkl_1 == config_pkl_2\n assert config_pkl_1 != config_pkl_diff\n\n config_unpkl = searcher._unpickle_config(config_pkl_1)\n config_unpkl_diff = searcher._unpickle_config(config_pkl_diff)\n\n assert config_unpkl == config_1\n assert config_unpkl == config_2\n assert config_unpkl_diff == config_diff\n" }, { "path": "core/tests/unittests/test_feature_selection.py", "content": "import numpy as np\nimport pandas as pd\nimport pytest\nfrom numpy.core.fromnumeric import sort\n\nfrom autogluon.core.utils.feature_selection import *\nfrom autogluon.core.utils.utils import unevaluated_fi_df_template\n\n\ndef evaluated_fi_df_template(features, importance=None, n=None):\n rng = np.random.default_rng(0)\n importance_df = pd.DataFrame({\"name\": features})\n importance_df[\"importance\"] = rng.standard_normal(len(features)) if importance is None else importance\n importance_df[\"stddev\"] = rng.standard_normal(len(features))\n importance_df[\"p_value\"] = None\n importance_df[\"n\"] = 5 if n is None else n\n importance_df.set_index(\"name\", inplace=True)\n importance_df.index.name = None\n return importance_df\n\n\n@pytest.fixture\ndef sample_features():\n return [\"a\", \"b\", \"c\", \"d\", \"e\"]\n\n\n@pytest.fixture\ndef sample_importance_df_1(sample_features):\n return evaluated_fi_df_template(sample_features, importance=[0.2, 0.2, None, 1.0, None], n=[10, 5, 0, 5, 0])\n\n\n@pytest.fixture\ndef sample_importance_df_2(sample_features):\n return evaluated_fi_df_template(sample_features, importance=[-0.1, -0.1, 0.1, None, None], n=[5, 10, 10, 0, 0])\n\n\ndef test_add_noise_column_df():\n # test noise columns are appended to input dataframe and feature_metadata\n X = pd.DataFrame({\"a\": [1, 2]})\n args = {\"rng\": np.random.default_rng(0), \"count\": 2}\n X_noised, noise_columns = add_noise_column(X, **args)\n expected_features = X.columns.tolist() + noise_columns\n assert expected_features == X_noised.columns.tolist()\n\n\ndef test_merge_importance_dfs_base(sample_features):\n # test the scenario when previous feature importance df is none\n prev_df, curr_df = None, unevaluated_fi_df_template(sample_features)\n assert merge_importance_dfs(prev_df, curr_df, using_prev_fit_fi=set()) is curr_df\n\n\ndef test_merge_importance_dfs_same_model(sample_features, sample_importance_df_1, sample_importance_df_2):\n # test the scenario where previous feature importance df exists and its importance estimates come from the same fitted model\n prev_df, curr_df = sample_importance_df_1, sample_importance_df_2\n result_df = merge_importance_dfs(prev_df, curr_df, using_prev_fit_fi=set())\n assert [score if score == score else None for score in result_df[\"importance\"].tolist()] == [\n 0.0,\n 0.1,\n 0.1,\n 1.0,\n None,\n ]\n assert result_df[\"n\"].tolist() == [15, 15, 10, 5, 0]\n\n\ndef test_merge_importance_dfs_different_model(sample_features, sample_importance_df_1, sample_importance_df_2):\n # test the scenario where previous feature importance df exists and its importance estimates come from a different fitted model\n prev_df, curr_df = sample_importance_df_1, sample_importance_df_2\n using_prev_fit_fi = set(sample_features)\n result_df = merge_importance_dfs(prev_df, curr_df, using_prev_fit_fi=using_prev_fit_fi).sort_index()\n assert len(using_prev_fit_fi) == 2\n assert [score if score == score else None for score in result_df[\"importance\"].tolist()] == [\n -0.1,\n -0.1,\n 0.1,\n 1.0,\n None,\n ]\n assert result_df[\"n\"].tolist() == [5, 10, 10, 5, 0]\n\n\ndef test_merge_importance_dfs_all(sample_features, sample_importance_df_1, sample_importance_df_2):\n # test the scenario where previous feature importance df exists and its importance estimates come from both same and different fitted models\n prev_df, curr_df = sample_importance_df_1, sample_importance_df_2\n using_prev_fit_fi = set([sample_features[0]])\n result_df = merge_importance_dfs(prev_df, curr_df, using_prev_fit_fi=using_prev_fit_fi).sort_index()\n assert [score if score == score else None for score in result_df[\"importance\"].tolist()] == [\n -0.1,\n 0.0,\n 0.1,\n 1.0,\n None,\n ]\n assert result_df[\"n\"].tolist() == [5, 15, 10, 5, 0]\n assert using_prev_fit_fi == set()\n\n\ndef test_sort_features_by_priority_base(sample_features):\n # test the ordering of feature importance computation when no prior feature importance computation was done\n sorted_features = sort_features_by_priority(\n features=sample_features, prev_importance_df=None, using_prev_fit_fi=set()\n )\n assert sorted_features == sample_features\n\n\ndef test_sort_features_by_priority_same_model(sample_features):\n # test the ordering of feature importance computation when prior feature importance computation from the same fitted model was done\n prev_importance_df = evaluated_fi_df_template(sample_features)\n sorted_features = sort_features_by_priority(\n features=sample_features, prev_importance_df=prev_importance_df, using_prev_fit_fi=set()\n )\n assert sorted_features == prev_importance_df.sort_values(\"importance\").index.tolist()\n\n\ndef test_sort_features_by_priority_different_model(sample_features):\n # test the ordering of feature importance computation when prior feature importance computation from a different fitted model was done\n prev_importance_df = evaluated_fi_df_template(sample_features)\n using_prev_fit_fi = sample_features[-2:]\n sorted_features = sort_features_by_priority(\n features=sample_features, prev_importance_df=prev_importance_df, using_prev_fit_fi=using_prev_fit_fi\n )\n sorted_prev_fit_features = (\n prev_importance_df[prev_importance_df.index.isin(using_prev_fit_fi)].sort_values(\"importance\").index.tolist()\n )\n sorted_curr_fit_features = (\n prev_importance_df[~prev_importance_df.index.isin(using_prev_fit_fi)].sort_values(\"importance\").index.tolist()\n )\n expected_features = sorted_prev_fit_features + sorted_curr_fit_features\n assert sorted_features == expected_features\n\n\ndef test_sort_features_by_priority_all(sample_features):\n # test the ordering of feature importance computation when feature impotance computation comes from mix of current and previous fit models,\n # and some feature are unevaluated\n length = len(sample_features)\n using_prev_fit_fi = set(sample_features[: length // 3])\n evaluated_rows, unevaluated_rows = (\n evaluated_fi_df_template(sample_features[: length // 2]),\n unevaluated_fi_df_template(sample_features[length // 2 :]),\n )\n prev_importance_df = pd.concat([evaluated_rows, unevaluated_rows])\n sorted_features = sort_features_by_priority(\n features=sample_features, prev_importance_df=prev_importance_df, using_prev_fit_fi=using_prev_fit_fi\n )\n unevaluated_features = unevaluated_rows.index.tolist()\n sorted_prev_fit_features = (\n evaluated_rows[\n (~evaluated_rows.index.isin(sample_features[length // 2 :]))\n & (evaluated_rows.index.isin(using_prev_fit_fi))\n ]\n .sort_values(\"importance\")\n .index.tolist()\n )\n sorted_curr_fit_features = (\n evaluated_rows[\n (~evaluated_rows.index.isin(sample_features[length // 2 :]))\n & (~evaluated_rows.index.isin(using_prev_fit_fi))\n ]\n .sort_values(\"importance\")\n .index.tolist()\n )\n expected_features = unevaluated_features + sorted_prev_fit_features + sorted_curr_fit_features\n assert sorted_features == expected_features\n" }, { "path": "core/tests/unittests/test_import_version.py", "content": "import autogluon.core\n\n\ndef test_import_version():\n assert isinstance(autogluon.core.__version__, str)\n assert len(autogluon.core.__version__) != 0\n" }, { "path": "core/tests/unittests/test_parallel_local_folding.py", "content": "import math\nimport time\nfrom unittest.mock import patch\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common import space\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.core.models.ensemble.fold_fitting_strategy import FoldFittingStrategy, ParallelLocalFoldFittingStrategy\nfrom autogluon.core.ray.resources_calculator import CpuResourceCalculator\nfrom autogluon.core.searcher import LocalRandomSearcher\n\nNUM_CPU = 8\nNUM_GPU = 1\n\n\nclass DummyBigModel(AbstractModel):\n def _estimate_memory_usage(self, **kwargs):\n return 1e9\n\n\ndef _prepare_data():\n # prepare an all numeric data so that we don't need to clean labels and features\n data = [[1, 10], [2, 20], [3, 30]]\n df = pd.DataFrame(data, columns=[\"Number\", \"Age\"])\n label = \"Age\"\n X = df.drop(columns=[label])\n y = df[label]\n return X, y\n\n\ndef _construct_dummy_fold_strategy(num_jobs, model_base_cls=AbstractModel, time_limit=None, num_folds_parallel=8):\n dummy_model_base = model_base_cls()\n dummy_bagged_ensemble_model = BaggedEnsembleModel(dummy_model_base)\n train_data, test_data = _prepare_data()\n args = dict(\n model_base=dummy_model_base,\n model_base_kwargs=dict(),\n bagged_ensemble_model=dummy_bagged_ensemble_model,\n X=train_data,\n y=test_data,\n X_pseudo=None,\n y_pseudo=None,\n sample_weight=None,\n time_limit=time_limit,\n time_start=time.time(),\n models=[],\n oof_pred_proba=np.array([]),\n oof_pred_model_repeats=np.array([]),\n save_folds=True,\n num_cpus=NUM_CPU,\n num_gpus=NUM_GPU,\n num_jobs=num_jobs,\n num_folds_parallel=num_folds_parallel,\n time_limit_fold_ratio=1,\n max_memory_usage_ratio=1,\n )\n return ParallelLocalFoldFittingStrategy(**args)\n\n\ndef _test_resource_allocation_and_time_limit(num_jobs, num_folds_parallel, time_limit):\n num_cpus = NUM_CPU\n num_gpus = NUM_GPU\n time_start = time.time()\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n num_jobs=num_jobs, time_limit=time_limit, num_folds_parallel=num_folds_parallel\n )\n for i in range(num_jobs):\n fold_fitting_strategy.schedule_fold_model_fit(dict())\n resources, batches, num_parallel_jobs = (\n fold_fitting_strategy.resources,\n fold_fitting_strategy.batches,\n fold_fitting_strategy.num_parallel_jobs,\n )\n time_elapsed = time.time() - time_start\n time_remaining = time_limit - time_elapsed\n time_limit_fold = fold_fitting_strategy._get_fold_time_limit()\n num_cpus_per_job = resources.get(\"num_cpus\", 0)\n num_gpus_per_job = resources.get(\"num_gpus\", 0)\n assert batches >= 1\n if batches > 1:\n assert num_jobs <= num_parallel_jobs * batches <= (num_jobs + num_parallel_jobs)\n assert num_cpus_per_job * num_parallel_jobs <= num_cpus\n if num_gpus != 0:\n assert num_gpus_per_job * num_parallel_jobs <= num_gpus\n else:\n assert num_gpus_per_job == 0\n assert math.isclose(time_limit_fold, (time_remaining / batches), abs_tol=0.5)\n\n\ndef test_resource_allocation_and_time_limit():\n num_iterations = 100\n\n search_space = dict(\n num_jobs=space.Int(1, 100),\n num_folds_parallel=space.Int(1, 200),\n time_limit=space.Int(60, 60 * 60 * 24),\n )\n\n searcher = LocalRandomSearcher(search_space=search_space)\n\n for i in range(num_iterations):\n config = searcher.get_config()\n _test_resource_allocation_and_time_limit(**config)\n\n\n@patch(\"autogluon.common.utils.resource_utils.ResourceManager.get_available_virtual_mem\")\n@patch(\"autogluon.core.ray.resources_calculator.ResourceCalculatorFactory.get_resource_calculator\")\ndef test_dynamic_resource_allocation(resource_cal, mock_get_mem):\n mock_get_mem.return_value = 2.5 * 1e9\n resource_cal.return_value = CpuResourceCalculator()\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n model_base_cls=DummyBigModel, num_jobs=8, num_folds_parallel=8\n )\n assert fold_fitting_strategy.num_parallel_jobs == 2 and fold_fitting_strategy.batches == 4\n mock_get_mem.return_value = 7.5 * 1e9\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n model_base_cls=DummyBigModel, num_jobs=8, num_folds_parallel=8\n )\n # If memory is not sufficient to train num_folds_parallel, reduce to max power of 2 folds that's smaller than folds_can_be_fit_in_parallel.\n # Here memory can only train 7 folds, therefore we train 4 folds instead in two batches\n assert fold_fitting_strategy.num_parallel_jobs == 4 and fold_fitting_strategy.batches == 2\n mock_get_mem.return_value = 6 * 1e9\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n model_base_cls=DummyBigModel, num_jobs=10, num_folds_parallel=10\n )\n # Here memory can only train 10 folds, therefore we train 4 folds instead in three batches, the last batch would train 2 folds in parallel\n assert fold_fitting_strategy.num_parallel_jobs == 4 and fold_fitting_strategy.batches == 3\n" }, { "path": "core/tests/unittests/test_search_space.py", "content": "from autogluon.common import space\nfrom autogluon.core.scheduler import LocalSequentialScheduler\n\n\ndef test_search_space():\n search_space = dict(\n a=space.Real(1e-3, 1e-2, log=True),\n b=space.Real(1e-3, 1e-2),\n c=space.Int(1, 10),\n d=space.Categorical(\"a\", \"b\", \"c\", \"d\"),\n )\n\n def train_fn(args, reporter):\n a, b, c, d = args[\"a\"], args[\"b\"], args[\"c\"], args[\"d\"]\n assert 1e-2 >= a >= 1e-3\n assert 1e-2 >= b >= 1e-3\n assert 10 >= c >= 1\n assert d in [\"a\", \"b\", \"c\", \"d\"]\n reporter(epoch=1, reward=0)\n\n scheduler = LocalSequentialScheduler(train_fn, search_space=search_space, num_trials=10, time_attr=\"epoch\")\n scheduler.run()\n scheduler.join_jobs()\n\n\ndef test_search_space_dot_key():\n search_space = {\"model.name\": space.Categorical(\"mxnet\", \"pytorch\")}\n\n def train_fn(args, reporter):\n assert args[\"model.name\"] == \"mxnet\" or args[\"model.name\"] == \"pytorch\"\n\n scheduler = LocalSequentialScheduler(train_fn, search_space=search_space, num_trials=2)\n scheduler.run()\n scheduler.join_jobs()\n" }, { "path": "core/tests/unittests/utils/__init__.py", "content": "" }, { "path": "core/tests/unittests/utils/decorators/__init__.py", "content": "" }, { "path": "core/tests/unittests/utils/decorators/test_presets.py", "content": "import unittest\n\nfrom autogluon.common.utils.decorators import apply_presets\n\n\nclass TestPresets(unittest.TestCase):\n def test_presets(self):\n presets_dict = dict(preset_1=dict(a=2, b=3))\n\n @apply_presets(presets_dict, None)\n def get_presets(**kwargs):\n return kwargs\n\n # assert no presets works\n out = get_presets()\n assert len(out) == 0\n\n # assert no presets works with user-specified values\n out = get_presets(a=5)\n assert out[\"a\"] == 5\n assert len(out) == 1\n\n # assert ValueError raised if unknown preset\n self.assertRaises(ValueError, get_presets, presets=\"invalid_preset\")\n\n # assert presets == None works\n out = get_presets(presets=None)\n assert out[\"presets\"] is None\n assert len(out) == 1\n\n # assert presets as str works\n out = get_presets(presets=\"preset_1\")\n assert out[\"presets\"] == \"preset_1\"\n assert out[\"a\"] == 2\n assert out[\"b\"] == 3\n assert len(out) == 3\n\n # assert presets as list works\n out = get_presets(presets=[\"preset_1\"])\n assert out[\"presets\"] == [\"preset_1\"]\n assert out[\"a\"] == 2\n assert out[\"b\"] == 3\n assert len(out) == 3\n\n # assert custom preset works\n custom_preset = dict(a=4, c=7)\n out = get_presets(presets=custom_preset)\n assert out[\"presets\"] == custom_preset\n assert out[\"a\"] == 4\n assert out[\"c\"] == 7\n assert len(out) == 3\n\n # assert that multiple presets can be specified, and later ones overwrite earlier ones in shared keys\n out = get_presets(presets=[\"preset_1\", custom_preset])\n assert out[\"presets\"] == [\"preset_1\", custom_preset]\n assert out[\"a\"] == 4\n assert out[\"b\"] == 3\n assert out[\"c\"] == 7\n assert len(out) == 4\n\n # assert ValueError raised if unknown preset in list of presets\n self.assertRaises(ValueError, get_presets, presets=[\"preset_1\", \"invalid_preset\"])\n\n # assert that multiple presets can be specified, and later ones overwrite earlier ones in shared keys, but user-specified keys override presets\n out = get_presets(a=1, presets=[\"preset_1\", custom_preset], d=None)\n assert out[\"presets\"] == [\"preset_1\", custom_preset]\n assert out[\"a\"] == 1\n assert out[\"b\"] == 3\n assert out[\"c\"] == 7\n assert out[\"d\"] is None\n assert len(out) == 5\n\n presets_alias_dict = dict(\n preset_1_alias=\"preset_1\",\n preset_invalid_alias=\"invalid_preset\",\n )\n\n @apply_presets(presets_dict, presets_alias_dict)\n def get_presets(**kwargs):\n return kwargs\n\n # assert preset alias works\n out = get_presets(presets=\"preset_1_alias\")\n assert out[\"presets\"] == \"preset_1_alias\"\n assert out[\"a\"] == 2\n assert out[\"b\"] == 3\n assert len(out) == 3\n\n # assert ValueError raised if alias points to invalid preset\n self.assertRaises(ValueError, get_presets, presets=\"preset_invalid_alias\")\n" }, { "path": "core/tests/unittests/utils/test_augment_rare_classes.py", "content": "import logging\n\nimport pandas as pd\nimport pytest\n\nimport autogluon.core.utils.utils as utils_mod\nfrom autogluon.core.utils.utils import augment_rare_classes\n\n\n@pytest.fixture\ndef utils_log_records():\n \"\"\"\n Bulletproof log capture: attach a handler directly to the module logger used\n by augment_rare_classes (utils_mod.logger).\n \"\"\"\n records: list[logging.LogRecord] = []\n\n class ListHandler(logging.Handler):\n def emit(self, record: logging.LogRecord) -> None:\n records.append(record)\n\n logger = utils_mod.logger\n\n handler = ListHandler()\n handler.setLevel(logging.DEBUG)\n\n old_level = logger.level\n old_handlers = list(logger.handlers)\n\n # Ensure our handler sees everything\n logger.setLevel(logging.DEBUG)\n logger.addHandler(handler)\n\n try:\n yield records\n finally:\n logger.removeHandler(handler)\n logger.setLevel(old_level)\n # (We don't mutate existing handlers; nothing to restore besides our removal.)\n assert logger.handlers == old_handlers\n\n\ndef _messages(records: list[logging.LogRecord]) -> list[str]:\n return [r.getMessage() for r in records]\n\n\ndef test_no_augmentation_returns_same_df_object(utils_log_records):\n X = pd.DataFrame({\"y\": [\"a\", \"a\", \"b\", \"b\"], \"f\": [1, 2, 3, 4]})\n out = augment_rare_classes(X, label=\"y\", threshold=2)\n\n assert out is X\n assert out.equals(X)\n\n msgs = _messages(utils_log_records)\n assert any(\"did not need to duplicate any data\" in m for m in msgs)\n\n\ndef test_augment_single_rare_class_adds_expected_rows_and_meets_threshold():\n # a occurs 2, threshold=5 -> add 3\n X = pd.DataFrame({\"y\": [\"a\", \"a\", \"b\", \"b\", \"b\"], \"f\": [10, 11, 20, 21, 22]})\n out = augment_rare_classes(X, label=\"y\", threshold=5)\n\n assert len(out) == len(X) + 5\n vc = out[\"y\"].value_counts()\n assert vc[\"a\"] == 5\n assert vc[\"b\"] == 5\n\n # New rows must be duplicates of existing rows from 'a'\n orig_a = X.loc[X[\"y\"] == \"a\"].reset_index(drop=True)\n new_rows = out.loc[out.index.difference(X.index)].reset_index(drop=True)\n new_a = new_rows.loc[new_rows[\"y\"] == \"a\"].reset_index(drop=True)\n\n for _, row in new_a.iterrows():\n assert ((orig_a == row).all(axis=1)).any()\n\n\ndef test_augment_multiple_rare_classes_total_added_and_threshold_met():\n # b occurs 2, threshold=7 -> add 5\n # c occurs 3, threshold=7 -> add 4\n X = pd.DataFrame(\n {\n \"y\": [\"a\"] * 10 + [\"b\", \"b\"] + [\"c\", \"c\", \"c\"],\n \"f\": list(range(10)) + [100, 101] + [200, 201, 202],\n }\n )\n out = augment_rare_classes(X, label=\"y\", threshold=7)\n\n vc = out[\"y\"].value_counts()\n assert vc[\"a\"] == 10\n assert vc[\"b\"] == 7\n assert vc[\"c\"] == 7\n assert len(out) == len(X) + 9\n\n\ndef test_augmented_indices_unique_and_start_after_max():\n X = pd.DataFrame({\"y\": [\"a\", \"a\", \"b\"], \"f\": [1, 2, 3]}, index=[10, 20, 35])\n out = augment_rare_classes(X, label=\"y\", threshold=4)\n\n new_rows = 5\n\n assert len(out) == len(X) + new_rows\n assert out.index.is_unique\n\n start = X.index.max() + 1 # 36\n new_idx = sorted(set(out.index) - set(X.index))\n assert new_idx == [start + i for i in range(new_rows)]\n\n\ndef test_missing_classes_zero_count_are_warned_and_ignored(utils_log_records):\n # Categorical with unused categories -> value_counts includes 0 counts for them.\n y = pd.Categorical([\"a\", \"a\", \"a\"], categories=[\"a\", \"b\", \"c\"])\n X = pd.DataFrame({\"y\": y, \"f\": [1, 2, 3]})\n\n # Sanity: ensure our pandas produces 0-count entries\n class_counts = X[\"y\"].value_counts()\n assert (class_counts == 0).any()\n\n out = augment_rare_classes(X, label=\"y\", threshold=2)\n\n # Only invalid classes were 0-count; function should return X unchanged.\n assert out is X\n assert out.equals(X)\n\n msgs = _messages(utils_log_records)\n assert any(\"0 training examples\" in m for m in msgs)\n assert any(\"These classes will be ignored\" in m for m in msgs)\n" }, { "path": "core/tests/unittests/utils/test_time.py", "content": "from pandas import DataFrame\n\nfrom autogluon.core.utils.time import sample_df_for_time_func\n\n\ndef test_sample_df_for_time_func():\n df = DataFrame({\"a\": [1, 5, 3, 8, 12]})\n\n df_out_1 = sample_df_for_time_func(df=df, sample_size=1, max_sample_size=10)\n df_out_2 = sample_df_for_time_func(df=df, sample_size=3, max_sample_size=10)\n df_out_3 = sample_df_for_time_func(df=df, sample_size=5, max_sample_size=10)\n df_out_4 = sample_df_for_time_func(df=df, sample_size=7, max_sample_size=10)\n df_out_5 = sample_df_for_time_func(df=df, sample_size=14, max_sample_size=10)\n df_out_6 = sample_df_for_time_func(df=df, sample_size=14, max_sample_size=None)\n df_out_7 = sample_df_for_time_func(df=df, sample_size=6, max_sample_size=4)\n\n assert (df.head(1) == df_out_1).all().all()\n assert (df.head(3) == df_out_2).all().all()\n assert (df == df_out_3).all().all()\n assert 7 == len(df_out_4)\n assert [\"a\"] == list(df_out_4.columns)\n assert 10 == len(df_out_5)\n assert [\"a\"] == list(df_out_5.columns)\n assert 14 == len(df_out_6)\n assert [\"a\"] == list(df_out_6.columns)\n assert 5 == len(df_out_7)\n assert (df == df_out_7).all().all()\n" }, { "path": "core/tests/unittests/utils/test_utils.py", "content": "import unittest\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, MULTICLASS_UPPER_LIMIT, REGRESSION\nfrom autogluon.core.utils import infer_problem_type\nfrom autogluon.core.utils.utils import generate_train_test_split\n\n\nclass TestInferProblemType(unittest.TestCase):\n def test_infer_problem_type_empty(self):\n with self.assertRaises(ValueError):\n infer_problem_type(pd.Series([], dtype=\"float\"))\n\n def test_infer_problem_type_nan(self):\n with self.assertRaises(ValueError):\n infer_problem_type(pd.Series([np.nan]))\n\n def test_infer_problem_type_inf(self):\n with self.assertRaises(ValueError):\n infer_problem_type(pd.Series([np.inf]))\n\n def test_infer_problem_type_ninf(self):\n with self.assertRaises(ValueError):\n infer_problem_type(pd.Series([-np.inf]))\n\n def test_infer_problem_type_binary(self):\n inferred_problem_type = infer_problem_type(pd.Series([-1, -1, 99, -1, -1, 99]))\n assert inferred_problem_type == BINARY\n\n def test_infer_problem_type_binary_with_nan(self):\n inferred_problem_type = infer_problem_type(pd.Series([-1, -1, 99, -1, -1, 99, np.nan]))\n assert inferred_problem_type == BINARY\n\n def test_infer_problem_type_str(self):\n inferred_problem_type = infer_problem_type(pd.Series([\"a\", \"b\", \"c\"], dtype=str))\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_category(self):\n inferred_problem_type = infer_problem_type(pd.Series([\"a\", \"b\", \"c\"], dtype=\"category\"))\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_object(self):\n inferred_problem_type = infer_problem_type(pd.Series([\"a\", \"b\", \"c\"], dtype=\"object\"))\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_multiclass_with_nan(self):\n inferred_problem_type = infer_problem_type(pd.Series([\"a\", \"b\", \"c\", np.nan]))\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_big_float_data_regression(self):\n big_float_regression_series = pd.Series(np.repeat(np.linspace(0.0, 1.0, MULTICLASS_UPPER_LIMIT + 1), 2))\n inferred_problem_type = infer_problem_type(big_float_regression_series)\n assert inferred_problem_type == REGRESSION\n\n def test_infer_problem_type_small_float_data_multiclass(self):\n big_float_multiclass_series = pd.Series(np.repeat([1.0, 2.0, 3.0], MULTICLASS_UPPER_LIMIT - 1))\n inferred_problem_type = infer_problem_type(big_float_multiclass_series)\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_small_float_data_regression(self):\n small_float_regression_series = pd.Series(np.linspace(0.0, 1.0, MULTICLASS_UPPER_LIMIT - 1))\n inferred_problem_type = infer_problem_type(small_float_regression_series)\n assert inferred_problem_type == REGRESSION\n\n def test_infer_problem_type_big_integer_data_regression(self):\n big_integer_regression_series = pd.Series(np.repeat(np.arange(MULTICLASS_UPPER_LIMIT + 1), 2), dtype=np.int64)\n inferred_problem_type = infer_problem_type(big_integer_regression_series)\n assert inferred_problem_type == REGRESSION\n\n def test_infer_problem_type_small_integer_data_multiclass(self):\n small_integer_multiclass_series = pd.Series(\n np.repeat(np.arange(3), MULTICLASS_UPPER_LIMIT - 1), dtype=np.int64\n )\n inferred_problem_type = infer_problem_type(small_integer_multiclass_series)\n assert inferred_problem_type == MULTICLASS\n\n def test_infer_problem_type_small_integer_data_regression(self):\n small_integer_regression_series = pd.Series(np.arange(MULTICLASS_UPPER_LIMIT - 1), dtype=np.int64)\n inferred_problem_type = infer_problem_type(small_integer_regression_series)\n assert inferred_problem_type == REGRESSION\n\n\ndef _assert_equals_generate_train_test_split(X, y, test_size, problem_type=None, test_equals=True, train_size=None):\n X_train, X_test, y_train, y_test = generate_train_test_split(\n X=X, y=y, problem_type=problem_type, test_size=test_size, train_size=train_size\n )\n assert len(X_train) == len(y_train)\n assert list(X_train.index) == list(y_train.index)\n assert len(X_test) == len(y_test)\n assert list(X_test.index) == list(y_test.index)\n assert len(X_train.index.intersection(X_test.index)) == 0 # No shared indices\n if test_equals:\n if isinstance(test_size, int):\n assert len(X_test) == test_size\n else:\n test_size_int = round(len(X) * test_size)\n assert len(X_test) == test_size_int\n else:\n if isinstance(test_size, int):\n assert len(X_test) <= test_size\n else:\n test_size_int = round(len(X) * test_size)\n assert len(X_test) <= test_size_int\n if train_size is not None:\n if isinstance(train_size, int):\n assert len(X_train) == train_size\n else:\n train_size_int = round(len(X) * train_size)\n assert len(X_train) == train_size_int\n if train_size is None or test_size is None:\n assert len(X_train) == len(X) - len(X_test)\n\n if train_size is None:\n if isinstance(test_size, int):\n train_size = len(X) - test_size\n else:\n train_size = 1.0 - test_size\n X_train_v2, X_test_v2, y_train_v2, y_test_v2 = generate_train_test_split(\n X=X, y=y, problem_type=problem_type, train_size=train_size\n )\n assert X_train.equals(X_train_v2)\n assert y_train.equals(y_train_v2)\n assert X_test.equals(X_test_v2)\n assert y_test.equals(y_test_v2)\n train_size = None\n\n if problem_type is not None and problem_type in [\"binary\", \"multiclass\"]:\n X_train_v3, X_test_v3, y_train_v3, y_test_v3 = generate_train_test_split(\n X=X, y=y, test_size=test_size, train_size=train_size, stratify=y\n )\n assert X_train.loc[X_train_v3.index].equals(X_train_v3)\n assert y_train.loc[y_train_v3.index].equals(y_train_v3)\n assert X_test.equals(X_test_v3.loc[X_test.index])\n assert y_test.equals(y_test_v3.loc[y_test.index])\n\n\ndef test_generate_train_test_split_edgecase():\n \"\"\"\n Test rare edge-cases when data has many classes or very few samples when doing train test splits.\n \"\"\"\n data = pd.DataFrame(index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])\n data[\"label\"] = [0, 1, 1, 2, 3, 4, 5, 5, 5, 5, 5, 5]\n\n for test_size in range(1, 12):\n _assert_equals_generate_train_test_split(X=data, y=data[\"label\"], test_size=test_size)\n _assert_equals_generate_train_test_split(X=data, y=data[\"label\"], test_size=test_size / len(data))\n for problem_type in [\"regression\", \"softclass\", \"quantile\"]:\n \"\"\"\n Normal Case: Regression should always work\n \"\"\"\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size\n )\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data)\n )\n for train_size in range(1, len(data) - test_size + 1):\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], test_size=test_size, train_size=train_size\n )\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], test_size=test_size / len(data), train_size=train_size / len(data)\n )\n\n for test_size in range(1, 12):\n _assert_equals_generate_train_test_split(X=data, y=data[\"label\"], test_size=test_size)\n _assert_equals_generate_train_test_split(X=data, y=data[\"label\"], test_size=test_size / len(data))\n for problem_type in [\"regression\", \"softclass\", \"quantile\"]:\n \"\"\"\n Normal Case: Regression should always work\n \"\"\"\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size\n )\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data)\n )\n\n for problem_type in [\"binary\", \"multiclass\"]:\n for test_size in range(1, 6):\n \"\"\"\n Edge-case: There are fewer test rows than classes\n This only works because of special try/except logic in `generate_train_test_split`.\n \"\"\"\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size, test_equals=False\n )\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data), test_equals=False\n )\n\n for test_size in range(6, 7):\n \"\"\"\n Normal Case\n \"\"\"\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size, test_equals=False\n )\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data), test_equals=False\n )\n\n for test_size in range(7, 12):\n \"\"\"\n Edge-case: There are fewer train rows than classes\n Error due to not enough training data to have at least one instance of every class in train.\n Note: Ideally this shouldn't raise an exception, but writing the logic to avoid the error is tricky and the scenario should never occur in practice.\n \"\"\"\n with pytest.raises(ValueError):\n X_train, X_test, y_train, y_test = generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size\n )\n\n # FIXME: Different for fractional inputs, because there is an inconsistency between float test_size and integer test_size in the internal logic.\n # We should fix this eventually. Once it is fixed, this test will fail.\n for test_size in range(7, 10):\n _assert_equals_generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data), test_equals=False\n )\n for test_size in range(10, 12):\n with pytest.raises(ValueError):\n X_train, X_test, y_train, y_test = generate_train_test_split(\n X=data, y=data[\"label\"], problem_type=problem_type, test_size=test_size / len(data)\n )\n" }, { "path": "core/tests/unittests/utils/test_version_utils.py", "content": "from autogluon.core.utils import show_versions\n\n\ndef test_show_versions():\n # Only verify this function does not crash\n show_versions()\n" }, { "path": "docs/.gitignore", "content": "_build\nmy_model\n.DS_Store\n**/.DS_Store\n*/*/.DS_Store\n*/*/*/.DS_Store\n**/AutogluonModels\njupyter_execute\n_autogen\ntabular/*.csv\n*.zip\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n" }, { "path": "docs/LICENSE", "content": "Attribution-NonCommercial-ShareAlike 4.0 International\n\n=======================================================================\n\nCreative Commons Corporation (\"Creative Commons\") is not a law firm and\ndoes not provide legal services or legal advice. Distribution of\nCreative Commons public licenses does not create a lawyer-client or\nother relationship. Creative Commons makes its licenses and related\ninformation available on an \"as-is\" basis. Creative Commons gives no\nwarranties regarding its licenses, any material licensed under their\nterms and conditions, or any related information. Creative Commons\ndisclaims all liability for damages resulting from their use to the\nfullest extent possible.\n\nUsing Creative Commons Public Licenses\n\nCreative Commons public licenses provide a standard set of terms and\nconditions that creators and other rights holders may use to share\noriginal works of authorship and other material subject to copyright\nand certain other rights specified in the public license below. The\nfollowing considerations are for informational purposes only, are not\nexhaustive, and do not form part of our licenses.\n\n Considerations for licensors: Our public licenses are\n intended for use by those authorized to give the public\n permission to use material in ways otherwise restricted by\n copyright and certain other rights. Our licenses are\n irrevocable. Licensors should read and understand the terms\n and conditions of the license they choose before applying it.\n Licensors should also secure all rights necessary before\n applying our licenses so that the public can reuse the\n material as expected. Licensors should clearly mark any\n material not subject to the license. This includes other CC-\n licensed material, or material used under an exception or\n limitation to copyright. More considerations for licensors:\n\twiki.creativecommons.org/Considerations_for_licensors\n\n Considerations for the public: By using one of our public\n licenses, a licensor grants the public permission to use the\n licensed material under specified terms and conditions. If\n the licensor's permission is not necessary for any reason--for\n example, because of any applicable exception or limitation to\n copyright--then that use is not regulated by the license. Our\n licenses grant only permissions under copyright and certain\n other rights that a licensor has authority to grant. Use of\n the licensed material may still be restricted for other\n reasons, including because others have copyright or other\n rights in the material. A licensor may make special requests,\n such as asking that all changes be marked or described.\n Although not required by our licenses, you are encouraged to\n respect those requests where reasonable. More considerations\n for the public: \n\twiki.creativecommons.org/Considerations_for_licensees\n\n=======================================================================\n\nCreative Commons Attribution-NonCommercial-ShareAlike 4.0 International\nPublic License\n\nBy exercising the Licensed Rights (defined below), You accept and agree\nto be bound by the terms and conditions of this Creative Commons\nAttribution-NonCommercial-ShareAlike 4.0 International Public License\n(\"Public License\"). To the extent this Public License may be\ninterpreted as a contract, You are granted the Licensed Rights in\nconsideration of Your acceptance of these terms and conditions, and the\nLicensor grants You such rights in consideration of benefits the\nLicensor receives from making the Licensed Material available under\nthese terms and conditions.\n\n\nSection 1 -- Definitions.\n\n a. Adapted Material means material subject to Copyright and Similar\n Rights that is derived from or based upon the Licensed Material\n and in which the Licensed Material is translated, altered,\n arranged, transformed, or otherwise modified in a manner requiring\n permission under the Copyright and Similar Rights held by the\n Licensor. For purposes of this Public License, where the Licensed\n Material is a musical work, performance, or sound recording,\n Adapted Material is always produced where the Licensed Material is\n synched in timed relation with a moving image.\n\n b. Adapter's License means the license You apply to Your Copyright\n and Similar Rights in Your contributions to Adapted Material in\n accordance with the terms and conditions of this Public License.\n\n c. BY-NC-SA Compatible License means a license listed at\n creativecommons.org/compatiblelicenses, approved by Creative\n Commons as essentially the equivalent of this Public License.\n\n d. Copyright and Similar Rights means copyright and/or similar rights\n closely related to copyright including, without limitation,\n performance, broadcast, sound recording, and Sui Generis Database\n Rights, without regard to how the rights are labeled or\n categorized. For purposes of this Public License, the rights\n specified in Section 2(b)(1)-(2) are not Copyright and Similar\n Rights.\n\n e. Effective Technological Measures means those measures that, in the\n absence of proper authority, may not be circumvented under laws\n fulfilling obligations under Article 11 of the WIPO Copyright\n Treaty adopted on December 20, 1996, and/or similar international\n agreements.\n\n f. Exceptions and Limitations means fair use, fair dealing, and/or\n any other exception or limitation to Copyright and Similar Rights\n that applies to Your use of the Licensed Material.\n\n g. License Elements means the license attributes listed in the name\n of a Creative Commons Public License. The License Elements of this\n Public License are Attribution, NonCommercial, and ShareAlike.\n\n h. Licensed Material means the artistic or literary work, database,\n or other material to which the Licensor applied this Public\n License.\n\n i. Licensed Rights means the rights granted to You subject to the\n terms and conditions of this Public License, which are limited to\n all Copyright and Similar Rights that apply to Your use of the\n Licensed Material and that the Licensor has authority to license.\n\n j. Licensor means the individual(s) or entity(ies) granting rights\n under this Public License.\n\n k. NonCommercial means not primarily intended for or directed towards\n commercial advantage or monetary compensation. For purposes of\n this Public License, the exchange of the Licensed Material for\n other material subject to Copyright and Similar Rights by digital\n file-sharing or similar means is NonCommercial provided there is\n no payment of monetary compensation in connection with the\n exchange.\n\n l. Share means to provide material to the public by any means or\n process that requires permission under the Licensed Rights, such\n as reproduction, public display, public performance, distribution,\n dissemination, communication, or importation, and to make material\n available to the public including in ways that members of the\n public may access the material from a place and at a time\n individually chosen by them.\n\n m. Sui Generis Database Rights means rights other than copyright\n resulting from Directive 96/9/EC of the European Parliament and of\n the Council of 11 March 1996 on the legal protection of databases,\n as amended and/or succeeded, as well as other essentially\n equivalent rights anywhere in the world.\n\n n. You means the individual or entity exercising the Licensed Rights\n under this Public License. Your has a corresponding meaning.\n\n\nSection 2 -- Scope.\n\n a. License grant.\n\n 1. Subject to the terms and conditions of this Public License,\n the Licensor hereby grants You a worldwide, royalty-free,\n non-sublicensable, non-exclusive, irrevocable license to\n exercise the Licensed Rights in the Licensed Material to:\n\n a. reproduce and Share the Licensed Material, in whole or\n in part, for NonCommercial purposes only; and\n\n b. produce, reproduce, and Share Adapted Material for\n NonCommercial purposes only.\n\n 2. Exceptions and Limitations. For the avoidance of doubt, where\n Exceptions and Limitations apply to Your use, this Public\n License does not apply, and You do not need to comply with\n its terms and conditions.\n\n 3. Term. The term of this Public License is specified in Section\n 6(a).\n\n 4. Media and formats; technical modifications allowed. The\n Licensor authorizes You to exercise the Licensed Rights in\n all media and formats whether now known or hereafter created,\n and to make technical modifications necessary to do so. The\n Licensor waives and/or agrees not to assert any right or\n authority to forbid You from making technical modifications\n necessary to exercise the Licensed Rights, including\n technical modifications necessary to circumvent Effective\n Technological Measures. For purposes of this Public License,\n simply making modifications authorized by this Section 2(a)\n (4) never produces Adapted Material.\n\n 5. Downstream recipients.\n\n a. Offer from the Licensor -- Licensed Material. Every\n recipient of the Licensed Material automatically\n receives an offer from the Licensor to exercise the\n Licensed Rights under the terms and conditions of this\n Public License.\n\n b. Additional offer from the Licensor -- Adapted Material.\n Every recipient of Adapted Material from You\n automatically receives an offer from the Licensor to\n exercise the Licensed Rights in the Adapted Material\n under the conditions of the Adapter's License You apply.\n\n c. No downstream restrictions. You may not offer or impose\n any additional or different terms or conditions on, or\n apply any Effective Technological Measures to, the\n Licensed Material if doing so restricts exercise of the\n Licensed Rights by any recipient of the Licensed\n Material.\n\n 6. No endorsement. Nothing in this Public License constitutes or\n may be construed as permission to assert or imply that You\n are, or that Your use of the Licensed Material is, connected\n with, or sponsored, endorsed, or granted official status by,\n the Licensor or others designated to receive attribution as\n provided in Section 3(a)(1)(A)(i).\n\n b. Other rights.\n\n 1. Moral rights, such as the right of integrity, are not\n licensed under this Public License, nor are publicity,\n privacy, and/or other similar personality rights; however, to\n the extent possible, the Licensor waives and/or agrees not to\n assert any such rights held by the Licensor to the limited\n extent necessary to allow You to exercise the Licensed\n Rights, but not otherwise.\n\n 2. Patent and trademark rights are not licensed under this\n Public License.\n\n 3. To the extent possible, the Licensor waives any right to\n collect royalties from You for the exercise of the Licensed\n Rights, whether directly or through a collecting society\n under any voluntary or waivable statutory or compulsory\n licensing scheme. In all other cases the Licensor expressly\n reserves any right to collect such royalties, including when\n the Licensed Material is used other than for NonCommercial\n purposes.\n\n\nSection 3 -- License Conditions.\n\nYour exercise of the Licensed Rights is expressly made subject to the\nfollowing conditions.\n\n a. Attribution.\n\n 1. If You Share the Licensed Material (including in modified\n form), You must:\n\n a. retain the following if it is supplied by the Licensor\n with the Licensed Material:\n\n i. identification of the creator(s) of the Licensed\n Material and any others designated to receive\n attribution, in any reasonable manner requested by\n the Licensor (including by pseudonym if\n designated);\n\n ii. a copyright notice;\n\n iii. a notice that refers to this Public License;\n\n iv. a notice that refers to the disclaimer of\n warranties;\n\n v. a URI or hyperlink to the Licensed Material to the\n extent reasonably practicable;\n\n b. indicate if You modified the Licensed Material and\n retain an indication of any previous modifications; and\n\n c. indicate the Licensed Material is licensed under this\n Public License, and include the text of, or the URI or\n hyperlink to, this Public License.\n\n 2. You may satisfy the conditions in Section 3(a)(1) in any\n reasonable manner based on the medium, means, and context in\n which You Share the Licensed Material. For example, it may be\n reasonable to satisfy the conditions by providing a URI or\n hyperlink to a resource that includes the required\n information.\n 3. If requested by the Licensor, You must remove any of the\n information required by Section 3(a)(1)(A) to the extent\n reasonably practicable.\n\n b. ShareAlike.\n\n In addition to the conditions in Section 3(a), if You Share\n Adapted Material You produce, the following conditions also apply.\n\n 1. The Adapter's License You apply must be a Creative Commons\n license with the same License Elements, this version or\n later, or a BY-NC-SA Compatible License.\n\n 2. You must include the text of, or the URI or hyperlink to, the\n Adapter's License You apply. You may satisfy this condition\n in any reasonable manner based on the medium, means, and\n context in which You Share Adapted Material.\n\n 3. You may not offer or impose any additional or different terms\n or conditions on, or apply any Effective Technological\n Measures to, Adapted Material that restrict exercise of the\n rights granted under the Adapter's License You apply.\n\n\nSection 4 -- Sui Generis Database Rights.\n\nWhere the Licensed Rights include Sui Generis Database Rights that\napply to Your use of the Licensed Material:\n\n a. for the avoidance of doubt, Section 2(a)(1) grants You the right\n to extract, reuse, reproduce, and Share all or a substantial\n portion of the contents of the database for NonCommercial purposes\n only;\n\n b. if You include all or a substantial portion of the database\n contents in a database in which You have Sui Generis Database\n Rights, then the database in which You have Sui Generis Database\n Rights (but not its individual contents) is Adapted Material,\n including for purposes of Section 3(b); and\n\n c. You must comply with the conditions in Section 3(a) if You Share\n all or a substantial portion of the contents of the database.\n\nFor the avoidance of doubt, this Section 4 supplements and does not\nreplace Your obligations under this Public License where the Licensed\nRights include other Copyright and Similar Rights.\n\n\nSection 5 -- Disclaimer of Warranties and Limitation of Liability.\n\n a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE\n EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS\n AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF\n ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,\n IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,\n WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR\n PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,\n ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT\n KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT\n ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.\n\n b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE\n TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,\n NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,\n INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,\n COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR\n USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN\n ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR\n DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR\n IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.\n\n c. The disclaimer of warranties and limitation of liability provided\n above shall be interpreted in a manner that, to the extent\n possible, most closely approximates an absolute disclaimer and\n waiver of all liability.\n\n\nSection 6 -- Term and Termination.\n\n a. This Public License applies for the term of the Copyright and\n Similar Rights licensed here. However, if You fail to comply with\n this Public License, then Your rights under this Public License\n terminate automatically.\n\n b. Where Your right to use the Licensed Material has terminated under\n Section 6(a), it reinstates:\n\n 1. automatically as of the date the violation is cured, provided\n it is cured within 30 days of Your discovery of the\n violation; or\n\n 2. upon express reinstatement by the Licensor.\n\n For the avoidance of doubt, this Section 6(b) does not affect any\n right the Licensor may have to seek remedies for Your violations\n of this Public License.\n\n c. For the avoidance of doubt, the Licensor may also offer the\n Licensed Material under separate terms or conditions or stop\n distributing the Licensed Material at any time; however, doing so\n will not terminate this Public License.\n\n d. Sections 1, 5, 6, 7, and 8 survive termination of this Public\n License.\n\n\nSection 7 -- Other Terms and Conditions.\n\n a. The Licensor shall not be bound by any additional or different\n terms or conditions communicated by You unless expressly agreed.\n\n b. Any arrangements, understandings, or agreements regarding the\n Licensed Material not stated herein are separate from and\n independent of the terms and conditions of this Public License.\n\n\nSection 8 -- Interpretation.\n\n a. For the avoidance of doubt, this Public License does not, and\n shall not be interpreted to, reduce, limit, restrict, or impose\n conditions on any use of the Licensed Material that could lawfully\n be made without permission under this Public License.\n\n b. To the extent possible, if any provision of this Public License is\n deemed unenforceable, it shall be automatically reformed to the\n minimum extent necessary to make it enforceable. If the provision\n cannot be reformed, it shall be severed from this Public License\n without affecting the enforceability of the remaining terms and\n conditions.\n\n c. No term or condition of this Public License will be waived and no\n failure to comply consented to unless expressly agreed to by the\n Licensor.\n\n d. Nothing in this Public License constitutes or may be interpreted\n as a limitation upon, or waiver of, any privileges and immunities\n that apply to the Licensor or You, including from the legal\n processes of any jurisdiction or authority.\n\n=======================================================================\n\nCreative Commons is not a party to its public\nlicenses. Notwithstanding, Creative Commons may elect to apply one of\nits public licenses to material it publishes and in those instances\nwill be considered the âLicensor.â The text of the Creative Commons\npublic licenses is dedicated to the public domain under the CC0 Public\nDomain Dedication. Except for the limited purpose of indicating that\nmaterial is shared under a Creative Commons public license or as\notherwise permitted by the Creative Commons policies published at\ncreativecommons.org/policies, Creative Commons does not authorize the\nuse of the trademark \"Creative Commons\" or any other trademark or logo\nof Creative Commons without its prior written consent including,\nwithout limitation, in connection with any unauthorized modifications\nto any of its public licenses or any other arrangements,\nunderstandings, or agreements concerning use of licensed material. For\nthe avoidance of doubt, this paragraph does not form part of the\npublic licenses.\n\nCreative Commons may be contacted at creativecommons.org.\n\n" }, { "path": "docs/README.md", "content": "# How to perform a new release\n\nRefer to `release_instructions/ReleaseInstructions.md`\n\n# How to locally build AutoGluon docs\n\nInstructions apply to Linux and Mac. Windows has not been tested for doc builds.\n\nEnsure you have a local AutoGluon install for development. If not run the following in package root:\n\n```shell\npip install -U pip wheel\n./full_install.sh\n```\n\nThen run in package root:\n\n```shell\ncd docs/\npython3 -m pip install -r requirements_doc.txt\n```\n\nNow you are ready to run the doc build:\n\n```shell\n# Note: GPU & CUDA is required to build tutorials\n# To skip running tutorials, manually edit `docs/conf.py` and set `nb_execution_mode=off`\nbash build_doc.sh\n```\n" }, { "path": "docs/_static/custom.css", "content": ".highlight-bash .highlight {\n border: 1px #ccc solid;\n}\n\ndiv.cell div.cell_input, .highlight-bash .highlight {\n border-left-color: var(--color-brand-content);\n border-left-width: medium;\n border-radius: 0.2rem;\n}\n\ndiv.cell_output table {\n color: #2b8cee;\n font-size: 0.8rem;\n}\n\ndiv.cell_output {\n overflow-x: scroll;\n margin-top: 0;\n}\n\ndiv.sidebar-scroll {\n scroll-behavior: auto;\n}\n\n.dataframe {\n margin: 1em 0;\n}\n\ndetails.toggle-details summary:hover, details.toggle-details summary:active, details.toggle-details summary {\n background: transparent;\n}\n\n\ndetails.sd-dropdown summary {\n padding: .5rem;\n}\n\ndetails.sd-dropdown .sd-summary-up, details.sd-dropdown .sd-summary-down {\n top: .5rem;\n}\n\nbody[data-theme=\"dark\"] {\n --mystnb-source-bg-color: #202020;\n --mystnb-stdout-bg-color: #202020;\n --mystnb-stderr-bg-color: #ee0000;\n --mystnb-traceback-bg-color: #202020;\n}\n" }, { "path": "docs/_static/custom.js", "content": "let sidebar_scroll_element = document.querySelector(\".sidebar-scroll\");\n\nlet saved_top = sessionStorage.getItem(\"sidebar-scroll-top\");\nif (saved_top !== null) {\n sidebar_scroll_element.scrollTop = parseInt(saved_top, 10);\n}\n\nwindow.addEventListener(\"beforeunload\", () => {\n sessionStorage.setItem(\"sidebar-scroll-top\", sidebar_scroll_element.scrollTop);\n});\n" }, { "path": "docs/_templates/autosummary/base.rst", "content": "{{ objname | escape | underline}}\n\n.. currentmodule:: {{ module }}\n\n.. auto{{ objtype }}:: {{ objname }}\n" }, { "path": "docs/_templates/autosummary/class.rst", "content": "{{ objname | escape | underline}}\n\n.. currentmodule:: {{ module }}\n\n.. autoclass:: {{ objname }}\n :members: \n\n {% block methods %}\n\n {% if methods %}\n .. rubric:: {{ _('Methods') }}\n\n .. autosummary::\n :nosignatures:\n\n {% for item in methods if item != '__init__' %}\n ~{{ name }}.{{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n {% block attributes %}\n {% if attributes %}\n .. rubric:: {{ _('Attributes') }}\n\n .. autosummary::\n {% for item in attributes %}\n ~{{ name }}.{{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n" }, { "path": "docs/_templates/autosummary/module.rst", "content": "{{ fullname | escape | underline}}\n\n.. automodule:: {{ fullname }}\n\n {% block attributes %}\n {% if attributes %}\n .. rubric:: {{ _('Module Attributes') }}\n\n .. autosummary::\n {% for item in attributes %}\n {{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n {% block functions %}\n {% if functions %}\n .. rubric:: {{ _('Functions') }}\n\n .. autosummary::\n :toctree: \n {% for item in functions %}\n {{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n {% block classes %}\n {% if classes %}\n .. rubric:: {{ _('Classes') }}\n\n .. autosummary::\n :toctree: \n {% for item in classes %}\n {{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n {% block exceptions %}\n {% if exceptions %}\n .. rubric:: {{ _('Exceptions') }}\n\n .. autosummary::\n {% for item in exceptions %}\n {{ item }}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n{% block modules %}\n{% if modules %}\n.. rubric:: Modules\n\n.. autosummary::\n :toctree:\n :recursive:\n{% for item in modules %}\n {{ item }}\n{%- endfor %}\n{% endif %}\n{% endblock %}\n" }, { "path": "docs/_templates/custom_class.rst", "content": "{{ fullname | escape | underline}}\n\n.. currentmodule:: {{ module }}\n\n.. autoclass:: {{ objname }}\n\n {% block methods %}\n .. automethod:: __init__\n\n {% set exclude_methods = ['__init__', 'get_indptr', 'assign', 'sort_index'] %}\n\n {% if methods %}\n .. rubric:: {{ _('Methods') }}\n\n .. autosummary::\n :toctree: .\n :nosignatures:\n\n {% for item in methods if item not in exclude_methods %}\n {%- if item not in inherited_members %}\n ~{{ name }}.{{ item }}\n {%- endif %}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n\n {% block attributes %}\n {% if attributes %}\n .. rubric:: {{ _('Attributes') }}\n\n .. autosummary::\n {% for item in attributes %}\n {%- if item not in inherited_members %}\n ~{{ name }}.{{ item }}\n {%- endif %}\n {%- endfor %}\n {% endif %}\n {% endblock %}\n" }, { "path": "docs/api/autogluon.common.space.rst", "content": ".. role:: hidden\n :class: hidden-section\n\nautogluon.common.space\n==================\n\nSearch Space\n------------------\n\n.. automodule:: autogluon.common.space\n.. currentmodule:: autogluon.common.space\n\nYou can use AutoGluon search space to perform HPO.\nFor a high-level overview, see this example:\n\n.. code-block::\n\n from autogluon.common import space\n\n categorical_space = space.Categorical('a', 'b', 'c', 'd') # Nested search space for hyperparameters which are categorical.\n real_space = space.Real(0.01, 0.1) # Search space for numeric hyperparameter that takes continuous values\n int_space = space.Int(0, 100) # Search space for numeric hyperparameter that takes integer values\n bool_space = space.Bool() # Search space for hyperparameter that is either True or False.\n\nFor how to use the search space to perform HPO, check out `Tabular Indepth Tutorial <../tutorials/tabular/tabular-indepth.html#specifying-hyperparameters-and-tuning-them>`_\n\n:hidden:`Categorical`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: Categorical\n :members: init\n\n:hidden:`Real`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: Real\n :members: init\n\n:hidden:`Int`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: Int\n :members: init\n\n:hidden:`Bool`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: Bool\n :members: init\n" }, { "path": "docs/api/autogluon.features.rst", "content": ".. role:: hidden\n :class: hidden-section\n\nautogluon.features\n==================\n\nFeature Generators\n------------------\n\n.. automodule:: autogluon.features.generators\n.. currentmodule:: autogluon.features.generators\n\n.. autosummary::\n :nosignatures:\n\n AbstractFeatureGenerator\n AutoMLPipelineFeatureGenerator\n PipelineFeatureGenerator\n BulkFeatureGenerator\n AsTypeFeatureGenerator\n BinnedFeatureGenerator\n CategoryFeatureGenerator\n DatetimeFeatureGenerator\n DropDuplicatesFeatureGenerator\n DropUniqueFeatureGenerator\n DummyFeatureGenerator\n FillNaFeatureGenerator\n IdentityFeatureGenerator\n LabelEncoderFeatureGenerator\n CategoryMemoryMinimizeFeatureGenerator\n NumericMemoryMinimizeFeatureGenerator\n RenameFeatureGenerator\n TextNgramFeatureGenerator\n TextSpecialFeatureGenerator\n\n\n:hidden:`AbstractFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: AbstractFeatureGenerator\n :members:\n :inherited-members:\n\n .. rubric:: Methods\n\n .. autoautosummary:: AbstractFeatureGenerator\n :methods:\n\n:hidden:`AutoMLPipelineFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: AutoMLPipelineFeatureGenerator\n :members: init\n\n:hidden:`PipelineFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: PipelineFeatureGenerator\n :members: init\n\n:hidden:`BulkFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: BulkFeatureGenerator\n :members: init\n\n:hidden:`AsTypeFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: AsTypeFeatureGenerator\n :members: init\n\n:hidden:`BinnedFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: BinnedFeatureGenerator\n :members: init\n\n:hidden:`CategoryFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: CategoryFeatureGenerator\n :members: init\n\n:hidden:`DatetimeFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: DatetimeFeatureGenerator\n :members: init\n\n:hidden:`DropDuplicatesFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: DropDuplicatesFeatureGenerator\n :members: init\n\n:hidden:`DropUniqueFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: DropUniqueFeatureGenerator\n :members: init\n\n:hidden:`DummyFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: DummyFeatureGenerator\n :members: init\n\n:hidden:`FillNaFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: FillNaFeatureGenerator\n :members: init\n\n:hidden:`IdentityFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: IdentityFeatureGenerator\n :members: init\n\n:hidden:`LabelEncoderFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: LabelEncoderFeatureGenerator\n :members: init\n\n:hidden:`CategoryMemoryMinimizeFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: CategoryMemoryMinimizeFeatureGenerator\n :members: init\n\n:hidden:`NumericMemoryMinimizeFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: NumericMemoryMinimizeFeatureGenerator\n :members: init\n\n:hidden:`RenameFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: RenameFeatureGenerator\n :members: init\n\n:hidden:`TextNgramFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TextNgramFeatureGenerator\n :members: init\n\n:hidden:`TextSpecialFeatureGenerator`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TextSpecialFeatureGenerator\n :members: init\n" }, { "path": "docs/api/autogluon.tabular.models.rst", "content": ".. role:: hidden\n :class: hidden-section\n\nautogluon.tabular.models\n========================\n\n.. note::\n\n This documentation is for advanced users, and is not comprehensive.\n\n For a stable public API, refer to TabularPredictor.\n\nModel Keys\n-------------------\n\nTo fit a model with TabularPredictor, you must specify it in the `TabularPredictor.fit` `hyperparameters` argument.\n\n`hyperparameters` takes in a dictionary of models, where each key is a model name, and the values are a list of dictionaries of model hyperparameters.\n\nFor example:\n\n.. code-block:: python\n\n hyperparameters = {\n 'NN_TORCH': {},\n 'GBM': [\n {'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n 'CAT': {},\n 'XGB': {},\n 'EBM': {},\n 'FASTAI': {},\n 'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\n 'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\n 'KNN': [{'weights': 'uniform', 'ag_args': {'name_suffix': 'Unif'}}, {'weights': 'distance', 'ag_args': {'name_suffix': 'Dist'}}],\n }\n\nHere is the mapping of keys to models:\n\n.. code-block:: python\n MODEL_TYPES = {\n \"RF\": RFModel,\n \"XT\": XTModel,\n \"KNN\": KNNModel,\n \"GBM\": LGBModel,\n \"CAT\": CatBoostModel,\n \"XGB\": XGBoostModel,\n \"EBM\": EBMModel,\n \"REALMLP\": RealMLPModel,\n \"MITRA\": MitraModel,\n \"TABICL\": TabICLModel,\n \"TABPFNV2\": TabPFNV2Model,\n \"NN_TORCH\": TabularNeuralNetTorchModel,\n \"LR\": LinearModel,\n \"FASTAI\": NNFastAiTabularModel,\n \"TABM\": TabMModel,\n \"AG_TEXT_NN\": TextPredictorModel,\n \"AG_IMAGE_NN\": ImagePredictorModel,\n \"AG_AUTOMM\": MultiModalPredictorModel,\n \"FT_TRANSFORMER\": FTTransformerModel,\n \"ENS_WEIGHTED\": GreedyWeightedEnsembleModel,\n \"SIMPLE_ENS_WEIGHTED\": SimpleWeightedEnsembleModel,\n\n # interpretable models\n \"IM_RULEFIT\": RuleFitModel,\n \"IM_GREEDYTREE\": GreedyTreeModel,\n \"IM_FIGS\": FigsModel,\n \"IM_HSTREE\": HSTreeModel,\n \"IM_BOOSTEDRULES\": BoostedRulesModel,\n\n \"DUMMY\": DummyModel,\n }\n\nHere is the mapping of model types to their default names when trained:\n\n.. code-block:: python\n\n DEFAULT_MODEL_NAMES = {\n RFModel: 'RandomForest',\n XTModel: 'ExtraTrees',\n KNNModel: 'KNeighbors',\n LGBModel: 'LightGBM',\n CatBoostModel: 'CatBoost',\n XGBoostModel: 'XGBoost',\n EBMModel: 'EBM',\n RealMLPModel: 'RealMLP',\n TabMModel: 'TabM',\n MitraModel: 'Mitra',\n TabICLModel: 'TabICL',\n TabPFNV2Model: 'TabPFNv2',\n TabularNeuralNetTorchModel: 'NeuralNetTorch',\n LinearModel: 'LinearModel',\n NNFastAiTabularModel: 'NeuralNetFastAI',\n TextPredictorModel: 'TextPredictor',\n ImagePredictorModel: 'ImagePredictor',\n MultiModalPredictorModel: 'MultiModalPredictor',\n\n FTTransformerModel: 'FTTransformer',\n GreedyWeightedEnsembleModel: 'WeightedEnsemble',\n SimpleWeightedEnsembleModel: 'WeightedEnsemble',\n\n # Interpretable models\n RuleFitModel: 'RuleFit',\n GreedyTreeModel: 'GreedyTree',\n FigsModel: 'Figs',\n HSTreeModel: 'HierarchicalShrinkageTree',\n BoostedRulesModel: 'BoostedRules',\n }\n\nModel Name Suffixes\n-------------------\n\nModels trained by TabularPredictor can have suffixes in their names that have special meanings.\n\nThe suffixes are as follows:\n\n**\"_Lx\"**: Indicates the stack level (x) the model is trained in, such as \"_L1\", \"_L2\", etc.\nA model with \"_L1\" suffix is a base model, meaning it does not depend on any other models.\nIf a model lacks this suffix, then it is a base model and is at level 1 (\"_L1\").\n\n**\"/Tx\"**: Indicates that the model was trained via hyperparameter search (HPO). Tx is shorthand for HPO trial #x.\nAn example would be **\"LightGBM/T8\"**.\n\n**\"_BAG\"**: Indicates that the model is a bagged ensemble.\nA bagged ensemble contains multiple instances of the model (children) trained with different subsets of the data.\nDuring inference, these child models each predict on the data and their predictions are averaged in the final result.\nThis typically achieves a stronger result than any of the individual models alone,\nbut slows down inference speed significantly. Refer to **\"_FULL\"** for instructions on how to improve inference speed.\n\n**\"_FULL\"**: Indicates the model has been refit via TabularPredictor's refit_full method.\nThis model will have no validation score because all of the data (train and validation) was used as training data.\nUsually, there will be another model with the same name as this model minus the \"_FULL\" suffix.\nOften, this model can outperform the original model because of using more data during training,\nbut is usually weaker if the original was a bagged ensemble (\"_BAG\"), but with much faster inference speed.\n\n**\"_DSTL\"**: Indicates the model was created through model distillation\nvia a call to TabularPredictor's distill method.\nValidation scores of distilled models should only be compared against other distilled models.\n\n**\"_x\"**: Indicates that the name without this added suffix already existed in a different model,\nso this suffix was added to avoid overwriting the pre-existing model.\nAn example would be **\"LightGBM_2\"**.\n\nModels\n------\n\n.. automodule:: autogluon.tabular.models\n.. currentmodule:: autogluon.tabular.models\n\n.. autosummary::\n :nosignatures:\n\n AbstractModel\n LGBModel\n CatBoostModel\n XGBoostModel\n EBMModel\n RealMLPModel\n TabMModel\n MitraModel\n TabICLModel\n TabPFNV2Model\n RFModel\n XTModel\n KNNModel\n LinearModel\n TabularNeuralNetTorchModel\n NNFastAiTabularModel\n MultiModalPredictorModel\n TextPredictorModel\n ImagePredictorModel\n\n:hidden:`AbstractModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: AbstractModel\n :members:\n :inherited-members:\n\n .. rubric:: Methods\n\n .. autoautosummary:: AbstractModel\n :methods:\n\n:hidden:`LGBModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: LGBModel\n :members: init\n\n:hidden:`CatBoostModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: CatBoostModel\n :members: init\n\n:hidden:`XGBoostModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: XGBoostModel\n :members: init\n\n:hidden:`EBMModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: EBMModel\n :members: init\n\n:hidden:`RealMLPModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: RealMLPModel\n :members: init\n\n:hidden:`TabMModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TabMModel\n :members: init\n\n:hidden:`MitraModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: MitraModel\n :members: init\n\n:hidden:`TabICLModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TabICLModel\n :members: init\n\n:hidden:`TabPFNV2Model`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TabPFNV2Model\n :members: init\n\n:hidden:`RFModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: RFModel\n :members: init\n\n:hidden:`XTModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: XTModel\n :members: init\n\n:hidden:`KNNModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: KNNModel\n :members: init\n\n:hidden:`LinearModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: LinearModel\n :members: init\n\n:hidden:`TabularNeuralNetTorchModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TabularNeuralNetTorchModel\n :members: init\n\n:hidden:`NNFastAiTabularModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: NNFastAiTabularModel\n :members: init\n\n:hidden:`MultiModalPredictorModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: MultiModalPredictorModel\n :members: init\n\n:hidden:`TextPredictorModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: TextPredictorModel\n :members: init\n\n:hidden:`ImagePredictorModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: ImagePredictorModel\n :members: init\n\nEnsemble Models\n---------------\n\n.. automodule:: autogluon.core.models\n.. currentmodule:: autogluon.core.models\n\n.. autosummary::\n :nosignatures:\n\n BaggedEnsembleModel\n StackerEnsembleModel\n WeightedEnsembleModel\n\n:hidden:`BaggedEnsembleModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: BaggedEnsembleModel\n :members: init\n\n:hidden:`StackerEnsembleModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: StackerEnsembleModel\n :members: init\n\n:hidden:`WeightedEnsembleModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: WeightedEnsembleModel\n :members: init\n\nExperimental Models\n-------------------\n\n.. automodule:: autogluon.tabular.models\n.. currentmodule:: autogluon.tabular.models\n\n.. autosummary::\n :nosignatures:\n\n FTTransformerModel\n\n:hidden:`FTTransformerModel`\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. autoclass:: FTTransformerModel\n :members: init\n\n" }, { "path": "docs/api.rst", "content": "API\n===\n\n\n.. currentmodule:: autogluon.tabular\n\n.. autosummary::\n :toctree: api\n :template: custom_class.rst\n\n TabularPredictor\n\n.. currentmodule:: autogluon.core\n\n.. autosummary::\n :toctree: api\n :template: custom_class.rst\n\n TabularDataset\n\n.. currentmodule:: autogluon.multimodal\n\n.. autosummary::\n :toctree: api\n :template: custom_class.rst\n\n MultiModalPredictor\n\n\n.. currentmodule:: autogluon.timeseries\n\n.. autosummary::\n :toctree: api\n :template: custom_class.rst\n\n TimeSeriesDataFrame\n TimeSeriesPredictor\n\n\n.. currentmodule:: autogluon.common.features.feature_metadata\n\n.. autosummary::\n :toctree: api\n :template: custom_class.rst\n\n FeatureMetadata\n\n" }, { "path": "docs/build.yml", "content": "dependencies:\n- python=3.10\n- pip\n- pip:\n - ruff\n - sphinx\n - furo\n - myst_parser\n - myst-nb\n - sphinx-material\n - sphinx-togglebutton\n - sphinx-copybutton\n - sphinx-design\n - sphinx-inline-tabs\n - sphinxcontrib-googleanalytics\n" }, { "path": "docs/build_doc.sh", "content": "#!/bin/bash\n\nrm -rf _build/\n\nsphinx-build -b html . _build/html/\n" }, { "path": "docs/build_pip_install.sh", "content": "#!/bin/bash\npython3 -m pip uninstall -y autogluon\npython3 -m pip uninstall -y autogluon.fair\npython3 -m pip uninstall -y autogluon.eda\npython3 -m pip uninstall -y autogluon.timeseries\npython3 -m pip uninstall -y autogluon.multimodal\npython3 -m pip uninstall -y autogluon.tabular\npython3 -m pip uninstall -y autogluon.core\npython3 -m pip uninstall -y autogluon.features\npython3 -m pip uninstall -y autogluon.common\n\ncd common/\npython3 -m pip install -e .\ncd ..\n\ncd features/\npython3 -m pip install -e .\ncd ..\n\ncd core/\npython3 -m pip install -e .[all]\ncd ..\n\ncd tabular/\n# Python 3.7 bug workaround: https://github.com/python/typing/issues/573\npython3 -m pip uninstall -y typing\npython3 -m pip install -e .[all,tests]\ncd ..\n\ncd multimodal/\npython3 -m pip install -e .\ncd ..\n\ncd timeseries/\npython3 -m pip install -e .[all,tests]\ncd ..\n\ncd eda/\npython3 -m pip install -e .[tests]\n# Resolve awscli and tox conflict\npython3 -m pip install \"colorama<0.4.5,>=0.2.5\"\ncd ..\n\ncd autogluon/\npython3 -m pip install -e .\ncd ..\n" }, { "path": "docs/cheatsheet.md", "content": "# Cheat Sheet\n\n## Tabular\n\n```{image} https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/main/docs/cheatsheets/stable/autogluon-cheat-sheet.jpeg\n:width: 900\n```\n\nDownload the PDF version with clickable links [tabular-cheatsheet].\n\nLooking for a different version? Refer to the [autogluon-doc-utils] repo to view all versions of the cheatsheet.\n\n## Time Series\n\n```{image} https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/main/docs/cheatsheets/stable/timeseries/autogluon-cheat-sheet-ts.jpeg\n:width: 900\n```\n\nDownload the PDF version with clickable links [ts-cheatsheet].\n\n## Multimodal\n\n```{image} https://automl-mm-bench.s3-accelerate.amazonaws.com/cheatsheet/stable/automm.jpeg\n:width: 900\n```\n\nDownload the PDF version with clickable links [multimodal-cheatsheet].\n\n\n[autogluon-doc-utils]: https://github.com/Innixma/autogluon-doc-utils/tree/main/docs/cheatsheets\n[multimodal-cheatsheet]: https://automl-mm-bench.s3-accelerate.amazonaws.com/cheatsheet/stable/automm.pdf\n[tabular-cheatsheet]: https://nbviewer.org/github/Innixma/autogluon-doc-utils/blob/main/docs/cheatsheets/stable/autogluon-cheat-sheet.pdf\n[ts-cheatsheet]: https://nbviewer.org/github/innixma/autogluon-doc-utils/blob/main/docs/cheatsheets/stable/timeseries/autogluon-cheat-sheet-ts.pdf\n" }, { "path": "docs/conf.py", "content": "import os\nimport sys\n\nsys.path = [\".\", \"..\"] + sys.path\n\nproject = \"AutoGluon\"\nrelease = \"1.5.1\"\ncopyright = \"2025, All authors. Licensed under Apache 2.0.\"\n\nauthor = \"AutoGluon contributors\"\n\nextensions = [\n \"myst_nb\", # myst-nb.readthedocs.io\n \"sphinx_copybutton\", # sphinx-copybutton.readthedocs.io\n \"sphinx_design\", # github.com/executablebooks/sphinx-design\n \"sphinx_inline_tabs\", # sphinx-inline-tabs.readthedocs.io\n \"sphinx_togglebutton\", # sphinx-togglebutton.readthedocs.io\n \"sphinxext.opengraph\", # sphinxext-opengraph.readthedocs.io/en/latest/\n \"sphinx.ext.autodoc\", # www.sphinx-doc.org/en/master/usage/extensions/autodoc.html\n \"sphinx.ext.autosummary\", # www.sphinx-doc.org/en/master/usage/extensions/autosummary.html\n \"sphinx.ext.napoleon\", # www.sphinx-doc.org/en/master/usage/extensions/napoleon.html\n \"sphinx.ext.viewcode\", # www.sphinx-doc.org/en/master/usage/extensions/viewcode.html\n \"sphinxcontrib.googleanalytics\", # github.com/sphinx-contrib/googleanalytics\n]\n\n# See https://myst-parser.readthedocs.io/en/latest/syntax/optional.html\nmyst_enable_extensions = [\n \"colon_fence\",\n \"deflist\",\n \"dollarmath\",\n \"html_image\",\n \"substitution\",\n]\n\nautosummary_generate = True\nnumpydoc_show_class_members = False\n\ngoogleanalytics_id = \"G-6XDS99SP0C\"\n\nnb_execution_mode = \"force\"\n# nb_execution_raise_on_error=True\nnb_execution_timeout = 3600\nnb_merge_streams = True\n\nnb_execution_excludepatterns = [\"jupyter_execute\"]\n\n# Sphinx creates a \"tags\" object from the arguments specified in the \"-t\" option of the \"sphinx-build\" cmd\n# This line allows AutoGluon's CI to execute a subset of our tutorial notebooks by setting the \"nb_dirs_to_exec\" variable\nnb_dirs_to_exec = [os.path.join(\"tutorials\", tag) for tag in tags if os.path.isdir(os.path.join(\"tutorials\", tag))]\n\nif len(nb_dirs_to_exec) > 0:\n nb_dirs_to_exclude = [\n dirpath\n for dirpath, _, filenames in os.walk(\"tutorials\")\n if any(map(lambda x: x.endswith(\".ipynb\"), filenames)) and not dirpath.startswith(tuple(nb_dirs_to_exec))\n ]\n\n for nb_dir in nb_dirs_to_exclude:\n nb_execution_excludepatterns.append(os.path.join(nb_dir, \"*.ipynb\"))\n\ntemplates_path = [\"_templates\"]\nexclude_patterns = [\n \"_build\",\n \"_templates\",\n \"README.md\",\n \"ReleaseInstructions.md\",\n \"jupyter_execute\",\n]\nmaster_doc = \"index\"\nnumfig = True\nnumfig_secnum_depth = 2\nmath_numfig = True\nmath_number_all = True\n\n# suppress_warnings = ['misc.highlighting_failure']\n\nhtml_theme = \"furo\" # furo.readthedocs.io\nhtml_theme_options = {\n \"sidebar_hide_name\": True,\n \"light_logo\": \"autogluon.png\",\n \"dark_logo\": \"autogluon-w.png\",\n \"globaltoc_collapse\": False,\n}\n\nhtml_sidebars = {\n \"**\": [\n \"sidebar/brand.html\",\n \"sidebar/search.html\",\n \"sidebar/scroll-start.html\",\n \"sidebar/navigation.html\",\n # 'sidebar/ethical-ads.html', # furo maintainer requests this is set if docs are hosted on readthedocs.io\n \"sidebar/scroll-end.html\",\n \"sidebar/variant-selector.html\",\n ]\n}\n\nhtml_favicon = \"_static/favicon.ico\"\n\nhtml_static_path = [\"_static\"]\nhtml_css_files = [\"custom.css\"]\nhtml_js_files = [\"custom.js\"]\n\nogp_site_url = \"https://auto.gluon.ai/\"\nogp_description_length = 300\nogp_site_name = \"AutoGluon\"\nogp_image = \"https://auto.gluon.ai/dev/_static/autogluon-logo.jpg\"\nogp_image_alt = \"AutoGluon Logo\"\nogp_type = \"website\"\nogp_custom_meta_tags = [\n '',\n '',\n '',\n '',\n]\n" }, { "path": "docs/index.md", "content": "---\nsd_hide_title: true\nhide-toc: true\n---\n\n# AutoGluon\n\n::::::{div} landing-title\n:style: \"padding: 0.1rem 0.5rem 0.6rem 0; background-image: linear-gradient(315deg, #438ff9 0%, #3977B9 74%); clip-path: polygon(0px 0px, 100% 0%, 100% 100%, 0% calc(100% - 1.5rem)); -webkit-clip-path: polygon(0px 0px, 100% 0%, 100% 100%, 0% calc(100% - 1.5rem));\"\n\n::::{grid}\n:reverse:\n:gutter: 2 3 3 3\n:margin: 4 4 1 2\n\n:::{grid-item}\n:columns: 12 4 4 4\n\n```{image} ./_static/autogluon-s.png\n:width: 200px\n:class: sd-m-auto sd-animate-grow50-rot20\n```\n:::\n\n:::{grid-item}\n:columns: 12 8 8 8\n:child-align: justify\n:class: sd-text-white sd-fs-3\n\nFast and Accurate ML in 3 Lines of Code\n\n```{button-link} tutorials/tabular/tabular-quick-start.html\n:outline:\n:color: white\n:class: sd-px-4 sd-fs-5\n\nGet Started\n```\n\n:::\n::::\n\n::::::\n\nQuick Prototyping\n: Build machine learning solutions on raw data in a few lines of code.\n\nState-of-the-art Techniques\n: Automatically utilize SOTA models without expert knowledge.\n\nEasy to Deploy\n: Move from experimentation to production with cloud predictors and pre-built containers.\n\nCustomizable\n: Extensible with custom feature processing, models, and metrics.\n\n## {octicon}`rocket` Quick Examples\n\n:::{dropdown} Tabular\n:animate: fade-in-slide-down\n:open:\n:color: primary\n\nPredict the `class` column in a data table:\n\n```python\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\ndata_root = 'https://autogluon.s3.amazonaws.com/datasets/Inc/'\ntrain_data = TabularDataset(data_root + 'train.csv')\ntest_data = TabularDataset(data_root + 'test.csv')\n\npredictor = TabularPredictor(label='class').fit(train_data)\npredictions = predictor.predict(test_data)\n```\n:::\n\n\n:::{dropdown} Time Series\n:animate: fade-in-slide-down\n:color: primary\n\nForecast future values of time series:\n\n```python\nfrom autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor\n\ndata = TimeSeriesDataFrame('https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_hourly/train.csv')\n\npredictor = TimeSeriesPredictor(target='target', prediction_length=48).fit(data)\npredictions = predictor.predict(data)\n```\n:::\n\n\n::::{dropdown} Multimodal\n:animate: fade-in-slide-down\n:color: primary\n\n:::{tab} Text Classification\n```python\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.core.utils.loaders import load_pd\n\ndata_root = 'https://autogluon-text.s3-accelerate.amazonaws.com/glue/sst/'\ntrain_data = load_pd.load(data_root + 'train.parquet')\ntest_data = load_pd.load(data_root + 'dev.parquet')\n\npredictor = MultiModalPredictor(label='label').fit(train_data=train_data)\npredictions = predictor.predict(test_data)\n```\n:::\n\n:::{tab} Image Classification\n\n```python\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\ntrain_data, test_data = shopee_dataset('./automm_shopee_data')\n\npredictor = MultiModalPredictor(label='label').fit(train_data=train_data)\npredictions = predictor.predict(test_data)\n```\n:::\n\n:::{tab} NER\n```python\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.core.utils.loaders import load_pd\n\ndata_root = 'https://automl-mm-bench.s3.amazonaws.com/ner/mit-movies/'\ntrain_data = load_pd.load(data_root + 'train.csv')\ntest_data = load_pd.load(data_root + 'test.csv')\n\npredictor = MultiModalPredictor(problem_type=\"ner\", label=\"entity_annotations\")\n\npredictor.fit(train_data)\npredictor.evaluate(test_data)\n\nsentence = \"Game of Thrones is an American fantasy drama television series created\" +\n \"by David Benioff\"\nprediction = predictor.predict({ 'text_snippet': [sentence]})\n```\n:::\n\n:::{tab} Matching\n```python\nfrom autogluon.multimodal import MultiModalPredictor, utils\nimport ir_datasets\nimport pandas as pd\n\ndataset = ir_datasets.load(\"beir/fiqa/dev\")\ndocs_df = pd.DataFrame(dataset.docs_iter()).set_index(\"doc_id\")\n\npredictor = MultiModalPredictor(problem_type=\"text_similarity\")\n\ndoc_embedding = predictor.extract_embedding(docs_df)\nq_embedding = predictor.extract_embedding([\n \"what happened when the dot com bubble burst?\"\n])\n\nsimilarity = utils.compute_semantic_similarity(q_embedding, doc_embedding)\n```\n:::\n\n:::{tab} Object Detection\n```ipython\n# Install mmcv-related dependencies\n!mim install \"mmcv==2.1.0\"\n!pip install \"mmdet==3.2.0\"\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.core.utils.loaders import load_zip\n\ndata_zip = \"https://automl-mm-bench.s3.amazonaws.com/object_detection_dataset/\" + \\\n \"tiny_motorbike_coco.zip\"\nload_zip.unzip(data_zip, unzip_dir=\".\")\n\ntrain_path = \"./tiny_motorbike/Annotations/trainval_cocoformat.json\"\ntest_path = \"./tiny_motorbike/Annotations/test_cocoformat.json\"\n\npredictor = MultiModalPredictor(\n problem_type=\"object_detection\",\n sample_data_path=train_path\n)\n\npredictor.fit(train_path)\nscore = predictor.evaluate(test_path)\n\npred = predictor.predict({\"image\": [\"./tiny_motorbike/JPEGImages/000038.jpg\"]})\n```\n:::\n\n::::\n\n\n## {octicon}`package` Installation\n\n\n\n\n\nInstall AutoGluon using [pip](https://pip.pypa.io/en/stable/installation/):\n\n```bash\npip install autogluon\n```\n\nAutoGluon supports Linux, MacOS, and Windows. See {doc}`./install` for detailed instructions.\n\n## Managed Service\n\nLooking for a managed AutoML service? We highly recommend checking out [Amazon SageMaker Canvas](https://aws.amazon.com/sagemaker/canvas/)! Powered by AutoGluon, it allows you to create highly accurate machine learning models without any machine learning experience or writing a single line of code.\n\n## Community\n\n[](https://discord.gg/wjUmjqAc2N)\n[](https://twitter.com/autogluon)\n\nGet involved in the AutoGluon community by joining our [Discord](https://discord.gg/wjUmjqAc2N)!\n\n## Citing AutoGluon\n\nAutoGluon was originally developed by researchers and engineers at AWS AI. If you use AutoGluon in your research, please refer to our [citing guide](https://github.com/autogluon/autogluon/blob/master/CITING.md).\n\n```{toctree}\n---\ncaption: Get Started\nmaxdepth: 1\nhidden:\n---\n\nInstall
\n
\n"
},
{
"path": "docs/tutorials/cloud_fit_deploy/autogluon-cloud.md",
"content": "# AutoGluon Cloud\nAutoGluon-Cloud aims to provide user tools to train, fine-tune and deploy AutoGluon backed models on the cloud. With just a few lines of code, users can train a model and perform inference on the cloud without worrying about MLOps details such as resource management.\nTo learn more, check it out [here](https://auto.gluon.ai/cloud/dev/index.html)\n"
},
{
"path": "docs/tutorials/cloud_fit_deploy/autopilot-autogluon.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"3e701bf3\",\n \"metadata\": {},\n \"source\": [\n \"# New managed AutoGluon-Tabular experience on Amazon SageMaker Autopilot\\n\",\n \"\\n\",\n \"[](https://colab.research.google.com/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/autopilot-autogluon.ipynb)\\n\",\n \"[](https://studiolab.sagemaker.aws/import/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/autopilot-autogluon.ipynb)\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"[Checkout managed AutoGluon-Tabular experience on Amazon SageMaker Autopilot](https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-autopilot-is-up-to-eight-times-faster-with-new-ensemble-training-mode-powered-by-autogluon/)\\n\",\n \"\\n\",\n \"\\n\",\n \"## What is Amazon SageMaker Autopilot\\n\",\n \"Amazon SageMaker Autopilot automatically trains and tunes the best machine learning models for classification or regression, based on your data while allowing to maintain full control and visibility.\\n\",\n \"\\n\",\n \"To learn more, checkout it out [here](https://www.amazonaws.cn/en/sagemaker/autopilot/)\"\n ]\n }\n ],\n \"metadata\": {\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}"
},
{
"path": "docs/tutorials/cloud_fit_deploy/cloud-aws-lambda-deployment.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"222aa95a\",\n \"metadata\": {},\n \"source\": [\n \"# Deploying AutoGluon models with serverless templates\\n\",\n \"\\n\",\n \"[](https://colab.research.google.com/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/cloud-aws-lambda-deployment.ipynb)\\n\",\n \"[](https://studiolab.sagemaker.aws/import/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/cloud-aws-lambda-deployment.ipynb)\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"After learning how to train/deploy models using AWS SageMaker [Cloud Training with Amazon SageMaker](cloud-aws-sagemaker-train-deploy.ipynb), in this section we will learn how to deploy\\n\",\n \"trained models using AWS Lambda.\\n\",\n \"\\n\",\n \"## Reducing the model size to minimize AWS Lambda startup times\\n\",\n \"\\n\",\n \"When the Lambda service receives a request to run a function via the Lambda API, the service first prepares an execution environment. During this step, the service \\n\",\n \"downloads the code for the function, which is stored in Amazon Elastic Container Registry. It then creates an environment with the memory, runtime, and configuration \\n\",\n \"specified. Once complete, Lambda runs any initialization code outside of the event handler before finally running the handler code. The steps of setting up the \\n\",\n \"environment and the code are frequently referred to as a \\\"cold start\\\".\\n\",\n \"\\n\",\n \"After the execution completes, the execution environment is frozen. To improve resource management and performance, the Lambda service retains the execution environment \\n\",\n \"for a non-deterministic period of time. During this time, if another request arrives for the same function, the service may reuse the environment. This second request \\n\",\n \"typically finishes more quickly, since the execution environment already exists and itâs not necessary to download the code and run the initialization code. \\n\",\n \"This is called a \\\"warm start\\\".\\n\",\n \"\\n\",\n \"Because AutoGluon containers are larger than a typical Lambda container, it might take some time (60+ seconds) to perform steps required for a \\\"cold start\\\". \\n\",\n \"This could be limiting factor when used with latency-sensitive applications. To reduce start up times with AWS Lambda it is important to reduce model size to a minimum. \\n\",\n \"This can be done by applying deployment-optimized presets as described in section \\\"Faster presets or hyperparameters\\\" of [Predicting Columns in a Table - In Depth](../tabular/tabular-indepth.ipynb):\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e1a9af96\",\n \"metadata\": {},\n \"source\": [\n \"```python\\n\",\n \"presets = ['good_quality_faster_inference_only_refit', 'optimize_for_deployment']\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"60767770\",\n \"metadata\": {},\n \"source\": [\n \"If the cold boot latency cannot be tolerated, it is recommended to reserve concurrent capacity as described in this article:\\n\",\n \"[Managing Lambda reserved concurrency](https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html).\\n\",\n \"\\n\",\n \"More details on the lambda performance optimizations can be found in the following article: \\n\",\n \"[Operating Lambda: Performance optimization](https://aws.amazon.com/blogs/compute/operating-lambda-performance-optimization-part-1/)\\n\",\n \"\\n\",\n \"## Creating a base project\\n\",\n \"\\n\",\n \"To start the project, please follow the setup steps of the tutorial: \\n\",\n \"[Deploying machine learning models with serverless templates](https://aws.amazon.com/blogs/compute/deploying-machine-learning-models-with-serverless-templates/).\\n\",\n \"\\n\",\n \"To deploy AutoGluon, the following adjustments would be required:\\n\",\n \"\\n\",\n \"- the trained model is expected to be in `ag_models` directory.\\n\",\n \"\\n\",\n \"- `Dockerfile` to package AutoGluon runtimes and model files\\n\",\n \"\\n\",\n \"- Modify serving `app/app.py` script to use AutoGluon\\n\",\n \"\\n\",\n \"When building a docker container it's size can be reduced using the following optimizations: \\n\",\n \"\\n\",\n \"- use CPU versions of `pytorch`; if the models to be deployed don't use `pytorch`, then don't install it.\\n\",\n \"\\n\",\n \"- install only the AutoGluon sub-modules required for inference - specifically `autogluon.tabular[all]` will deploy only all tabular models \\n\",\n \"without `text` and `vision` modules and their extra dependencies. This instruction can be further narrowed down to a combination of \\n\",\n \"the following options are: `lightgbm`, `catboost`, `xgboost`, `fastai` and `skex`.\\n\",\n \"\\n\",\n \"The following `Dockerfile` can be used as a starting point:\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e39dc75f\",\n \"metadata\": {},\n \"source\": [\n \"```\\n\",\n \"FROM public.ecr.aws/lambda/python:3.8\\n\",\n \"\\n\",\n \"RUN yum install libgomp git -y \\\\\\n\",\n \" && yum clean all -y && rm -rf /var/cache/yum\\n\",\n \"\\n\",\n \"ARG TORCH_VER=1.9.1+cpu\\n\",\n \"ARG TORCH_VISION_VER=0.10.1+cpu\\n\",\n \"ARG NUMPY_VER=1.19.5\\n\",\n \"RUN python3.8 -m pip --no-cache-dir install --upgrade --trusted-host pypi.org --trusted-host files.pythonhosted.org pip \\\\\\n\",\n \" && python3.8 -m pip --no-cache-dir install --upgrade wheel setuptools \\\\\\n\",\n \" && python3.8 -m pip uninstall -y dataclasses \\\\\\n\",\n \" && python3.8 -m pip --no-cache-dir install --upgrade torch==\\\"${TORCH_VER}\\\" torchvision==\\\"${TORCH_VISION_VER}\\\" -f https://download.pytorch.org/whl/torch_stable.html \\\\\\n\",\n \" && python3.8 -m pip --no-cache-dir install --upgrade numpy==${NUMPY_VER} \\\\\\n\",\n \" && python3.8 -m pip --no-cache-dir install --upgrade autogluon.tabular[all]\\\"\\n\",\n \"\\n\",\n \"COPY app.py ./\\n\",\n \"COPY ag_models /opt/ml/model/\\n\",\n \"\\n\",\n \"CMD [\\\"app.lambda_handler\\\"]\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"cff71541\",\n \"metadata\": {},\n \"source\": [\n \"Lambda serving script (`app/app.py`):\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"609500f9\",\n \"metadata\": {},\n \"source\": [\n \"```python\\n\",\n \"import pandas as pd\\n\",\n \"from autogluon.tabular import TabularPredictor\\n\",\n \"\\n\",\n \"model = TabularPredictor.load('/opt/ml/model')\\n\",\n \"model.persist(models='all')\\n\",\n \"\\n\",\n \"\\n\",\n \"# Lambda handler code\\n\",\n \"def lambda_handler(event, context):\\n\",\n \" df = pd.read_json(event['body'])\\n\",\n \" pred_probs = model.predict_proba(df)\\n\",\n \" return {\\n\",\n \" 'statusCode': 200,\\n\",\n \" 'body': pred_probs.to_json()\\n\",\n \" }\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"37a0b3c2\",\n \"metadata\": {},\n \"source\": [\n \"Once the necessary modifications to the projects are done, proceed with the steps described in \\\"Deploying the application to Lambda\\\" section of the \\n\",\n \"[tutorial](https://aws.amazon.com/blogs/compute/deploying-machine-learning-models-with-serverless-templates/).\\n\",\n \"\\n\",\n \"## Conclusion\\n\",\n \"\\n\",\n \"In this tutorial we explored how to deploy AutoGluon models as a serverless application. To explore more, refer to the following documentation:\\n\",\n \"\\n\",\n \"- [Deploying machine learning models with serverless templates](https://aws.amazon.com/blogs/compute/deploying-machine-learning-models-with-serverless-templates/).\\n\",\n \"\\n\",\n \"- [Operating Lambda: Performance optimization](https://aws.amazon.com/blogs/compute/operating-lambda-performance-optimization-part-1/)\\n\",\n \"\\n\",\n \"- [Managing Lambda reserved concurrency](https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html)\\n\",\n \"\\n\",\n \"- [AWS Serverless Application Model (AWS SAM)](https://github.com/aws/serverless-application-model)\"\n ]\n }\n ],\n \"metadata\": {\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"
},
{
"path": "docs/tutorials/cloud_fit_deploy/cloud-aws-sagemaker-train-deploy.ipynb",
"content": "{\n \"cells\": [\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"e19aeec9\",\n \"metadata\": {},\n \"source\": [\n \"# Cloud Training and Deployments with Amazon SageMaker\\n\",\n \"\\n\",\n \"[](https://colab.research.google.com/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/cloud-aws-sagemaker-train-deploy.ipynb)\\n\",\n \"[](https://studiolab.sagemaker.aws/import/github/autogluon/autogluon/blob/master/docs/tutorials/cloud_fit_deploy/cloud-aws-sagemaker-train-deploy.ipynb)\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"To help with AutoGluon model training and deployment, AWS developed a set of training and inference [deep learning containers](https://github.com/aws/deep-learning-containers/blob/master/available_images.md#autogluon-training-containers).\\n\",\n \"The containers can be used to train models with CPU and GPU instances and deployed as a SageMaker endpoint or used as a batch transform job.\\n\",\n \"\\n\",\n \"The full end-to-end example is available in the [amazon-sagemaker-examples](https://github.com/aws/amazon-sagemaker-examples/tree/master/advanced_functionality/autogluon-tabular-containers) repository.\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"d562bebc\",\n \"metadata\": {},\n \"source\": [\n \"### Training other types of predictors\\n\",\n \"\\n\",\n \"The examples are focused on `TabularPredictor`. For training other types of AutoGluon Predictors, i.e. `MultiModalPredictor`, the training script you provide will be similar to the one above. You would need to replace `TabularPredictor` with `MultiModalPredictor`.\\n\",\n \"\\n\",\n \"To ensure the container can load the model without external network access (SageMaker containers might have issues getting the models from HuggingFace), the model artifacts of type `MultiModalPredictor` need to be saved with `standalone=True`: `predictor.save(path='MY_PATH', standalone=True)`.\\n\",\n \"\\n\",\n \"Keep in mind that the specific Predictor type you want to train might not support the same feature sets as `TabularPredictor`. For example, `leaderboard` does not exist for all Predictors.\\n\",\n \"\\n\",\n \"### Note on image modality\\n\",\n \"\\n\",\n \"To do inference on image modality, you would need to embed the image info, as bytes for example, into a column of the test data.\\n\",\n \"Then in the inference container, if you are using the `MultiModalPredictor`, you just need to decode the aforementioned image column and feed the test data to it.\\n\",\n \"\\n\",\n \"For example, to encode the image:\\n\",\n \"\\n\",\n \"```python\\n\",\n \"def read_image_bytes_and_encode(image_path):\\n\",\n \" image_obj = open(image_path, 'rb')\\n\",\n \" image_bytes = image_obj.read()\\n\",\n \" image_obj.close()\\n\",\n \" b85_image = base64.b85encode(image_bytes).decode(\\\"utf-8\\\")\\n\",\n \"\\n\",\n \" return b85_image\\n\",\n \"\\n\",\n \"\\n\",\n \"def convert_image_path_to_encoded_bytes_in_dataframe(dataframe, image_column):\\n\",\n \" assert image_column in dataframe, 'Please specify a valid image column name'\\n\",\n \" dataframe[image_column] = [read_image_bytes_and_encode(path) for path in dataframe[image_column]]\\n\",\n \"\\n\",\n \" return dataframe\\n\",\n \"\\n\",\n \"test_data_image_column = \\\"YOUR_COLUMN_CONTAINING_IMAGE_PATH\\\"\\n\",\n \"test_data = convert_image_path_to_encoded_bytes_in_dataframe(test_data, test_data_image_column)\\n\",\n \"```\\n\",\n \"\\n\",\n \"To decode the image:\\n\",\n \"\\n\",\n \"```python\\n\",\n \"test_data[image_column] = [base64.b85decode(bytes) for bytes in test_data[image_column]]\\n\",\n \"```\\n\",\n \"\\n\",\n \"Note that if you are using the `TabularPredictor`, you would need to save the image to disk and update the test data with the image paths accordingly.\\n\",\n \"\\n\",\n \"For example, to decode the image and save to disk in the inference container:\\n\",\n \"\\n\",\n \"```python\\n\",\n \"image_index = 0\\n\",\n \"\\n\",\n \"\\n\",\n \"def _save_image_and_update_dataframe_column(bytes):\\n\",\n \" global image_index\\n\",\n \" im = Image.open(BytesIO(base64.b85decode(bytes)))\\n\",\n \" im_name = f'Image_{image_index}.png'\\n\",\n \" im.save(im_name)\\n\",\n \" image_index += 1\\n\",\n \"\\n\",\n \" return im_name\\n\",\n \"\\n\",\n \"\\n\",\n \"test_data[image_column] = [_save_image_and_update_dataframe_column(bytes) for bytes in test_data[image_column]]\\n\",\n \"```\\n\"\n ]\n }\n ],\n \"metadata\": {\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"
},
{
"path": "docs/tutorials/cloud_fit_deploy/index.md",
"content": "# Cloud Training and Deployment\n\nThis section provides an overview on how AutoGluon can be trained and deployed on cloud instances.\n\n::::{grid} 2\n :gutter: 3\n\n:::{grid-item-card} AutoGluon-Tabular on Amazon SageMaker Autopilot\n :link: https://aws.amazon.com/blogs/machine-learning/amazon-sagemaker-autopilot-is-up-to-eight-times-faster-with-new-ensemble-training-mode-powered-by-autogluon/\n\n Checkout managed AutoGluon experience on Amazon SageMaker Autopilot\n:::\n\n:::{grid-item-card} AutoGluon Cloud\n :link: autogluon-cloud.html\n\n A tutorial on using AutoGluon Cloud module to train/deploy AutoGluon backed models on SageMaker.\n:::\n\n:::{grid-item-card} Cloud Training and Deployment with Amazon SageMaker\n :link: cloud-aws-sagemaker-train-deploy.html\n\n A tutorial on fitting an AutoGluon model using AWS SageMaker.\n:::\n\n:::{grid-item-card} Deploying AutoGluon Models with Serverless Templates\n :link: cloud-aws-lambda-deployment.html\n\n A tutorial on deploying an AutoGluon model with serverless templates.\n:::\n\n::::\n\n```{toctree}\n---\nmaxdepth: 1\nhidden: true\n---\n\nAutoGluon Cloud \n \n
\n ![](\"./_static/sagemaker_autopilot.png\"/)
\n - \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"a062a26c\",\n \"metadata\": {},\n \"source\": [\n \"```\\n\",\n \"hyperparameters = {\\n\",\n \" \\\"optim.lr\\\": tune.uniform(0.00005, 0.005),\\n\",\n \" \\\"optim.optim_type\\\": tune.choice([\\\"adamw\\\", \\\"sgd\\\"]),\\n\",\n \" \\\"optim.max_epochs\\\": tune.choice([\\\"10\\\", \\\"20\\\"]), \\n\",\n \" \\\"model.timm_image.checkpoint_name\\\": tune.choice([\\\"swin_base_patch4_window7_224\\\", \\\"convnext_base_in22ft1k\\\"])\\n\",\n \" }\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"fbff8bb5\",\n \"metadata\": {},\n \"source\": [\n \"This is an example but not an exhaustive list. You can find the full supported list in [Customize AutoMM](customization.ipynb)\\n\",\n \"
- \\n\",\n \"a. Specifying how to search through your chosen hyperparameter space (supports `random` and `bayes`):\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b37b4f87\",\n \"metadata\": {},\n \"source\": [\n \"```\\n\",\n \"\\\"searcher\\\": \\\"bayes\\\"\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"a6603d76\",\n \"metadata\": {},\n \"source\": [\n \"
- \\n\",\n \"b. Specifying how to schedule jobs to train a network under a particular hyperparameter configuration (supports `FIFO` and `ASHA`):\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"024d0740\",\n \"metadata\": {},\n \"source\": [\n \"``` \\n\",\n \"\\\"scheduler\\\": \\\"ASHA\\\"\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"89b80d99\",\n \"metadata\": {},\n \"source\": [\n \"
- \\n\",\n \"c. Number of trials you would like to carry out HPO:\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"854016ae\",\n \"metadata\": {},\n \"source\": [\n \"```\\n\",\n \"\\\"num_trials\\\": 20\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"8ea99c2a\",\n \"metadata\": {},\n \"source\": [\n \"
- \\n\",\n \"d. Number of checkpoints to keep on disk per trial, see Ray documentation for more details. Must be >= 1. (default is 3):\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"bee92903\",\n \"metadata\": {},\n \"source\": [\n \"```\\n\",\n \"\\\"num_to_keep\\\": 3\\n\",\n \"```\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"6d782e6e\",\n \"metadata\": {},\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" target\\n\",\n \"item_id timestamp \\n\",\n \"T000000 2013-03-10 00:00:00 5207.959961\\n\",\n \" 2013-03-10 00:30:00 5002.275879\\n\",\n \" 2013-03-10 01:00:00 4747.569824\\n\",\n \" 2013-03-10 01:30:00 4544.880859\\n\",\n \" 2013-03-10 02:00:00 4425.952148\"\n ]\n },\n \"execution_count\": 3,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"data = TimeSeriesDataFrame.from_path(\\n\",\n \" \\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/australian_electricity_subset/test.csv\\\"\\n\",\n \")\\n\",\n \"data.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next, we create the [TimeSeriesPredictor](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.html) and select the `\\\"chronos2\\\"` presets to use the Chronos-2 (120M) model in zero-shot mode.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {\n \"tags\": [\n \"hide-output\"\n ]\n },\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Sorting the dataframe index before generating the train/test split.\\n\",\n \"Beginning AutoGluon training...\\n\",\n \"AutoGluon will save models to '/fsx/ansarnd/repos/autogluon/docs/tutorials/timeseries/AutogluonModels/ag-20251214_125317'\\n\",\n \"=================== System Info ===================\\n\",\n \"AutoGluon Version: 1.4.1b20250910\\n\",\n \"Python Version: 3.11.11\\n\",\n \"Operating System: Linux\\n\",\n \"Platform Machine: x86_64\\n\",\n \"Platform Version: #38~22.04.1-Ubuntu SMP Fri Aug 22 15:44:33 UTC 2025\\n\",\n \"CPU Count: 96\\n\",\n \"Pytorch Version: 2.7.1+cu126\\n\",\n \"CUDA Version: 12.6\\n\",\n \"GPU Memory: GPU 0: 39.38/39.38 GB\\n\",\n \"Total GPU Memory: Free: 39.38 GB, Allocated: 0.00 GB, Total: 39.38 GB\\n\",\n \"GPU Count: 1\\n\",\n \"Memory Avail: 1030.21 GB / 1121.80 GB (91.8%)\\n\",\n \"Disk Space Avail: 1758.43 GB / 11459.15 GB (15.3%)\\n\",\n \"===================================================\\n\",\n \"Setting presets to: chronos2\\n\",\n \"\\n\",\n \"Fitting with arguments:\\n\",\n \"{'enable_ensemble': True,\\n\",\n \" 'eval_metric': WQL,\\n\",\n \" 'hyperparameters': {'Chronos2': {'model_path': 'autogluon/chronos-2'}},\\n\",\n \" 'known_covariates_names': [],\\n\",\n \" 'num_val_windows': 1,\\n\",\n \" 'prediction_length': 48,\\n\",\n \" 'quantile_levels': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\\n\",\n \" 'random_seed': 123,\\n\",\n \" 'refit_every_n_windows': 1,\\n\",\n \" 'refit_full': False,\\n\",\n \" 'skip_model_selection': True,\\n\",\n \" 'target': 'target',\\n\",\n \" 'verbosity': 2}\\n\",\n \"\\n\",\n \"Inferred time series frequency: '30min'\\n\",\n \"Provided train_data has 172320 rows, 5 time series. Median time series length is 34464 (min=34464, max=34464). \\n\",\n \"\\n\",\n \"Provided data contains following columns:\\n\",\n \"\\ttarget: 'target'\\n\",\n \"\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'WQL'\\n\",\n \"\\tThis metric's sign has been flipped to adhere to being higher_is_better. The metric score can be multiplied by -1 to get the metric value.\\n\",\n \"===================================================\\n\",\n \"\\n\",\n \"Starting training. Start time is 2025-12-14 12:53:21\\n\",\n \"Models that will be trained: ['Chronos2']\\n\",\n \"Training timeseries model Chronos2. \\n\",\n \"\\t6.94 s = Training runtime\\n\",\n \"Training complete. Models trained: ['Chronos2']\\n\",\n \"Total runtime: 6.97 s\\n\",\n \"Best model: Chronos2\\n\"\n ]\n }\n ],\n \"source\": [\n \"num_test_windows = 3\\n\",\n \"prediction_length = 48\\n\",\n \"train_data, test_data = data.train_test_split(num_test_windows * prediction_length)\\n\",\n \"\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length).fit(\\n\",\n \" train_data,\\n\",\n \" presets=\\\"chronos2\\\",\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"As promised, Chronos does not take any time to `fit`. The `fit` call merely serves as a proxy for the `TimeSeriesPredictor` to do some of its chores under the hood, such as inferring the frequency of time series and saving the predictor's state to disk. \\n\",\n \"\\n\",\n \"Let's use the `predict` method to generate forecasts.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Model not specified in predict, will default to the model with the best validation score: Chronos2\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"| \\n\",\n \" | \\n\",\n \" | target | \\n\",\n \"
|---|---|---|
| item_id | \\n\",\n \"timestamp | \\n\",\n \"\\n\",\n \" |
| T000000 | \\n\",\n \"2013-03-10 00:00:00 | \\n\",\n \"5207.959961 | \\n\",\n \"
| 2013-03-10 00:30:00 | \\n\",\n \"5002.275879 | \\n\",\n \"|
| 2013-03-10 01:00:00 | \\n\",\n \"4747.569824 | \\n\",\n \"|
| 2013-03-10 01:30:00 | \\n\",\n \"4544.880859 | \\n\",\n \"|
| 2013-03-10 02:00:00 | \\n\",\n \"4425.952148 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" mean 0.1 0.2 \\\\\\n\",\n \"item_id timestamp \\n\",\n \"T000000 2015-02-26 00:00:00 5223.812012 5153.143066 5178.589355 \\n\",\n \" 2015-02-26 00:30:00 5001.890625 4940.849609 4967.337891 \\n\",\n \" 2015-02-26 01:00:00 4759.131348 4684.923340 4712.408691 \\n\",\n \" 2015-02-26 01:30:00 4560.188477 4505.166016 4523.577637 \\n\",\n \" 2015-02-26 02:00:00 4439.416992 4369.610352 4390.421875 \\n\",\n \"\\n\",\n \" 0.3 0.4 0.5 \\\\\\n\",\n \"item_id timestamp \\n\",\n \"T000000 2015-02-26 00:00:00 5193.954102 5210.103027 5223.812012 \\n\",\n \" 2015-02-26 00:30:00 4982.128906 4991.323242 5001.890625 \\n\",\n \" 2015-02-26 01:00:00 4729.202637 4743.586914 4759.131348 \\n\",\n \" 2015-02-26 01:30:00 4535.823730 4550.487793 4560.188477 \\n\",\n \" 2015-02-26 02:00:00 4412.242676 4428.110352 4439.416992 \\n\",\n \"\\n\",\n \" 0.6 0.7 0.8 \\\\\\n\",\n \"item_id timestamp \\n\",\n \"T000000 2015-02-26 00:00:00 5234.564453 5248.638672 5265.144531 \\n\",\n \" 2015-02-26 00:30:00 5012.041504 5026.311523 5047.328125 \\n\",\n \" 2015-02-26 01:00:00 4770.625977 4784.588379 4803.948242 \\n\",\n \" 2015-02-26 01:30:00 4580.130859 4591.911133 4615.944336 \\n\",\n \" 2015-02-26 02:00:00 4456.724121 4474.257324 4496.027832 \\n\",\n \"\\n\",\n \" 0.9 \\n\",\n \"item_id timestamp \\n\",\n \"T000000 2015-02-26 00:00:00 5295.290527 \\n\",\n \" 2015-02-26 00:30:00 5078.305664 \\n\",\n \" 2015-02-26 01:00:00 4828.080078 \\n\",\n \" 2015-02-26 01:30:00 4636.415039 \\n\",\n \" 2015-02-26 02:00:00 4509.391602 \"\n ]\n },\n \"execution_count\": 5,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"predictions = predictor.predict(train_data)\\n\",\n \"predictions.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We get a dataframe with the point forecast (`mean`) and nine quantiles which capture the uncertainty in the forecasts. Custom quantile levels can be specified as follows:\\n\",\n \"```py\\n\",\n \"TimeSeriesPredictor(..., quantile_levels=[0.05, 0.1, 0.5, 0.9, 0.95])\\n\",\n \"```\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"AG-TS also makes it easy to generate predictions for multiple backtest dates and to visualize the models' predictions.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAB9sAAAGTCAYAAACIxHJ2AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3Xd8nXX5//HXmdmjGc1o0zbde1BaKHuUFijKEkVBQREcRQUUEX7Kl6EiKKAigihDZCjKkCW0pWy6994zafZeZ96/Pz5J2tCkGT3JOUnez8cjjybn3Oc+n5NcPee+7+vzuS6bZVkWIiIiIiIiIiIiIiIiIiIi0mH2cA9ARERERERERERERERERESkt1GyXUREREREREREREREREREpJOUbBcREREREREREREREREREekkJdtFREREREREREREREREREQ6Scl2ERERERERERERERERERGRTlKyXUREREREREREREREREREpJOUbBcREREREREREREREREREekkJdtFREREREREREREREREREQ6Scl2ERERERERERERERERERGRTlKyXUREREREREREREREREREpJOUbBcREenjbDZbh74++OCD5sc8+eSTjBs3jujoaEaNGsUjjzzS6r7z8vL48pe/THJyMomJiVx88cXs3r271W17yz5FREREREREOqq/n3Nv27aNm2++mVNOOYXo6GhsNht79+499i9NRESkD3GGewAiIiLSvf7xj3+0+PnZZ59l4cKFR90+btw4AP7yl7/w3e9+l8svv5xbbrmFjz/+mB/+8IfU1dVx2223NW9fU1PD2WefTWVlJXfccQcul4uHH36YM888k7Vr15Kamtq8bW/Zp4iIiIiIiEhn9Pdz7iVLlvDHP/6R8ePHM27cONauXRuS36uIiEivYYmIiEi/Mn/+fKutQ4C6ujorNTXVmjdvXovbr7rqKisuLs4qKytrvu3++++3AGv58uXNt23ZssVyOBzW7bff3uv2KSIiIiIiInK8+ts5d2lpqVVVVWVZlmX99re/tQBrz549bf5+RERE+hqVkRcREZFm77//PqWlpXz/+99vcfv8+fOpra3lrbfear7tP//5DzNmzGDGjBnNt40dO5Zzzz2Xl156qdftU0RERERERKQ79Zbz487sMyUlhYSEhM7+KkRERPoMJdtFRESk2Zo1awA48cQTW9w+ffp07HZ78/3BYJD169cftR3AzJkz2bVrF9XV1b1qnyIiIiIiIiLdqbecH3d0nyIiIqJku4iIiBzh0KFDOBwOBg4c2OJ2t9tNamoq+fn5AJSVleHxeMjKyjpqH023NW3bW/YpIiIiIiIi0p16y/lxR/cpIiIiSraLiIjIEerr63G73a3eFx0dTX19ffN2AFFRUa1ud+Q2vWWfIiIiIiIiIt2pt5wfd3SfIiIiomS7iIiIHCEmJgav19vqfQ0NDcTExDRvB+DxeFrd7shtess+RURERERERLpTbzk/7ug+RURERMl2EREROUJWVhaBQICioqIWt3u9XkpLS8nOzgYgJSWFqKgoDh06dNQ+mm5r2ra37FNERERERESkO/WW8+OO7lNERESUbBcREZEjTJ06FYCVK1e2uH3lypUEg8Hm++12O5MmTTpqO4Bly5YxfPhwEhISetU+RURERERERLpTbzk/7ug+RURERMl2EREROcI555xDSkoKjz32WIvbH3vsMWJjY5k3b17zbV/60pdYsWJFi5Pvbdu2sXjxYq644opet08RERERERGR7tRbzo87s08REZH+zmZZlhXuQYiIiEjPufHGG3n00Udp6xDgz3/+M/Pnz+dLX/oSc+fO5eOPP+bZZ5/lV7/6FXfccUfzdtXV1UybNo3q6mp+8pOf4HK5eOihhwgEAqxdu5b09PRet08RERERERGR49HfzrkrKyt55JFHAPj000955513+PGPf0xycjLJycnceOONIfvdioiIRCIl20VERPqZ9k78Af7617/y4IMPsmfPHnJycrjxxhv50Y9+hM1ma7HdwYMHufnmm1mwYAHBYJCzzjqLhx9+mJEjR/bafYqIiIiIiIh0VX875967dy+5ubmtvs6hQ4eyd+/eY/26REREej0l20VERERERERERERERERERDpJPdtFREREREREREREREREREQ6Scl2ERERERERERERERERERGRTlKyXUREREREREREREREREREpJOUbBcREREREREREREREREREekkJdtFREREREREREREREREREQ6yRnuAfQGwWCQ/Px8EhISsNls4R6OiIiIiIiICJZlUV1dTXZ2NnZ7751Lr3NuERERERERiTQdPedWsr0D8vPzycnJCfcwRERERERERI5y4MABBg8eHO5hdJnOuUVERERERCRStXfOrWR7ByQkJADml5mYmBjm0fQOPp+PBQsWMGfOHFwuV7iHIxFIMSLtUYzIsXi9Xh588EEAfvjDHxIXFxfmEUmkUYxIexQj0hGRfjxSVVVFTk5O8zlrb6Vz7s6L9NiU8FOMSHsUI3IsOlaW9ihGpD2KEemISD8e6eg5t5LtHdBUxi4xMVEn/h3k8/mIjY0lMTExIv+DSPgpRqQ9ihE5Fq/XS3R0NGA+n3XALp+nGJH2KEakI3rL8UhvL72uc+7O6y2xKeGjGJH2KEbkWHSsLO1RjEh7FCPSEb3leKS9c+7e29RNRERE+rWYmBgcDke4hyERTDEi7VGMiIiIiIi0TsfK0h7FiLRHMSL9hZLtIiIi0uu43W5uvvlmJk2ahNvtDvdwJAIpRqQ9ihERERERkdbpWFnaoxiR9ihGpD9Rsl1ERERERERERERERERERKSTlGwXERERERERERERERERERHpJGe4ByAiIiLSWT6fj+eee47S0lJ8Ph8ulyvcQ5IIoxiR9ihGRERERERap2NlaY9iRNqjGJH+RMl2ERER6XUsy2L//v3N34t8nmJE2qMYERERERFpnY6VpT2KEWmPYkT6E5WRFxERERERERERERERERER6SQl20VERERERERERERERERERDpJyXYREREREREREREREREREZFOUs92kV5q+Z4ybDaYmpOMy6F5MyIiIiIiIiIiIiIiIiI9Scl2kV7ow+3FXPPUcgDi3A5m5qaQmxbPgFgXGUnRzJuURVyU/nuLiIiIiIiIiIiIRKRgAPwN4PdAwAsxA8AZFe5RiYhIJykbJ9LLBIMW9729BQC3006tN8D724p5f1tx8zZ7Smq57fyx4RqiiEiPcLlcBAKBcA9DIphiRNqjGBERERERaZ2OlbtJfTnUFIOnCry1Jske8IIVhNh0SB8NsSnhHmWHKEakPYoR6S9Ue1qkl/nvujy2FlSTEO1k6e3n8vYPT+f/vjCe7501gjNHpwOwbHdpmEcpItK93G43t956K5MnT8btdod7OBKBFCPSHsWIiIiIiEjrdKzcTSwLSnZC0WaTdAeIioeETEjIgoZyyF8N5fvMthFMMSLtUYxIf6KV7SK9iMcf4MEF2wH43lkjSIlzkxLnZnx2IgC7i2s458EP2ZhfhS8QVC93ERERERERERERkUjgrYGGCogfCK6YlvfZMEn3hkoo3gbRSRCTHIZBiohIZykTJ9KLPL90PwfL68lIjOKbp+QedX9uWhxJMS68/iBbD1WHYYQiIiIiIhIJhg0bhs1mO+pr/vz5ADQ0NDB//nxSU1OJj4/n8ssvp7CwsMU+9u/fz7x584iNjWXgwIHceuut+P3+Ftt88MEHnHDCCURFRTFy5EieeeaZnnqJIiIiIr1LfQX46o9OtB8pOsn0cPfo2q6ISG+hZLtIL9HgC/Cn93cCcNPs0cS4HUdtY7PZmJKTDMDaA+U9OTwRkR7l9/v517/+xe7du4+66C8CihFpn2JE+roVK1Zw6NCh5q+FCxcCcMUVVwBw880388Ybb/Dvf/+bDz/8kPz8fC677LLmxwcCAebNm4fX6+Wzzz7j73//O8888wx33nln8zZ79uxh3rx5nH322axdu5abbrqJb3/727z77rs9+2JFREQkpHSs3A0sC6oLwBnV/rZOF9RFdptQxYi0RzEi/YnKyIv0Eh9sK6as1kt2UjRXTB/c5nZTByfx0fZi1h6o5OuzenCAIiI9KBgMsmvXrubvRT5PMSLtUYxIX5eent7i59/85jeMGDGCM888k8rKSp588kleeOEFzjnnHACefvppxo0bx9KlSzn55JNZsGABmzdvZtGiRWRkZDB16lTuvfdebrvtNu666y7cbjePP/44ubm5PPjggwCMGzeOTz75hIcffpi5c+f2+GsWERGR0NCxcjdoKiEfldD+tq5Y09Pd7+lYcj4MFCPSHsWI9CdKtov0Em9vOATAvMlZOI/Ri33qkGRAK9tFRERERMTwer0899xz3HLLLdhsNlatWoXP52P27NnN24wdO5YhQ4awZMkSTj75ZJYsWcKkSZPIyMho3mbu3Ll873vfY9OmTUybNo0lS5a02EfTNjfddNMxx+PxePB4PM0/V1VVAeDz+fD5fCF4xX1f0+9Jvy9pi2JE2qMYkWM5Mi70+Rwi1aXQUAfRKRBoJ/Foj4a6IqitgNiUHhleZylGpD2KEemISD8e6ei4lGwX6QUafAHe22L6J144KeuY204ZnAzAruJaKut9JMW4unt4IiIiIiISwV577TUqKiq49tprASgoKMDtdpOcnNxiu4yMDAoKCpq3OTLR3nR/033H2qaqqor6+npiYlrvR3rfffdx9913H3X7ggULiI2N7fTr68+a2gOItEUxIu1RjEhrAoFA8/eLFy/G4Ti6naV01cGOb7pzafcN4zgpRqQ9ihHpjEg9Hqmrq+vQdkq2i/QCH24vptYbYFByDFMbe7K3JTU+iiEpsewvq2PDwUpOG5XWM4MUEREREZGI9OSTT3LBBReQnZ0d7qEAcPvtt3PLLbc0/1xVVUVOTg5z5swhMTExjCPrPXw+HwsXLuS8887D5dIEazmaYkTaoxiRY/F6vWzYsAGAc845h7i4uDCPqJfz1sCBFeCOBVfrkxGPUlcGUfGQPR1stu4dXxcoRqQ9ihHpiEg/HmmqwtYeJdtFeoG31psS8hdMzMTWgYOrKTnJ7C+rY+2BciXbRURERET6sX379rFo0SJeeeWV5tsyMzPxer1UVFS0WN1eWFhIZmZm8zbLly9vsa/CwsLm+5r+bbrtyG0SExPbXNUOEBUVRVTU0f1HXS5XRF5giWT6nUl7FCPSHsWItMayrObvFSMhUFUOwQaITu/4Y6JjwVcL+MAVeUlKxYi0RzEinRGpMdLRMbXd+FlEIkKLEvKTj11CvknT6ve1Byq6aVQiIiIiItIbPP300wwcOJB58+Y13zZ9+nRcLhfvvfde823btm1j//79zJo1C4BZs2axYcMGioqKmrdZuHAhiYmJjB8/vnmbI/fRtE3TPkRERET6vboyKNsFMcmde5wzBnwN4KnulmGJiEjoKNkuEuGaSshnJ0UzrZ0S8k0OJ9srW8wgExERERGR/iMYDPL0009zzTXX4HQeLmyXlJTEddddxy233ML777/PqlWr+OY3v8msWbM4+eSTAZgzZw7jx4/n61//OuvWrePdd9/l5z//OfPnz29elf7d736X3bt389Of/pStW7fy5z//mZdeeombb745LK9XREREJKL4PVCyHYI+iEro3GNtNvNVX9EtQxMRkdBRsl0kwr29obGE/KSsDpWQB5iQnYjTbqOkxkNeRX13Dk9EJCzcbjd33HEHU6dOxe12h3s4EoEUI9IexYj0B4sWLWL//v1861vfOuq+hx9+mIsuuojLL7+cM844g8zMzBal5h0OB2+++SYOh4NZs2Zx9dVX841vfIN77rmneZvc3FzeeustFi5cyJQpU3jwwQf529/+xty5c3vk9YmIiEj30LFyCFgWlO2BmiKIG9i1fbhiobYYgoHQji0EFCPSHsWI9Cfq2S4SwfyBIIu3mLKNF07K7PDjol0OxmUlsiGvktX7Kxg8ILa7higiIiIiIhFqzpw5bVa6io6O5tFHH+XRRx9t8/FDhw7l7bffPuZznHXWWaxZs+a4xikiIiLS59SWmGR7XCrYHV3bhzsOakuhoRJiU0I7PhERCRmtbBeJYFsOVVPt8ZMQ5WRqzoBOPXbWiFQA3tl4qDuGJiIiIiIiIiIiIiKfZ1lQlW++dx3HIiiHC4J+lZIXEYlwSraLRLBle0oBOHHYABz2jpWQb3LJ1EEALNpcRGWdL+RjExEJJ7/fzyuvvMKePXvw+/3hHo5EIMWItEcxIiIiIiLSOh0rH6eGSqgphJik49+XOxaqD0VcKXnFiLRHMSL9iZLtIhFs+Z4yAGbmpnb6seOzExmbmYA3EOTNDfmhHpqISFgFg0G2bt1KZWUlwWAw3MORCKQYkfYoRkREREREWqdj5eNUUwR+D7hijn9fUQngqTIJ/AiiGJH2KEakP1GyXSRCBYMWK/Y2Jdu71pPn8hMGA/DK6ryQjUtEREREREREREREWuGtg8q80Kxqh8ZS8oGIS7aLiMhhSraLRKidxTWU1/mIcTmYNKhrB2cXT83GboNV+8rZW1Ib4hGKiIiIiIiIiIiISLPaYvBWgzs+dPt0RUPVIdDqYBGRiKRku0iEWtZYQv6Eocm4nV37rzowMZrTRqUD8MoarW4XERERERERERER6RYBP1QeMH3WbbbQ7TcqATyV5ktERCKOku0iEaq5X/uwzvdrP9LlJwwC4JXVBwkGreMel4iIiIiIiIiIiIh8Tn25KfceHaIS8k0cbgj6ob4itPsVEZGQULJdJAJZlsXyPaVA1/u1N5kzPpM4t4OD5fWsO1gRgtGJiIiIiIiIiIiISAv15YAFdmfo9+2KgZpCsLSYSkQk0ijZLhKB9pfVUVjlweWwMW1I8nHtK8btaE7Yb8qvCsHoRERERDAXeSoOQN5q8HvCPRoREREREZHwCQZMMtwV1z37d8aAtw78Dd2zfxER6TIl20UiUFO/9imDk4l2OY57f6MzEgDYXlh93PsSEYkELpeLn/zkJ0yaNAmXyxXu4UgEUox0M78XirfCoXUm4V5xINwj6jTFiIiIiIhI63Ss3AUNleCpgaj47tm/Mwr89SbhHgEUI9IexYj0J0q2i0SgZbsb+7UfZwn5JqOUbBeRPsZms+F2u3E4HNhstnAPRyKQYqQbNVSZJHvJDohLhdgBUL6nsWRi76EYERERERFpnY6Vu6ChEqxA95SQB7A7THUxX2Qk2xUj0h7FiPQnSraLRJhN+ZW8sS4fgFNGpIVkn6MzzIzKHYU1IdmfiIiI9FPVhZC/BmoLITELnNEQlQABL5TuNqUTRURERERE+pNgEKoLwBXdvc9jd5jJzyIiElHCmmwPBAL84he/IDc3l5iYGEaMGMG9996LZVnN21iWxZ133klWVhYxMTHMnj2bHTt2tNhPWVkZV111FYmJiSQnJ3PddddRU9Myqbh+/XpOP/10oqOjycnJ4YEHHuiR1yjSGXVePz94cQ3eQJDZ4wZy6sjUkOx35ECTbC+t9VJao56qItL7+f1+3njjDfbt24ff7w/3cCQCKUZCLBiEsj1waC0EPJCQ3XLFRlw6VOdDVX7YhthZihERERERkdbpWLmTvNXgqQJ3N5WQb+KMhvoyc34WZooRaY9iRPqTsCbb77//fh577DH+9Kc/sWXLFu6//34eeOABHnnkkeZtHnjgAf74xz/y+OOPs2zZMuLi4pg7dy4NDQ3N21x11VVs2rSJhQsX8uabb/LRRx9xww03NN9fVVXFnDlzGDp0KKtWreK3v/0td911F0888USPvl6R9tzzxmZ2F9eSkRjFA1+aErLyKrFuJzkpMQBs1+p2EekDgsEgGzZsoLy8nGAEnGRK5FGMhJCvAYo2Q+EmcMWYxPrnj1EcLnDFQtlu8PeOiX2KERERERGR1ulYuZMaKk21L2dU9z6PKxr8DRFRSl4xIu1RjEh/0k0NRDrms88+4+KLL2bevHkADBs2jBdffJHly5cDZlX773//e37+859z8cUXA/Dss8+SkZHBa6+9xpVXXsmWLVt45513WLFiBSeeeCIAjzzyCBdeeCG/+93vyM7O5vnnn8fr9fLUU0/hdruZMGECa9eu5aGHHmqRlG/i8XjweA5fJKyqMqVZfD4fPp+vW38nfUXT70m/L8PrD/K7hTsoqfGQHOsmJdbF6Ix4puYkkx7vZn1eFa+vO8Q/VxzAZoPfXT6JBLctpL+/kelxHCirZ+uhCk4ckhiy/XaVYkTaoxiRYzkyLvT5LK1RjIRIQ4XpzV5TDPHp5uJRoI2TZHciVOZDeR4k5/ToMLtCMSIdEenHI5E6LhEREZF+w7JMu63uTrQDOKLAX2qS7VHdvIpeREQ6LKzJ9lNOOYUnnniC7du3M3r0aNatW8cnn3zCQw89BMCePXsoKChg9uzZzY9JSkripJNOYsmSJVx55ZUsWbKE5OTk5kQ7wOzZs7Hb7SxbtoxLL72UJUuWcMYZZ+B2u5u3mTt3Lvfffz/l5eUMGDCgxbjuu+8+7r777qPGu2DBAmJjY0P9a+jTFi5cGO4hRIQPDtl4da+j1fuiHBaewOHVYedlBynbupS3t4Z2DPZqO2Bn0YrNpJRuDO3Oj4NiRNqjGJHWBAKH+0IvXrwYh6P191jpvxQj3aG4Y5vt3gBs6NaRhIJiRDojUo9H6urCv6pJREREpF/z1YOnEtxx3f9cTRXGvLXd/1wiItJhYU22/+xnP6OqqoqxY8ficDgIBAL86le/4qqrrgKgoKAAgIyMjBaPy8jIaL6voKCAgQMHtrjf6XSSkpLSYpvc3Nyj9tF03+eT7bfffju33HJL889VVVXk5OQwZ84cEhPDvyK4N/D5fCxcuJDzzjsPl8sV7uGEVXWDn7se/hjw8aUTBpEW76ao2sPm/Cq2F9XgCdiIczs4a3Q6F07K4LxxA0NWPv5I3rX5vPfyRrwxqVx44YyQ77+zFCPSHsWIHIvX62XDBpPMO+ecc4iL64GTWulVFCNdEAyai0Q1RVB9CLz1EJvcuYtGlmX6tg8cDwOGdttQQ0ExIh0R6ccjTVXYRERERCRMvLWm9VZMSs88n8MF9RU981wiItIhYU22v/TSSzz//PO88MILzaXdb7rpJrKzs7nmmmvCNq6oqCiioo4u++JyuSLyAksk0+8M/v7BHsrrfAxPi+M3l0/G6bA331fj8bO3pJaRA+OJdnXvaqpx2ckA7Cyqwel0dktCvysUI9IexYi0xrKs5u8VI9IaxUgn1RRD+W6oLQMCEJ0ECWld21dMAtTkwYDBpqdghFKMSGdEaoxE4phERERE+hVfrZl03FPXWl0x4KmCgB8cYU3viIhIo7C+G99666387Gc/48orrwRg0qRJ7Nu3j/vuu49rrrmGzMxMAAoLC8nKymp+XGFhIVOnTgUgMzOToqKiFvv1+/2UlZU1Pz4zM5PCwsIW2zT93LSNSHcoqfHwt493A/CTuWNaJNoB4qOcTByU1CNjGZEej80G5XU+Smq8pCf0QB8hERERiWyWBZUHoXgrWAGISwGHu/3HHUt0ktln9SFIyW1/exERERERkd6qrgycx3kO1RnOKKgrN0l+R89cVxYRkWOzt79J96mrq8NubzkEh8NBMBgEIDc3l8zMTN57773m+6uqqli2bBmzZs0CYNasWVRUVLBq1armbRYvXkwwGOSkk05q3uajjz7C5/M1b7Nw4ULGjBlzVAl5kVD60+Kd1HoDTBmcxAUTwzuxI8btYEhKLAA7CqvDOhYRERGJAMEglO2Bwo1mRUR8RqcT7Qt21bNodwPewOFV4thspvR8VZ5ZbSEiIiIiItIXBXzQUGVWm/cUhxuCPvDW9dxziojIMYU12f6FL3yBX/3qV7z11lvs3buXV199lYceeohLL70UAJvNxk033cQvf/lLXn/9dTZs2MA3vvENsrOzueSSSwAYN24c559/Ptdffz3Lly/n008/5cYbb+TKK68kOzsbgK997Wu43W6uu+46Nm3axL/+9S/+8Ic/tOjLLhJqB8vreH7ZPgBuO39sRJRtHzUwAYDtSraLSC/ncrn40Y9+xMSJE1VCV1qlGDmGgB+qDkHeKijaDFHxEJ3c6d0s2t3ADW+V8+03y5j5ZAE/f7+C/OqAuTM6ERoqob48tGMPIcWIiIiIiEjrdKzcQd5a8Nf3bLIdABt4anr4OVtSjEh7FCPSn4S1jPwjjzzCL37xC77//e9TVFREdnY23/nOd7jzzjubt/npT39KbW0tN9xwAxUVFZx22mm88847REcf7v/4/PPPc+ONN3Luuedit9u5/PLL+eMf/9h8f1JSEgsWLGD+/PlMnz6dtLQ07rzzTm644YYefb3Svzz6/k58AYtTR6Zyysgu9jwNsdEZ8SzaUsj2ovAejImIHC+bzUZcXBxOpzMiJjNJ5FGMtKG+HIq2Ql0pOFwQP9D820n+oMV9n1YBEOWAigaL5zbUsfSgl/99LR2XwwnYoKYQ4tND/CJCQzEiIiIiItI6HSt3kLcWggGw93CaxRkFnsqefc7PUYxIexQj0p+ENdmekJDA73//e37/+9+3uY3NZuOee+7hnnvuaXOblJQUXnjhhWM+1+TJk/n444+7OlSRTjlQVse/Vx4E4ObZo8M8msPGZJqV7SojLyIi0g/VV0DBRrMCIiHjuC4I/WtTHbvK/QyItrP46wPZUOTlpncr2Fnu57kNtXxzarxZ3V5TCN5cU1ZeRERERESkL/FUmzZaPc3hPiLR7+j55xcRkRbCWkZepK96ZPEO/EGL00elceKwlHAPp9nhMvI1WJbVztYiIpHL7/fzzjvvcPDgQfx+9YSWoylGPqehEgo2mItBx5lor/EGeXiZmbj3g5nxDIixc8bQaH48yxxnPLysmrL6gEmwe2vNKvoIpBgREREREWmdjpU7wLLMuU6Pl5DHrGwPeMFX3/PP3UgxIu1RjEh/omS7SIjtK63l5dV5ANx8XuSsagcYnh6H3QaV9T7yKsJ3MCYicryCwSCrV6+mpKSEYDAY7uFIBFKMNAoGoboADq0HTxUkZB73yosnVtdQUhdkaJKDqycdXrF+5YRYxqY5qfJYPLy0sYqOKwYq88w4IoxiRERERESkdTpW7gBfnflyRre/bag53BDwgN/T88/dSDEi7VGMSH+iZLtIiP1+0Q4CQYszR6dzwpAB4R5OC9EuR/OY3t1UGObRiIiISLeqL4eCdZC3Cvz1kJB13In2otoAf11dC8BPT0nE7Ti8P4fdxv+dkQTA8xvr2Fbqg+gkU76+vvy4nldEuiYvL4+rr76a1NRUYmJimDRpEitXrmy+/9prr8Vms7X4Ov/881vso6ysjKuuuorExESSk5O57rrrqKmpabHN+vXrOf3004mOjiYnJ4cHHnigR16fiIiISNh468zq8nAk2202sDDneSIiEnZKtouE0NOf7uHVNZG5qr3JvMlZALy1Pj/MIxEREZGQsyyoLYVDG+DACqjKh9hUiEsPSS/Bh5dWU++3mJbp4sKRR19UmjU4ivNHRBO04Jm1teBwgRVQsl0kDMrLyzn11FNxuVz873//Y/PmzTz44IMMGNByQvD555/PoUOHmr9efPHFFvdfddVVbNq0iYULF/Lmm2/y0UcfccMNNzTfX1VVxZw5cxg6dCirVq3it7/9LXfddRdPPPFEj7zOnvTZrhLufXMzCzdr4rKIiEi/560x51/h6NkOYAO8SraLiESCrjdrFJEWXl+Xzz1vbgbgJ3NGMzUnObwDasOFk7K4583NrN5fQV5FPYOSw9BXSEREeoeAz/T4jk0J90ikPZYFdWVQvhdqioAgxCSDKzZkT7G91Me/NtcB8P9OS8TWxkWlL46O4Z1dDWws9pkbXNFQWwypI8J3IUqkH7r//vvJycnh6aefbr4tNzf3qO2ioqLIzMxsdR9btmzhnXfeYcWKFZx44okAPPLII1x44YX87ne/Izs7m+effx6v18tTTz2F2+1mwoQJrF27loceeqhFUr4v+GxnKU9+soeaBj/njc8I93BEREQknOrLzeTicHFEgacyfM8vIiLNlGwXCYFPdpTw45fWYllwzayhzD97ZLiH1KaMxGhmDkth2Z4y3lqfzw1njAj3kOQ4ldd6eW9rERsOVrAhrxK7zcbEQUlMGpTE7PEZJMWE8cBfRHovvweKtpoLCIOmmXLgEpk81VC2F6rywAqayRHOqJA/zW8+rSJowdwR0ZyY3fb+x6Wbz51tpT78QQunOw48NWblR1RCyMclIq17/fXXmTt3LldccQUffvghgwYN4vvf/z7XX399i+0++OADBg4cyIABAzjnnHP45S9/SWpqKgBLliwhOTm5OdEOMHv2bOx2O8uWLePSSy9lyZIlnHHGGbjd7uZt5s6dy/333095eflRK+kBPB4PHs/hHqNVVVUA+Hw+fD5fSH8PoTQ+Mx6AdQcrwj7OpucP9zgkcilGpD2KETmWI+Mi0j+fw8JbDzXl4IyBQLh6UbugvgY8DWB39PizK0a6UU0RYEF8757cqRiRjoj045GOjkvJdpHjtDGvku/8YyW+gMW8yVnc+YUJba70ihQXTclm2Z4y3lx/SMn2Xq6s1su8P37MocqGFrev3GfK9Q5JieWNG08jKVYJdxHpBF8DFG2GyoPm54oDkJGoVcmRyFcPBRuhrsSUi3d1T8Wazw54WLzXg9MOt52SeMxthyY5iHXZqPNZ7Cn3Myo1GmpLTMJdyXaRHrN7924ee+wxbrnlFu644w5WrFjBD3/4Q9xuN9dccw1gSshfdtll5ObmsmvXLu644w4uuOAClixZgsPhoKCggIEDB7bYr9PpJCUlhYKCAgAKCgqOWjGfkZHRfF9ryfb77ruPu++++6jbFyxYQGxs6CpyhFqFB8DJ9oIqXnvjbdw9f137KAsXLgz3ECTCKUakPYoRaU0gEGj+fvHixTgcEfChF7HC3DJr87theVrFiLRHMSKdEanHI3V1dR3aTsl2keOwr7SWa59eTq03wCkjUnnoy1Nw2CM/EXH+hEz+778bWX+wkv2ldQxJjdwLWr3N9sJqXl51kBi3g+zkGAYnx5CdHENWcjRRztAeUFiWxU//s45DlQ1kJ0Vz4aQsJg1OwrJg/cFK3tqQz/6yOn7877U88fUTsfeC2BSRMPN7zUr2ygOm13diliklX3kQErNVTj7SBANQstOUaE8aBDZ7tzyNZVn85lOz6vRrE2MZPuDYpxB2m40xqU7WFPjYXOJjVKrLjK2+3MSUiPSIYDDIiSeeyK9//WsApk2bxsaNG3n88cebk+1XXnll8/aTJk1i8uTJjBgxgg8++IBzzz2328Z2++23c8sttzT/XFVVRU5ODnPmzCEx8dgTesLJsiz+tP1Dimu8DJ1yCtOGJIdtLD6fj4ULF3LeeefhcmlirRxNMSLtUYzIsXi9XjZs2ADAOeecQ1xcXJhHFEEsC/LXmPOb+PQwjiMI1QUweGZYztUVI92kbK9Z/OCONbGWMQHiB7b7sEikGJGOiPTjkaYqbO1Rsl2ki4qrPXzjqeWU1HgZn5XIX74+PeTJ1O6SnhDFrBGpfLqzlDc35PP9syK37H1vUVTVwMOLtvOvFQcIWq1vMzYzgUevOoER6fEhec5/LN3Hoi1FuB12/nbNDMZnH74wecm0QVx2wiAue+wzFm0p4omPd/PdM1XFQPoOl8vF97//fd5///2IPBDrdQI+c0JXnW9KfWMzSVG703zVV0DFPohOBnv3JHRDrV/ESMV+83eJH9htiXaAD/d5WF/kI8Zp44czO7YyfVyaizUFPraW+Ll4DOZCQV2JmSAQhhKHrekXMSL9WlZWFuPHj29x27hx43j55ZfbfMzw4cNJS0tj586dnHvuuWRmZlJUVNRiG7/fT1lZWXOf98zMTAoLC1ts0/RzW73go6KiiIo6uh2Fy+WK+P+Pkwcn897WIjYX1DBzRBgvsDfqDb8zCS/FiLRHMSKtcTqdzcfKsbGxipEj1VeAtwLiksERzvNjO9htgA/C8PdRjHQDby1UH4S4JNPKr7YESrdBVEyvXPygGJHOiNTjkY6OqXdcLRWJMA2+ANc/u5J9pXXkpMTwzLdmkBAdeW8Ex3LR5GwAXl2dh8cfaGdrOZbthdWc++CHvLjcJNpnj8vgyhk5nD4qjeHpcUS7zFvt1oJq5j+/mgbf8f++N+dX8cu3tgBw+4VjWyTam0wclMRdX5gAwG/f3cbHO4qP+3lFIoXNZiM5OZmoqKiIb90R8fweKNwMxVto7gnWlGhvEpsCVYdMsrSX6PMxUlMEJdshOrFb+rM3sSyLP62oAeCqSbGkxXYsUT4uzRwXbSlp7G3ligNvnekvHyH6fIxIv3fqqaeybdu2Frdt376doUOHtvmYgwcPUlpaSlaWqUIxa9YsKioqWLVqVfM2ixcvJhgMctJJJzVv89FHH7XoZbdw4ULGjBnTagn53m7S4CQA1udVhnkk/UcwaPHW+kPsL+1YCUcRETl+OlY+hroyCHi79Tysw2yY/vHheGrFSOhV5oG3GqIar/PGpYGvziTdeyHFiPQnSraLdJJlWdz+ygbWHqggKcbFs986iYEJ0eEeVqedPyGTOLeDHUU1fO+51Uq4H4eHF26n2uNnXFYi//nuLP52zYn85vLJ/OO6k1j847PYcs/5fHTr2aTFu9laUM09b27u9HNYlkWDL8CqfWX84MU1fPFPn+D1Bzl7TDrXnjKszcd9dWYOl00bRCBoce3TK3h44XZ8geBxvFoR6fUsCwJ+UzLeUw0FG8zq6IRMiE7Cstk5UOXn9e313PdJJS9tqsNnc5t+7WW7KSqr5LU1edR4/OF+Jf1XfYWZIGGzdXsP9GV5XlYe8uK2w/XTOl6ZZXz655LtDpepoBBByXaRvu7mm29m6dKl/PrXv2bnzp288MILPPHEE8yfPx+Ampoabr31VpYuXcrevXt57733uPjiixk5ciRz584FzEr4888/n+uvv57ly5fz6aefcuONN3LllVeSnW0m737ta1/D7XZz3XXXsWnTJv71r3/xhz/8oUWZ+L5kclOy/aCS7T3l4UXbmf/Cam74x0osq40yYiIiIj0h4IeqPHBHSDlsRxR4dEzSJ9SXQ+V+iB1gzvWbuOOgptDEnohELJWRF+mkxz/czatr8nDYbTx21QnkpkXIwVUnDYhz89dvnMi3/r6CxVuL+P5zq/nz1Sf0mlL4kWJ7YTX/21gAwB+vnMqojKOTHjabjSGpsTz8lal846nlvLBsP6eMSG2uLtDEsiw8/iAef5DqBh9Ld5fxwbYilu8po7zOiy/Q8sLSqSNT+d0VU445M9Bms/GrSycRsCz+uzafP7y3gw+2F/PXr09nYGLvmyQi0iQQCPDee++Rl5dHIBCIyDJDEScYML29Kw+CpwawTI83b13zSvZqT5Br/lvK6gJfi4f+aUU135sex/qDxby8qxRvEL5yYg73f2lyeF5LB/TZGPHWQdEW8NVCYnb72x+nRxtXtX95QiwZ8R0/RhiTak4zCmuDlNUHSIlxgNMFdaWQnNMtY+2sPhsjkSYYBH+DaSUgPWrGjBm8+uqr3H777dxzzz3k5uby+9//nquuugoAh8PB+vXr+fvf/05FRQXZ2dnMmTOHe++9t0WJ9+eff54bb7yRc889F7vdzuWXX84f//jH5vuTkpJYsGAB8+fPZ/r06aSlpXHnnXdyww039Phr7gmTBiUDsKu4hhqPn/goXVbpTos2F/LI4p2AqRS2en8F04f2vYoJIiKRRsfKbagvB09V5PTQdrhN6fEwtOtSjISI3wNV+aa1n98Dsakt74+Kh5oSE3e9rJS8YkT6E50VinTCq2sO8sC7WwG464sTOGVkWphHdHxOGZnGk9fM4FvPrOC9rUX833838ZvLIzdxEon+/L658HP+hMxWE+1HOn1UOt8/awSPvr+LW/+9nl1FtXzrtGHYbDaeXbKXpz7ZQ0mN95j7iHbZ+cLkbK49dRgTspM6NMYYt4M/XDmNc8YO5OevbWTdgQoeWbyTey+Z2LEXKRKBAoEAy5Yta/5e2lFTBKW7TRl4hxNcsY09vm0Qkww2u6ncsriC1QU+nHYYn+ZiXJqL9/Y0sL8qwO3vV3NkUaQ31+dz1xcnEOOOzElafTJGAj4o3mr+jj2QaF9T4OXjAx4cNvjO9GOsam+ogs3/hZ2LYNRcmPY14t12hiY52FcZYEuJn1NzHCbu6itMVQWnu9vH354+GSORpqESynaDrwEGzzDvP9KjLrroIi666KJW74uJieHdd99tdx8pKSm88MILx9xm8uTJfPzxx10aY2+TnhBFdlI0+ZUNbMqr5KThqe0/SLpkb0ktN7+0FoDEaCdVDX5eXL5fyXYRkR6gY+U21BSZf+0RclzrjDLVw3z1JinbgxQjIVBfAYUbzSSOqITWz/PtTrACZttemGxXjEh/ESGfCiKRzbIsHvtwFw+8Y3oefv3koXz95LZ7HfYmp45M409fO4Hrn13JW+sP8etLJ2G3q4dKR+wtqeX1dfkA3HjOyA495ubZo1l/sJKPd5Tw8KLtPPPZHgDK61quIrXZYEJ2ImeNHsgZo9MZPCCGxBgXcW5Hl3vcXDx1EG6Hne89v5oVe8u6tA8R6YUq86Bos1nFnpDR5kWB5zbU8eaOBpx2+NflaUzPMsnQWm+QZ9bV8vKWOnKTndww1c1PFlVxoCbAwi2FfHFK9yd9pVH1IVOZIDGrcbJEaHn8Fp8e8LB4bwOrD3nZWmrK1F0yNoacxCPiJuAzCdTSnSa2dr5nVi4DrHgCUkfAkJMYm+YyyfZiH6fmRJlke02huRjkVHKqTwsGoGK/iRNvjfnb+xvA0bMXAEW6y6TBSeRXNrBByfZus6eklu/+YxXVDX6mDx3ArXPHcOUTS3lzfT6/uGg8STFaGSUiIj0s4DcTnyOlhDw0tuvymBXRPZxsl+MUDEDpLpNET8w+9jm+K9qcSw8YBnZ1hhaJREq2i7QjGLT4v9c38Y+l+wC4/vRcbr9gXJhHFVpnjUnH7bRT7fFzoLyOoakRdNAYwR77YBdBC84ek87EQR1bZe502Pn7N2fy1oZDPLxwO7tLagHITYvjB+eMZPb4DKKdDlwOW5eT6sdyQuMqkO2F1dR6/MSp7KVI32VZJtlVtNnMdo9puxrLhiIv935k+rz97JTE5kQ7QJzbzvwZCcyfcbh6xyWj6nlkjZ/XVu1Xsr2nBHzm7+mODfkqioKaAA98VsXC3Q1Ue1u2LBmb5uTmk46o3FJ9CN76sSlzd6T4gRCTYlbef/Br+MpzjEtz8e6uBraUNk4osztMXNaVQZySU31WMAAl282Fo6h4SBoM1QUQOHb1HpHeZPLgZN7dVMg69W0POV8gyBMf7eYP7+3A6w+SFh/Fo187gYzEKEZnxLO9sIbX1+bx9VnDwj1UERHpb7zV4Ks7usx3ONnsYAH++nCPRDqrphBqCsy5dHuT6d3xpmqYtxqiO3YNWkR6lrIsIu14ftk+/rF0HzYb3HnReL55am64hxRyLoedsZkJrD9Yyca8KiXbO6CoqoGXVx8E4MZzRnXqsXa7jS9MyeaCiZn8b2MBdpuNuRMycDq6f2ZiRmJ0c9nL9QcrmTUigk4QROT4NfVlb6gyvbHry8xJWXRimw+xLIsfL6jAG4Tzhkdz3bT2PwMuHj+AR9YU8+GOUkprPKTGR7X7GDlOtSVmxntCZkh3u7fCz1WvlpJXbUq6ZcTZmTsimlmDo5iW6SbzyD7ttcXw5o+hOh8cUWb2fdJgyJoGI2ebnuz/+aZJrC7+JeNG/R8AW4qPqN4SFWcenzLMrMKQviXgh+JtZkV7fBo4o83twaBZbSPSR0webC5ybjhYEd6B9DGlNR6ufXoFG/LMJIbTR6Xx60snkZlk3ku+OnMId7+xmeeX7efqk4d2y+RkERGRNnlqzPFupJ3H2GxmbNJ7+D3mnMnh7lg8OaPMYxoqlWwXiVCqOSFyDFUNPh5etAOAn8/rm4n2Jk39vzfla3VGR7y7qQB/0GJqTnKXewY6HXa+MCWbeZOzeiTR3mTaEDPeNQfKe+w5RaQHBINQsgPyVkHpDvDVmhn3x0i0A2ws9rG9zE+008ZvZyd36ML1yFQXk9PtBCx4c83+UL0CaUswCJUHzEm43dH+9h20pcTHl/5TQl51gNxkB//+UipLvpXBPWclc8HImJaJ9royePMWkyiPToa5v4TLn4Q598KkyyAmEVwxcO5dZlb+gWVMr/kAgJ1lfnyBxhXz7gRTRr5en0F9jt9jKhuU7YL49MOJdjAXAJtaDYj0AZMaq1rtLa2j8nPtoKRriqob+Opfl7Ihr5LkWBcPf2UKz35rJjkpsc3bXDptEG6nna0F1aoqICIiPa++HBwRuHbRFWMm21tW+9tKZKjMM+fYnenB7oyCmuLuG5OIHBcl20WO4fEPdlFW62VEehzfmNU3erS3ZeIgk4zZmF8V5pH0Dgs2FwJwwcTQrjDsCdOGJAOwZn9FWMchIiFkWVC+1/TQjhlgVhzHppqTsXa8ud2Umzt3WBTJ0R0/NLxkrOkH9+pqJdu7XV2pWdke07XJXZ+3u9zPfZ9W8eX/lFBSF2RcmpOXvpTGjOwo7K1NtijbA2/8yCT8oxLhjJ/C4Bmt94obOBamfBWA1O0vkeC24Q3CrnLT/x27wyTjdZGgb6mvgENrG1e0Dzz6vcfuAG9dOEYm0i2SY90MaUwCN63Clq4rqGzgyieWsr2whozEKF7+3ilcOm3wURMAk2PdzJuUBcC/VhwIx1BFRKS/CvhMst0V2/62Pc0ZbcrI+3S83St4asz1m5jk9svHHykqHhrKzeR1EYk4SraLtCGvop4nP9kDwM8uGIerB1ceh8PEppXteZVYmgl5TJV1PpbsKgVgzoTenWzX31qkDwj4TRK0ZBvEJJlZ7R1kWRZv7jCrTS8a3fHHAXxhTAwOG6w91MCeQyWdeqx0gmU19ke3HXe5wrL6AF9/rZRz/lHEX1bVUO21mJ7l4p9fTCC9brdJlFYfMqXpAo0rNbcvgNe+15hoT4DTfwzDTjn2E024BLBhq9jLKQMqALOKvllUPNQUgU99BXs9yzKrMvJWQ22pmejT2iQfh8v0FxTpQ5pKya/Zr0odx2v+C6vZXVzLoOQYXvrOLEakx7e57eUnDAYaK40Fgj01RBER6e881WbyqDtCk+2+Bk1u7S1qi8FbY86vO8MZY86hi7fpXFokAkVg3RORyPDgu9vw+IOclJvC7HEDwz2cbjcmMwGH3UZprZeCqgaykjqXdOlP3t9WhD9oMWpgPLlpva+//YTsJFwOGyU1HvIq6hk8IAJPFETa4XK5uP766/noo49wuSKsX1p3C/jBU2W+6srNv94ac9LvbvvidGvWFPjIqw4Q57Jx9rDo9h9whPRYB6flRPHhfg/3vLGJJ799BnZ75PRO7TMxUl8ONQUQm3xcuymuC3DVK6XsKvMw3p7HFWn7mB2/l8Gendhe3A1W4OgH2Z0QbFyRPiAXTr8FMie1/2Rx6ZAxHgo3can9E95lHmsLfFw6tvF+d7xJ0NaXd2pySKj1mRgJl2DArMgo2W4S7IlZbW/rcJsy8wF/ZJbeFOmCmbkpvLn+EMv2lPGDcA+mFztQVseqfeXYbfDi9SczJPXY5yYnD09hQKyLslovy/aUcerItB4aqYhI/6Jj5c/x1oDlN+dIkaapEoy3BkjvsadVjHRBwAeVB80E9M6y2SAh00yQtyzImBCZkz+OoBiR/qRvL9UV6aIth6p4ZU0eAP9v3rgO9a/t7aJdDkYNNB/0G/NUSv5YFmwuAGDOhIwwj6Rrol0OxmeZtgEqJS+9lc1mIz09nZiYmH7xHk0waPp5FW2FfZ/C/qVQsAFqi8z9cWmmj3YnvbnDzIaePTyaaGfnf4+3npJIlAPe313Dwwu3dfrx3alPxEjTquGgv2X/604qrKzn9y8t4IbqP7Em+ru87f4Z36x6jJz8/2Er3WES7a4YUxLxyItHQT9gg9yz4ILfdizR3mTEuQDM9HwGwIp8z+H7bDZwuqCmsMuvKRT6RIyEi98LRVvMV1RC+y0OHG4IeNW3XfqUWcNTAVi5rwyPv5UJS9IhTe25ZuamtJtoB3A67MxtrC729oZD3To2EZH+TMfKn1NXdtyVxrqVw2UmM/cgxUgX1JWZBRNRiV17vN0JCVlQXQCFG815WQRTjEh/EoFTsUTC79H3dwIwb1IWkwcnh3cwPWhCdhJbC6rZlF/JeeN7ZyK5uzX4AnywzfSZndsLS8g3mZqTzLqDlazZX8EXpmSHezgi0hbLMgnJigNQV2KSou54iE8/7hn1QcvircZk+0Wjura6eOJAF785O4GbF1XzyPu7mDAoifMnHmN1q3ROfTlU5UFsSuce563hpaW7OLBzA9N8azmJDfzK5gFH4/0ONyRkw4BhkD4aMiZDylCTcA8GTELUUw3eWtNrO2kION2dG0PuGfDZI6TU7SGTUraUpFLlCZIY1TjXNyrB9KH3VHe+fJ6EVzBoSheW72m9P3trHC6TbA9E9sUgkc4YOTCe1Dg3pbVe1h+sZMawTr5XC2DKwQPMGd/xc6sLJ2XxzxUHeHdTAfdcPBFHBFXWERGRPsjvhfqKyOzX3sQVAw1VZuV0JE8K6O+qD5nJ53ZH+9u2xe6AxEzTms1TBU5V+RGJBEq2i3zOruIa3mqcIX/jOSPDPJqeNXFQIi+v1sr2Y/lsVwl13gCZidFMGpQU7uF02bQhA/j7kn2sOaAek9I7BQIBPvroIw4dOkQgEOib5ah8DVC6Cyr2mZOpmAEdS2p10Ip8L4W1QRLcNs4Y0vX9Xjo+gQ15VTy1xcaPX1rHxEFJEdGeotfHiGWZSRZWoP1V7ZYF+WtgyxtwcAV4a/hy032N+YcSkonKHEfCyFkw5BSTwLe3UuTK7jRxFn2cn3FxaaasXeFGropdxoN1F7LqkPdwuwJXLNSWQXVh2JLtvT5GwqWmACr3dzzRDmCzmzj1e9rfVqSXsNlsnDw8lbc2HGLprlIl27ugtMbDyr1lAJ2a7D1rRCrJsS5Karws31PGrBGp3TVEEZF+S8fKR/DWgK/OTHqPVK4YM5nZW9N+1akQUYx0UkOV6dfehaqER7E7zfmVp9qce0coxYj0JyojL/I5f35/F5YFs8dlMC6riyVdeqmJjcnjTfmVYR5J5Hp3oylzOGdCRq8ufzNtSDIAm/KrVPZSeqVAIMAnn3xCYWEhgUAfjOG6MshbCWW7ICa5c0mtVlR6ggQtq/nnel+Qv62pBWDOiGiiulBC/kh3nBrHtIF2ar0BXl6Vd1z7CpVeHyN1ZVCdD7HHSCBYQdj+Drz0DXjrFtj9fmOfPiixEjngGk7pkAuoOO1OUq5+koQv3gfjvwjxaa0n2kNtxDkAXORYApgJHi1EJ0LlAfDVd/9YWtHrYyQcvHVQstO8H3X6PcmCgJLt0recPNwk2JfsLg3zSHqn97YWEbRgQnYiOSkdn6jnctiZ05icVyl5EZHuoWPlI3iqTAWwSOzX3sTuNG3AvHU99pSKkU6qLTHnvq6uVRY8ijPK7DOCKUakP1GyXeQIB8rqeG2tSRL0t1XtAOOyErHZ4FBlAyU1uhj6eYGgxaItjcn2TpQ5jERDUmJJiXPj9QfZcqg63MMRkSPVV5h+7J4qSBp03Cdib++oZ8pfCjjr70X8aUU1b+2o57zni1m42/ROvmLc8a9Cd0Yn8LVRQfN8uuh9/CzLrBy2rLYTmgXr4bXvwQe/MQlrhxtv1gy+57ibcQ1PccvAvzLoG38l9fzbSB5/DvZjJe27S+4ZgI1c3y6yKGXl55PtUQlmJn51Qc+PTTrPsqBsNzRUQEwXVvDaneCpDfmwRMKpaUX1qn3lmsDaBQs2df3c6oJJpm3NO5sKCAStdrYWERHpIl89lO8Hd/irt7XL5jDXESTyBHymRVwoq7q5Ys35dJgmr4tIS0q2ixzh8Q93EQhanD4qjak5yeEeTo+Lj3KSmxoHmBXP0tLq/eWU1npJjHZy0vDeXSbSZrNxQuPq9n8s2RfewYjIYd5aKNxs/o3PNKWXj9NfVpuVzvurAvxuSTXz/1fOwaoA2fEO/nZRCicPDkFperuDOUPAZYdthdXsLKo5/n32Z95aM0M9Jrn1+9c8D6//0PTNdrhh5By45C/c5r6d/9WOYkB8HL8/Pw274zj6wIVCUyl54ALHctYWevH4j0iI2GzmYkPFfpUX7w1qCs3Ejvh087frLIcLvJrgJ33LiPR40uLdePxB1h1QdbDOqPP6+XhHMWCqhnXWqSPSSIx2Ulx9uBS9iIhIyJXva5xs2jOl2Y+LK9pUSLM0CS3iNFSaiRBR8aHbpyvatDfw6PqLSCRQsl2kUY3Hz79XHQTgxrP736r2JhNUSr5NCzaZlXfnjsvA5ej9b5/fPn04dhu8vPog/2mMfREJI7/HJNrryyAho2vJrM/ZXOxjXaEPlx1+eXYSM7PdJEXZuG5qHAuvTmf28HZ6gXdCUlISp2ab90atbj9Ofg8EvEevarcsWPEkrPir+Tl7Glz4OzjrZ3xSm82r2xqw2+CRCwaQEhPmRHuTxlLyFzuX4A3A+qJWSsk3VEJNURgGJx3m90LpbjMByNnF9w2HuzG2/aEdm0gY2Ww2ThpuVrcv2aVS8p3x0fZiPP4gOSkxjM3s/Covt9POnAlmRfy/Vh4I9fBERERM4rpiP8SmhOT8vNs5G5Ovvp4rJS8dVF9hzudD2YqgaXGGRxOaRSJB788WiYTIyr1leP1BBg+Iab5g0h9NzDZ96jflaWX7kSzLYsHmpjKHnV95EYlOHp7KTbNHA/CL1zayo1AHZyJhVb4Pqg9BQmhWtAP8a5M5yT5veDRXT4rjpS+lse47WfzijCTi3CE+DHTFcuEQk0RTsv04+evNifiRcWBZsOxxWPMP8/PIuTDnV5A1mYagjV98UAHANybHMT3L3fNjbktjKfkptp1kUcryvM8l2212U5KxYp8prSeRqfIg1JWYagVd5XCbSSTq2y59zKzGc8el6tveKe9sNBOZ547PxNbFBMbVJw8F4I11+RRWNYRsbCIiIgQDULbH9EF3x4V7NB3jjDbH2161booowQDUFHRPKwJXNNQWqZqBSARQsl2k0dLdpvTcrH6caAcY3biqQCWAW9peWMO+0jrcTjtnjE4P93BCZv7ZIzl9VBr1vgDfe341DT71mhQJi4ZKk2yMHQD20KxIbvBbvLrNJNuvnNAD/eVsNuYMd+O0w9aCanYV63Oky7x1R6+c2PIGrP+X+X7MRXDGzc0n639ZVcOeigDpsXZuOTmEPeBC4YhS8hc6lrHi833bAaKTzUz/2uIeHZp0UEMllO8xbQ2OZyKQw2Uu/qllgPQxJzeeP67eX65j6Q6qrPfxTmPVsAsnZ3V5P1NzkpkxbAC+gMUzn+0N0ehEREQwlbdqCo5vsmlPs9lM0rVO7VUiiqfKrD53h7CEfBNXDHhr1LddJAIo2S7SqGklwsn9PNk+Mt188O8pqSUQ1Ky4Ju82Xgw6fWQacVEhLPkTZg67jYe/MpW0+Ch2FtWweKvK+PYnH24v5jf/28r3n1/FxY9+yvXPruQPi3bw/tYi/IFguIfXf1gWlO83qz1DePL1v531VHksBiU4OG1ICPqyd0ByYhKnNl4zf3u9Vrd3WUNFyxLyVYdg6Z/N96PmUjF9Pr9e6uGejyp54LMqHl1pKpPceUYSiVEReHjfWEr+QscyVh3yHn18YXeYRGzlATPrXyJHMAhle83Fm6jjnMhhs5v3O78HfyDIws2FfLarhLyKeh1zSq82Ij2OtPgoPP4gzy3dF+7h9Ar/XZtHgy/ImIwEpuUkH9e+vn36cACeX7qPWo/aVIiISAhYFlTlHz5P6U2i4qGmUFXDIklDpWml1R2x5IwBX4NJuItIWEXg1TiRnlfj8bMhz/QoP2l4SphHE16DkmOIctrxBoIcKFOPnyYLNjeWOWzsC9iXpMVHccFE87pW7ysP82j6rkDQYsXeMn799hYu+/OnvBTm3pJvrs/nmqeW8/iHu3h7QwHrDlSwcHMhDy/azjefWcEji3eGdXztcTqdXHvttYwePRqns5dPgKktMSWaY0M7Y/7FjeY9/CsTYrH3VH85ZxTzhpqk2VthLiXfa2Mk4DMr2x2NpeCtIHz0APgbIGkInPwd7vmsgSdW1/LU2lr+vLIGbwBOz4niolFd7KXd3YafiYWN6fYdxHtL2FbaSjIkZoD5v1DXc2WYe22M9KSaQqg6GNoVPQEPv12wjeufXcnX/rqMU3+zmMl3vcsnO0pC9xwiPchms3HNLFPO/JdvbeFvH+8O84gim2VZvLBsPwBXzszpcgn5JrPHZTAsNZaqBj//WXUwFEMUERH6+bGyp8qcl0QnhXskneeON4nX+opuf6p+HSMdZVlQXWDKvXcHmw1sRGzfdsWI9CdKtotg+rUHghY5KTEMHtADpXYjmN1uY3jj6naVADbyKurZmFeF3QbnjhsY7uF0i2lDkgFT/lJCb09JLWf+9n2ueHwJT3y0m9X7K/jpf9bz/17dgNcfxBcI8tH2Yv65fD8ef/ev6iyv9fJ//90EmAuUv7hoPI9ffQK/uGh8c5uEj3ZEdjlnu91OdnY2sbGx2O29+HAm4DflmW20XMl8nDYUeVme78VugyvG9ezn2pwRsc2l5N8PY7WMXhsjvnpT5aApHjb/F/LXgN0FM29gfVU8r2w1JeK+OTWOayabr9+dl3zcCYtuE5uKLXMiYFa3ry5opZS8w2UuFFTl9Vi/uV4bIz3FVw+lO83Ej1C9P9kdbM8v48mP9wAwJCUWl8NGrTfAr9/egqVeg9JL3XjOSL531gjAJNwffT+yJy2G07qDlWwtqCbKaefSaYOOe38Ou43rTssF4MlP9qhShohIiPTrY+W6MtP6yBmhk5mPpaktXQ9MYu7XMdJRnmpoqDIVB7qLM9q0PYjAcynFiPQninARDvdrPzm3f5eQbzIiPQ5Qsr3JwsYS8icOTSE1vmdKMfe0E4YMAGBjXlWPJHv7E8uyuOOVDRwsrycx2sklU7O57rRcbDZ4ftl+LvjDR0y/dyHfeGo5P3tlA1/+y1LyKrq319I9b26mtNbL6Ix4Hr1qGtedlsv5E7O47rRcfnmxSYhtzKtU39GeUFNgVo6GcNXozjIf1/7XfK7NGR5NVkJoesB3VHJSAleNNid5N7+0loPlqpLSKf4Gs7rd7jIrvZf9xdw++nysIbP45cdVAFw6Job/OyOJu88yXxnxPft37rThZwMwz7GUdYWtJNsBYlLMrP96TfwKO8uCsj3mbxEbuuNjyxHFzxccwh+0mDMiho/mT2L5HbOJj3Ky+VAV724qDNlzifQkm83GT+eO4ebZowH47bvb2F+qz7/W/HO5WdV+4aQskmPdIdnnl6bnkBzrYn9ZHX9X73YRETkeAb+ZAOzuxYux3HEqJR8pGirNOX53TtxwxZpqBt7a7nsOEWmXku0iqF/7540caGbb7SxSsh1ovvA7Z0JGmEfSfYamxpIS58YbCLIpvyrcw+lTXlubx5LdpUS77Lz1w9P5/ZXT+MVF43nqmhkkRDvZVVxLVYOftHg3idFO1h2o4KI/fsxH27tnZfn7W4t4dU0edhvcf/lkopwtE3Q5KTGkxUfhC1hsbGyvEYkCgQBLly6lqKiIQKCXTgrwe0wyyxUD9vbLafmDFgerjt2LdE+Fn6+9WkppfZAJ6S7uPzc5NGP11sBnj8DO99qfLW13cscMG5Mz3FTU+Zj//OqwTOLptTHiqwcss8p7/b/MiXniIJjxTRbsNRULohxw6ymJ4R5p5zSWkj/BvpP8/PzWt3FGmbL5VW3cH2K9NkZ6Qm0xlO8zE4FCWDHhtf1RLC+EaAfcOa0OirYwwB3k2lOGAfCH93YQ1KpU6aVsNhs/mj2KqY09yNcc0MShz6vx+Hl9nXmP/+rMISHbb4zbwY/OHQXAr97ewmc71ZZCROR49dtj5fpykyCN7mXnW0dyx/VIKfl+GyMdZVnmvCqEVQxb5Yox1w08kXc9VzEi/YmS7dLvqV/70UY0l5HXjLi9JbUs2V2KzdY3+7U3sdlsTGu6MLi/Iqxj6Usq63z86q0tAPzgnFHkpByeGX322IG8+YPTuHXuGF76ziyW3TGbt354OpMGJVFe5+Pap5eHvO/knpJafvbKegC+dWou0xorGhzJZrNx4lBz+8p9kXuROBAIsHjxYvLz83vvAXtVvjmRjzn67/B51Z4gl/+7hNOeKeKZda1PhFq0u4ErXy6hqDbI2FQnz12SSlJ0iA71Vj8LG1+GxffCwjvb7QcWFZfIo2fZSI5xsu5gJb98c0toxtEJvTZGPDWm9F9DBWx5w9w29gt43QO47xNz8nz9CfFk93DFguMWm4ov3VTOmF69mBpvsPXtopNMCTxv968I7bUx0t38HijZCXa7uXATIpWeIL9qjOEfzExgcFaWeQ+sPsS3T88lPsrJlkNVLNis1e3Su00ebPq7bjgYuZMWw+WNdfnUeQOMSI9jxrD2j38649pThnHptEEEghbff2G1KguIiBynfnusXNPYCq0DE+Ijlt0JWN1eSr7fxkhH1RSar+ik7n8umwPqIu8anmJE+hMl26XfU7/2ozUl23cW1fT73pn/WLoPgLNGp7dIlPZFJzQmWNW3PXR+u2ArJTVeRqTHcf3pw4+6f2hqHPPPHsnM3BQcdhs5KbH8+7uzuPyEwQQt+Mm/1/GPJXtDMpaNeZV86bHPKKzyMCI9jlvmjG5z2+mNsbAqgpPtvZ6nBsr2mtnytmMfjtX7glz3RhnrCk0JuLs/rOJ/Ow+3GjhQ6efbb5Tx7TfLKKwNMirFyXOXpjIgJkSHeXVlsOm/h3/e+zG8dI3p5dwWVxw5UXU8fNFgbDbzXvrOxkOhGU9fZllmNrojCja8bGanx2fCmAt5bVs9eysDpMXa+e70buz31o3cky4G4GvOxWwsaKNdhivWlL9rUJIqbKryoa4kpO0tLMvip4sqKKkLMnyAk+tPiDfvfdEJUL6XZKefb546DIDfL9qu1e3Sq00c1Jhsj+AKQeHy8Q5TuenSaYOwhbBqBpgJo/ddNokpg5OoqPNx/bMr1RJJREQ6x1dvku1RCT3/3Mv/Bs99CV78GvznOvj0D8fXg1ul5MPL12Cumdgd3b+yHczfu65Ef2+RMFKyXfo99Ws/2vD0OGw2qKz3UVbbRl/VfqDW4+ellQcA+EZjedO+rHlluxKsIbFibxnPLzM9Ke+9ZCJuZ8c+cqNdDn53xeTmpMMv/ruJxz/cdVxj+WxnCVc+sZTSWi8TByXyr+/MItbd9izt6Y0rjVbvK+/3E266TeUB8NW0O8PZG7D47tvlLM/3kuC2ccGIaCzgR++W89yGWr73VhlnPVvEoj0NOO3w3enx/PcraaTFhnDV8/p/QcADCdlw/gMQlw71ZfDBb9p+jM0GrhjOTq/mO40TTW57eQP5FW0kWMUIeM1JedAPm14xt40+Hys6gafXmooG10+LJ94doYfw3jqzeiLQxrFD7hlU2xPJtJVTuXlx69vYbOBwmH710vO8dVC+1yTB25kI1BlPrqnl3V0NuO3w+znJuB2NSbaoJFMpozKP607LJSHKydaCaj5RCWjpxSY1Jts35Vdp4sjnNE1AaK26UihEuxz85esnkp4QxbbCav6xZF+3PI+IiPRRtcWm/Lq7hyc3++pg/T9NsrQ6H8p2waZXIX9d1/fpjjevpSHySov3C+V7zcKF2B7KN7gbJ623U4VQRLpPhF6pE+k56td+tGiXg8EDTNnQ/ty3/bW1eVQ3+BmWGsuZo9LDPZxuNyUnGbsN8isbKKhsCPdwerUaj59bXlqLZcEV0wdzyojOrQ602WzcedF4fnDOSAB+87+tPLRgW5uJ72MlxP+1Yj/feGo5NR4/s4an8uL1J5MWf+xZtROyE3E77ZTWetmrEpyh560zK0fbKR9/oMrP114p5cN9HmKcNp7+Ygp/umAAs3Oj8Qbg5+9X8r9dDQQsOC3Hzf++ms7PTk0k1hXCw7v6ctjcuKp97EUwZCZc8mdToqx0J+Svbfux0UlQV8aPT0tjyuAkKut93PSvtQSUeGibrx4CDbD9HXOiHJsGE77IsjwvW0r8RDttfGVCBFZZCQbMqglvDbjioLYMKvMa+88fweFm58C5AAw99L+29+eON8l2nz6LelxlnrlAExW6UoerDnn5zWfmIt8vzkhicob78J02G8QkQ+V+ku0NXDgpC4BPlWyXXmzUwHiinHZqPH72lqotV5PKOh8HysznwsTs7iunmpkUza1zxwDw5w92Ut2gFV4iItIBAR9U7Ad3jDlG7Ul5q82E6+hkOOsOyJhkbt/xTtf3aXealfFeJV97XG2JSbbHpoZ0AvMx2Z1gBSKyb7tIf6Fku/RrRVUNrDtYAcApI5VsP1J/79tuWRbPfmZWQnx91jDs9h4+0A6DuCgnYzITAVijUvLH5ZdvbuZAWT2DkmO48wvju7QPm83Gj+eM4bbzxwLwx8U7+eVbW/D6g2w4WMk/lu7jJ/9ex+yHPmTcne9wy0trW1SiCAQtfv32Fm57eQP+oMVFk7N4+pszSIh2tfvcUU4HkxtXZa3cW9al8csx+BvM1zF6Ib+2tY4LXyhm5SGzov2JiwZwYnYUDruNR85P5uRBbuJcNq6eFMu7V6Xz3KVpjEpt/2/baetfMmNNyDLJdjAr20eeY75f81zbj3W4wAriqivij1+dRpzbwfI9ZTz2wTHKz/d3/gbw+8wqBoCRsyEmhWfWmc/iy8bGkBz9ucP3oP9w2fWGCqivMP82VJjbvDVmv0F/aMdqWWYFRl2pmTwSlQSDpkPOSTDkJEge1urqdGvcFwhYNsb6NkPJjtb37Y4Dn0rJ9zhPNVTuN8nvEF1grPEGmf92Gf4gfGF0DFdPamWySFSCmYRUcYCThqcAsHSPPntCJS8vj6uvvprU1FRiYmKYNGkSK1eubL7fsizuvPNOsrKyiImJYfbs2ezY0fL/ZllZGVdddRWJiYkkJydz3XXXUVPTckLu+vXrOf3004mOjiYnJ4cHHnigR15fJHI67IzLMsfUKiV/2MZ887sYkhJLUmw3HLMc4bJpgxieHkd5nY+nPtnbrc8lIiJ9RG2xOZeKTu75596/zPybPgZGz4GJl5mfD644vlLyTjfUFB//+KTjLMsk2rHMavOe5Iw2bRBUoVIkLJRsl37t3U0FWBZMG5JMVlLbSY/+6HCyvX+ubF+6u4xthdXEuBx8afrgcA+nx0wbkgyob/vxWLS5kH+uOIDNBg9+eUqHktvH8r2zRnDPxRMAePKTPYy/8x2+8KdP+MVrG/nPqoPsLKqhwRfkldV5zH7oQ578ZA93vb6Jcx/8gCc+2g3Aj84dxSNfnUa0q+OlxZtLySsWQs9Xb05+2pjh/Idl1dy0oIJqr8X0LBdvfzWd04dEN98f47Lz4mWpbPhuJr88O5kx3ZFkByhYfzjpO2YexCQevm/yV8y/eaug/EDb+4hJgppChibYuPviiQD87ZM9KqvbFl89VOWZigIOF0y4hANVfhbsNiu8r50Sd3jbulKoOGgS2n6fiSebExxOM6vd5jS3BfzgqTEn3a2tNu8Kb93hfbnjIWMiDDrB9Pi22yE2BVKGmgkln3u+MUMHszh4gnkJq/7V+v5tdvOlUvI9q+KA+duGsEflq1vrKagNkpPo4L5zktru0RyXAlUHOSnLfE5tzKukxhPiCSL9UHl5Oaeeeioul4v//e9/bN68mQcffJABAw5XVnnggQf44x//yOOPP86yZcuIi4tj7ty5NDQcrixx1VVXsWnTJhYuXMibb77JRx99xA033NB8f1VVFXPmzGHo0KGsWrWK3/72t9x111088cQTPfp6I0lTKfmNSrY3a5p40PS76U5Oh52bZ48G4G8f76airv+2RhMRkQ4IBkyrN4fL9NjuSZYFB5aa7wedaP4dcpIZS10pHNrQ9X27Ys2E2lCcA0rHeGvM+Xw7LQO7hTvWrGz3qUKlSDi03bBVpB94a8MhAC6cmBXmkUSepmR7fy0j/8Jy02v70hMGkRTTvSsvIskJQwbwwrL9rNlfEe6h9Eq+QJA7XjUnQt8+LTdk7Sm+MWsYMS4Ht728Hn/QIinGxZScZKYOTmJKTjIxLgd3v7GZbYXV3Pvm5ubHxbgc/ObySVw8dVCnn3N6Yy/NVfuUbA85bx20kW96Zl0NDy8zZd5+MCOeH52UgLOVyho2m62tXRw/KwjrXoQVT5rvk3Jg3BdabpM6ErKnQf4aWPMPOOeO1vflioO6g1BXwsVTB/OL1zZSUedjZ3ENozNCl9DrMzxVUL7HfJ+QDXEDee6zaoKNrQJGN02sqC02yfRB00yy2xUDdpdZjdyUzLQs8xX0Q9BnSrJX5UP1IZPEttlMQtvuNMlxewdPC3x15uJB+lhIyjYXcFpLoLrjISbF9B08oopDnNvOB7FzOc+zCteBT8BTB1GtzPh3x5nX6feaFRnSvaoLoGKfiYUQsSyLFzaaqgzXTokj3n2Med6uWKivYJBVwOABMRwsr2fVvnLOHN332/h0p/vvv5+cnByefvrp5ttyc3Obv7csi9///vf8/Oc/5+KLLwbg2WefJSMjg9dee40rr7ySLVu28M4777BixQpOPNFcAH7kkUe48MIL+d3vfkd2djbPP/88Xq+Xp556CrfbzYQJE1i7di0PPfRQi6T8kTweDx6Pp/nnqipT8tLn8+Hz9f6y3+MyzbnU+oMV3fZ6mvbbW35f6w+YY8pxmfE9MuY5Y9MYm5nA1oJq/vz+Dm6dM7rbnzPS9LYYkZ6nGJFjOTIu+srnc5tqi6GqGOLTIRDs2ecu242rthjL7sQ/aJZ5fns0jkEzse//lMD2BQQzJnZt3/YoaCiH2gqIC30aqF/FSEfVlEFDHUSn9Hws2aKgvhRqyiEhMs6hFSPSEZF+PNLRcSnZLv1WcbWH5Y0lKs+fmBnm0USekQP798r2FY2xcfGU7DCPpGc1rWxfn1eJxx8gytnDM3p7ubUHKiiq9jAg1sWP54wJ6b6vODGHGcNSCFoWuWlxR60OfOMHp/HXj3ezYHMh4zITOGtMOqeMTCOxiyvrpw81yfbthTVU1vm6vdxnZzmdTq666iqWLl2K09nLDmc8leCIOurmV7fWcdeHJtlw00kJ3HRSGJLRnmpY/Es40FjGLmMSnHaLWaH+eVO+apLtez6A+h9ATCvjtdnM7OrKPFyJg5g2JJnPdpWyfE9Ztyfbe12MBAPQUAVlpiIFycOo8MKLm8ys9GunmM/l5kR75iRzMagtTYl3uxtwm+R1XCok55gS81YQAgFzMl5VYP5OUfEmKe+vN+NpTsi7wNVYXaG2zJQ3TB1hVrEf6/kTMsxK/c8JZJ/I7l2ZDKcA1v4DTvrO0Y93x5kEcEPlsV/nceh1MdJd6sqgcItZPRPCUofrCn1sKfHjdsDl4zqw39hUqC7gpJwBHCyvZ9nuUiXbj9Prr7/O3LlzueKKK/jwww8ZNGgQ3//+97n++usB2LNnDwUFBcyePbv5MUlJSZx00kksWbKEK6+8kiVLlpCcnNycaAeYPXs2drudZcuWcemll7JkyRLOOOMM3O7DF/Xmzp3L/fffT3l5eYuV9E3uu+8+7r777qNuX7BgAbGxPVxysxtU1AI4Wbe/jDffepvu7Ei1cOHC7tt5CC3f4QBs1B3cyttvb+mR5zw92cbWAgdPf7qHnLqdJEbGdece11tiRMJHMSKtsSyLESNGAPDBBx+0XaGoTyno8WccWbiACUBh/ASW7Q4CBwEYbJvEdD6lbv9qFq8+eHxPsmv5cY+zNf0zRjrqOP9mx2P3qvA99+coRqQzIvV4pK6uY9Ui+vFVJenvFmwuIGjBlMFJ5KT0/gs6oTYi3ZSqzauop94bIMbdf5KuRdUNFFQ1YLPBxB4ocxhJhqfFMTAhiqJqD3/7eA/zzx4Z7iH1Kh9vN72wThuV3qmS7R01LC2uzfvcTjvzzx4Zsr9ZanwUuWlx7CmpZfX+cs4eOzAk+w0Vu93O0KFD2bRpE/ZjJfwiTaCxv7azZbJ9Q5GXnyysAMwK0B/NjO/5sZXvhXf/n0mO2p0w7otw4jfbLik9eAYkDzWrYde/0HrCFEz5tOpCqClkxrAUPttVyoq9ZVx98tBueynQC2OkodKUey81vZKD6WP48YJyqjwWIwY4OXtYVOOKdEf7ifZjiUk2X00CuWa1e9luqCs3q9ATssEZY1bFW42TAHx1plx92sj2E+1NopMbS8nXmZXLjSZnRfGnbZfwkPtx2PwaTLsa3HHsKPXxiw8qmTU4ih81TTapK+22ZHuvi5Hu4KmBwk0QqIeE0FZ6enGjOSGdNzKG5OgO/H6dUWCzcVKal5eBZerbftx2797NY489xi233MIdd9zBihUr+OEPf4jb7eaaa66hoMBcUM7IyGjxuIyMjOb7CgoKGDiw5TGA0+kkJSWlxTZHrpg/cp8FBQWtJttvv/12brnlluafq6qqyMnJYc6cOSQmJh61fW/jCwT5/ebFNPiDTDz5TIaltn0M1+Xn8PlYuHAh5513Hi5XZE2K/Lyqeh8lS94H4NpLzmVAbM9kvS+wLJY/sZx1ByvZGz2Cn50f2smwka43xYiEh2JE2tMvYqS+HA6uNOdIzqMnxXc3x1tbAUgfNIILTziijaX3Qqzn/0ZCQz4XDq6GgeO69gR1ZWZSdfb01iuSHad+ESMd5a2Dg8tN7/Se7tfepKEKsEHOSabFXARQjEh7Ij1GmqqwtScy/seJhMHbjSXkL5ikEvKtSYlzkxzroqLOx+6SGiZk95+kc1NvxRHp8cRF9a+3SZvNxm3nj+XH/17HHxbtYO6EDEYOVKnnjvpwh+kvfPqotDCPJDROyk1hT0ktb6zLj7hke6/lrwe/B6JbJhL+sKyGgAVzhkdz5xmJpkR8fbnpi11TYE6YvDWm15rN3rhiPB4GDIOU4RCXfvjEORgwyU1fndnW4TYne21dOPDWwu73YcmfzWOiEuHk78KoOccuLW6zmd7tHz0A2/4HM65rfXu70zx32W5mDjEzmlcoiXa0+grzNy7fB8BzlZN5b68HtwP+MHcADk+F2S5jYmiTzw6nWe0elw4Bjyn9//mTcssysedvMJMnOtrHMKqplHxxi2T7lAw3vwieyg+s/5LrOwSrnuHdgd/ilgUV1Pos1hR4+e70eKLccaYMfTDYseS+dI7fC0VbzESPxNBW8qn2BHl9u+kN+bVJnbjQFJvKycmFgI31Byv63YTPUAsGg5x44on8+te/BmDatGls3LiRxx9/nGuuuSasY4uKiiIq6ujPJZfLFZEXWDrL5YJxWYmsO1DBlsI6RmUmd+NzRf7vbNs+c341eEAMA5NCP/HgWG4+bzTXPr2C55cf4LtnjSI9oecTKeHWG2JEwksxIu3p0zFSWgx2C6Ji2t821DzVULgRAEfuKTgcR5zzxCSaHu4HluLatQCyJnTtOaJjwVcL+Fuck4Van46RjqqvhWADRKd2y8SGDomJh5oSCNabUvYRRDEi7YnUGOnomPpXFkmkUWmNh6W7zYV+9Wtvnc1mY2R6PCv3lbOruLZfJdvXHzQXgyb3s1XtTS47YRBvrM/ng23F/PQ/6/n3d0/B0Z21L/uI8lov6w9WAHDGqL5R9varM4fwzxUHeGN9Pj+7YCwDE6PDPaRmgUCAlStXUlxcTCAQiMiDsVb5GiDgNQnwRpuLfSza04AN+NlJLuxr/gEb/mP6d3eGzW6+gv7W749OgsRBkJBpku8OtylJfnA5BBr7DyUPgTNuNSunO2LkubDsMVOWfPsiGHt+69vFpkBlHtNSsnDabeRXNnCwvI7BA7rvZL9XxUgwaEqmVx0ELDyuZO7eYE6M7zoziYmJ9eDzQOZkU5q9O7iiD5eK/7ymdgBdmZ2fkGl6xVtW8wWH0SlOXE4nD/su54/uP1Gz4S1ubZhDLSYB4wnAxmIf09Ojzcprf70pKx9ivSpGQs2yoGyXqWqQmBXyi0Gvbaun3m8xcoCTE7M6sYLV7iQn2UVWXJBDtUFW7y/n1JF9YwJbOGRlZTF+/PgWt40bN46XX34ZgMxM00qrsLCQrKzD50SFhYVMnTq1eZuioqIW+/D7/ZSVlTU/PjMzk8LCwhbbNP3ctE1/NGmQSbZvzKvki/2sNdXnbWiczDwpDOdXZ45OZ2pOMmsPVPDER7v4f/PGt/8gERHpH8fKnhqoOWTOlcMhb5Vp7xWbBgNb+XwacRYcWAr7l3b9OZwxpoKZtybkq637RYx0Rk2xWWwQzlLpdidYfnM9KTb8yXbFSDcq32cWRCQPMRUFJey0RET6pQWbCwkELSYOSmRIqkrIt2VEuiljvLOof/Vtb1rZ3t9KyDex2Wz8+tJJxEc5Wb2/gr9/tjfcQ+oVPt1VgmXB6Ix4MpMiJyl9PKbkJHPi0AH4AhbPLtkX7uG0EAgEWLBgAXl5eQQCgXAPp+P8ZqXnkSdfj66sBizuyV7C8He/CSufOpxoj0o0pdoHToBBM2DY6TD0dBh6GmRNgbiBJsEO5iT9yES7zXH4PjCrV4s2w67FsO1tU75736cm0R6TCqPmwgUPdDzRDmbF+rgvmu83/aft7Wx2iE4gtnY/E7LMZ8uKvd27ur1XxYin0vzNK8z/s098owhYcOmYGL46CnNhZOA4SBoU3nF2RUwyuGPMyvhGLoeN/3daIuvjTmVbcDDx1PEt5//45tQ4Zuea989leR5wRJnJKd7abhlar4qRUKvKN60D4tKOXcGiCyzL4sWN5m/2tUmxne7LZ4tN4aQM8/dYtrs0pGPrb0499VS2bdvW4rbt27czdKhp45Gbm0tmZibvvfde8/1VVVUsW7aMWbNmATBr1iwqKipYtepw78fFixcTDAY56aSTmrf56KOP8Pl8zdssXLiQMWPGtFpCvr9oSixvaJzI259tCOP5lc1m40ezRwHwj6X7KKnx9PgYRER6o35xrFxTZEp/d8PE3g7Z85H5N31M64nwoaeaqmI1hab1U1fYbGDDrKIPsX4RIx3lqzct0Npqw9eTnNEmti0r3CNRjHQXb605ny/cDAdWQOVBU+VSwkor26VfWrDJ9Be8QKvaj6mpP/SBsrowj6RnNa9sH9w/k+0A2ckx/OyCsfz8tY38+u0txLodXDlzSLiHFdE+auzX3ldWtTe57rRcVu4r5/ll+7jxnJHd0ou+X/HUtCiHvbPMx9s7GviJ8yW+XvZfc2N0Eoy9CMbOMzORne1M3vDUQm2RObAO+syKdXeCmdlqd5hS0b46c/Bdsc+Upfc1mMS83WF6r2dPN2XqumL8xbDuRSjdCQWbIbONFWNRSVB5kJmZKazLg+V7yrl02uDWt+1vGirNpIfi7QB85hvNgGg7vzwzDlt9IaSPNZMueiN3HMSmmpX7R1xA+vrkOL4+OY6Sjd+Cz+7hRvdbOKddyVO7Ylm0p4HleV6+f6LNXCDopmR7v1VfYWLNFdMtM+BXHvKyucRPlAMuG9uFSa12JydlOXltd4ClSrYfl5tvvplTTjmFX//613z5y19m+fLlPPHEEzzxxBOASULedNNN/PKXv2TUqFHk5ubyi1/8guzsbC655BLArIQ///zzuf7663n88cfx+XzceOONXHnllWRnm9XaX/va17j77ru57rrruO2229i4cSN/+MMfePjhh8P10iNCU2J5Y34llmV1euJJX7IxjCvbAc4anc6UnGTWHajgiY92c8eFXex7KyIifYffC5UHwpccLdxkJsIDDDut9W2iEmDwTNi/BLa+DRldLCXflHxNGR7eVdd9WX2Fue4SkxzukZjzbk+VGU+4JpJI96rMNxNoBgwxLSgPrQNsvXOBRh+ile3S71iWxer9FUDfS4qF2pAUc4F0X2n/uchdWNVAUbUHuw3GZ3cx8dRHfG3mEC6Zmo0/aPGzVzbwq7c2EwiGf1ZkJLIsi4+b+rWP7lvvK3MmZDJ4QAzldT5eWZ0X7uH0fg2VZrVuoz+vrOEGxxvc6GxMtA8/By5+DGZeb3oot5doB4iKg5RcSBtpVj+njoCEgRCdYE6yYpMhKRuGzITJV8ApP4Azb4Wzb4czfwojzu56oh0gfqBZcQ+w7oW2t7PZICaZGSlmhXN3r2zvNYJBqCowJdyLNgOwNjiS84ZHE+cpNnGQktu7L4rEZ5jJHa3MrE+bcBakjcEZ9MD7v2JmY8nxVYe85jPH4TInjxIaAT+U7AB/HcR0z4rjp9ea48ZLx8aSHN21082Zw0xCbu2BChp8mqHfVTNmzODVV1/lxRdfZOLEidx77738/ve/56qrrmre5qc//Sk/+MEPuOGGG5gxYwY1NTW88847REcf/vx5/vnnGTt2LOeeey4XXnghp512WnPCHiApKYkFCxawZ88epk+fzo9//GPuvPNObrjhhh59vZFm1MAE7DaobvBT3I9XU1c1+NhbaiZvhyvZbrPZuOlcs7r9xWX78fj1viIi0u/VFkNDFUSH4dpf0A+fPGS+z5xizsnbMuo88+/+z7q+UtnV2Lf9iGpjEkLBoKkcZrdHxnm7Mwa89Sa+pe/xVEPlfjOxw2Y3ixvsDvOeJmGlZLv0O/tK66is9+F22hmTGQGlXSJYU7J9f1n/ORhrKvM4cmA8se7+XfzDbrfx8FemclNj2cW/fryHX/x3Y5hHFZl2FtVwqLKBKKedk3LD3xMplBx2G9eeMgyApz7dgxUBZah6Lb/HrCh3mmT7zjIfMTve5HbXi+b+kXPgrJ+axHhvM+lL5t8DS6H2GEl0dzwzUkzCYWdRDWW13h4YXITzVpsy8r46qC/HbznYaOVyQY7XzEJPG20Szr1ZdHLjBZ5WjidsdjPxw+6EgnWML3mXBLeNaq/FlhK/WXndUGVW/svxq843VQbiB3bL7vOq/by7qwGAa6d0fRXF8JQosuLAG7B4cemeUA2vX7rooovYsGEDDQ0NbNmyheuvv77F/TabjXvuuYeCggIaGhpYtGgRo0ePbrFNSkoKL7zwAtXV1VRWVvLUU08RHx/fYpvJkyfz8ccf09DQwMGDB7ntttu6/bVFOrfTTnayqR6xr7R/VQo7UtOq9kHJMQyIc4dtHGeOTmdgQhTVHj+f7VLVDBGRfi0YhKqD4HS3bL3WUza9CqW7zOT6md8+9iT7oaeYawj15XBwRdeez+ECvw8C/XfyX7eqKzGl/mNTwz0Sw2Yzif86LXDokyoPmutHR1blcMWa9wi//o+Hk5Lt0u+sO1gBwPisRNxO/Rc4lqZke0mNhzqvv52t+4Zw9hOMRDabjZtmj+bhr0wB4D+rDlLv1UqQz/uocVX7zNyUPllm/SszcoiPcrKzqIYlujjZdb56CJhke9CyeObdZdzjfMrcN+w0OO2mjq1kj0QZE82K+qAfNh6rd7uNAbEORqWYyUxa3Y4pNxfwmostwGZrCC6Xm1MG+k3vvnCstAg1d6y58OBpY2b9gGEw7WoA7Msf46wMk5Rfnu8xyXZ/vUrJh4KnxsRZVHzI+7Q3eXZ9HQELThnsZmxa1yeJ2Gw2bpxhLh48tGgnpf14VbD0bsNSzaSTvSX99z1sY/P5VXg/z+x2G3MnZALw7saCsI5FRETCzFMJdeXhKfldWwwrG68DjL4QBrZTGt4ZfbiS3Lb/de057U6wAprA3B2CASjfZxLcjvBNKjyKO85MAtDfvG/wNZhKmVX5Jtn++Sp1rlhzzcRTHZ7xCaBku/RD6w6Yk/2pOcnhHUgvkBTrIjHaXIw90E9Wtzcl2ycr2d7CJVMHMSg5Bq8/qP6preir/dqbJES7uHCSuTi5YHNhmEfTi/kbzImY3cm/15fzrco/4bBZ1KVNhtN/2qKfda9js8G4L5rvd7137PJ2UYnMGGjuX7FHyXZqisxKhcYS8muCIzlniIOomHhTfr2viB/YZil5wCTbBwwDXx0/8PwNgOV5XnNhKOA3M7el6ywLyveahHs3XVSs8wV5caNJKH5ranw7W7fvyonxjE+Bak+A3y3Ydtz7EwmHoalNlcL673vYusbKYZMHJ4d3IMD5E83x7MLNhWqPJSLSnzVUmiRkTydHq/LgrR+bifiJg2H61WYFcntGzjb/Hlxuzqm6KqDKciFXWxxZq9qbuOOUfO0LggEo3Q37lpivvNVm4oz7c1Xk7I0Lvxoqe36M0kzJdul3mla2Tx6sZGpHDOlHF4gsy2pOtk9SfLRgs9k4c4xJJH+4XT1gjlTvDTRPQDijj/VrP9J54w9fnFQp+S7y1QMWRbUB6pc+yXB7AbWORGJP+/7x9UyPFCPPNeXpagrh4Oq2t3PFMCPNzK5ee6CiZ8YWqXz15uTXFYtVuAkw/dovGGqZmcr2PlQpo7mUfBvHE3YnnPUzAEZVLSWNSlbke837jd1uksTSdbXFUHEA4tO67Sle2VpPlcdiaJKDc3Kjjnt/DruNu88wSft/rjjQ3OpHpDdpSrbv7cdl5Nc3nn9PiYBk+8zcFJJjXZTWelmp6joiIv2TZZm2Su6Ynn3eQ+vg1e9BxX5T/nnmtyGmg20IB59oHuOthd0fdX0MKjEdWgE/VOwz10EirfVbUzWDtqrLSeTze6BoCxRuMtdE4lIhaVDbizJcMWYxRzDYs+OUZmFNtg8bNgybzXbU1/z58wFoaGhg/vz5pKamEh8fz+WXX05hYcsVdfv372fevHnExsYycOBAbr31Vvz+ljO8PvjgA0444QSioqIYOXIkzzzzTE+9RIkwvkCQTfnmQt0UrWzvkMN92/v+BaLCKg/F1R7sNhifpWT7553VmEj+YFtRmEcSWZbsLsHjD5KdFM3ojONfyRepThuZRrTLTl5FPVsOhX9mrNPp5Mtf/jK5ubk4nd1TDjnkPFXgcPHswuV8g7cBiJ7yJRg4NswDCxF3POSeab7f8mrb29lsjEszf7NthdXdNnmjV8SIt9aUSK8txla0maBlY41tHGdm+MNT0rA7NZeSP8b7R/pYSBuNjSCXuD6jtD7IrnK/KZ1YV3rsigld0CtiJBSCASjfA3Zbt7aqeG69WdV+zZQ47DZbSPY5Y0gil+QGsSy47eX17C7WpAvpXYY2lpHfV9o/y8iX1XqbK6RFwmRml8POuWPNBcp3NqmUvIjIsfTZY2VPFTRUmfPXnrJ/mVnR7qmChCw4924YflbHH293wohzzPc7FnRtDA4neEN7bbfPxkhH1RRCTTHEdnDSRE9zRkFteKuT9vsY6Sq/Bw6th7LdEJ9u2gu21wrOHWeut3iPOGdW4r1HhTXZvmLFCg4dOtT8tXDhQgCuuOIKAG6++WbeeOMN/v3vf/Phhx+Sn5/PZZdd1vz4QCDAvHnz8Hq9fPbZZ/z973/nmWee4c4772zeZs+ePcybN4+zzz6btWvXctNNN/Htb3+bd999t2dfrESE7YXVNPiCJEQ5yU2Na/8BwpAU83s60A+S7U2r2kcNTCDG3YdWE4bIKSPTcNpt7C2t69d9Jz/v/a1mpf/ZYwdiC1FyIRLFuB2cNtJMuFgYAaXk7XY7I0eOJCkpCXtHyq6FW8APDVXsrXHyxYI/YbdZVKROwzH1y+EeWWiNnWf+PbACPG1/bgwfmIDTBtUNfvIrG7plKL0iRrw1JoG8+XUA3guewJghWcRG2Xv24k9Paa+UPMDo8wG40vUJAMvzveCKNivifaFtadMrYiQUaoqgtqRbSxseqg6wtdSP3QaXjQ1tS4yfnRxNgtvG5kNVzHn4I+56fRPltSqBKb1Df+/Z3lRVbnhaHEkxkbHiq6mU/LsbC1StSUTkGPrssXJDpWnx5jz+SkwdUrEf3rvHnAelj4Xz74PBJ3R+P00T2ws3dW0Sst0JvtAej/TZGOkIbx2U7jLnqu0lQcPFGW2Sr/7wnTv16xg5HtUFUH0IErM7/l7ljDJJ+qYFDrWlkLfKLPKQHhHWd4L09Jbldn/zm98wYsQIzjzzTCorK3nyySd54YUXOOccM3Pr6aefZty4cSxdupSTTz6ZBQsWsHnzZhYtWkRGRgZTp07l3nvv5bbbbuOuu+7C7Xbz+OOPk5uby4MPPgjAuHHj+OSTT3j44YeZO3duq+PyeDx4PIfLqlRVmXIbPp8Pn8/XHb+KPqfp9xRpv681+0ypuEmDEgkE/AQCYR5QLzAoybyh7y2pCenfMxJjZO1+Ex8TshMialyRIsoO04cms2xPOYu3FPD1k4d06/NFYox8nmVZLN5qEs9njEqN6LGGwjljUlm0pZCFmwv4/pnDwj2cXhEjzWqLobaCDzYUcK09jwaiiDvlBnw2NwT60EzTgZNwxmdgqynEv+V1rEmtTyawuWIYnljB9kobmw6WMzCuew5JIz5GqorB78e5/R1swLOB87h4qAOfzQm4IVLH3VWOeLDHQEPN0T3Gmgw7E+eSPzEyuIfhtnyWHozmirGJ4CmH+iqwhTZZE/ExcrwCfijdA5YDLHu3vd98uM9MhJiY7iLOZapJhUpqUgL/Ob+U32xO4v2dlTzz2V425lXw4rdnhuw5jiXSYyRSxyVGU5WwqgY/FXVekmN7uDdsmK0/0NSvPfyr2pucPiqNWLeD/MoGNuRVRkQveRER6SGWBdWFJkHaE7w18O7/M0nupBw49/8gMatr+8oYb9qMeauhZBekj+zc4+0uk4gLBvpWu7JwsCwo2wMNFZA0ONyjaZsrBmpKTPw5+9cxaK8W8JlJOu64zv9fdThNVUB3LBRuhPpyU3q+resvElIRM+3G6/Xy3HPPccstt2Cz2Vi1ahU+n4/Zs2c3bzN27FiGDBnCkiVLOPnkk1myZAmTJk0iI+Nwn4K5c+fyve99j02bNjFt2jSWLFnSYh9N29x0001tjuW+++7j7rvvPur2BQsWEBsb2pUafV1TtYJI8dYuO2AntqGEt99+O9zD6RUOVdgAB1sOFHfL7yySYuSz7SY+gmUHePvt/eEeTkTKCJh4+M+nm0kt29gjzxlJMfJ5h+ogr8KJ02ZRuX0Fb+8K94i6V8ALNhxszK/ihVffJrmHJoK3xrIsysrMBJkFCxb0iqoCQQtqdy0FO+yJmcj2g3Fw8GC4hxVyoxNOZ1zNf6jY9D6f+k5pc7t4px2w8fpHK6nfFfqVZb0lRoYVL2aKr45dwSxWMIGLamt4eyuw9b1wD60b1QLlbd57UsIkMqvWcYnjE54/kMXbaxrLoO1cFtJR9JYYCZ3ue7/59w5zDJXh8vD26sPPY7MCjCx8i5roLA4lzziu57gkvZRxbhuPbbazcl9Fj38ORerxSF1d368+1ZvFuB1kJEZRWOVhb2kdU/tbsr1xZXskJbSjXQ7OGpPO2xsK+N/Ggogam4hIJAkEAqxfv57S0lICgQAuV2RUKDku3hqTIO2JKmJWEBb/GioPmH7rp/+464l2MKuUU0dB8VbIW9n5ZLvDZSqFBbxmAnQI9MkY6YjaYpMMjUuDSD6HtDvB8ptV+DEDwjKEfhsjx6O22FTgSMjs/GPdcVBXZh7vqwOHG+rKzQp56XYRk2x/7bXXqKio4NprrwWgoKAAt9tNcnJyi+0yMjIoKCho3ubIRHvT/U33HWubqqoq6uvriYk5+sPl9ttv55Zbbmn+uaqqipycHObMmUNiYuJxvc7+wufzsXDhQs4777yIehN97NElQDWXnDGNuRMy2t1eYGJZHX/e8gkVPgfnnz8Huz00BxGRGCNPHVgGVDJ71gmKjzYML6jm9UeXsKfWybnnnU2Uq/tmw0ZijHzeEx/vAXZwysg0Lv3C9HAPp0e8UrSMNQcqsQZN4sKZOWEbh9fr5Xe/+x0Al112GXFxETxL01MFB1bySbGbYawFIDd3JCNPiOAZ0Mej9gqsf75MWu02LsyphfQxrW6231fG6lIvtqQsLrxwasiHEfExUlcGB5bj3PMBAP8InMeXJiRw2ZgaU14wZVhYh9dtvHWQvwaCvjZ729mSLoIP1nGp/VMerr+CsyZlE9tQDImDzIqOUA0l0mPkeAX8kL/aXFSMS+u2p7Esi3vXFgNBvnFiKicPPpwBty/5I45Dr2HZ7PgvfgJSh3ftSTw1EAxw4aAT+aRyHevzqojNncqFU7v/okGkH480VWGTyDU0NY7CKg/7SmuZmpMc7uH0GMuyWHfQrGyfkhM5K9sBLpiYxdsbCnjykz2cOTqdk4d3X5sNEZHeKhAI8OabbzZ/3yc0lZDvxmPjZsv/Cvs/MytTZ34Hsqce/z6zJptke8EG4MrOPdbhMq8/4DUrnkOgT8ZIe/weUz7ebg/Z77Fb2eyHy4qHQb+MkeMRDJgJOg5X1ypQuGKh8qB5fEKWuR5ZV2KuDTgiJhXcZ0XMb/jJJ5/kggsuIDs7/LMsoqKiiIo6epmEy+WKyAsskSySfmf13gA7iszKqOm5qREzrkg3JC0Bh92Gxx+kwhMkIzG0pZYiKUaa+gYPTUuImDFFmomDBzSvzlmTV83po9Lbf9BxiqQY+byPdpQCcO64zIgdY6jNmZDFmgOVLN5WwrWndjFpEgJH9tiM5BgBoKIM8LJoey332kz5g+gRp4LjGP2qPNXmJNgd33O95EIlMQOGnQp7P8G1+mm48IFWNxufEQN42V5Y0y1/v4iPkWADlG6Bin3UWlG8HDiDVyfH4XLUQmwiRNp4Q8WVBANHmYS75Ws9voefDp9Ek+Mv5gTbdnaVp3NCcjT4a8DpDNnqgYiPkeNVX2zK7ydkmItB3WRbqY+S+iDRThszB0XjcjT+fTa9CptfA8BmBXF98lu49PGu/f1iEsxFg7oCTh2Vzvq8KpbsKeeKGUND90LaEakxEoljkpaGpcayfE8Ze0v6VxWCQ5UNlNR4cNhtTMiOrGT7hZOyeH1dPgs3F3L931fyz++cHHFjFJG+pbzWy97SWkYMjCcxWp/dYVNdaFZ6dretb8K6F833E66AMeeHZr+Zk2D9S1C6o/OPtTtNIi+M/bt7vaby8bXFpjR3b+CMNhP9LSuyV+GLUVdqeq13dUKQ3WFi0+Ywf293nGkl4Klqc7GDhE73XXXphH379rFo0SK+/e1vN9+WmZmJ1+uloqKixbaFhYVkZmY2b1NYWHjU/U33HWubxMTEVle1S9+1Kb+SQNAiPSGKzBAnjPsyl8NOdrL5fe0v67sXiBp8AYqrPQAMGqD3hrbYbDbOHG0S7B9sKw7zaMKrst7Hyn2mDPLZYwaGeTQ957zx5rUu2VXS/H9GjsHvgao8qmzx+Pcuw26zqI/JgvTRrW8f9ENVfuNs8zior4CKA1BTaA666yvMPsPFsswJetBvvm/LCdeYfw+uMP2iWzEm3SRZd5XU4fX3ob71HVVXCrs/AODVwGlMzUlmZFLAJJ/7ej+thCxIHgI1xa3HkTMacs8E4FLHJ2wu8YEjysR+OOO/NwkGD89ot3fv/OqP95u/ycxsN1HOxgs4B1fAZ4+Y73NONj0iS7bB1i62JLLZIHYAVO7ntCHmOO2znaUtJkyIRKqhqeY9fV9ZbZhH0rPWHagAYExGAtHdWA2rKxx2G498dRozc1Oo9vi55qkV3PX6Jn780jpueWktH2wr0vuLiITMgbI6LvjDx1z658+YfNcCTrnvPe5+Y5PeZ3paQxXUl5mS7t0pbxV8/LD5fvjZMOPa0B2PZ0wy/9YWmYRvVwSUbO+y6gIo39NYPj4i0mrtc0WDv860EJDIZllQlQc2zHl8V9mPWKBgd5rrd2GsbtCfRMS7wtNPP83AgQOZN29e823Tp0/H5XLx3nuH+1Vu27aN/fv3M2vWLABmzZrFhg0bKCoqat5m4cKFJCYmMn78+OZtjtxH0zZN+5D+Y23jyf6Uwcn9oCdnaA1JiQVgf2nfTbYfalzVHuNyMCBWs4yP5bTG1ewr95aFeSTh9fGOYgJBixHpcQxJjQ33cHrMiPR4Jg5KxBewuPlfawkEdYHgmGqLwVvNm/ucnG5bA0B01liTTDxSMGBmG1cXQHwGDJoOOSfBkJPN7PWEQRCdZA6UGypNAr6utGdPlGtLzIF/TVHj9/ltJ9zTRkHOTMCClU+2usmgBAcJLvAHYXdJTfeNOxL5vVBfiVWwEYD/Bk7lminx5gTYFWNKf/VldjukDIfoRBPHrRlxFgCn2jeyudgHTndjsl0XCTqkvtzMiI9J7van+vSASbafNqSxSsHej2HBL0yfyszJcNYdMO0qc9/yv5iy9vUVZhJRZy4yu+PB18D0hArcTjsFVQ3sKu5fyUvpnYY2Hifu68PnUq2J1BLyTaJdDv52zYmMy0qkpMbDM5/t5eXVB3lldR7XPr2CL/zpExZsKgj3MEWklyusauCqvy2joKqBKKe5DJ9f2cDTn+5le2E/OwcKt/oyU0K+O0t/l2yHhXeCFTCJ8VNvOvrc/3jEJJvWWgAHVnRtH0q2d42nGoq3mSRobygf38QZDb4G079bIlt9ubkmGBPiFeiuKHMNT7pd2JPtwWCQp59+mmuuuQan8/Asr6SkJK677jpuueUW3n//fVatWsU3v/lNZs2axcknnwzAnDlzGD9+PF//+tdZt24d7777Lj//+c+ZP39+cxn47373u+zevZuf/vSnbN26lT//+c+89NJL3HzzzWF5vRI+WwvMDJ5JgyLzZD+SNSfb+/DK9rxyc/F+0IAYTcZox9TByQBsOVTdP1ejNlq81Uz0Omds/1nVDqa6wcNfnkqMy8EnO0v48/s7wz2kyFZ1CMvu4j+bqjnDvh4AW85Jh+8P+k3CqbrAlLPLnARZUyFmgElIxiRDSi5kTYLBM2DYaSYBnzXZJJ7qyqDioElcdefKiAZzwZzsE2DoLBg80zx/Q0Xbj2la3b7/Myg/cNTdNpuNMSnmUHRbfj/rOeytgdId2DyVNFguSmJHcPawKHMSHJPSrSW/I0ZUvOlNbwVbn2WdMRELG7n2Qg4WFJqJJlbA/I6kfdUFQLDby2R6AxbL8swFu1MHu2H1sybR7m+AAcPhzJ9BTCJM+SrEZ5rydc9cBP+4BF74CrxzW+cu/MSlEl2bz4mD4wH4bJcuGkjkG9a0sr20f00OWX+wAoDJjecOkSgx2sVz183klvNG8/2zRnDb+WO59pRhxLgcbMyr4oZ/rOKt9YfCPUwR6aXKa718/cll7C+rY0hKLB/99GzW3TmHU0emArBoS2E7e5CQCQagMr+bE+074K0fg7cWknLgrJ9BTDdcg86aYv49tK7zj7U7tMK5KwJ+KN5uzuNDnQjtbjY7YJm4lMhlWWZRjRUMfStJV5y5dqf/+90u7FfyFi1axP79+/nWt7511H0PP/wwF110EZdffjlnnHEGmZmZvPLKK833OxwO3nzzTRwOB7NmzeLqq6/mG9/4Bvfcc0/zNrm5ubz11lssXLiQKVOm8OCDD/K3v/2NuXPn9sjr62kb8yr598qWF7TLa73Mf2E1/1y+P0yjigyHKs0bSk5KL5p9FiFyGpPtB/pwsv1guXltg5IVH+3JSYlhQKwLbyDI1oJ+liBr5A8Eeb852Z4R5tH0vFEZCdx7yUQAHl60nSW72liZ2t9568BTxdKSaNwlm0i01RN0xsKgE839QT9UHTKJ9UHTTRJ9wDBwHKPEnM1mVrgPGGYS3kNOhowJph9TZR4EfKF/Hb468NRA2mjT+ylmAMSlQuoIc8IW9Lf+uIwJZuKAFYRVT7W6yZg0U0lka34/q5ThrYGizQCsCo7myinJOOw2k0yO7keTAhMyTFw1VJnk7JGiEvAmmn7ciRVb8ActU05NK9vb562FmgJTOaCbrS3wUuezGB1dxfg198DKxv/rg2fCBfdBUrb52RkFZ/wE80dsYoMDy+G173e8DKYzGmwWp2YGAPhkh5LtEvmaKiCV1Hip8bTxmdnHBIMWGxpXtk8eHNmfa6nxUfzw3FH89PyxfO+sEdz1xQl8+rNz+NL0wQD89t2t+AL9d4KxiHTdbS+vZ3thDRmJUTz/7ZPISIwmKdbFBROzgMMT+KUH1JebZFN3nWuV7TaJdk+1WXl+7p3d19M7s7GUfPG2zj/W4VLStStqi6D6kDl/7Y0LtByuYy+UkPCrLzcx1h191V3RJtGuUvLdLuzJ9jlz5mBZFqNHH927NDo6mkcffZSysjJqa2t55ZVXmnuxNxk6dChvv/02dXV1FBcX87vf/a7FCnmAs846izVr1uDxeNi1axfXXnttd76ksNlZVMOXHv+Mn72ygc92mgtPlmXxs1fW89b6Q9z9xmaqG7rhInwvcajCXMTNSlIytbP6xcr2CnPxfrD6tbfLZrMxqXGFSlN5yP5m5b5yyut8JMe6mDFsQLiHExZfmj6YL00fTNCCG19Yzca8/hkLx+SpBl89j671cbbdlJC3p42E+LTDifbEbFNmOTGr8z2Z7HaT+E4dDoOnQ/Jgs0reUx2aVe5+j1nRXlsKqSNNj+0jJWZDQrbpu92W/8/emcc3VpX//32zJ02TdN+32fed2Ri2AWZkBlBBBRRUQBQFFVDxhwuKqCyKqIgCoqBfREQ22ZmBYZ997+xLp/u+pE3b7Lm/P07STpm207Rps/S+X6++kjY3NyfJaXLO83mez7Pwq+LyxPvQcPCUm6eli4zdw+Mtccfdib9RvB5bAtNZNSlJbH40htHvIRhr2IogpUTMo08kbujy5gAwj8OU232i77dT+aw5LV1NIoimTRr1h/qo0sUV6nd5SfoeUsVHonJixmfggp+Llhgnk78IrnwaPvMX+OJ/Ye1vhUNGWzk8f4PoMT8UTGmcmSbWpJvKWpR2Jgoxj8WgJS1JuEyMl+r2Ey1dONw+9BoVU7Li73stNUnHXZfOJN2so7ylm2e3n+rQo6CgoDAYJ5q7WHdAVK4/8dXFPUUsAOdPF+54OyvbaOl0R2V8446uJkCOXO/0k2k9Aa/eJhycknPh/J+LtmqjRUhsb68Cd5iJyCqNSF4OKElkQ8bvg7YK0dZsNObPWKAxCDfEgD/aI1HoD1kWxTMBX2TbToSQghKwEksZdaIutitEjokZSVw0Kwd/QOamp3dS1drNM9uqeGu/WNw5vX7+t7s2yqOMDrIs9/TkzrGOwodWgjMuxPaTbOQVTs/cYIXK3ip7dAcSJdYFP1fPn5aFRj1+v0p/8emZzMqz0NLl4arHNrOlbOwq3DUaDZ/97GcpLi4+JckuZnB1sLsJtlV1sla9Rfwtd4Go9O6oExnvWbMiY2WnSxI94TKmix5sHbWiv7rTHt55Aj4RiLBXi2pjSSWE9rRJp2Zwq9TC4l6lGTg7PnceFC0Xz/ntO0W1/0mEKtsPN0ZefIjpOeLpxN9wCICj2qkUW9XivUrKHJNq5JhCpYKMKWDNF/8XJwnuUjCQtEh1hP2hvu0eR8SCQzE9R4aL3ydEa13SqFddBLqaOXv/T7lP+1dMchckZ8M5t8Pym0Fn6v9OlhzInA7mDOHocdlfITlHVFp8+LuhPbBax+y0AMl6FQ6Xj1Il2UshDigcZ33bQ9br8wpsaON0rZyk13DzeZMA+MPbR3F6lAC1goLC0PnHxnJAtJ2bkdt3fZ9jNTIz14Isw7uHh+juEwUSZq3sdYkWS6OR1NxWAa/dJtayydmioj1jFIV2EHEEg1W4otXuCO++Kq3YL0Sob3vCzJHB6GqE7hZR6BCvaAzCSS4KrgbjYo6MFGcbOGpHp6o9hM4EXQ3i/19h1IjPXY9Cv0iSxD2XzWZ2npW2bi/XPrmNX7wiLEpnBhd2/x6nVvLtTi9Or9gcZytie9iExPZGhzthgwzVwcp2xUZ+aIR6L+4dh5Xtsiyz7kA9ABfOGH8W8idj0ml4+oalLC5JxeH28eW/b+Xdw2NjhadSqZg+fTo2mw1VLPa3DgSgq5GH96u5Qf0a+VKzqOCceJ7Y4OgtkDVD2DlFCrUG0idBwVLR3z1jusiQ7RxCL0A5AJ2N4kdngbxgb/aiM8U4B7K2N6UKS/vu1oGr6c/5f+K4riZ4754+N01NE2J7TYePjgi778TsHAkEoOUYOo8dt6zFnDsFKSQwW3KjO7ZoodZC5gwhunbUi/kIPVUbM6VyjtR1gFovAkOftJwfJjE7R0aCs1Ukbox20kbFRrzPXseiwF6csg7XxIvgkj/BlE+FV/FhyYFPBT8XaneKXohDQG20sCxbJBN8fEyxkleIfUJ928vHQWW7xxfgqc0VAHxxSeFpjo5trlpSSJ7NSKPDzT82lUd7OAoKCnFCh8vb0+LzujNL+j3m/OkilvBODPdtT5i1srNV7MF15sie114Jr94qhDJzFqy8EzKnRfYx+kOShDseQN2u8O6r1kDAGzGxPWHmyED4feJ9jueqdhAtvXwe0SJwjEn4ORIJQlXtkSjEGQh9svisso9PbXCsUGZ4gmHQqnnsywtJN+s51tiJ0+tnxaR0/nndYnRqFftrO3p6p40nQlXtqUk6DFp1lEcTf1iNWpINYlER6m2eaIQq2xUb+aERqmw/2uig2zO+suIO1jmobnOi16g4e0p6tIcTdSwGLf+8bjEXTM/E7Qvwkxf3IUfCwjze8XZxqL6Tg5UN3Kx5Sfxt+qWQUiT6nydljN5CWm8WvcTSJkLOXFAbRO+ngd6XgE9UwuvMotK04AzRX06fLKrXT4c1XyQPuAZYXxgswlIaCco/hIOv9N7VoCInSQhmh6vHSd92vwfq9gCwS57EGcWpohLClDq6mcyxjtYgEjuS0kWFO4A5i25tKlrJj6dufzBI4I6Y2J6QdLeIANxoBYN8bvj4j/DWj9B7OzgQKOI/hT/DcN4PRIuM4ZBSDMUrxPVtjw/tPjozZ2aJBJ3NY+iqoqAwXIpCle3NibmXOpk39tXR6HCTkazv6Uscr+g1am67ULQ8/PO7x2jvHr9t+RQUFIbOs9uq6PL4mZJl5sxJaf0ec/40YSX/wZEm3L7ELGqJCVwd0Fouknsj6frU3QKv/0AI+UmZoqI9a0bkzn86QlbyDfvDu59KGxTble+zIdHVFFtV7fV7Ye9/wD6M9jaSFL7zocLo4+kW7gkG6+g+jkojHqPlGHQpyeqjhSK2JyA5ViOPXL0AnUZFulnHA1+YS5pZz0WzRb/7p8dhdXtduxBSsy1KVftwkCQpoa3kff4A9R0icJ9nG8D2VKEPmRYD2RYDARn21YyvXsvrg33XzpqcgUkXx5mtEcSgVfPQVQvQqVXU2J2Uj4FFaiAQ4ODBg9jtdgKx2G/M7eCvpV7u1PwTg+QVfannXCkE74B/7ETVpDTImQNas+jp5qgXVehuhxAsfW4hbJqzheV7cvbQBPaT0ZkgtVicc6AeYNlzYP414vqWR8TjBglZyR+qjaxgFrNzxO/GV1sKiH7ty/KDArI1P/zXPtHQJQmLcY1ebDolCXemCCSldBxERhL/Q94wexMOQMzOkeHi8wTdKSJctROi9QS89E3Y/wIAf/NdxFWBX/Dpc5eJdgAjYcFXxGX1Nmg5cfrjJYmFOaIH9t7qdiXJSyHmCVW2V7QmfmX7k0Hr5KuXFKHTxH/I6TPz85iSZabD5ePRD45HezgKCgoxjj8g9zhhfHV5CdIAAu/sPCsZyXq6PH62lMVm0nHcr5WddiFOOltFQm+k8HbDm3cIBzljKqz8MWTNjNz5h0LufHHZehy8YVSpSxLIRKyyPe7nyGAE/GCvECJlLFS1Nx+F174Pm/8Cz14Dz10Ph14f+v31SWLOjnGiRULPkUjgsovPFO0Y6BH6ZECGpkMRi6ko9CX+dz4K/bKoOJUPbz+Pd247l6ygwPzFxcLC7eXdNXS6x1claqiyPdemiO3DpSRdBIgSUVhtcLjxB2S0aonMZH20hxM3zAn1ba+2R3cgY0zIQn7VzPFtIf9JjDo18wttAGw8PvpZkj6fjxdffJHy8nJ8vtj7TvM6WnFV7WaVegcBVLDwK2C0iAWtzjT6WasnY0oVQnrOXLAWikV8ICAq7J1tIhEge7YQOodLci6Y0kUgYyAWflkc4+mC/S/2/HlquhDMjtZH9vslZueIz4O/QbT5Oa6bSoGhWzgDJGVEeWAxgtEmqkOCTgnmAmGROCNwlIaugAgQRWhjGLNzZLi42kXSy0j+l/sj4IPd/4YXvwGtZcjaJO7W38bdvmu4Yo6NFGMEtpTpk6FgMSDD9r8N6S5TcqzoVKJdVFWrEixQiG1CPdsrE7xn++4qO7sq7WjVUtxbyIdQqyS+v2oqAE98XE5jh+KuoqCgMDBvH2ygqtWJzaTls/PzBjxOpZJ6qts3HBqbVmzhEtdrZacd6vaKynZLLkgRkkACfnjnl9B8ROyrV9wq9tljTdpEsYf0e0Wyarj43ac/ZgjE9Rw5HZ0NogI4FtznXO2w/qciSUJvEXvi1uPwwf3CuWEo6Mzg6RzYkXCUSOg5Egk6G0UyRySdNwYjKV0U4DQfG9j9UmHYKGJ7ApNlMWA1aXt+X1ySyoSMJLo8fl7ZUxvFkY09dXaxIVb6tQ+fMyeJLND3j8TmJmAkhCzkc6xGVKox+nJLAOYW2ADYM45aU1S3dbO/tgOV1Gv7ptDLsonCIm/T8XFu6ev3se1EA18maJdeuBRKzhbXPZ1gsAnBfSwxWIRVc/ZM0Yu9eIXox164TGThj7R3vEYHqSVisz9QprRKA7M/J64f+F/Pwn6CTVRzn0hwAaKHlmPoPW24ZQ3mnKki6cFaIKq5FQSWHNG3PeBDmyvE9gWqIxxocIJaJ7K/FU6lO/jZG0mHhPp98MLXYeujIriTOoHtC+7lb+2L0Knha/MjWEUfqm6v3Ahtp7dG1On0TEsRnyN7a+yRG4eCwiiQbxOtY+o7XPj8iVvV849gVfslc3LJSKAk5gtnZDG/0IbT6+ehDceiPRwFBYUY5qnNFQBceUYhRt3ga7JQ3/Z1++vxBxTRI6J01IK7XTi3RUrEkmXY+JBYq6rUcMYNUHJWZM6NcEUIDFX8klSQu0Bcr9oc3gNJKvAqiWOD4vdCW0Vs9GoP+GHDL4VLoTEFLrofrnoW0iaJ24+8ObTzqDSALIRWhdjA0y328PrksXtMSQXmTOio7o0fKEQMRWwfR0iSxJVnFADwemldlEcztoQq23OsSj/u4XLOFFFxt7vKjr07MnZDsUKoD32eTZkf4TAeK9tDFvKLilNJMydOADFSLJ8oknI2l7WMb0tfj4OPjrWyUDoCgGrWZb0bNJ9HLGyjjVojBH+jTVyPBOZMMGdB9yDOBtPWClHZUQcnPgSg2CYev6ItMtn1MU/lJgD2yBNZVGQDSS3eB4VejKnC/cHtgNQSXJIBi+SkvvyQENs9XeBXsuL74PeK6gt9BKraZRlqd8Ebt8PLN0NrGWiMMPNzsPZBHq3MBeCKGSYykyIo7GfNhJx5ItFi21+HdJfZmSKxuLTKHrlxKCiMAulmPTq1ioBMT/uqRKO928ure0VS/1eWF0d3MBFGkiRuXz0NgH9vraSiJfHbASgoKIRPVWs3Hx5tRpLgS0Nw9zhrcjo2k5badhfvxmh1e1wiy6J6V2uKbLXojifhwEvi+pyrYMYlIz5lU7ef329xcM2LLcx9tJ45j9Tzx60OnN4hJOblLxSXDfvCe1C1RthWKwxMqKo92r3a/R748AHhXqDSwtJvQuY0MGfA1DXimKotQz+fLmQlr+ylY4KxtJA/GY1etJNoqxCulwoRQxHbxxkhIWRPlZ3AOMqarO8IVS4rle3DJddmZEqWmYAMHx4dfYvosSRU2Z6foojt4TAnzwZARUt3wiVgDMTuYDA/lHyi0Je5BVYMWhXNnR6ONnZGezhRQ3Z10FG1D40UoEuXDpkzxA0+t8iMNliiO8DRQqUWlvQyfXqy90GfDFMuEtdL/wNAsVWI7dWOAB5XYgoQJ+Op3gPA9sBUluVIYmM1lpnM8YBaI3rYuztBpaHJLAQOVUOp2Bj6PeBL/LkSFq52kYQw0n7t9aXwv5vg1VuhaisgCQF87W/hzJup8SWxoVz8f3913ij0hj/jenFZ8RG0Vp728DnZYm1fWq1UaCjENiqVRE6wpVmtPTE/v443d+L1y+RaDT0OWInEsolpnD0lA19A5sH1R6I9HAUFhRjkmW1i7XLW5AwKUk8vnhi0aq5YJIqi/hmsiFeIAD63aDuliWAMeN8LsPMf4vq0S2DBNSOueO72Brji+RZ+v8XBh1VuOr0ynV6Z3212cN4/G3nz2GnaJOUFK9vtFeAKI/6i0oK3S7GQHgifB9rKQWeMblV7ZwOBl78Dh0Vf9rKSq2DiBb23F58pLtvKwTHEZB2dGdwdY24lrzAAY20hfzJJqdBZD11NY//YCYwito8zpmYno9eo6HD5ODGOsrFDNvJKZfvIOHeqqMZ873BifRDX2MUCNk8R28PCatJSHOw/uXecWMmXN4vPzYkZoyAwJAB6jZozikU/q43HEispJxyO17Uyw70XAF3G5F7LeE+n6K+lS2BhNSldWIAPZk02+3OABA37oeU4mUkqjBoIyFDdbB+rkUYNR5MIpHXossjRu4LuAtrB7zQeSUoHrRG83fgzZwGQ1nVUVLb73IrY/km620RF+HADQh218PZd8PK3ofGAOE/eIlj9K2FVmDUTgP/s7yYgw7J8HRNTRiH4lD1bWGLKAdj26GkPn5Ut1m6ltY5xlUisEJ/kBveitfbTBM/jlN4E5jGuzhlDbl8terf/b08t7c4B2uYoKCiMS7z+AM9urwbgqqCr6FC4emkRkgQfHGmirGn8JqxHFG+32CtodJE5X9l7sPGP4vqE82D5TRFpAfbLDzsoa/ORmaTi7nOtvH5VBg99KoV8i5r6rgDfeqONyvZBKpCTcyEpU6ybKz8e+gOrtaKyeaD2b+OdznoRzzDYojeG6m3Iz9+AqukQdjmJaz0/4LuNawmcLMqas4JW8jIcXT+086o0IslCsZKPPp5u8T5Eq/BCHWyRYC9XnA4iiCK2jzO0ahWz84T18+5Ke3QHM0bIsnySjbxS2T4SQtW87x9pSqiAZo/YrtjIh82cfBswfqzky4M9pYvTEzeIOFKWTgj2bS8bv71/3j7SxtlqIbZr8+f13uBxig2RKoGXX5IE1kJxOZAYas2HwiXi+q6nkCSJoqCVfHljgifuBPxIDtHKJyUtQwQ4TKlRHlSMok+GpAxwtWMrmA5Asb8cp08WAQLF+rAXv08EhXRhWsh7naLH36u3wTNfgrJ3xd9zF8Ca38JF90LR8p5Apdcv88x+8bp/aVYE7OoHIlTdXrkRWssHPXRKqgadGhzuABWtypxQiG1yg3uNmkQV24PPK9eWuHvuWXlW8mxGZBkO1HZEezgKCgoxxDsHG2lyuEk367lgRtaQ71eQamJlsLDlqc2nd/VRGAI+18iSUE+m5Ti8d6+4nr8Yzrw1IhXz6447eXqfWLs+uCqFa+YkMSNDyyVTjLx9dSZL8nQEZPj3vkHWt5IE+YvE9ertQ39wlQYCPuEWptAXTxe0ngC9WTj3jTVyAHb8A/n125HcHewNlPBZ76/YrFpAaZOfV49+IsZScra4rNw49MfQJYm9oyKwRpdoWcifjClVVNd3NkRvDAlGAkd7FQZiXtDSbfc46W3Y7vTi9PoByFbE9hGxqDgFk05Nc6ebA3WJE1wIVWEole3hMyNX2GEfaUj8DOy2Lk9PBUtR6iiKDHHO8olCbN9c1oo/gZJyhozfx4FjJ8iXmvGjhqKl4u8+t9jUjofe3KZUkWXfNUjCxfRLxWWdsFQvCVrJl7ck+GeJ14nFK9xh0tKyQNKM3PY7kUnOhoAfa66oJJwo1XG8rhU0WnC2RXlwMYS7Q/S314cxl058CM9eI4KHtTsBGVInwLk/gk/dC7nzTglQvlPuorErQLpRxaqJo7imzpoJ+WeIpIqtg1e3a9USM9LElra0cvwmeQ3Ez3/+cyRJ6vMzbdq0ntvPPffcU26/8cYb+5yjsrKStWvXYjKZyMzM5Ac/+AE+X9/g3HvvvceCBQvQ6/VMmjSJJ598ciyeXtwR2mskrNg+TvZUM4P7n/21CZ4gqKCgEBb/3iqE8s8vykerDi/cfs2yIgD+u6OKbo8igI0Yd1dkbJldHbDup0K8T5kAZ98OxpG3hGvs8vPDd8R3yNcXJHFmQd8qeYNG4tq5Iub03wPdePyDxFVCVvIN+4c+ALVWCO2K2N6XQACaj4n3PRpV7d0t8OYdsOMJJGSe9q3ki/6fcefF0/jmIrHP+83GDty+k+ZD8Qpx2Xxo6K0E9Gaxd1Ss5KNLZ5NI6IiGhXwIlUZUuDvqozeGBEMR28ch8wptwPgR20NV7alJOgzaKGSlJRB6jbpHSHv/SGJYycuy3BPwyrcp1crhUpIuNgAnmhO/LUWo9Ua2xYBRp3yWDMTsPCtmvYZ2p5eDo5iUo1arufjiiykoKECtjp33o6W9gzT7bgB8lkKw5IkbutvAkg3GlOgNbqyQJEgpFIt2t6P/Y3LmABI4W8Fe01vZHsHPkpicI/YKNPhxyxrS01KETbrSr31gdGbQGJAMZlpVqagkmcYT+0X2d3fbiK0PY3KODIeulqFX73Q1w7qfwPqfiusGK0y6EC76DXzmLzBl1YCWm/8qFZU1n59hQqce5aBAT3X7Zmg6Nuihs7NEgLK0cvy2LxmMmTNnUldX1/Pz0Ucf9bn9hhtu6HP7/fff33Ob3+9n7dq1eDweNm7cyD/+8Q+efPJJ7rzzzp5jTpw4wdq1aznvvPPYvXs3t9xyC1/72td46623xuw5xgt5wYrvkCidaPS6hSX2nmpmrnAK3K9UtisoKASpau3mg6MiRnZlGBbyIc6enEFxmgmHy8dLu2ojPbxhE7drZVeb2IuOhIAf3v0lOGrFevns74E5PSLDu+v9dtpcAWaka/je0v7F+/NLDGSYVDQ7A7xdNkj7rNz54tJRC51DXAtLKpHUGgGxPW7nSH84aqGjGswZYyuABnyw97/wn2ugagt+ScP3vd/gx76v8cc12ZxXbOBr85PIMKmo6vDzVOlJMZOUEjCLBHVObBja46k0Yu/YXi0SDEaZhJojkcLVAV2NYBh58s6I0ZlE8r5PSb6JBIrYPg4JVbYfrOvAFaz4TmTq2sWmP9uiVLVHgnOC9lbvJ0jf9uZOD25fAElSnA+Gw8SMXrFdlhO7irkiKLYrFvKDo1GrWFwibLE3HR+9KkO1Ws2cOXNIS0uLqQX7u4cbOUsSFvL6nOliI+NzixutBdHNWh1LjCmif5irQ1ixfRKdGVJLxPXqLZTYxHtY3uoUG/8IEItzRG4+DkC1nEFRshpMKUq/9sHQmoJ92500mSYB4Gk83NPLfcBkjiESi3MkbMKxkLdXwkvfgvKPRJCt+Gy49E+w8sdQcMagvScr7D4+rHQjAVfNGoPvwYxpULgckGHTHwc9NCS2761RqjP6Q6PRkJ2d3fOTnt43UGwymfrcbrH0Bn3WrVvHgQMHeOqpp5g3bx4XXXQRd999Nw8//DAejwjIPPLII5SUlPDAAw8wffp0br75Zj73uc/x4IMPjunzjAdCInSi92xP9Mr2WXlKZbuCgkIv/oDMT17ahyzDmZPSKEoL3wVPpZK4eqmobn9uR1Wkhzhs4nKt7PeKXsgj6akuB+CjB6Fqq9jPL7lROC9FgG21bl475kIlwW8vTEGv6T8+oFVLfGGGWDc8vW+QhHRTGqQUi+vlH4Y3mAiJ7XE3R/rD7YDmo2KfOZK5Ew4BHxxdB89/DTY/DN5uXEl5fM57N8/5z+Hbi82sLBFxapNWxa1LRZL+Q9sctLuCIrkkQclZ4vqJMN7/pHToqIGu0Y/tJ8wciSTdLeB1RddCPoTGINrLeRLcZXKMCLt5SWVlJQUFBUifCBbLskxVVRWFhYURG5zC6JBnM5Ju1tPc6WZ/bTsLixK7V2iosj2Re8eNJecG+7bvqGyjw+XFYohvkSBUgZGVbECnUfKPwqUg1YRKgk63jyaHm8wETmo50Rzs1z6MzfN4Y9mENDYcamRTWQs3nD0h2sMZU3aWNfIT1UHxS/4Z4rK7FSw5YiM8nkgtEYGKpkOAJDJmTyZ7DrSWQd1eiqavBaDc7heb/rHa4I4xjvpjWIAqOYPlZnl8OB2MBJUKTOnQehxv6hTo3EqS40Swz6BfbAjHe897V7vIRDdnDn5c81F4/QeiN5wpDZZ8EyaeO+Reln/fLYJ8ZxXqKbRGoP/lUFh6I1Rthvq9ULkFCpf0e9icTLEW3V/vJBCQUanGSVLTEDl69Ci5ubkYDAaWLVvGPffc02fP/q9//YunnnqK7OxsLrnkEn76059iMonP602bNjF79myysnp7z65evZpvfvOb7N+/n/nz57Np0yYuuOCCPo+5evVqbrnllkHH5Xa7cbvdPb93dIgqYa/Xi9c7MteKWCXTLP53au1OPB7PKTGVcAm9TrHwesmyTLVdrJWzzNqYGNNoMTVT/H8ca+yko8sV045XsTRHFGITZY6MnHvfPMz7R5owaFX84MLJw34tz5+azi9fgz3V7dg7nSTpx2i9dRribo64O8DtFEnN/mFU7coB1B8/iOrwa8hI+Gd+AXnChcM71ycIyDJ3fyDWO5+fbmRyqhrvIOf93HQDf97eyUdVHo61eHrc4D6JKncR6rZyAhUb8U//9FAHAy4nROB9jbs58kmcbdB6HJydYM2NyHs9KJ4uVIdfR7X/eaSuRgBkjRH3xDVcVnkZB3wSS/J0fGthUp/58dmpBv62q5PjbX5++WE7v14pnHakguVoSv+LXF+Kr7sT9EMQcCUtyCpoPg7a5FEvAIj7ORJJ/F5orQKNafTn2pBQgc8P3R1iLkSJWJ8jQx1X2N/cJSUl1NXVkZnZN6DT2tpKSUkJfn/iV0rHO5IkMa/AytsHG9lVaU98sd0uxHalajkyFKSaKElP4kRzFzvK2zhv2mmCuzFOdZsICiV6BcZoodeoKUg1UdHSTVlzV0KL7b2V7YrYfjrOCFa276xsQ5blEQeT+yMQCHDs2DHa29sJjIH11VAxNOzEJLlxqZMx5C4QVe2SBNbC8VPVHkKSRA9oOQDNh0Gib+Zu9mw48BI0H6EkGDio7pTxuLrQmUcutsfiHOlsLMcCtKoz0GmUfu1DwmCBQABz7jSohDxvhfhc0ehEIott+Im+sThHwqa7RbhBDCaaN+yHN24XLhPmbFHJnj17yA9R5/Dz72BFzY0Lx3DO2gph6ho49CpselhU30unJkZOStVg0ECnV+ZEfQsTcyNj8ZkILFmyhCeffJKpU6dSV1fHXXfdxVlnncW+fftITk7mi1/8IkVFReTm5rJ3715++MMfcvjwYV544QUA6uvr+wjtQM/v9fX1gx7T0dGB0+nEaOx/jX3PPfdw1113nfL3devW9Yj9iYbHD6Chy+PnuZffIClCcc3169dH5kQjoNsHXW7xOVS6+X0Ox67+PGJkGcxaNZ1eiSdefIviOOgGEwtzRCG2UebI8NjWJPHUMfGBd0Wxl4rdH1Gxe/jnS9WraXXDo8+vZ5ot+s6Bsiz3JMOtW7duVPb1o0dD+HeRA8yt+gfFLe8iI7Gz6BtUa5bD7vCs/WW5/63/tiaJvY1q9CqZWUYHr+88vUvXNJuKg3YV97zbyKVF/e9XUj1TOQvw1+3jze0nCKiGuMA4dgg4NLRjByC+50h/VI/amfVeOxMb36K45V3UfhGLdmmslGVcyPHU83msLJkDbRJmjcza7G7e2t19yjnW5sIf2zT896CTDKmTqVYZ5BTO12dhdjdw4L3/Up5xfhijaoK9o/v5n3hzJJLEUDX58T3AnmiPImbXI93dp/4/9kfYYvtAQfPOzk4MhsQVWRKNeQU23j7YOC76tocq23OsipgaKWbnWTnR3MWBuo64F9tre3oLKvNjuJSkJwmxvamLpRMSt3I31Eu6OC0xA8CRZEaOBb1Ghb3bS1lzFxMzIi/O+Hw+nn322Z7ren1sVEIXOnYC0G2djMFohY46sOSO3+pblUrYyQM0HwEjvRXuIbHPUUemyoFBI+HyyVQ3dzDBPPLXKxbniK+1AgCnLlW8Dkq/9tOjSwK1luzCybAZiqmjqaWVzGQjOO2it9gAPcZPRyzOkbDwe4WFvH6QJLDWE/Dm/xNCu7UAVt4JGZPDepi/7HDgCcDiXB3L8kfY/zJcFl0HR9dDeyUcegOCLhgno1FJzEjXsrPeS2lFkyK2n8RFF13Uc33OnDksWbKEoqIinn32Wa6//nq+/vWv99w+e/ZscnJyOP/88zl+/DgTJ04c1bHdcccd3HbbbT2/d3R0UFBQwKpVq/pY2Sca9+5/j5YuDzMXr2BGzsiep9frZf369Vx44YVotdF1GztQ1wHbNpOapOUzl6yK6ljGghdadvDB0RasJbNZszj8/sxjRSzNEYXYRJkjw2dPdTs/+Ns2IMA3zynhtgvCW1/1x/uufbywqxY5YxJrLhz5+UaKx+Pht7/9LQDf/e53SUqKg8KDtnJoPCQqlMPB50H9wT2oWt7vqWifs+gzzAlzn1HR7uNrr7Rh1Ep85wwz55fokSSJ5m4/95S2AAFuXpzMlUNMYNXaXHzrDTs7W9X84ZKc/m3n5Vzk6r+gdbZykeUo8pQhfA93twnL9ILFYT2/TxKXcyRE6wloPChcEIbSkmu4yDLS4ddQb30EySsEO9mYhn/CeainX8qE5DweXt9OaZsLnRoeuTiVJXkD70tbVR08VdrNy9U6XluRhkmrQmX4PGz+E3PaXmfGqmv6TVDuF6cdJDXkLQTt6Oh6cT1HIo0sQ0MpOBogOev0x48VrnaQNFCwRMTwokCsr0dCCSOnY8hie2gjLElSH2s5AL/fz5YtW5g3b154o1SIGvMKhG3pnmp7dAcyBtR3CDE1R6lsjxjTcpJ5eQ8crBvaB00s09Ah7CuV+TF8JqSbee9wEyeaYygjbxQobwnayCuV7adFp1ExJ9/KtvI2dla0jYrYHot0e3zk+apADcbMCcLmWg6AJW/8VbWfTI/gLkHLEUAWm1lzJpizoLMBqWYbxda5HGrxUdHUwYTiKI95lNB2iqoI2ZQKRpvSr30oaJNAa8SghSZSyZBaqT5WSubiFdDZJKzkNeM0mcXVLvoLDmQh39kAb/xAHGPJg1W/gJSSsB6izuHnmX3i+++WJcljX4lgSoW5V8HOJ2H74zDh7H6TVOZk6YTYXt3GZ8Z2hHGFzWZjypQpHDt2rN/blywRVv3Hjh1j4sSJZGdns3Xr1j7HNDSISrHs7Oyey9DfTj7GYrEMWNUOoNfr+01w0Wq1MRlgiRR5KUZaujw0OLzMLYzM84yF16yx0wdAfoop6mMZC2bl2fjgaAuHGzrj4vnGwhxRiG2UORIeDR0uvvX0bjy+ABdMz+IHq6dHpI3NsonpvLCrlq3lbTHxfshyb3V93MwRbyfodKAOQzBytcNbP4aGfSCpkGZdgeaMa8NO6G3s8nPdy21UdgjX32++YWdelhaVBLvqvchAXrKaGxYko1UPbb6smmgk1+ygttPPm2VuPje9v+IPFUw4F/a/gObEuzD9U6c/sU4Hsg9UEqiH37IgLudICI8dDCYwjmICfGcjfPAbqN4mfk/OhalrkKavQWNMJSDL/L+37bx2zIVWBY+uTWVF4eDx6f93poUNJ9xUd/j549Yufnq2FaatgR1/Q+pqQlv+AUxaObTxJaVAezV4O8A0Oq9DXM+RSONqB2cLJNnC+4wabfQmcHUAnqhayUPszpGhjmnI7+quXbvYtWsXsixTWlra8/uuXbs4dOgQc+fO5cknnxzueBXGmDkFViQJqlqdtHS6T3+HOCZkI69UtkeO6cEqjEP1p7c8inUaOsT8yEiOs2q2GKIkQ4jPZU1dUR7J6NHW5aHdKfqzFKUqYvtQWFAokrp2VtqjO5AxpLyxgwlSHQDG9ELwdgubcIM1yiOLAVQqSJsI6VNFTzR/sN9R9hxxWbeb4qCVfHmiJu4E/JjdwnZZb8kAgy2644kX1BrR297npM4oKm2d9Yd7+7a7438tMmy6moEBLORddtGjvatZ9L0/70dhC+0Af94uqtqX5OlYXhDBtZK3WwSfOhvA7xn82LlXiOfgbIM3/h8EfKccMivYt720dhzPhyHQ2dnJ8ePHycnJ6ff23bt3A/TcvmzZMkpLS2lsbOw5Zv369VgsFmbMmNFzzDvvvNPnPOvXr2fZsmWj8Azin5CbVshdK1GoCbXmGiduYbPyxNpuX038J58rKCiEh8vr5+v/t4NGh5spWWYevGJuRIR2oMcpcG91O92eU9c7Cqch4Bc92zVhFNN0NsL/bhZCu0YPS26EJdeHLbS3uwN8+X8tVHb4KbSo+cZCMwaNxO4GLzuDQvvMDC0PfSoFQ3/V6QOgUUlcPUcI7E/s7uojXPZhwrnisn43eE+ztobgXsp7+nV4ouJzC+evUarmBoTA/vzXxKVKA1Mugkv+CAuuBqNIFv/L9k6eP+hELcFDF6VwXvHpx2PWqXr6tT+xp4vmbr9wzZsadAArfXboY5QkUQDQ1RT201MYBl3N4HcLV4lYQqPv/Z9QGBFDTl169913Abj22mv5wx/+kNDWbuMBi0HLxAwzxxo72V1l5/zpMWRdEUFkWT7JRl6pXI4U07PF/39ZUycurx+DNn6b8jU6RLJJVgL3Gh9tJgYrvU80J+6XcnmwX3u2xYBRF7/zfSxZUBQU2yvaojySsaOivoULpKAgYSsUFbfWwmFbXCccKhWklgjBzNkiqtqzZ8Ox9dB4mKKcoNje0jVwo7t4pquF5IAIyttSs8ILAo13jDawV+K0TQHnNvTtJ8TfNTrRszylKKrDiwp+nxCqdf04h7ja4bXvgb1SVIGffTtkzQz7Iao6fPxnf29Ve0TwOkWQQWsUAjqI5wGiil3Tj6CvNcJF98JL34LG/fDefSJ54KTPiDlBsX1fkw+/24laH2MBjCjx/e9/n0suuYSioiJqa2v52c9+hlqt5qqrruL48eM8/fTTrFmzhrS0NPbu3cutt97K2WefzZw5IhFq1apVzJgxg2uuuYb777+f+vp6fvKTn3DTTTf1VKXfeOON/OlPf+L222/nuuuuY8OGDTz77LO89tpr0XzqMUtuSGwP7lEThZpx1pprZq7YDx+ud+D1B9DGUnWSgoLCqCHLMj96oZQ9VXZsJi1//fIikg2Rq8ArSDWRZzNSY3eyo6KNsyZnROzc4wJvtxCPh9qqy1EHr94mLvUWWP5tmHh+2DbKzd1+bni1lUPNPjJMKp76bBqFVg3Xz0viuYPdWPQqzi82kJN8mliSLIsqY5VKOMHpzKDScOVME7/f4mB/k5cddR4W5fazXs6eJRKUnW1w4j04nZW8WguukNg+DlslervB6wL9KLSfkgOw6ynY/gQggzkblnwDSs7qkyS9qdrNA5tFovAvz7PyqYlDX0OdW2xgerqGg80+Nld7uHiKEWZdDvueh6ZDooVf+pShnUxvhu5WsU+LNRE4kfC5xf93rLYSlCRwd0KMDi9eCHtH8MQTT2CxWDh27BhvvfUWTqfYVA2YWaUQs8wrsAGwJ4H7trc7vTi9wr4nWxHbI0aWRU+KSUtAhqMN8V2B2BisbFfE9uETqmyvbO3G6w9EeTSjQ0hsL04fh5uQYRKqbD/S6KDD5Y3yaMaGltrjaCU/XjRgKxCZ9eO1V/tAqNQiESHgF9Xtob7t9gommMX3dbndB77EEiEAaD0uLmQzuWk2JQkjHHRmkCQMWaJ3Zaa7QvxdaxTCsi+xXZr6xecSz/uTSRuuDnjt+9ByXATpzrwVCsPvxegLyNzyVhueACzL17EsP0JV7U67qLAvXAr5CyF3vugVackRlUX9VK0DohXFyjsBSSTo7Pl3n5snpmgwaqDbByfqWiIz1gSgurqaq666iqlTp/KFL3yBtLQ0Nm/eTEZGBjqdjrfffptVq1Yxbdo0vve973H55Zfzyiuv9NxfrVbz6quvolarWbZsGVdffTVf/vKX+cUvftFzTElJCa+99hrr169n7ty5PPDAAzz++OOsXr06Gk855gmJ7TVtCVbZHhLbU8ZHkLYw1USyQYPHH+BYY3zvhxUUFIaGLMvc9coBXthVg1ol8fAXF1CUFnnXuyUTxP5xc5myngkbr1OIx+oh7LPaq+Hl7wih3ZgC5/0YJl8YttC+r9HDpc80s6veS7JO4p+fEUI7QGaSmm8tSubq2UmnF9qht+I1ZaLon9xRDz43qUY1n5kqYlFP7hmgyEVS9Va3H99w+sdSaYQo7B8fsZpT8HQHbfSHb6HfL13N8MYPYfvfARlyF8DaB2DieX0eq7HLz3febCMgw+XTjVw5M/xY4/Lg/mxjdXAvbMmBouXi+q5/Df1E2iSRfOBqD3sMCmHQ1SRe41gV27UG6G6O9ijinrA/UVpbW/n85z/Pu+++iyRJHD16lAkTJnD99deTkpLCAw88MBrjVBgFZgStwA83JK7dYqiqPTVJF9fV17GGJElMz7Gw8XgLB+s6mJ0fnxbJsiz39GzPsig28sMl22LAqFXj9Pqpau1mQgL25z7RHOzXPgqb6UQlI1lPQaqRqlYne6rs4yIr391wFIB2TQbpKi1oZJEhr9CXpAxR1d7VJCqSdWbwdDLdfwgoorw9kJBZ1d0NxzABVXImE1MNSmV7OOiSQGMiK78EdkO+XI+7sw29ySKqot2d/VdEJzJ+j7B+VJ9UTeV1wuvfh5ajImhy5m1D79f3CR7c7GBHnQga3n++LTJjBpD9kJTeG2RQqcTvoXYb9iqw5omA4ScpWQGLb4Ctj4lKkckXiT6DgFolMTNDx/Y6D3urW5hUnB+5MccxzzzzzIC3FRQU8P7775/2HEVFRbz++uuDHnPuueeya9eusMc3HglVftcknI28eD6546SyXZIkZuRY2HKilX017T1t1hQUFBITWZb5xasHeHJjOZIE91w2mzMnjUJFLMJK/oWdNWwpax2V8ycsPo9wdYLTO6R1t8Crt4r9qCkNzv2RSAINk9eOOvneejsun8wEm5rHLk5lUuoInA68LrEHzpgK8mSo3SlaQ2ky+MrcJJ490M0bx1zUOfz9i/cl58D+F6Fut7CS1w4h6WC82sh7uvrfb4yEsvfgw9+JVgYqDUz/NJxxvbB4Pwl/MKm5qTvAlFQNd59rRRqGq9/yAj1/293FpuqTEs9nXQ4VH0P1VvD5QDME6U+SxGvR1QLJ2WGPQ2EIBPxin6s1Rn7eRQqNUSShJGAsbiwJ+9295ZZb0Gq1VFZWYjL1flhcccUVvPnmmxEdnMLoMjlLCGJHEzgTu65dbPqzlarliDMtaCV/sD5++9Q53L4e54PMZGWODBdJkihJcCv5ip7KdkVsD4eFwer2HePESl5lF9bWHmO62LQarEIkVOiLSiWq22VZVLFmzQKguFMINdWdMl534lW2t9eJyvYGKR2zyQDqcSYOjwSNHgzJZFmMNMipqCSZuqO7hVOCHBAtG8Ybfk+w3cJJ27n9LwnLQK0JVtwCk88f1qk/qnTz5+3iNb33fBsF1ghVfPhcIsmkv2x+tRbSp4pkHEfDwOeYe5Wocg94Yec/+tw0O0sEN/dW2yMzXgWFUSBhxfZxZiMPvX3b99fG735YQUFhaNzzxiGe+LgcgHsvm80XFhWM2mMtC/Zt31NtV/q2D4TfK1ycXMHP30AAmo9CRy2YM09/37d/3iu0r7xzWEL7swe6ufmNNlw+mXOL9Lx4RcbIhHYAnxMMKWK/rNaIBHWv2BfPzNCyOFeHX4Z/7Rsg7pY9Gww2seY+/OoQHlAen2K7LIuEi0gla3s64d17xLxydwjb+PN+BMtuOkVoB3iqtJuN1R5MWok/r0nBpB2e+Lo4V4daghN2P7UOEdsmZ64o+PA6heg+VPRm8T8xHh3jxoLuFnC2ivZ4sYrWID6DPN3RHklcE/Z/87p167jvvvvIz+9bLTB58mQqKioiNjCF0WdKlgh0VbR04/b5ozya0SFU2Z5rU4TUSDM9R8yfg3XxG1wIWchbDBqlD/cImRC0ki9rSkyxvTyYRFCcptjIh0NP3/ZKe8TPrVarWbVqFXl5eajVsfH/m9RdDYDGnCE2xebMxOs7HilM6SJ40N0COcJKPtl+GINGwi9LVDeNvJIj1uaIu1mskx2adCGGhmlROO4xpSH5vVTqJgLQUHFQ/F0d7Ns+DGJtjoTFJwMhPjeU/ldcn3axsMEcBm3OALeua0MGrpplYu3kCApnni7hZDFQEpLOBJnTRSb9QO+pJMH8a8T1o2+Bq3fd0dO3vd4pqgcUFGKQkM16k8OdMHtwl9dPc6cI1uePExt56O3bvr9WsV1VUEhkXtpVw2MflAHw68/O5oozCkf18fJTjOTZjHj9Mjsr7KP6WKcjZtfKbodIMK3eBq0noK0c2k6I/ffpbME3PQz1pSLxednNkDs37If/V2kXt79tRwa+NNvE3y5JxaqPwN4u4AfDSU4pBqvY6wTX/V+eI9bQLx8eIGFPpYaZnxXXt/1N9G8fDEk9YmEtZufIYPjcQozWRGDNUrcHnrte7EuQoPhsuPhBmLiy3/1+jcPH/RtFHP2OMy0jStBI1quYHdz/9FS3q9RQvEJcP/He0E+mNYmkgVGwko/LORJJZBk66gAp8m0LIomkChYyJK4D9lgQ9jdBV1dXn4r2EK2trej1SoVOPJGZrCfZoMEfkBO2GjVkEZ6pVLZHnJBV3sE6B7IsR3k0w0OZH5FjQrDiuyxBP0vKW4I28kple1iE+rbvqmwjEIjs54RarWbRokVkZGTExIK92+Mj21cLgDk1SyyiDfHZYmNMUKnAmi+qENKnAiC1V1JsFe9leX2zuG0ExNockdqrAPAa05T2AsNBJxyZvFkiIOapP8jmaveI+rbH2hwJC5+7bzLPkbdEtrze0htkGwa//ridpu4Ak1I03HlWhOfpUJKQjDZInyKe30CVNsUrIDlXZN7vfbrnz6Fg0/5mPz7nOHQ7UIgLUkxaDMEKpvr2xHBxCVW1J+nUWI0jrOqLI+YW2ADYU91Op1upPlVQSESONXbyoxdLAfjO+ZP54pLRFdpBOAcuKYmNvu0xu1b2e4QwrdII4bzpoBCpB6tUlmXY9zwceEn8PvcqYbseBgFZ5uFtDn78rhAkr52XxC/PtaJWRSDBPuAXYtfJSal6i/gJunidU6RHLUFlh5+q9gG+d+Z9UbjIebvg3V8P/phqrThuBMTsHBkMb7fYR4ykst3tgI9+D6/cItqaGWyw7Nuw8ieid3o/yLLMT95tp8srszBHy5dmj7yYZ3nBJ/q2Q6/YXrNTzPuhoAq+d92Rb18Rl3MkkrjaxRyJ5ar2EFqjSAxQEteHTdhi+1lnncU///nPnt8lSSIQCHD//fdz3nnnRXRwCqOLJEk91e1HGhIzINXkCIqpyUoiSKSZlGlGrZJod3qp74jPQFGjQ4xb6dc+ckp6KtsT77OkrctDu9MLQFGqIraHw7TsZEw6NQ6Xj2MJODdOpryxgwmqOgBM1nRhw6UIqoOjNwurKlswaOVsY4ZZzJMDja4Rb/xjDaNTJGNoktLE81YID10SaA0sXSRsHs+QDvHt1+o56tCJgIknsebLafF09vZrD/hgT7A3d/E5AwZ4TsfGKjf/PeBEAu67wIZxmJaG/SIHk2eG8rmYnAOWXOhs6v92lRrmf0lcP/hKjyhfYtOQpJVw+uB4vdLnVCE2kSSpp695qM95vBN6HnkpxmH1HI1XJqQnMSE9CY8vwIZDjdEejoKCQoRxevzc9K+ddHv8LJ+YxnfPnzxmj7082A/+/SMDrIXGO6GETINFrBmNKf23KQrRVgGv3QYbHxK/TzgP5l0VltOY3RXghlda+c0mUfX5jYVm7jzLErnvPZ9LVFprTxJgVSrRQ9sjvmeT9SrmZ4s+7B9VDZBorNYKwVdSicr/w4O0/FVpxOOOMMk97vB0CRFaNQzhV5bh2Nvw7JeDiRsy5MyDtQ/A7MtAoxvwri8fcfJuuRudCu4734YqAnNneb6IZ2+q8vQWwuUtFIkE7nYhuA8VfZIQhf3eEY9L4SRcdvC746MPusEqHDGc9miPJG4JO4Jy//3389hjj3HRRRfh8Xi4/fbbmTVrFh988AH33XffaIxRYRSZnCmqhI41JKZFREhsz1DE9ohj0KqZGBRY49VKPlTZnqX0ax8xE9LFZ0kiumSUB/u1Z1sMSruBMNGoVczJF9XdOyPctz0QCFBRUYHD4SAQA5vDqvpGcglWHpgyRd/h4WzexhNaE2iTxOtkyQVgrfkoAG+U+cA9sgSNmJojgQA2rwjEm23Zom+1QnhojaBNQpWUgWxMwyB5meY9yFdfsdPukkV1QZjE1BwJB1kWCQaqoNhe9j44asX/1JzLh3VKl0/mRxvsgLDDXJgzcKBoWHi7xf/7YIHQECoVpJSIpJSB3tfJF4rAqrsD9r0AgFolMTNDvCalVYrYrhC7JFrf9vHYrx1E4sRFs7MBeKO0LsqjUVBQiCSyLPPjF0s53OAg3azn91fOi0z18hA5Z0oGkgSlNe00RLG4JWbXyl5Xr1OSSj2wiOV1wtbH4PnroXaXEJcnXQBnfnfAquZ9jR7eOOZkfZmL9WUu/m9vF7/+qJ21/27inXI3OjX8eqWV/7c8ObIJZj6XaKn0yaRso02MOyAq2c8sOI3YDsIlau5V4vrGPw68nlZpwe8bUd/2mJ0jg+F2iGSEcKnbAy99Czb8UgiSxjRYchNcdC+kTRz0rq1OP3e9L2LnNy9OHpF9/MkszNGiU0Ftp5+K9mA1skYP+UvE9eMbhn4ybZL4n/FGtmd3XM6RSNLVItpBxAMqjUiS72yI9kjilrA/WWbNmsWRI0dYsWIFn/70p+nq6uKyyy5j165dTJw4+AeLQuwxOVjZfrQxMSsOmzqDYrtZEdtHg2nZvVby8Uho06LYyI+cUGV7o8ONw5VYWZAhsb1I6dc+LEJW8rur7BE9r8/n41//+hfHjx/H54u+dWdr9WFUkoxTMgibZINS1X5aJEn0bve6IGMaAMvUh1BLsK9V4kTdyCo5YmqOOGrR4cUnq8jIzIifzVYsIUlgSgOfG6lgMQBr9Hupcfh5q1o9rL7tMTVHwsHvFT9qrRDe9wSt1AuXQUrxsE75p20Oytv9ZCapuH35KHx+ebrBaB26q4PRBikTRFZ9fzZ2ah3MvVJcP/hKz59DVvKlNfYRDVdBYTRJOLH9pMr28cZFs4STyHuHm+j2xNH3iIKCwoDIsszdrx7khV01qCT445XzyBzjAo2MZD1z820AvBtF54yYXSuf7PDUH7IM5R/Cf78Ku58WQnX6FFh1N5z7owHtnP++u5OLn2nmm6+3ccOrrdzwais/fa+dx3Z2UePwU2xV8+IXMvjirKTIO7n43EK8/SQGq3CECyain1UYtA2v8hAYzCJ84VeFW5S3e+DqdrUGAt4Rie0xO0cGQpaFUB6O01xbObz5I3jlu6JlgVoLE8+HSx+CuZ8fUiL9Lz/soNUVYEqqhhsXmoc//k9g1KqYF3Q76GMlXxKykt8+9JOpteJ/xTf8+dAfcTdHIonPDa4OkUgTLxgs4GgQiRcKYTMsb0Cr1cqPf/xjnn32WV5//XV++ctfkpMzPLtChegSqmw/kqCV7c3ByvZ0pbJ9VOjt2x6fle2Nocp2xUZ+xFgMWtKDSS3lzZHNgow2oQBiYWocLY5iiNl5orJ9f218fk4MFU/jMQDaNRliM6xULg8NQ7DKNdi33dR+rKfv2Kv7GhPG0s7TKCr2a+U0CtIsyvwYLqGq6PxFAFyg2Q3A1kZVsG97ZAMDMYvfI8R2lQZqd0LLcVGZMvvzwzrd+xUu/rJdBPB+cY4Viz6C9vEh/B5ISg/vPtZ8SM4CR33/gvukCwEJOmrAXgHAnCwReN3b4BaJPAoKMUhIbK9NFLG9p7J9/K2VZ+ZayE8x4vT6ef+wYvesoJAI/P7to/z94xMA3P+5uT2W7mPN+dMyAXhHaVPRF1kWNuCDie37X4B1Pw320rbCouvg4t+LxNQBrOPfPObk7g9EzGJ2ppZ5WVrmZGq5oMTAdfOS+OV5Vl65MqPHRSniyLKw8f4kKjWYs3tarM3N0pGklWhzBTjQNIhwGRKEASo39X+MShsU2xOrYGZQvM5gv/Yh7Me7muGD38Bz10HlRpH8nXcGrHkAzvsxWHOH9JAfVLh44ZBo1XXv+TZ06sgmaoTiJ5uqT9oLFy4DSS3+BxqPhHdCn7KHihjuTvB1iRYR8YLODB7HsIoZFEAT7h327t3b798lScJgMFBYWIherwhX8UKoZ3t5SzceXwCdZhSCa1FClmWlsn2UmZ4j5k+8iu09le2KjXxEmJCeRHOnm2NNDmYHrcMTgVC7gWyrMk+Gw8xcMRcO1zvw+gNo1YnzPXMy6vZyALyGdJElrYipQ0NnFjZnqSXi9/ZKLllk5MNKN68ec/NtT2dCuAS0VR8mC6ghk6UWw4C2hQqnQZckAkfZswGJDG8tGbSxtT4t2Le9EzSp0R7l6OP3iMCYWgv7XxJ/y50nKnb6YcMJF/d+3EGXV1S/WPUS316czKcmGjja6uPmN9rwy3D5dCOrJ47CZ1coMWAoFvIno9FB1ixgH3TUiV70J7fnMKVC1gxo2A9H1sHiG5idKSo7DrTI+FydaMKpWlFQGCNye8T2xAhmjufKdkmSWDM7h8c+KOONffVcNFspQlFQiGee/PgEf3hHJMnedelMPrcwP2pjWTk9kwfWH+Gjo824vH4MWqVFGdCbdDrQfsrVAdv/Lq4XLIVl3wJb4aCn3FHn4btvtSEDV882cfe51shXrg9GaK2sHSBpzZQCbVpor0GrS2JpnpZ3yj18VOViVuYg4n/RMtj9FDTuB58XNJ84VpJAZkSV7XGHtzvoIpAy+HEVG+HdX4v9JQgnvnlXQ9FS8V4NkS5PgB+92w7AV+clsSCSrbrkAPhcLM/X8fst8HGlG69fRquWxL4rZ65IzD6+HjL73yeegoSSsBxJPA7xPxZPLSYlSXy+dtRBcu6ACUoK/RP2qzVv3jzmz5/P/PnzmTdvXs/v8+bNY9q0aVitVr7yla/gcin/mPFAlkVPsl6DPyAnXK/lDpcPj09UxCk920eHUGX7ieYunJ5+Ko5inEaHUtkeSeYV2gB4+0BiZV4r7QZGRkGqkWSDBo8/wLEEbVkCkNxdBYDWnCbsjRWxfWjokkRQwZIHSOBq51MZbWhVcLgNjtQkRpVYV6Oouu1QpyDpk3t7DCqEhy5J9GTU6CB9MgBnq0up7AhQ3xkYVt/2uMTvEdUvXc1Q8bH427SL+90IH2z2ctMbbRxp9VHj8FPj8HOg2ddjj3ndK604PDKLc3X8+jxb5AOLnm5RmW5MAV2YYjuI9zxrNiRng6Pu1Ar3knPFZeVmAIptasxaCbcfjtYq2fgKsUluotnIj9Oe7SE+NUv0bd9wqBGXN/72xAoKCoKNx5u5+7WDAHx/1RS+srw4quOZkWMh22LA6fWzuUxZ0/Tgc4u18ECV7bv/JSrfkzLhrO8PKrT7AzJP7+viq/9rwe2H84v1/PycMRbaQVQSawxi3dsfpjTInS/2PyoNKzJEjOrjytOI5BnTQG8Rr9lA1e2QmGK73wf2yr7CsdclLOFleeCe7QEfbHkU3vqRENqTc2DFrXDJ74U1exhCe0CWufvDDqo7/OQlq/n+0mHshfrD0y32RR214LSzINVNuklFqyvAu+UnPd+Ss8Rl1dahn1utEwKxQmToahnchSNWMVhFZbvLHu2RxB1hi+0vvvgikydP5rHHHmPPnj3s2bOHxx57jKlTp/L000/zt7/9jQ0bNvCTn/xkNMarEGEkSWJylrCSP9qYWB+mTUEhNdmgUTJAR4nMZD3ZFgMBGTafiK/FvyzLPSJqliKiRoRL5woLpfUHG+hIoL7tDaGkDCVpZ1hIksSMYGJOolrJd3t8ZPvrALCkpAtBScn+HBqSJKylJUlYRgOWlr2cHexF9+re+miOLmJ422vFpc4afnWvQi8qtehl6HVCvujbfrF+DwBbmzXQ3RrN0Y0dvmA/voMvi4oGWxEULjnlMLsrwDdebcXpkzmrQM//rkjnf1ek8+0zzGhV8PYJN9Udfoqsah5Zm4JeE8HAos8tsuE9naLiPmeu6As5HHQmyJ4lLDQd9SJAFiLUj7C1DDqbUUlST4XPnorESNZRSDzyU3rFdnmwfqtxgD8gUx/cU41XsX1evo0cq4FOt4+PjjZHezgKCgrDoNbu5Oand+EPyFy2II+bzpsU7SEhSRIrpwsr+Q2KlXwvfo8QRPsTPTsbhIU8wIzPgLn/FgCyLLO1xs1l/23mRxvacXhkFmRreeiiFDSqKCRFe53CzW0gUU6SwJwJGVMhbwErgnvlrbVuXL5B1hEqtbATB9HDfqBzh/YWiYTHAU2HoX4POO3iOTYeEAJ1ctapx8syVG2BF78Je/4t/pa/BNb+DmZ8Ouxiii5PgG+81sYz+0WbzV+dZyVJN8IYkc8t2md5uyE5D/LPAFsRGm8nl08Ta7BnD5yUyFl0pri0VwhxfiiodWL/liDt/KJKPPZrD6HRg+wDe1XfvbfCaQn7v/xXv/oVf/jDH7j++uuZPXs2s2fP5vrrr+fBBx/kgQce4Etf+hIPPfQQL7744miMV2EUmJwpgr5HGhKr4jAktitV7aPHyYv/9Qcaojya8Ohw+nArzgcRZWauhUmZZjy+AG/uSwyBDKBRScoYMSEr+f217VEeyehQ3txNiSTmvMmWJbJAFYaO3iIW8BnTxO9NB7l4itgsvnqoA9k/SC+6OEHdKeZHwJAi2gwoDB+jFfz+nr7tZ7APiQDbGtXjp2+7zy2CjIdeE79POLcnANTS7Wd/k5ctNW6+/WYblR1+CixqHroohblpAeaa2/nejA5ev0RmeZ6aIquav1+aRqoxQompnm5orwFnmwgK5p8BmdOEI8FI0BohczoYbeA4ac2ZnAOpEwAZjq4DYHGesGdcf9QhgpcKCjFGlsWAJIHHF6C5M74/s5o73fgDMmqVNG73VCqVxOqZorr9jQTaAykojBdcXj83PrWD1i4PM3Mt/Pqzs8e+snkAevq2H2yM++SsiOFz8151gG11/Xx/bv87+L34rEX8n7yarTXCVhvA65fZ1+jh91scnPfPRr7wfAt7Grwk6yTuPNvCs59Lx6SNQsK83yMq209nax5Cl8SkLBtZJnD7YWd/r8PJFAXF9tpd/d+u1gongETD5xaV7F3N4rk37If26mBbqpMSNeSAENlf+S688UNoOQpqPcz7Eqy6Sxz/CXbWeXhwcwdfeK6Z2Y/Ucd3LLT3vgy8gs73WzeXPNbO+zIVOBb+70Ma5xcOIAXi6RXVxZ6MQ2btbwVIg9le5c4XzlzkLZJnPTxNrsHfLXTR2BV12zJm9bcaOrBvaY6q1oq2BPwETMMaaeOzXfjKmdDHvOpVkr3AIu7ygtLSUoqKiU/5eVFREaWkpIKzm6+qGmDGjEHVCle3HEq2yXenXPiZcOCOLp7dU8s7BBgKfnoUqGlmgw6DBIQRUm0mrOB9ECEmS+Oz8PH7z1mFe2lXDFxYVRHtIIyYQkE9qN6AIZMNlZm5iV7bX1NUxQ7KLX2xFipgaLrokkUGdNhGOrYeWY1yw2IBeDWXtAQ5WNTCjOC/aoxwRBreosFUlpYjNu8Lw0SWJAEnGVNAYMPsczJAq2Vo/EbxdwkpekxbtUY4unk6o2SEEbV0yTFsDwLZaN198oQXvSYUIRo3EY58yY3PVigx1ow3MWUxO6eTpjAZxLkMXEIEkIbcD3F2QUiICU8aUyLZM0JshaybU7hGbfrMIQFNyjqhsr/gY5n+Riycb+ePWTj6oCdBub8OaEacBDoWERadRkZVsoL7DRa3dGdcidV17sN1Ssh51nOwDR4NVM7J4cmM5Hx9rRpblmBHqFBQUBkeWZX784j72VreTYtLyyNULYyo+tHxiOnqNihq7kyMNnUzNVhyyGlvbyP/whyBJNJ77HTKnLhFtho6u6xEUf+y5lv985AbcJGklJqVqONzi61MFbtRIXDrFyPeWJZOZNMbvuSwLe2Z3lxA3k7LAlDrku0vJWZyZ3cgLZRLvV7pZXjDIOiJ/EUhq6G6G5iO94msIlVZUSicaPrfYh1hyhWDdXi3E6ZDQ7vfC/hfhwEui2h2EE0D+Yph/NWRMP8WtsKHTz10ftPP6sb6tkzeUu9lQ7mZ2ppaKdh8dbjHP0k0qHl2bysJw+7TLAehqAiQwpoqkao1e7K1CroAhjDYwWJmkc7EgW8vOei8vHXby9QVC66HkHPG+V26ChV85/WOrdeDsEAkgI02WHu/EY7/2k9Hoxdhby8TnUzza4UeBsFO2pk2bxr333ovH05s55fV6uffee5k2TVQk1dTUkJXVjyWHQkwyOSsxK9ubgwJZehwHL+KB5RPTSNKpaehwU1oTP1WrPX24lfkRUUJW8pvKWqhvd53m6NinpcuDPyAjSZBuDnOBrNDDzDwhth+o7SAQiExGvlqtZuXKleTm5qJWR3fx2lEjevs5pGSx+VH6tYeHziw2crZgMmd7Fcl6VU/QYPOx4VlBx9IcsXjEc9CZ08WmRWH4aIN92wM+0bsQOFe1m8OtfuwuOay+YrE0R4aMLIuA2NH14veCxWDOwuWTuf1tO94A2AwSE1I0LMzR8sgFGqYb7ZBSLKzm88+AlCLInil+z54tbALbq0dmIel1CrE9c5o4tyk1skJ7CGMKZM3ofR2gtx9h8yFwdTIlTcuUVA3eALy1rzbyY1BQiAC5NrFWqI3zvu2h9f54T0qdX5iCVi1R3+GisjUBRQsFhQTlsQ/KeH5nNWqVxENXLaAgNbbsfo06NWdOElboNz+9c8yd4mJxrVxzbC+TVLVMkmrIfP+HsO4n8OLX4f37AJmd+jP4T9sUkrQSqQYVXV6ZPQ1eXD4Zi15iZbGeB1fZ2P61LO67wNa/0O73inWxzyUuI+kq4HaIdbekFm2SCpeK9Xw47nj6ZFYWCEnn5cPd+AeLsejMYr0PcPzdU29Xa4KVzMNrBRmLcwQQ1fohsdyUBraCvmLhB7+FzX8WQrtaDwVLYfV9sOpukdx7ktDe6vTz+M5OLniqkdePuVBLsHaygV+vtPLc59L4wgwTGhWUNnrpcIt5dukUI/+7Ij08oT3gD86PGtBZIG8hFJwBObMhYwqYM07dX6m1wunL08XnZ4jPr2f3d/c6YRQHW241HwXnED4/VBqQ/RF1i4vZOTLaxGu/9pMxpYnEj/bqaI8kbgi7sv3hhx/m0ksvJT8/nzlz5gCi2t3v9/Pqq68CUFZWxre+9a3IjlRh1JgSrGwvb+7C4wug0yRGn1mlsn1s0GvUnDM1g9dL63n7YANzC2zRHtKQaOxQqpVHg4JUE2cUp7CtvI2X99Tw9bMnRntIIyKUlJFu1qNRJ8ZnYzSYmGFGp1HR6fZR2dpNcXrSiM+pVqtZunQpra2tUV+wy83HALBrM0jWJSkZwOGiUolFfHKW2Dy6HWCvZFFOCu+Wu9lR0cZ1wzhtzMwRnxuLLFwdzClZitg+UjQ60dOwqwmKlkPlJtZod/Kw/zNsa9Vzoa1OCMtDyCCPmTkSDn4vuBzQJJJ8mH4xAH/Y4uCE3U9mkor1V2di1SF6puvMkD5TBGE+UZ2BLglSS0SSUGu52ETLPlHRoEsCrWlogrnfKywa0yf3Js2MJuZMkTDQcgysJlFJn5wjehEefxtmfoaLpxj53WYHrx5o4QvnBE597goKUSYvxcTOSjs1cS62h9bK2eN8T2XUqZmbb2N7RRtbylopShv5WldBQWF0eftAA/e+eQiAOy+ewYrJ/ff3jja3XTiFvdXtHG3s5DMPf8z3V03lhrMmjImrZCyulR31xwHokI0k4UJd/pG4QaNnq+VTXFP7WVQS/GVNCisK9Rxo8nHC7mN6uoYJKRpUg61t5YBoVyRJQnSUJJGUGrLU1ujFXl9jFIKk1yVuCyWPD0bAJ5yZNHpRNW3LH37cQG/hwslJpGzupK4zwPsVblaWDPI9XHQm1O2G6u2w5Bt9b1Npg8/DMyxhMBbnCCDcu9QDCN3lH8LRtwAJpl8Csz4L1qI++4WALLOuzMWz+7v5oNJNsAspc7O03LPSxoyM3tdqUa6e7yw282GlmylpWuZmadEM9v/p94K7Q7zuyCCpehM6dEnCfSCleOiOiaZUUKm5eKKauz6QONbmY1e9lwU5OrAVgiVP2IEffwdmXTa0c/oiVzwVs3NkNInnfu0no1KDIVnECsyZYn4qDErYUYfly5dz4sQJfvGLXzBnzhzmzJnDL37xC06cOMHSpUsBuOaaa/jBD34Q8cEqjA7ZFgPJeg2+gEx5S+L0aVF6to8dF0wXThbx1Lc9ZCOfmTy+A0OjwWfmC7vnF3fFfzVZo0NxQIgEWrWKaUHLu0S0ktd0VAHg1qWJzbNiEx4+RhuodL1CXe1usTkEdtY6RZZ3vOIQrZXcspbU1NSBN/0KQycpQ2xgC5YAMJ3jpNIR7NvuEL3bExW/pzezXGOE9Mnsa/Tw2E7hUPXLc61YtQForxVJLHkLwJo3uNisTw5W1gQr3U3p4vVtrxa2iwHfwPf1OsFRL6pF0iaPnahtLQCtWbzXkgQlZ4u/BwOuF08WwcuPq320trWNzZgUFMIgVNke72J7fUhstyp7qiUThAXw5hMtUR6JgoLC6dhX0853n9mFLMOXlhTy5WVjkCw4TGblWXnrlrO4cEYWXr/MPW8c4ouPb457Z5Rh4ffhaxPr4HWBM/i05252aubhK1zBo8UPcmXdFbjR8eMVFs4uMqCSJGZlarlkipFJqdrBhXa/V1QUG1OFlXjhMihcLnqe5y+GrFlgTANfUDTvDq4vDSmiYrizceA9q9cpkmCTMiFvEWRMHlmCvlqD3pLFZcHaln/vP42jSqhve+txsVc6GZUG/D6xx0gUAn7xmve373ba4cPfietFy2HZzSJxN7iHkWWZd8tdXPJMMze+1saGciG0z8zQ8qvzrLzw+fQ+QnuIfDNcNcnHQqsDjatV9FfvbAR7tZhXPT/VouJZmwSZM0QVfdoUyJwu9rZFy4VTWDitCQ02MKaQLHexZpK4338PBOdEP/uk0yKpxOunMHzcDvB1x2+/9pMx2IQlvtK7fUiEVdnu9XqZNm0ar776KjfeeONojUlhjJEkiUlZZnZV2jlc72BKVmL0AFLE9rFj5bRM1CqJQ/UOqlq7Y856qz96K9uV+RFp1s7O4ecv7+dgXQdHGxw9rSrikQbFASFizMy1sLe6nf217aydkzPi8wUCAWpra+nu7iYQCJz+DqOIoVuIqZLRJipulQrK8NEliYrljOnQVg61u5g7+WLUEtR1ydQ2NpObHV6LoliZI57WKnRAvZxCVqp1dKy1xxtGmwhQ6c2QOhFV63HOVe1mS935MNcvBNgh9D2MlTkSFn4POGrEdWMKHsnI7W+34JeFneGqEq0I5iXnCLv1oWafS5J4zUypopLC0y16O9qrxcZaloPV7kYRgJEkEUQKBCB1IqRNFDaUY4XeDKnFUF8qkgWKzoS9/4GG/eDzMSFFw4x0DQeafbxZWs0Xz00bu7EpKAyBPJsIvtW0xXcwM2Qjr4jtsKQkjYffPc6WstZoD0VBQWEQ9lbbufrxLXR5/CyfmMbPL52JFOPr8zSznseuWciz26u465UDbC5rZfXvP+Cey2Zz8ZzcUXvcmFsr+93ousQ6OCsthSONE7is83aya1TUd4nxfXGWievmhVF96XMLYczrFInfGVNOFcL1wZhWaok43tMlKj61SeKyqwlajgtLckkV/EFUysuI39MnizWzJkKJ10YbV02W+dsBiQ0nXDR2+QfuPW/NF62YnG1QvxeKz+y9TaUWVfrDtJGPuTkCoip7739EIcScK3qd5WQZPvqdeB2SMmDpTX3ej5ZuP7eus/NBpYgDJmklvjwnicuni2SNUx/HLSroPU7xOuqSQG8Jvu+yqGo2WIVbmCxDwCsSAfTJQsCMVNxIpRJ7v7o9fG56Ji8ccvLGcSd3n2cVFfbFK2DPv8U+yesB7WnmoEYnKu8jREzOkdGmq1lcxmu/9k+iTxaJIpa8yH2GJShhRUS0Wi0uV/z34FU4lVm5VnZV2tlVaeeSuaO3UBtLFLF97LCZdJxRnMLmslbWH2jguhUl0R7SaQlZHioiauSxmXQsKUnjo2PNbC1vjXOxPTRPlM+RkTIj1wpURayy3efz8eSTT/Zc1+uj9x6ZvSLDU59sA138zveoErLey5sPR96A2p2YNBIzMrSUNnrZcaIxbLE9VuaIo6GcNKCRFAptiuAXEXRmEbhw2UX2f+txVqp387/Gs3HIepId9SJYdpoARqzMkbDwe0QlOUBSOvds6uRAsw+bQeLnZ5mF0G7NF5US4VREfBKdCXRB20GnPVidUS+CkQG/CBgZrJA+SQSrohGkDlkidrcGEwvMIuBVvQ2Kl3HxFCMHmh28uq+JL5479sNTUBiMkNhe254gYruyp2JhUQpqlUSN3Ul1Wzf5KbGfgK6gMN7YXWXnmr9tweHysbAohUevWYg2TtrFSZLEFWcUsqQkjVv+s5vdVXa+8+9dTM+xMDHDPCqPGXNrZb8Hm1skumdlZvGVnCT+uquL+q4AmUkqfnqWlYsnG0TyRCBUrS0JsVutFZch3A6xxtXoxZo2Yyok554+eVSjP7UtmDlTiKedDUJQ9fvE42v0onJca4z8ellvYVKGkUXZXrbX+/nvgW5uOmOQWET6FKjaAo0H+ortIYZZ2R5zcwRE4sPuf4nrh16DFbeIZILS50SygaSCRdeBtVf/2FHn4abXW6nvCqBXw1fnJvGNhWZSjZ8QS33uXgt4jV6I66kTxRzSW8Y2+fhkTGmgt7BY4yDFoKLNFWBbrYdl+XpRNR9Ktih/HyZfOPi51DqRfBLwR0Qsjsk5Mpr43GK/rhudz+WooE8Wz6m7BSwjL55KZMJeUdx0003cd999+HyD2AkqxB0hy7MtCWR5pvRsH1vizUo+JKIq9uCjw7wCGwB7quxRHcdICVW2K+0GRs7MXAuQeDbyXn+AdL/IWjVb05V+7cNFpRa2fGlThQ2/qx1qd7MwaCW/oyJ+baCdzZUAtKlSkLTKd05EkCRIzhZBjkLRxupc9R4k2cc/D2vE/IlgNn5M4XOLXpLA0UAeT+wWLaB+c76NjEATJGeJgMpIhPaTUakhKU3YXRYuEz9FIUvNRSK4GK1qMI0OUieI6hVJBQWLxd/LPwB6reQ317hpsjsGOouCQlTIDYnt9vguZlASmHtJ0muYnWcFYOsJpbpdQSHWON7UyTWPC6F9cXEq/7huMcmG8HtUR5vi9CSeu3EZZxSnEJDhgyNN0R7SmOFyuciVRdJpenYBNy9O5uLJBr6xIIl3rs7kkinGXpeCjnqxV/B2C2G9o1Ykpbo6RLKm3yNc1QqWiB9b4ciEUo1OtFVKnSCq47NmCOenlKLRWS/rTGCwceVk0ev7mf3dBEJ9v/sjbbK4bDne/+2JZCPfeLD3emcDvHkHrL+zV2iffFGP4OwPyDy2s5Mrnm+mvivAhBQNL1+ZwR0rrKRqPH0t4O3VIkHDYIXceWIfWrBEOB6YUqMntINw/cqYiibg4/wiIZCvKwuuMSUVlJwjru9/6fTnUmvFfIhg3/ZxRXerSADXJ5DYrlKLedFRI5ztFAYkbLF927ZtvPDCCxQWFrJ69Wouu+yyPj8K8cniYiG2H6jroMM1POuYWMIfkGntEgsFpbJ9bLhwhhDbt5W30uWO/WScxqDzQaYSGBoV5vaI7fHdN7dRCSBGjOnZFlQSNHe6e17XRKDR4SZXEolqltScyAlc4xGjDdRqyD9D/H58Q2/f9pouUSEQh/jswuqwW2MDVfwF9GIWgxU0BtFjT2/BjJNFqiM8uttJe7dXVL0nIj63CBoBT9SJ/qLfWJDEhZl20FuDFe2jlPSj1oiggcEi/l9jwULOmCp6t3u6RBIAQO0uAAqtGuZkagjIsGFfdRQHqaBwKnkp4v+0tctDtyc+v99kWaZOsZHvw5KSYBGDYiWvoBBTBAIydzxfisPtY1FRCk9cewZmfRSFsRGiUatYOU3E4DYeT5yiqdNRXVNNqtQJQEp2MVa9ij9dlModK6wk60+SODxdoDNC7nzRd71wmdhjWvOFxbelQPyeMVmsaWO8jcCAmDNYW+AlWSdR1eHn7bJB4izpQbHdXnnqbSq1sEJPFFqOisu0STB1rXh/tUYoPgfW/BZWfBdUGqrafVz1Qgu//qgDX0C05Hr5inSmpmpEBa+rXSRP5MyB7NnisnAp5C8WyRkGa2y1ELTkQPpkVuWIePe64y7kUALG7M8J0b1xP1RtH/w8al1QbHeP8oATlM4G8T8lxc7c8Adk7vmonW+/2cYvPmjnkR0OKuxh7j8MVtEyI1HjLBEi7HfdZrNx+eWXs3r1anJzc7FarX1+FOKTTIuBkvQkZBm2l8f/xrCt24M/ICNJkJoUA4HAcUBRWhJ5NiO+gMzOytiuQJRlWenZPsrMLRDfB0caHXTGQfLFQDQ4FBv5SGHUqZkQtLfbVxvfSRgn09jUiEXqBkBlzRPin8Lw0CUJMTokmFVvY2GOEKf3Nwfo7ozTeeMQVocenU1sXBUig94iRF9vt6iyBi437qDDLfPYIY2wORusuiNe8XRCl2hdcciXzeJcHd9fgHCEyJrZ21NyvKA1gDkD3J0i8CVJwu6+tQKAc4vFZ/KHx5qjOUoFhVOwGLQkB4WeeK1u73D5cHr9gGIjHyIRHQMVFBKBZ7ZVsbW8FZNOze+vnEdSHAvtIZZPFO2pNpe14A8k4Jq3H5oqDwPQgg3JNEh7LmebsIQ3pYoKcL1ZuGLlzBF7zZw5QjiKd8zZGK0ZfG6S+PU7b9l5r3yANUVIbO9sOFVYV2vB2zV64xxrmo+JS1shnPMDuPI/8LknYdVdkLcANDq21rj51NNNbK31kKSFe1eo+dOZbsyuOmivEnGJnHnCMcxWKBwKUopiPzkjtYSzpudjUMvUOPwcaA7GY635MHm1uL71kcH3yZJK3K6I7eHj7oTuZhGriCH+uquTR3d28coRJ3/f3cW9Hzu47pXWwd0wPolGD7K/J/FfoX/CFtufeOKJQX8U4pdQdfuWBLA8C/VrTzXp4qb/UiIQyuSPdds8e7cXj1/YnijOB6NDZrKBPJsRWYbS6jgVyOi1kVcq2yPDwsIUAB77oKw3wzbOaa8vB6ATEySlKmL7SAj1bc+eLTZ4nfXkuivITlLhl2FvZXxaJGq7xWYkYEgVgQyFyCBJkJwj+skVCrF9jWYHAH/fH6DJ3iGE6URClsHVjtwlhONKOYs/fioFrdcB1kIRUByPJKWLKiV9MmTOFH8reweAswvFOu+jEw78fsXyTiG2CFnJ19jjs5osZCFvNWox6kbe0zMRWFScikqC8pbuntdHQUEhujR0uLjndWEr/f1VU8lPMUV5RJFhZq6FZIMGh8vH/gRKZh+MrgZhgd6qyRQien94u8We3JLb/+0afWyLpeGg0UHaJG5fAOcVanD5ZL72aiv/PdBNWZuPY61eOtzB9W9yDmiThFjWeKDveVQaYRmeKPbQbWXi0pIvLpMzRautIAFZ5qfvtdPllVmYpeKNz1u4cn4mkq0I0qcJkT1voagUj7e5olJjzJ7E2flCC1l3/KQ15qJrxXvdcgzKPzr9uRQb+fBxtoKne+DPpyhwtMXL7zaJlmpXzzbxjYVmknUSx9t8rB/MDaM/9JZgi4743LuMBYoKqdBDKAs71oXSoRAS2xUhdWxZHCe2eaFq5dQkHXqNEhgaLULV7Xuq7dEdyDDx+QO0dIbaDSifJZHg5pWTMGhVbC5r5YWdNdEeTkRwNZ0AoF2dAgZb/G3GYgm1Bkxp4jJ7NgDSsfW9fdvjtErM5BZVyJrkNEVsjzQGqwiYZc8BSU2Su5E1aQ04ffDwLo/ILE8kfG6wVyEh45CNJFusZOvdIqCYnBnt0UUPg1UED71dPS4HVG0DYG6WjmSdhN0ts7+iPoqDVFA4lZCVfG2ciu31IQt5JSm1B4tBy4xcUc2UCEUMo8FTmyt4o7Qu2sOIOA6XNyFaMsYC/91exdsHIlM5J8syd/5vHw63j7kFNr6yvDgi540FNGoVS0pEdfe4sJKXZeQ24VzkNmQMfJyzTQjLRtvYjCvaJKVjTC/i0bPcXDrFgC8AP3jbzsr/a+SCp5pY9vcGGrv8Ik4Rqm4/RWzXipZtidC33e/tscr/n72k30NePuzkcIuPZB38/fozKZxzlqh4z54J6ZNEBXs899vWJ7Nqig04qW87gDkTZlwqrm99bPDqdpVaiMYKQycQgI7a0WvpNgx8AZnvrbfjCcB5xXruPtfKHWda+PKcJAAe2dEZXiGULgl8XYrYPgjDEtufe+45vvCFL7B06VIWLFjQ50chfgkJpaXV7XHbNy6EIrZHhyUTxEJ/d5UdV9BSMBYJVStnKvNjVJmbbwNgT5U9quMYLi1dHgIyqFUSaUnKXIkEBakmvnv+FAB+9fpB2rrifzPna6sCgv24E8GKLtoYbeD3Q8k54veqLb1926sd8de3PRAg2SeC7XpLpmIjH2n0lqBFmywCJMAdqe8B8PRhic6O2G5rEzZ+t7A1BCrkLKak68FpB3P2+P780RohKQ3cjh6XA1qOgrsLrVpiWb74Dv/wsCK2K8QWuTYhUse72J6l9GvvQ0j8ev9wfDryjCYnmrv4yUv7uPnfu2hMgMr/qtZu7nn9IJf+6SPm3rWOBb9Yzy9eOUC7UxHdh0tVazc/eG4vNz61oyeuN1xcXj+3/Gc3b+1vQKOSuPey2ahViZUYfeYk8Xnz8Xhol+P3YOwWiToq8wBJpl6nEI4HqmpPRCQJUkvQJdn4/VkBvrEgCateIlknoVFBl1dmc00w7pIuYjE0H+17DrUGAt7EENu7mkXCBfCTgwXsqu/7nLx+md9t7gDgxmU5WC2xZfcdKc6fVYRKgoPNPqraT4qhzL9GtB9rr4I3ftjT8u4U1Dpwd4zNYBMFl13MvRhq6/bojk72Nnqx6CXuXWlDChYHfXVeEjo17Kr3sq02jP97SQUBWTiIKPRL2GL7H//4R6699lqysrLYtWsXixcvJi0tjbKyMi666KLRGKPCGJGfYurtuV1hj/ZwRkRzsBo1w6wIZGNJcZqJzGQ9Hn+A3TEssCrJGGPD3AIbEL9ie8j6McOsT7hNeTT52lklTM1KprXLwz1vHBz2edRqNStWrCArKwu1OnoOFWpHLQA+vU0RUiOBLkls9guWiN/bylliFZu8HQ1+5DA2fDExR7qa0OAnIEtYbRkiQ1whcqhUov+ixwmTVwFQ0PwhJVYVnkBQXA0MnPwXE3MkHHzunoBIhZzJ1FQVII2vgOJAJGWIRB1bMSRlQsAHJ94H4KyglfwHx5QqU4XYIs8mLCZr2uJUbA+ulXOUyvY+rJmdDcBrpbWK6PoJDtWJdZw/IPPs9qooj2ZkVLV2c9lfNvLoB2XsrW4nIIsKsr9/fILzfvse/9udGC5eY83hemF16wvIvLRr+K9hS6ebqx/fwv9216JRSdxz2Wym5ySeqLZ8YjoA28pb8fgibwEeU2tlv5dUr1gHm1MzoLVc9A4OBIVEVzt0tYhe7caU6I0zGuhMkDYBldfFHcuT2PONHEpvzOFLs0T16p6Q4JwebO5ur+h7f5U2KLaH/50VU3MEoOmQuJCtODDxu82OPjf/50A3lR0B0k0qrj1vVjRGOCakpGeyOEe8H28dPym5zZgCy24Somn1VvjPl2HXv049gVor9p7DmBOnnCrW5sho0V4jYg+a2NAauuqPMXfHj/g/7a/51Zkassy9r32GSc3npot9yKM7wnQDVEmK68EghC22//nPf+axxx7joYceQqfTcfvtt7N+/Xq+853v0N4+PnrEJDK9Pbfj24IoJKamK2LqmCJJUo9DQiy3IwhZg6cryRijyuw8KyoJattdcVm50NuvXZknkUSrVvHry8Sm5tnt1RysG162rFqt5uyzzyYnJyeqC3aDM5hdb0pTxPZIoEsGrUlkA6dNAmSmtb6DQSNhd8Mbe4YemI2JORJMxmjCSmZq7GQ4JxQGq0hiKFwqsvSdrVydKfr0vVPuGrRve0zMkXDwucAhbFUr5GymJruFyDzeAor9YbQFbe26oWi5+Fv5h0Bv3/addW46u7qiNEAFhVMJVbbHa8/2kNiuVLb3ZUFhClOzknF5AyMSCxORo42938n/3lpFIBCGdWkM0dLp5st/30qTw82ULDMPXjGXTXes5B/XLWZSppnWLg+3/Gc3ZU0J1s5mDDh+0mv27Paq8Oxtg+yraeczf/6Y7RVtJBs0/OO6xXx+UUEkhxkzTMkyk5akw+UdnYKXWFortzm6KEC4FGWkZQhLbINN9A+2VwmRK3sWZE4fn63dzNlgzoDu3ljs3CzRwmxPQ1AwTQvayDtqwXeSiCpJIDOsyvZYmiMANB0G4LgskpE/rHSzrVbE9pzeAH/cIuJP3z6nCJMhgeM3ai1rpotknD9td1DfeVIC+oxL4TOPCqeDgBe2/RUqt3zi/jpxm29kDiMQg3NkNHDaRVK8KQb25V4nbHkU48tfZ4WqlLPU+7i47O7exKQgNywwIwHvlLs50hJGUoXGIKr4FfolbLG9srKS5ctFAMNoNOJwiAyha665hn//+9+RHZ3CmBMSSjfHsFA6FJqUyvaoEbKS3xLDCRvNPWJ7Ai+sYoAkvYbJmUJc2lMdf8lYocr2TKVaJ+IsLEpl5TRh/fbh0fi22Ez2iH7cBqUfd2RQa8CYKnovTzgPAE3lh3xtvsjK//FbNTS2x48o4W0TQfZ6OZXM1LQojyZB0VtETz3ZD8UrAFjjfRuAd6sCBFyOwe4dX3i6kTuF2F4uZzHV4gNrvqjwH+9ojZCULqzkS84Wf6vbAwEfRTYNhRY13gBsOTK+hK+f//znSJLU52fatGk9t7tcLm666SbS0tIwm81cfvnlNDT07ZNbWVnJ2rVrMZlMZGZm8oMf/ACfr2+w5r333mPBggXo9XomTZrEk08+ORZPL+7Js4mejvEqtjcoPdv7RZIkrloshL2nt1QOSyxMVI409H4n19idfDDMfYAsy7R2eTjS4GDjsWZ2VLTR6HCNyWvd5fZx3ZPbONHcRZ7NyP9dv4TPzs8nx2rknCkZvPHdszh7SgayDH/76MSojyfROHZSQsbRxs5+BeQDtR3c8cJeXt1bi9fft5r72W1VXPaXjVS1OilMNfHit5Zz5qT00R521JAkiWUTQ33bE9tKvrK6kmTJSQAJoy1L7AHyFkLuXEidAPmLILVE7CfHI2oNpBQLQS0oms/NFjHPfU1evH4ZbAVCRPV7ofnIqedIBBv5FmGRXxbIwRScCg9ubKPR4eGa5+tp7JbJs2i58sypURzk2HDF0gnMTAO7S+a2dW34T05wy5gMn30UioP7piNv9r2zWivmQyLMibHAUSdeK60p2iOBd+6GPf9GRYAN/nl4JS1S/V746A99DiuxabhokljD/21XGAnpap2obI+A60EiEnZkJjs7m9ZWIcQWFhayefNmAE6cOKFsIhKAkNge6z23T4diEx49Qu4IOyraRsXGKhI0d4rFglLZPvrMLRA9ZOPRSj5Uja9Uto8OSyeEXDCG11NZlmWamppwOp1RW3/4/AEyAiKokZyi9OOOGOYMCAR6BbOWo3xnhoeZ6WraXDI/fG7XkN7zWJgjXY0iyNogp5JiiYGNVyKi1gjbcHcXTL4QgOyWLdh0AVpcEntONAx411iYI2Hh6cQXrGyvIYuiNKOo6FYQhKzks+cIdwxvN5z4CICzioJ9248mdiC6P2bOnEldXV3Pz0cffdRz26233sorr7zCf//7X95//31qa2u57LLLem73+/2sXbsWj8fDxo0b+cc//sGTTz7JnXfe2XPMiRMnWLt2Leeddx67d+/mlltu4Wtf+xpvvfXWmD7PeCQvRYjt9e2uvgHQOKEuKLbnKJXtp/DZBfkYtCoONzjYWTm8tW4icrRBCKkTM0QS5dNbKod0P1mWqbU7eWVPLbc9u5szfvU2C+5ez6oHP+CLj2/h8r9sZPGv3mHGnW9x23920+gYHVc1rz/ATU/vZE91OzaTln9ct5isTySbaNUqvnXuRACe21Hd46qnMDSOBSvbU5PEvurZ7dV9bt94vJkvPLqJf2+t4uand7Hivg38/OX9fPeZXVz80Ifc/vxePL4AK6dl8srNK5iUmfjOUiEr+Y3HI1/wEktr5ZYqUbHcLKUKNyNDshAEbYWiot1gjer4YoKkTFHh3iXmQolNTbJOwuWTOdLqA5VGJCYANOzve19JGlYVcyzNEQBajgOisv22pcno1BIbawOsfrqJ7Y2QrFfz2ysWotckaIX1SejNKfzxU+kYNbCx2sOjOztpdfp587iT/+zvwhsApqwWB9fuhJPfP0klfo+A2B5zcyTSuB3QURMb+3K3A6qEVnuz7xau895O7cIfitsOvQKlz/c5/Lp5Yj328hEnDvcQNRyNHvxuUUGvcAphi+0rV67k5ZdfBuDaa6/l1ltv5cILL+SKK67gs5/9bMQHqDC2lKQnkZGsx+MLUFoTf5WoIRSxPXpMzjSTGrSxitU5FKpsT1PE9lGnp297tT2q4xgOPTbyyUoAcTQ4oziUmNM6LAtJr9fLX//6Vw4fPozXG52MymaHi2xEAqI1LTtmejPFPcZUIZQZLCIYIMvoyt7iwdWp6FQy7x5t499bT28nHwtzxNUixulQW5GUZIzRw5QqAkR5C0BvQfJ2cUPaPgDeOdICfl+/d4uFOTJkAn5wdaDuFlWAKnMGWq1eWOcrCAxWEXz1u6H4LPG34+8AvX3bPyxrF8k84wiNRkN2dnbPT3q6CMq3t7fzt7/9jd/97nesXLmShQsX8sQTT7Bx48aehPp169Zx4MABnnrqKebNm8dFF13E3XffzcMPP4zHI4JvjzzyCCUlJTzwwANMnz6dm2++mc997nM8+OCDUXvO8UJmsgGNSsIXkHv2r/FEQ09iqrJW/iRWo5aL5wgL238NUVBOdHz+AGXNQkj9ydoZALxzqLFnHgG4fX6e31HNjf+3gy//fStXPbaZix/6kJk/e4vl927g2//exQs7a3qS520mLZMyzeTZjKgkcHr9vLCrhvMfeJ//21Qe0SQWWZb54fN7ee9wEwatir9/9QwmZZr7PXZJSSpz8q24fQGe2qy8/0NFlmWOByvbv71S9JZ+ZU8tTo8oBHpzXx1f/fs2Ot0+ZuRYSDfraehw8+TGcv63u5Z9NR1IEnx/1RQe//IirKbx4Tq2PFjZvquyLeJFU7G0VnY3ChG1Q5MhbIw1xqiOJyZRqUR1uySBtxuVJJ1kJR8UTTOCFd2frGxXacATfrulWJojAHKbSHY/LudwzqRUrlxcCECbW2JSRhL/u3lFjxtEwiNJTCwq5K4l4tffbHSw4K8N3PhaGz98p51ffdQhHCE0enC1Q83OU88RARv5WJsjEaejXgjPuqRojwTqdoMcoFOXwau+xeRb1BTOPx8WfFncvvnPUFfac/jCHB2TUjQ4fTKvHB2ieK7WiSQMX/y1ix0LwvZW+fGPf0xeXh5Aj+Xcxo0bufTSS/nUpz4V8QEqjC2SJDG/wMa6Aw3sqbL3iCHxRo+NvCK2jzmSJLG4OJU399ez5UQLC4tioF/JJwgFsxQb+dFnbr4NEJXtgYCMShU/vbMaHEoAcTSZmWvFoFXR1u3leFMnk7Pir+qgqaGabMlLAAm1LUexkY8UGh0k54gAwMSV0FoG5R8yZf6XuH2Rml9uDXDP6we5fGFezGek++3Csrpbm6I4H4wmegtoTCIYMHElHHiJS+T3+A1zeKfCy/c9jvjva+5zQUctKtmPS9aSbgsKy4qFfC86k0i8cNRByTlw+HWo3QUBP8vy9aglOG4PUFbbyIT87GiPdsw4evQoubm5GAwGli1bxj333ENhYSE7duzA6/VywQUX9Bw7bdo0CgsL2bRpE0uXLmXTpk3Mnj2brKysnmNWr17NN7/5Tfbv38/8+fPZtGlTn3OEjrnlllsGHZfb7cbt7g3gdXSIHpperzcxA3EDkG3RU213UdHsIM0U3nda6HWKxuvl9gVo6RKB+zSTely9Z0PlCwtzeW5HNa/ureOO1VOwRUH4i+Yc+STHm7rw+mWMWhXLS2wsKrKxvcLOHc/vZU6+lS63jxd31/YI6Z9Eo5KYmJHEiklpnDslg/kFVvTa3v8Zjy/AvtoOfvn6IUprOvjp//bz3+1V3HXJDGblWUY8/t+sO8ILO2tQqyT+cMVcZueYB31dr1texC3P7uUfm05w3fICDNrYXLPG0hxpcrjpcPlQSfC5+Tn8/aMTVLU5ueP5PdS0u9he0YYsw4XTM3nw87ORJIm3DjSwq6qdbIueolQTM3KTKUgx4ff78MevWWdY5Fq0pCZpae3yUlrVyrxgwUMkOHleRP37ua0CALchA6+sBkkDMTBvYw5tMpjzwV4Jqg5mZ+j5qAp21Xn4/HQjUspENECgrRx/nzYMGnB1hv2axtQc8blRt9ciARVyDrmpZm48K509VXZK0kzcefF0kg2amPi8GzP0KXxmdgbv1zTyWrmIx05IUVPW5ufJPV3MzdLw6bzFqCo+xH/sHQI583vvG5DA1T3i/7OYmiORxuuGtirQJIM/+gndqqrtqIGdkkhqXFWixxeQYd5XUDcdRVW1CXn9nfgu/3uPG8jnZxi552MHz+zr5vPTh5jE5JPB6QB95GIssbQe6Y+hjitssX3SpEnU1dWRmSl6rV555ZVceeWVtLS0kJmZiX+8rGYSmLkhsT0OeyyD2GTZu8U/gNKzPTosLgmK7WWtfOvcaI/mVEKBIcVGfvSZmp2MUaumw+Vj84mWHouzeCBU2Z6p2MiPCjqNivkFKWwqa2FbeVtciu0dDeUAtGElTT/yIJ7CSSSlCwu4ouWw7XEhvHe1cN38ZB7Z206zy8euSjtLJ8R2Vrqqsx4Arz5FScYYTbQGSEoT9m2TL4QDL1HQsRM9Hg626qhtaiW3MN7Fdje0C6eECjmLySlq0MXf5+aoY84Ur1PufNEzz9MJlZuxFp/JuUV63il38/hHJ/j1leNDbF+yZAlPPvkkU6dOpa6ujrvuuouzzjqLffv2UV9fj06nw2az9blPVlYW9fXis6u+vr6P0B66PXTbYMd0dHTgdDoxGvsP2txzzz3cddddp/x93bp1mEzjp+2G3q8GJF5/bxP16cOrwl2/fn1kBzUEWlwAGjSSzKb33kaKn3zaMUOWIdekprY7wA+ffIdPF0cvCBuNOfJJ9rRIgJp0nZ8333yDaVqJ7ajZcLiJDYd7e7dbdTLLMwOkGkAjgVYFmUaZdD2oVXYI2Gk9dJx3DvX/ONcVwMc6iVcrVeyt6eCyRzaxIlvmovwASadZinn8sKNZQpJgVoqMWQv13fBalYq9rSK57QslPlzHt/H68cHPFZAhVa+mtcvLL59ax/Ks2LbOjYU5crRdzJFUncyG9W8x2yxR1abmpT11PcecmRXgImst7wrk0MQAAPYhSURBVKyvBUANLJIAB/gdUFoBpf2ePbHJ0qpoRcV/1m+iNjtyc+3kGP+GDRtQq6OXNGLpFC4Rddoijh31wNGPTnMPBQBvlxdQ81FFN6/vdGDrtnAO4O1o4M2d1afeYd/rYZ0/luaI0d3EqoAHt6zBqc/g7U17APG9AC18uOH07niJyvlZMCEJso1g1vp4tVLF+hoVP3zbjqVoJufzId2Vu9lwypw4EvwZPrE0R0aX6OtoK8u2kgw82yWSJiz+Dl7fKRKaNSlf5ZzGMsyuBtpe/imbJn4fJBVmL6glNXsbvfz1w2ryhlqgX7YP2Bfx5xAL65H+6O7uHtJxYYvtA/VW6OzsxGAIv/qvpqaGH/7wh7zxxht0d3czadIknnjiCRYtWtTzeD/72c/461//it1u58wzz+Qvf/kLkydP7jlHa2sr3/72t3nllVdQqVRcfvnl/OEPf8Bs7rV02rt3LzfddBPbtm0jIyODb3/729x+++1hj3c8MCdfZLbsjUPbZ4CWLiGQaVQSVqMS2I4Gi4N923dVtsVcNXMgINMaFNsV54PRR6tWcfnCPJ7aXMkj75fFldjeqFhjjjpnFAuxfXt5K19cUhjt4YSNq0lk17erU0gzxqcTTMxisIEpBdwaSCmBthNw5E1U877Ismw7r5RLbDzWHPNiu84p+mtLpjRFbB9tTGmi4iVzBpjSkLpbuCblII+3zWXDwXquLpwY7RGODJ8bHELcrJCzmJYmgU6xzzwFg1WI7AGfSNY59rawki8+k28sNPNOuZvnSlu5Za2LzHHQJuaiiy7quT5nzhyWLFlCUVERzz777IAi+Fhxxx13cNttt/X83tHRQUFBAatWrcJiGT8JbBu6Szm+p46M4mmsObskrPt6vV7Wr1/PhRdeiFY7tt8x2yvaYNc2cmwm1q49a0wfO57QljRw07/3sKFOxRXnL2Tl1IwxffxozpFPUvbucThynEVT8lizZharAzK5G8ups7vw+AMEZFg+IZXVM7PQqkfm2nIxcKvDzT1vHObV0no+rJfY2abl2mVFXLu8CMsn4kRef4D/7qjhT+8epylYWa9WSczISWZ/bQcBGVQSfH/VZG5YMfT/0+aUCn79xmE+bEni1i8sJS0p9lyOYmmO/GtrFRw4yKyiDNasWcCSTjfV/9qFXqPmwumZXDg9k/wUZe3TH0f0xzj4XhlySgFr1syK2Hk9Hg+lpSJ9YeXKlSQlRccmWZZlTuz8MUgwI9tAxuwMKDgjKmOJK/w+FpXv5/HDDTQ4Jc6ZlUsSqXAY9D4Ha6ZowRxMmPS5wN0FBUvD2mPEyhwBkA6+DAegXM5mdpaaNResEM5XCtB0BFqOgTUPJIlV82SufbmNzTUe/tC2hJXSX0l21bKmoBMypon7uNqFg0ThUkaSVRlLcyTi1O8FRwMkZ53+2NGmqwntrjpkJD7wzSDTpOKbZ+ehOvm9m/gr5P99k0zHPi4OrCNwxtcA+KjNzhvHXdTJFm5YMIR9mLMd1BrxeRGhjNtYWo/0R8iF7XQMWWwPbYQlSeLOO+/sk23u9/vZsmUL8+bNC2uQbW1tnHnmmZx33nm88cYbZGRkcPToUVJSeitP7r//fv74xz/yj3/8g5KSEn7605+yevVqDhw40CPuf+lLX6Kuro7169fj9Xq59tpr+frXv87TTz8NiBdj1apVXHDBBTzyyCOUlpZy3XXXYbPZ+PrXvx7WmMcDc/JsAFS0dGPv9mAzxd6GYDB6LcL1MSXyjiemZiej16jocPk40dLFxIz+e5lFg7ZuT0/vttQY3OwmIl8/ayJPb6nkgyNN7KtpZ1aeNdpDOi2ek6wxFbF99DgjmJiztbw1yiMZHgG7yIx2aW3KJi7SqFRgyYXa3TDxPNh+Aso/gvlfYkWuxCvl8PHxFm473XmiTJKnGQCtOV2xkR9tDFZR4e5zCpH14Ct8WreFx5nLO8ccXH2BN74THnwu/I4G1IgA0hqbpPRr7w9dkmgZ0NUIE84RYnvNDpBlzsjVsTBLzY4GP098cIwfro1cMDpesNlsTJkyhWPHjnHhhRfi8Xiw2+19qtsbGhrIzhaV/9nZ2WzdurXPORoaGnpuC12G/nbyMRaLZVBBX6/Xo9efOoe1Wm1MBlhGi8I0EXCstruG/byj8Zo1d/kAyLEax9X7FS5r5+azraKdJzeWc/vz+3jtOyvITxn7NWMs/F8dbxYVQVOzLWI8wLfOmzJqj5eXquVPX1rIlUeb+dXrBzlY18Gf3ivj8Y/LmZqVzMRMMyadmsP1Dg7VO3C4xJzOTzFiNWrZX9tBaY0IrK6emcX3V00N24nrS0uLeWpLFZWt3dz87z3864YlMdsCKRbmSHmL6Bc7JThHslO0vHyzkswzFOYVpgJl7K91RPR9PLngLppzxO7opoBGANJT09Em2UD57jk9Wi15+YXkmOqp65Y43OpnSZ4ZjKngbEVrLwNrjjhW0oG7A1SBsF7bWJkjALSVAVAm5zAlTYfWmKy03AqRMQFczeBpB1MqWjU8dFEKn/pXE3vtetry5pLashPt8XcgW1iQo9WLhG+VLFr9DZOYmiORxN0J7jYwWWGESYIRoX43ANXqAjow8+mJhlPXHBmTYMVt8MF9qPc/h3rulWC0cNUsE28cd/G/I05+fJYVg+Y0eppeD14XSH7QRjYJLlbnyFDHNOSZsGvXLnbt2oUsy5SWlvb8vmvXLg4dOsTcuXN58sknwxrkfffdR0FBAU888QSLFy+mpKSEVatWMXGiqDyRZZnf//73/OQnP+HTn/40c+bM4Z///Ce1tbW89NJLABw8eJA333yTxx9/nCVLlrBixQoeeughnnnmGWprha3Qv/71LzweD3//+9+ZOXMmV155Jd/5znf43e9+F9Z4xwtWk5biNLEB3BuHVvIhsV2pWo4eWrWK2UFBdXelPbqD+QShHnA2k3bEGfMKQ6MwzcQlc3MBeOT90/jtxQhNneJzRKuWSIlCf8XxwvzCFFQSVLc5qWt3Rns4YaNyCIstv96mCKmjgTFVJDEULBW/Nx+GrhaWFwthYk+VnU63L4oDPA2uDgyymNdGWyaoYjO4mjDokkT7AVeHENuBqc5dgMy2ej8BtyO64xsp7k6c7SLI2CBlkGfRgEZZ6/aLOQt8XshfLF4jVztUb0eSJG5cJDL1n9paRYcrNvvBjSadnZ0cP36cnJwcFi5ciFar5Z133um5/fDhw1RWVrJs2TIAli1bRmlpKY2NjT3HrF+/HovFwowZM3qOOfkcoWNC51AYnOk5Yk4eqBtatUSs0BBygLIqSamn40drpjO3wEa708tNT+/C44t+T89ocKyxE4ApWWObiL9icjqvfXsFf/nSAqZmJePyBthT3c4LO2t4anMl28rbcLh8pJt13HXpTDZ871xe+85ZfPCD87j/8jm8cvMKHr1m0bBaXiXpNfz9q4tINmjYXtHG/3u+dEC3UAU43iTmyKQYKtaIF0IOpUcaHDg9idfetbmhGpMkYjQ6WzbolTkyZIypzM0SMa09DcG1r61AXLZV9B6n0oDsB38cr49bjwFwXM5lcoZJEdpPRmuEtIngdfa8xxkmNecUif3kdkMw5lJ9UpKtWgMBrxDcFU7F2Qae7tgpvKnZCcCr7rkArJk8gAg+9VPCPdLvgV3/B8CKQj15yWo63DL/2T8Eu3R1MBHDG39x3NFmyJXt7777LgDXXnstf/jDHyJi7fbyyy+zevVqPv/5z/P++++Tl5fHt771LW644QYATpw4QX19PRdccEHPfaxWK0uWLGHTpk1ceeWVbNq0CZvN1mM7D3DBBRegUqnYsmULn/3sZ9m0aRNnn302Ol1vIHz16tXcd999tLW19amkB3C73bjdvR8kIZsAr9eL1xvHXzphMDPXQnlLN7sqWllWYgv7/qHXKRqvV71dfCikJWnHzfsVi8zJs7C9oo2dFa1cOudUO5VozZGG9i4A0pJ0yvwYQ752ZhH/213L66V1HKtvpyjt9IuRaH6O1LSKjX5msh6fL4bFvDhHr4IZORb21Xaw+VgTF8/JGfJ9T54X0fp+NnYLS2eS0vDKKlA+UyKLSg/6VPB40KRMQGorw7//f2Qv+DIF5k6qOmHj0UbOG8CSNepzpLUSLdAum7BZLMp3zlhgzIS2GsiYjUatQ+dpZ666kj3eIg5XNTFpQt9AedTnSDh0tyN3iM8cvykDHzqQ1crnTn9okkBtAJ8XdcFyVCfexX/odQK5Czm7SMckKxxrD/DPj0/wjSHYdkdzPTIUBhvX97//fS655BKKioqora3lZz/7GWq1mquuugqr1cr111/PbbfdRmpqKhaLhW9/+9ssW7aMpUtFwG3VqlXMmDGDa665hvvvv5/6+np+8pOfcNNNN/VUpd9444386U9/4vbbb+e6665jw4YNPPvss7z22mtj8vzjnVm5QiA5VOfA6w/ETTJwfbsQ27MtStLP6dBpVPzpqvlc/NBH7Kmy89t1h/nRmunRHtaY4vMHKGsS+/DJmeGL1iNFpZK4aHYOq2dmU9bcxbFGB0cbOun2+pmSZWZatoVJmeY+/3+FaSYKh7BnPR2TMpP5y5cW8pUntvLirhqm5yTz9bPjvLXNKBFKyJiYmUAWw2NElsVARrKeJoebA3XtLCxKrBZn7fWiYrkVC6kGG2iUdgJDRq1hbmEqb55oZE+9KD7Cmg91e8DeT892v2dsxxdJWkRxT1kgh3OzbdEdSyySnAPWJmivEXbywBm5Ol445OQF5wJWAbRXQWeDSF5WaYUwH89zYrSQZdHiLVaS32UZX/UONMBHgVl8dqqRpXkDFARJEsz/Emz4JRx5HRZ/DZVGz1fnJvGrjzr4xQftFFjUrCwZJKFWpRbJOYrYfgph92x/4oknIvbgZWVl/OUvf+G2227jRz/6Edu2beM73/kOOp2Or3zlK9TXi4BSVlZfoS4rK6vntvr6ejIzM/vcrtFoSE1N7XNMSUnJKecI3fZJsf2ee+7hrrvuOmW869at62Ofn8hoOyRAzds7j1DcfWjY51m/fn3kBjVENlaLsbvsjbz++utj/vgKgkCzeB8+2F/J6+oTAx431nNkR3BcktuhzI8xZoZNxQG7ijuf/oArJg69oiManyN7WsQ80ficyjwZZdICKkDFCx/sQVW9a8j3CwQCZGQIkfXdd99FFYWs5RJ3A0hQHsih7L3NY/7444kJpmXMbivDcfgD3pcuIN+koqpT4ul3duA83v/nSbTnSEbHPpYD9XIqh2s76FQ+S8aQVhYnzSSnYxeX6newp7uIpz48zOJDfde00Z4j4aAKeFjjqgOgUVvI64cccOjtKI8q1ukmU1rAMt7FV7Wdt7aXI6s0LEmXONau5rH3jpDnOMhQu05FYz0yFLq7B64+qK6u5qqrrqKlpYWMjAxWrFjB5s2be+b9gw8+iEql4vLLL8ftdrN69Wr+/Oc/99xfrVbz6quv8s1vfpNly5aRlJTEV77yFX7xi1/0HFNSUsJrr73Grbfeyh/+8Afy8/N5/PHHWb169eg96QSiINVIskGDw+XjaEMnM3Ljo199XbCyPduqCB5DoSDVxG8+N4ev/98OHvugjLMnZ7Bicnq0hzVmVLR24/EHMGrV5NmiN2dUKolJmWYmZZr51Bh2ElkxOZ2fXTKDO/+3n8c/PMHXVkxQWh5+gk63j7pgEs+kjLFPyEgE5uRZeedQI3urIye2q9VqlixZQllZGWp19Fy6nM2iAtuuSiFVaxKtoxSGzNziLHi/kd0NQdHUki8uHXV9D5QIW1iNlTkCEGgtRwWUyblMyIyP9dSYolJD6gTobhVucAYLi3KEIPteYxJySi6SoxZq98CUVb29uEcotsfSHIkY7g5R2W6IjXnW3lCB1dmMW9YSSJvGvefbkAbrpT7hXNjyKHQ1Qel/Yf7VXD8/iQNNXl487OSbr7fyf59JY3HeYMkEkiK290PYYnskCQQCLFq0iF//+tcAzJ8/n3379vHII4/wla98JWrjuuOOO3p61IOobC8oKGDVqlURqeiPBzLK23jpb9to9BlZs+acsO/v9XpZv349F1544Zj3Wdj+6kGoqmL+9ImsuWDymD62Qi9z2pw8+bsPqXepOP/CC9Br+36hRmuONGysgKOHmVqYy5o1c8bscRUgY0YbX/zbNra3anj46+di1g/+FRTNzxH71io4cpDJ+ZmsWTN/TB97vKHa38D7z+yhCQtr1iwP677RnCOBgEzbzu8CcEZxKmlnny96WilEFq8LqrbCpIuQn3sGm7OcNXkOZEMWmxo91PmTWbPmzIHvHs05srsDjkOjbOOyFTNIyyke08cft7RVQMN+JNN58PEuVqt3cDeXodIZWPOps0+x84/mHBkybgfs/Q9qArTJZhYUmFmzsBiyZkR7ZLFLRx3U7QbzauTav6F32VljOYQ85VNc4PXyXHkLHV6JWUvPoTht8Aq6WJ8jIRe2/njmmWcGva/BYODhhx/m4YcfHvCYoqKi0yYennvuuezaNfSEOYVeJEliVq6VTWUt7KttjxuxvaGnsl0RPIbKqpnZfGlJIf/aUsltz+7mzVvOJjVpfLQhOtogWrlMzjKPW5H5yjMK+c1bh2l0uNlZ2cai4sSqPB4pZUEL+XSzHqvSxm1YzM4XYntpBNuBqtVqzj//fNxud1RFMr+9BoAujU0I7Uple1jMLslBopQaR4Dmbj/ptqDY3tXQ90BJLWyxwyBW5gg+N6ruJgC8xgxMJsUho18MViG415eCzsTEVA1WvUS7W6Y9eTI2Ry00HRJie4gR2sjHzByJJM428LlA07/L4ljz9kcfczlQKk3ioUty0J+u57pKA3OvhI0Pwf4XYe5VqFRq7r/ARoc7wDvlbq5/pZXXv5hBgWWA2L1GDy57pJ9K3BNVsT0nJ6en11uI6dOn8/zzzwOQnZ0NQENDAzk5vdayDQ0NzJs3r+eYk3vIAfh8PlpbW3vun52dTUND3y+Q0O+hY05Gr9f32OKdjFarjckAy2gwrygVlQQNDjetTj9Zw9xER+M1a+kWVopZFuO4eb9ikeIMDelmHc2dHg43OVlYlNLvcWM9R9qcwhI8w2JQ5scYs2xSBsVpJspbutlYZmftEC3Do/E5osyTsWPpRLE4PdzYydEm57CCzNGYI412B5nYAUjLykZrUHqCjQpaLVgyhZ1Z0TIo/wjtoZc4a8kP4N1mDjd20e4OkG4ePNEhGnOk3V6PHmjBypnWJFTKZ8nYYM2GjnLImwdAnrecdNopbbKild2gtfZ7t5he57v9YC8H4FCgkBlpGrTGZPH/odA/lkywJ4PshsmroPRZNMfWwfQ1aNV6ptlk9rRIHGl0MnmINpOxOkdicUwK4TErzyLE9pp2vrCoINrDGRKhCtRsq5JoGA4/WTuDLSdaOdbYye3P7eWvX144eOVRgnCkIdiLO3P89lnWaVRcOD2L/8/em4fHddZn/59zZl80i2a07973PbEdEsjixCSGQglLKUtpgTZp2hcCLbT9UQqEQl8KZWlJKYUS3rKEQCmEJJA4+2IncRzvqxZb+y6NttHs5/fHMzOybNmWpZk5M/LzuS5flmaO5jySHp05z3N/v/f9ywOdPHakR4rt55G2kC+RAtlcSeW2H+nMnNieL6hjXQBEzF4w2UWWtGTWFFlNNBRbaBkKc7Q3zI2pzPaJQYjFwJj8eRrMomO3EBkVBRmTmpkSjwuMshjworhrIDgI472omsbmUgNPt2s0qvVcw3Mw1DJ1rHrlBRgLnkRCFHbnSVb7ZDjKksGnQYHyqkX4nbMs5Fx+B7z2AzEXTj2eXCcrfPuOYt758wGO9kf5XVOIj266yL2b0QKRCYjH5DX5HHTdFX7DG97AqVOnpj12+vRp6urqAGFJV15ezlNPPZV+fnR0lFdeeYXt27cDsH37dgKBAPv3708f8/TTT5NIJNi6dWv6mOeff35ant3u3btZvnz5BRbyEoHdbExnaR1qD+g7mCukf0xUXJUUyTdWPVEUhQ01HgAO5tEcGkjOj1m/+UgyhqIo7FwtCpweP9aj82guzeC4sEm6nIAnmT8lRRZuWOpH0+AD338lvdFyOTRNIxAIEA6H0TQty6O8kMHus6iKRlgzYSoql0J7NnGWiDyoZW8Wn7ftwWdTWeEVv/c9zYMzfpnecyQ8LDaFggYXar5keV0NWJzgLBfV2r6lKGjcZDjA8UGNyOTYtEP1niOzJhYiOtwGwEmthuU+VdpnXg6TVWQNhkanOjN6jkAwAMAqv+iqON618DakJYXHmiohkBwtEIEkntDoTdrIV+poCV6I2MwGvvkHGzAbVJ480cuxrgIVNa6QxuT9/bKyq9se/Pa1otj8d0e78/u+Qwea+2VBxnxJvZc09Y8zEY5l5DXz5V7ZMika5jSrW3TmSq6YNdVC/zjaExTZ3SgQD0OykAEAo1l068ZmbxueL3OE5FqpS/OxxGeSYvulMBihYj3UbIWSFWypEOuil0NCj2Ok/ZxjTRCZ3R7dxcibOZIpwqMQGgFzfrxftT/3Q9YrzUxgpWrdTbP/QpMNVv+++Pjgj0AT8YxWo8LtS8Tfz+G+6MW+WhTnxCPimiFJo+vO8L333svLL7/Ml770JZqamvjJT37Cd7/7Xe655x5ACDMf//jH+eIXv8jDDz/MkSNH+OAHP0hlZSVvf/vbAdEJ/+Y3v5mPfvSjvPrqq7z00kv8xV/8BX/wB39AZWUlAH/4h3+I2Wzmwx/+MMeOHeNnP/sZ3/zmN6dZxUsuJFUVeTiDFkS5YEpslxvbepOPYvvghBRR9eS2pNj+zMk+wrG4zqO5OAPjqaIMOU9ywb/94SbWVLkYnIjwvu+9TOvgxGW/JhqNcv/993PixIlpxXS5YqxX5MYNKV6wycK9rGLzioVM2WqwekT17OknuL5adHPuaeyf8cv0niOxUVFUFDK6xCJVkjuKks5VtdsAeLNxP5GEwumO6YUZes+RWRMNERoUG0gdhmp8NuQG0mxwloqsQW+9+KfF4cTDAKwqEz+/453D+o1PIkmyulKsu493jxJP5P8mZP9YmFhCw6AqlMoC9ytmdaWbNy4Tzk4vNA7oPJrckLaRv8qF1BuW+nGYDXSNhDhUYPts2SZVcC3F9rlTWmSlwm1F08hYIU++3Cs7w8LR1uTw5U03aaGxNiW290VER6qzVDwxfE4Xs8EiLMNjs89hzpc5QmBKbF9a4pDNEJfDYAKHH/xL2LJY6GcPDybjBYKD4h+AahJiamLu+7d5M0cyRWgE4lHxd6Q3PUdYcvYnADxV/F6Uqk1X9vVr3yXcQsa64MRvph4uFftXR3ovUXhjSBXnzC9mYKGh65Xnmmuu4X//93/56U9/ypo1a7jvvvv4xje+wfve9770MZ/61Kf4y7/8S/70T/+Ua665hvHxcX73u99htU4t6n784x+zYsUKbrnlFu644w6uv/56vvvd76afd7vdPPHEE5w5c4bNmzfzyU9+ks9+9rP86Z/+aU6/30JjXVIoPdQR0HUcV4oU2/OHDTXiZu5ge/5sZEoRVV821ngoLbIwFo5dtBs1H5DzJLe4bSb++0+2srysiN7RMH/8g315v9kcCYgK8FGDG2yyuj6rGC3gKBX57akO1VO/5Q11wmbyhcb+vKyQVifEplDc6hELEUnusHnB4oLydQBsV46jkuBQ+4CwfSs0ImMYR0WHQcRZLTbC5Jy6PDYvWL1iQyTljNHyNACrSkU37rHusYt9tUSSMxr8DuxmA6FoIp1bnM90BsQmfLnLiuEqzd+eL9cv8QHwUtPCF9sTCY2WflFIm3JPvFqxmgzcvLIMgN8e6b7ksYPj4XS399VAc3KOLC6RYvt8SHW3Hy6wfdzL4Y2La6XDWyLz2ufI6ioR13dkILludiet5IfPTh1kMEEiVpji2UhKbPezZJYRURLBuvoyTCo0TjqJ2fziwZ4j4n+DCRLRwpwT2UDTYLwnP1zmwmMknroPlQT/E7+eRdt//8qLTKwuWP9e8fH+/yeKCIC1pWKv4exInJHwRfZPFAVQrqg452pA9zKft7zlLRw5coRQKMSJEyf46Ec/Ou15RVH4whe+QE9PD6FQiCeffJJly5ZNO6a4uJif/OQnjI2NMTIywn/913/hdE6/QVu3bh0vvPACoVCIjo4OPv3pT2f9eyt01p+T95OPm9gzEYzEmIiIaisptuvPuho3igLtQ5MMjufHG3PaRl7OD11QVYXbVosNhify2Ep+IG0jL8WMXOF1mPnRR7bisZtoGZjgxTzffIyPiA0y0bUsrydZx1kCaLB0p/i87xhbvUHMqkbnSJiWgcu7IeQaU0jMYcXqlZ3tuUY1CCt5ZxmYbNiZZKXSxuGeCEQLLHMukYCRTmyxERKags1XLYR22dl+eVQDuCpEzuCSHWJDYPgsDJ1hhd+EgkbfeDRdqCuR6IVBVVhVITbBj84QbRCKxhmZzJ9OoO4RsalW6ZHXobly/VKxmf3q2SFC0fx1+8oEgxMRIvEEigIVcs5w+xrhvvPboz0X3WfrGQnx5m++wO3feIG2wQK7b5kDiYSWdjZbJDPb58W6qoWX2x6PxSjRhgBw+8rzo5u0AEm56HROKAyPT4I72cU80nnhwQVoCx0fFoXJ3RRTX3J1F3ZdKdaiYtYkI7b6bYvFg30nxf8GE8Siwi5cAuExmBwR0XV6s+ffUCf6aE2U8n3LH7G6co7zfu07RTzH5CAc/hkAXptKjUvMiWOXspJXFIhIsf1cdBfbJfnLinIXFqNKIBjleHdhZIkNjImLv81kwGE26DwaictqSlcm54OVvKZpaRHV55Aiql6kctt3H+/N2+7lVFGGT3a255SSIgtvWy8srH6xv0Pn0VwadVzkxkVNLtlhmgusHmEl7/BB8WLQEtibH+HacnEr++zJXn3HNwO2iHDvMDm9QvST5BZ7ssihbA0A16onONQfn3fmXM6JR2CwGYBWrZRFfofYZDQYdR5YgeDwizw6sx0qNorHTjyCw6zS4BbXjxMyt12SB0zltk9fd2uaxjvu38Mbv/JM3ohuXYGU2C67C+fK4hInZS4LkViC/a354wKXDXpHhWjjd1owGeQW5I3LS7CaVNqGgjNafYeicf7sR/vpHwsTiSf43bFLd8AvBAbGw0TjGqoiHDMkc2dtqmlqAcUUDPecxagkiGoGvMVlUmyfI26bibpiYcF/tHsCPMnO9rHzrjGKKgpVC4zIkBDb+/BR7MoDIbSQUA1sqRVFn8e1evHYUHPyOaOI4pJiuyA0IopR9C58b3sZGh8ngcK90T/nphVlKMoc3aZMNtj8IfHxoQchKu7z1yWt5A/3XcpK3gQR6RR3LvJOV3JRzEaVm1eIDJeHD3XpPJrZ0T+eXMgVmed+kZFklPXVHiA/xPaxcIxIXNifSOcD/di2yIfLamRgPMLrbfm3uRSKxhkLxwAokWJ7znnnZrHoe/xYT151cZ2PKSQswrHIPO6cYDQLq7vQGCxLdre3PMeb6sVG/3On8kxsj0Wwx8Wiw+ry6zyYqxSrG8wOKFkBwLXqSU4HIDhWYJuPsRAMNQFwUqtlmRdx3ZHMDksR2H2iC2HJDvFY60sArPSLgoXjHfkbayO5epgS26dfo5r6xjnePcrIZJS///XRvHCc6wqINbcU2+eOoii8YYm4P8h3N6f50j0i5kuFW4qoAHazkRuXiX22bzzZyGRkytlA0zQ+++ujHDpn72T38Ty7x80CXck5UuayYpQFGfMi5ZJydnBiwbhmjHQL0a8fL0abU66958GaVDFGbxhcyc72if7pBxnNEC6Mhrtz0UZF0UDE6kPJB4vvAmNLQwkAz08kizACbdMPkDbySQv5Pv0LfiIT8MLXAPhhfCeva8vYtXSe9+Qr3gLOUtGY8KqI5l5blsptv8S+rMEsviaxMN5vMoG8i5FckrdtEB2GvznYRSJPO1DPJZ3XLgWyvGFjrQeAfWeH9B0IU93KTosRq0l2GeqFyaBySzKr7ndH889KfnBCVO2ZDSoum+wczDVrqlwsLysiEkvwyOH8LfSyhoU4Y3B4ZWd7rnCWiO7U2m3Cripwlh0+sUH9ytlAfm0oJTctopqBIrdP58FcpRhMwkbe2wDANvUkCQ2OtfWKhXKhEI8QHmgF4GSilmWuhCgikMweZxnEIlB/vXCZGO+BvhOsKhUbccc6A/qOTyJB3P8AHOsanbbufqFxSoh97nQ/v82De+d0Z7sUT+fF9Umxfc8CF9t7RqeEVIngg9fVYVAVnjzRy53/voe2wSD7W4f5u/89ykOvdaAq8JU71wGwv3U4byL5skV38poiCzLmT0mRhSKrkYQmBPeFQLDvDADDqlcWnM6TtcnCvmODgLtKPBgchOg51xiDWXS2xmO5H+A8MAXF/ZFi98mYvzmweYnQf54YrRMPjPdBKHkNUZCd7SBE7skh/S3kX/kPmOhn3FTM/42+h0UeQ7qIfM4YTLDtHvHx8V9D7wnWJXPbD1/KRt5oEYUYBRg9kS2k2C65JDcuL6XIYqRrJMT+POxAPZ+02C67lvOG7YuFyPB6a2Ba1bYeyBzu/GFnMrf96ZN9Oo/kQqYs5KVDhh4oisI7N4sq63y2knfGRAGRpcgvxfZcYSkCZwUoBqjYAEB952NUOiAc09jbkkfdqcmYgQHclHg9+o7lasZeDMWLwGDCq4yxWOniUHewsHLbY2GiyQzCfks1TrOifzV9oWHziAIF1QhV14jHTj3G6hJRrX+8p8CiBSQLkiUlTixGlfFwjNahqWtUquu52is6Vj7/m2OMhfR1/ukakTbymSDV2X64c4SRYP66Oc2XnhEppJ7PdYv9/PgjW/E5zBzvHuWN//wMd/77Hn76qugk/JvbV/Dua2pYXekiocFTebhmziSpzvYKeU2ZN4qisLRUCEGNvQvj/iY6LPYExg0eESsmmTNrkrntR4ZUsLjFujoRg5FzupgN5qR4NkMOcyIB3Udg6Ex+dTqHRjDFxb2TxVUq47bmgN/toMFroptiIiYXoEHfEfGkaiq8KLZsEBqBaAhMdv3G0PYynHgYgL8M3UUIC7+33J6ZvetFb4K6N4CWgGe+yOpkz0j7aJzhycTMX2Mwi0KMfLoe6IwU2yWXxGoysHONyFf+9cFOnUdzeaTYnn8s8juocFuJxBO81qpvd/vAuMzhzhe2L/ajKHBmYIK+0fyqgJuaJ1JA1Yu3b6zCoCocaAvQ1Ddz/o+qqmzatAm/34+q5v52xpsQBWgOt08u5nKJq0J0pi6+BQDl7Au8qVo4lTx3Xm67rnNkXGyK9mtu/G65KaQbVjfYvOBfDsBW9SSHemMQFpsFel9HZkV4HOuE2GTUXDVi/uudEVdomB2i8CIyBkvEtYPWvaxKdgC0DEUIhheu0CUpDIwGlRVJ+9/DHQEAIrEELycLyb75Bxup99npHQ3ztSdO6zVMQNrIZ4oyl5WlpU40Dfa2LNzu9pSNfLkU26exbZGP3/zl9axP2jo7LUbetqGS7//RFj56wyIAbl0lCtSfXOBW8t3SLSOjLEmK7U198xfH8uJeeVTsRYfMxWCSe3nzIeWi0zamMRKcFGtrgKGzUwddSjyLBoVDVM9haH8VRjryY46MiLXSkOakxFukzxgWAJtr3YBCl1k4w9F7QvxvMIn18xzd4fJijswXTRMNFUYdYyz6jsOTnwPgocQtPBNdw7YqM3dtzuB+0xv/ShQ1jXbiPvR96t1ir+3IxXLbFVX8bGRne5oCneGSXPJ764WVyGNHeojGL1LJkif0JzuXS5zyJj1fUBSF6xaLqv2XmvTtOkyJqLKzXX/cNlM6T+zlM/pHDJzL1DyRCzm9KCmycNNykRn1i/0zF3oZjUbe/OY3U11djdGYW7E7HAnj1USOmbu4LKfnvuqxecFRCmWrxKIv2M9bvWcBeP709K4fXefIiMiM69c8+N3S8ls3jBZwlIB/GQDXqCfZ0wOJSZGJrOccmTUDpzBqUYKaBa+/XAjtelbTFyqOUohHofa65LVjgNKRY/htChpwqmPhCl2SwmHbomIAfv6a2DQ+2B4gGInjc5jZWOPhC29bA8BPXmkjEtNnXR6KxhlKRi5VuqXYPl+uhtz23mRhdbm0kb+ASo+Nn991Hb++5w3s//sdfPMPNnLLyrJ0h9qOZPTaC40D+RWXlGFSBRkV8pqSEZaWCrGxqX/+Yns+3Ctbkvbgms0r7cHnicduTjvlHOuLgDuVz906dVCqQ3Ym8Sw6KYR4V5V4vv8UxkRE9zlCQLiAdWl+Kovd+oxhAXDNolIADsbqxQNDTeJ/g1lEcs3RSj4friPzJjwKEwOimF8Phlvht38DsRB7tbX8f5E/Yn2Zie+9tRirMYOOrDYv3PAJ8fHRX/DnticxEOfIpazkQVwbJIAU2yWz4LrFPvxOM0MTkbxfBMrO9vzkDUuE98hLOs+fKRt5OT/yga0NYl68kk/Wz8h5ki/cuUlYyf/mUBdanuUrD/d3oyoaMU3FVVyq93CuLhRF5MsZrUI0AzaPPIFR0WgZDNE2mB/24JODokhkCBdOqyzw0hWHD0qE2L5dPcFgCI60Fkhuu6ZBn+goOKXVsNxnALMdjHJOXTE2ryhSUIDa7eKxxt+yKmklf6wjvwr/JFcn799ah6oI4fVkzygvNvYDcN0SP6qqcMNSP3azgUg8QfuwPu93qbx2h9mAy1agG6Z5RCq3Xe+i9GwiO9svjdmosr7Gg8VouOC51ZUuqjw2JqNxXmzM7724+TAVTSHnSCZId7YvEBt5e1g4OxicPlEwKZkXqdz2I0OqEM0h3RmeRgEiM4hn0QmxPlENQnSMR4UNvd4kbfC7NR9VXlnoPlc2N4iGl2cnasUDw8kiDIMZ4uGr2yp8YkD8DPRwmAu0kXjsryE8ysHEYj4cvpfFfis/fJsPpzkL0u6im6D+BtASvDvwnzxq/jvC7a9f/HiDCUKjmR9HgSLFdsllMRpUdq0V1jK/Odil82guTb/sXM5LUhX7R7tGCATnVgmXCWTHcn6xNdm984rsbJfMwBuXlWBUFToDk7QNXbihrGkaExMTxGKxnIvxY/1CSB3Gherw5/TcEsDuF4v7OiG2W7r2sSlZ8/DcOd3tes6RSEDcLwUNbhQ9rcYkYHFB+VpQDJQrQ1QrAzx7ZgKiQV3nyKyIhdEGRUfByUQNy9wJsHp1HlSBYraD3QfhsXQMBW0vs9qfzG3vCug3NokkSU2xndvXiHX39184wwvJQuUbkmspRVFo8ItN5Jb+CV3GeK6FfEbyIa9yti4qTkdrpRoHFhKaptEju5bnjKIo7FgpbnKfPLFwreS7A3KOZJKU2H5mYILYPN1J8+Fe2RMT74U2d6kQ/STzYk1SbD86pEKRiI1lvGf6QQaz6OQ9n9DoVISeooKWQItHdZ8j2rAQ2zs1H5Ve6QA2VxaXOPDaDByK14sHxnsgGgHVKIoq5ii258N1ZF7EIqIgxZL7eMB42z4i/3M36kQfTYlK/jjy1+xa5eXBd/jxWLMk6yoK7PgHuOajRA12VqjtfGzgc9B5cObjjWaIjEMiv92wc4UU2yWz4vc2CCv53cd7SSTy98I4IDvb85Iyl5UlyTy6l3XsYk7ND7+cH3nBtfVCbG/qG08L3PnAVGe7XMjpicNiZFOtEJVmclWJRqN885vf5OjRo0Sjuc3aDQ4JIXVEdYksYEluMRhFFX7xElBNEB7lbX5h2/70iamNAj3nSGJMbIiGTS4xRol+mGzCQrxY5J9eq5zg2XaR267nHJkV8TDhgbMANFLNYpcmrznzwVkqNopqtiYr8ANcazkDwPGehdH9JSl8PnyDyMn89cEuDrUHALh+6VRh36ISsdHXkgF74LmQ6kCtkHntGaHIamJJ8nd6uCOg72CywFg4RjAi7M+ljfzcuHWVEMOePNGX13txcyUWT9A3lhTbZWd7Rqjy2LCZhAvKTEXrV4Lu98rREF5NxD+5vOUiIkoyL9Ji+yBQlIzEGztfbLckxbNz4isSCQgFpn4HqgG0BNFIRPf1VGRIiO1dmo/y4twLogsFRVHYXOOiVSsjppjFummw8dLRArNA9+vIfAkOioJti2vWXzIRSTA8mUgXF4RjGsf6ozx7NkQwOjtRuvnFX8DvPo05PsFriWV8yvIZ7n9HA/+8w3tlQnt8Dj9z1Qgb30foXT/hifhmDGhEX/g6aDOM3WAWEQPx/NnX1xMptktmxYYaLzaTgbFwjJaB/NyM0jRN2sjnMW9YLCzD9YwiSHcsO6SImg94HWZWlIs8sVfzqLs9XZQhO9t157pkBMWePLPWTHUtT6guaeesF44SsHugdCUAO9QDALzUPMR4WH8rO3VCdNjHLR5pd6g3ipLMbV8OwFb1JAf7YXh4WOeBzYJYhPiIcNIYt1ZiNhpE8YBkblg9wkpei0PZWgDWBPcBcHIgRjwytw0kiSSTbKr1srHWQySeIKHBohIHlecI24t072wXYnuVFMUyxrpqDwCHOkb0HUgWSHW1u20mbOYLbdIll+fahmIcZgMD42FO9ozpPZyM0zsWJqGByaDgd8j1dyZQVYXFpeK9oqkvP/dvZ0tkSGRxT2pm/H6/XFdlgFUVQjA8O5Jg0poU20MBmDxnT86YFM/OFVejQdHZfL6Ntqb/2js2LGzwx03FWC3y/mQ+bK73kUCl3VgjHug/Kf5XgOhVuFbSNBjtEg0fyuVl1HhC499fG2Pjd3vY+J89LL+/m+3/1cOqf+9m10/7+dDDQ+z4734ea5y8ZJd/94v/zeLj/4aBBL9JvIHD6/6eBz+4ku3VV/g+OTEAE30w0jmnXPUil4fvOv6Mcc2KabQVjj984UEGs+j+n2MxxkJDiu2SWWFQFdZUiTfkQ+35uQgcDcWIJC2SpEiWf6Ss5PUUzdIdy7IYI2/Y2pC0ks+j3PbBCSm25wupHMs9zQN51ckRHxWV3yGj7FrWDYsTnOVQtgqA0pEDNLg0InGNZ0/1XeaLs485JK5pit0rqoIl+mJxQdkKAG4wHkdD4fnTPfmf2x4ewzop5rPJXS42t0zSGnHOmO3gLIPQCFRvBsA/fBCbEUJxONO9cPNwJYXFR65flP44ZSGfYlGJEFDODOgrtldKu+eMsb5GdBkuxM72KQt5KX7MFbNR5ZrkmnlvHq2ZM0V38ppS7raiqjKaIlOkHDMaC1xsH+lpBkQWt9dVpPNoFgZ+p5lihxkNaJqwg01cX+g9MXVQqlP1XNvw6KT43HDePtm53e86oYwLl7u41SejBubJNYtEbvuhaFJsHxJ/gxjMEFl4BV+XJTQCwQFRtH0Z2kdjvPeXg/zfPWNEkg3gkTh0jyeIa+C2KPhtKl3jcf78t8O8/1eDPN48SSg2fU8iceDHVBz/PgC/Nr+FG97zKf7kuhrMhit4j9Q0GO8FxQAVG8BbL2IgRrtE4cwVUFNWwv2xt4lP9v/gQlFdNYpi9quxGGMGpNgumTWpiusjnfkptqe62l1WI1aTrJrON7Yu8qEq0DIwkd6kyTWDMos779i6SHQu51Nu+1RRhrxJ15v1NR4cZgPDwSjHu2fIDNMJZUJYhMfMrqnMMknuKSqHsnUAKINNvLlWLFJ+d7TnUl+VE2wRsRlqdhZP2a5J9MPihIpNoKhU0kcFgzx3dhIi87PWzDpDZzAQJ6hZKPd5RVe7SQoW86KoHFDEpgPi2rGqWCyJj3UWgNuB5Kpg5+oyqr1CzL5xeem05xb5kzbyOrnNdafEU2kjnzHWVqXE9pHCzDK9BCmxvUxayM+L65IugXubF15RWFfqmuKS15RMsrRMCNPNBS62j/e0ADCoFqPYi3UezcJAURSWlYl7iVMB0lFbDDSec5AqxLrzO9vFC0x/wYTOne2JOJZkcbLBWSLdD+bJmio3ZoMyJbYHWsX/hqs0l3tyOOnocGkdoXc8zu89OMCrXREcJoWv7PBw8s8reOGPSvnlu/y8/CdlHPzTcl74UCn/51onZgO81B7hzx4d5prv9fA3u4cYbj8Be/4Vdd9/AvBv8TvZ/va78HjdVz7usV4w2qBiPbiroWw1VF8DrkqxBxJoh8nArF5qbZmJ78dvZ0D1i+KDfd+f+UDZ2Q5IsV1yBayrFn/ch/K04lpayOc3bpuJtcmCjZd0sJKfjMSZSObFySzu/OHaZJX+yZ4xhiciOo9GZMYNB1OZ7fJaojcmg5ouyNiTR5tL5sl+ABSrR3a264nNC5UbwOyEeIS3u04D8OzJPsIxHSvsw+NYNLHQsLn8lzlYkhOMFnBXgVdkIW9Xj/FcR5xEvlfnJ237zmrlLPEaxJyXzA+bV/xzlIDZAfEwtzqaADjelZ8FxZKrD6NB5YE/vpZvvGcDNy4vmfZcQ7KzfWA8wuhk7nMvO1Od7dJGPmOsrHBhVBWGJiJ0DOtTlJ4tekZlZ3sm2L5I3E++0jJELL6whI5UZ7vMa88sixdIZ3s0mcU9avCASe7PZIrlyWKM0yMG8NaJB4fOTD9IMYjO2FQRWGhkhkYDTf/O9vE+DFqMmKZi95Rc/njJJbGaDKytcHBSqxUPjIoIRQwmkf19teVyh8dnFR35hedHGA4lWO4z8th7S3j3KjtWo0KN28imCjPlTgOKomAzqXxim4vd7yvlIxsdLHVM8ueJn/CJlj/B+9u74ej/APC16Dtxb3s/pR7nlY85EQc0KF0FDrGfKqL1fFC5EWq3Qfla8fsMX34/ZF2piTBmvq79oXjg2K8gdJ7DlsEoflYSKbZLZk+qs/141yjRPLzB75ddy3nP9an85ebc25+l8trNRhWnRXai5gt+p4XFyU3DfWf1724fmoigaaAq4LXLoox8INXJ8WIe5bbbwmIsBkexrJzWE9UAnhooWwPAsonXKbNpjEfiukaWkMxrD2oWPG6PfuOQTMfug9KVANxgPMZgSOF4W/4U8VyApqVt+1q0CupdqujQl8wP1SAKL2JhqNgIwPbE6wAc75nI/2gByVXDklInb99YhXJeB5nTYqTMJda7ZwZz686hado5me2yCzVTWE0GVlQI4ePwAstt75ad7RlhVaULl9XIWDjG0a78cfvKBGm3DBlNkVGWJjuXm/vH8yqO7UrRRkUWd8jku9C+XDJnlpWL95xTAVV0vQIkf9Zp7F7RHTs5LLqZQyMXdveqBpHVrCejnQD04qWieA4dwJIL2FJfzMlEsrN9chgmh67eXO7I+GUbbJ4+E+LRphAGBf7lNi91nstrDnXWCT7jfZInjPdyt/E3lCoBJjQLT8Q38+eR/8Pzxe/iD9d75jbmREzsU5ovEj9nKYLiBvAvhckRIbpfglUlJlQFfjy5lZijHBJROPPM9IMMFgiPyLU0UmyXXAH1PjtFViPhWILTvfnXCSQ72/OfNywWFdkvNQ3k3CIvVYxR4rRcsGkl0Zd8spJPzZNihxmDzIzLC65fKq4b+84M6dutfA5FMTFXLUXFMo9bb2zFUCUEM6X3MLfViVtbXa3kx4XY3q+5KfG69BuHZDoWZ7ow4wbjcQBe1KH4b9bEI4QHRTdPs1ZBvVuRee2Zwu4XjhjlYj4smjwMwPGBGFpEnxxsieRKaPDrk9seCEYJRUXRfbnsVM4oqcaGhZbb3is72zOCQVXYloduX5mgS7plZIW6Yjsmg0IwEqd7tHDFMdO4WNPFbLLIPZOkO9sDmrCVBhjrhnMLM4wWIdqNdAoL+VgIjOf9nSoGSOgsto+0A9Cl+ajyFek7lgXClkWlDONiAI94oPeE2PdKxPQvrsglqeKCS3S2B6MJ/v5ZUSj54Y0OVpecd52KR6HvOJx8BPb8Kzz6Sfjvd8D/exvs+VeU8CjY/UQ3/QnfXfY97op9kifYxj/e4p37nnQiKn5fhss0kHlqwVMt9q8uodHYTSpLi42AQrf3WvFg28vTD7paizFmQIrtklmjKEraSj4fK66l2J7/bKrzYjGq9I2Fae7P7ebQYCqHW1rI5x3bkxsHesQLnM/UPJHXkXxheVkRfqeZyWicA22B9OOqqrJ27Vq8Xi+qmtvbGY8mcn0d7lKZx603ZgfUXic+HmnnjmpRlbv7eA8aij5zZLwXgH48+N1SbM8bzEXCNk0x4E8MUq30c6gnwtrVK3W5jlyWWIjIsOgwGTKWYbNaRWa7ZP6Y7SK7vXgxAM6xM9iVMIMh6BvKvzWORHI+i0pSue25XU+lLOT9TgsWoyGn517orM/zyL65kupalsUZ82cqtz2PCwXngOxszw5Gg5ouzGqcR7OUnmtuAHtIrKtUp/+ymcmS2bM0KbZ3jycYMZYCihDURzqnH2gvFiL8WI9whTrfXUA1oGpRXecIgZTY7qfK68j9+RcgG2s9AByLJ63k+0VcH4oyJzFV7+vInImHhVh+ic72b74yRudYnKoiAx/fmiz2iIzDoQfh0b+CH74VfvXn8PxXhUV8537hFAAi2mzV78Pb78e05YPce1MNT76/lEf+oIQ1pfPQLuLJzvbLFSipBvAtEUXok5dufltbKl7rNeN68UDv0ekCvdEsrhHRhRWHNBcKaIZL8oGpiuv824hK2YRLsT1/sZoMXFMvMrr3tOS2i3lAxgzkLdcv8aMoIre9T+eqazlP8g9FUbgu6Yqx55yCDKPRyFvf+lbq6uowGnPXXR6bHMWBmKcen8wE0x1FEd3KznLQNK5JHMBt1hgKRjnQMarLHAkHuoFkZ7vcOMwfDEZw1wjLNERu+5FBjbfuuD7nc2RWxCKYxkVGXsRWJjpJzu8mkcwdZxm4qsDuQ9HivNUpNpGOd+rvsiORXI5FSQHl7EBubeSnLOTltSjTpPZZjnaOFrTl8/n0jIg5I8X2+bM9uR7ad3aISCz/Yh3nSndyjkj3g8yzpFQUZjXNI7ddrzU3AJqGJyocw6zusst3aUpmjdtmSv/NNY6bp7rb+09MP9Bkg3gEQgFAu7DRQDFg1OL6zREgPtQKQLfmo1KK7RnB77RQXmTiRCq3fahF/K+qMAcXMF2vI/MhFhHODRe59jQORvn+QfHz+MKNbuxaCA78CH76XnjlO9D5mihOMNmgeBHUvQHWvxdu+jt41wPwnh/D9R8DZ2n6NRd5jazwz9PFIxEF0yz/FixFULJMfK+X+N2uS4r/vwuuEMUHoQD0nZw6QDWCFped7UixXXKFrKtKdbYH9B3IDKQ726VIltdct0SfiuyB5Pzwyc72vMPrMKevLc836tvdPiW2y3mST6Ss5Hef6NN5JDDSL6q9JzQLXn+ZzqORAGB1QcU6AIxdr3FLjdgEeOakPvNlckgIpMO4cVjlPUleYfdC6SoAtqvH6ZtU6BvMvwJSAIIDWGNibGZXqah8l04amcPmEdeOcnHtuNWUtJLvDOg3JolklixOdrbn2kZedqBmj6WlTqwmlfFwjJaBuQtj+UQoGmc4KByHKlxyzsyXZWVOfA4zoWiCg+0BvYeTEcKxOANJZ7lKj5wjmWZJqeiynI/YrivBIazJIndHcYW0kc8wy8pSue2ki5EZbLzwQJsXxvtnjs9TVGEtntCvACg8JGK3eimm2CXF9kyxptLFqVRu+6hwD8BgFl3bVwuxEGjMuAbXNI3PPz9CLAE7GqzcYm+Ghz4A+74H4TERW7by7XDrffAHP4Z3fA92/iNs/TNYeht46y+eqT5f4lHhADlbiirAvwyCw6I7fQbWJDvb9w8Y0MrXigfPPHfhgVEptkuxXXJFrKvxAHCqZ4xQND+yc1NIG/nC4PolQjR75eww8RwW7cuO5fzmhqWiQ/j50/26jiO12PfJeZJX3LqyDKOqcKJ7NG2Dp2kakUiEeDyOdol8oUwzNiDE9iHcGG3SIjwvsLig+hrxcc9hrqkQC4HjnQFd5kg0IMT2oNE186aERD/MznRhxvWGY0CCw63dOZ8js2JAbHb1aR4qPHaROS/JHKpBdBGULAdgXfwIAMd6JiCRX2scieR8UtbAZ4eC5LIJeipbWYpimcZoUFlTmbSSb8/TIrArJJXXbjMZcNnk/dB8URSF7YsXVm57T7KAx2JU8dqlkJppliY720/Pw0ZerzU3gDZ0BoBezUO5z5vTc18NLC9P5raPGMBTJx4MtF54oNkuOpqtM+x9qAa0eJxIKKjbeioRELFbUUsxiowayBhrqos5mepsH+uBeEKI7dFJYVN+Beh5HZkX8ZmFZ4AnWkK82B7BbID/W/0S/ObjEBwEq0d0r7/9O3DDx6HhBrAVi7+hXKFpVxa7oSii4MZbL/LbZ1gLryoxYVBgIJhgrGyreLDz9ekHGUwQWhj3sPNBiu2SK6LSbcXnMBNLaJzoHtV7ONPol2JqQbC60o3LamQsFKM9hwVxAzKLO6954zIhtr/YNKCrdWLKAUHOk/zC6zDzpuQcefiQEDKj0Shf/epXOXLkCNFoNGdjmRwW5x9R3VdWLSrJHkYzLLpRLP5CI2w0io2Zxp6ALnNEGxPZghGTW3Zg5BtmB1RsAMVAKcPUK728+sILOZ8js6JP2Di2aBXUe41glOJWxrFNOR2UhDuwEub4YHxO9ogSSS6p9towGRRC0QSBSO7O25kW26XdczaYiuwL6DqOTNFzTl67Ip1ZMkI6WmuB5LZ3BcQcqfTY5BzJAitSYmrv+Jz3WPRacwME+5oB6NT8VJT4c3ruq4GpznZFRCvBhZntKZylYJqhC1c1EI1H+eo3vqXbesqYjN1K2Evl2juDrKly06xVEsMgOrwDZ8R+SzxySRF6JvS8jsyL8ISIojuPUEzjvhdGAY2flD+Ib9/XhHV7yQrY9XXRve7U+Zp1pbEbqgH8S6GoTBRXnIfVqLDMJ34Why0bxINDzRA6R9gxWiAydtUXrkuxXXJFKIrCuuqUlXz+VKvEExqDSbG9VHa25zUGdaoi+/RI7hZU6c52OT/yko21HpwWI0MTEY516VfIMzCRKsqQNvL5xu9tEDliDx/q0rUaNpLM455QXSKrSJIfuCqhTNhZLRp6AQWNgQl9FnJqUDh0JCweOUfyDdUgNpN8SwDYqp68zBfohKahDYkNxpZEBQ1eM5ikuJVxLC5w1YDVg0KCNcoZzo7C+PjcO8DynX/6p39CURQ+/vGPpx+78cYbURRl2r+77rpr2te1tbWxa9cu7HY7paWl/PVf/zWx2PSulmeffZZNmzZhsVhYsmQJDzzwQA6+o6sTo0GlzicK/vomc7eekjby2WV9TbKzPY/2WeZDT7Kzvdwl378yxbUNxYAoyIjFCz+3Xea1Z5cGvwOzQcRTpIqlComJHnEv3Icfm10WuWea5SmxfUhDKyoXD473QuwK1tCKqq+wFpnAGg0AYHTLiL9MsqbKTQQTzYkK8UDfCVHMEI9c1Gp8wREZn1G0/s7+cTpGY3zR9lO2DD4sHlx0M7z5n8DXkONBnoemgcKVi+0g9htKVooc94kLHXRSue0vj5eDo1RktLe+MHWA0SLmxlWe2y7FdskVM1VxnT+LwOFghISWdL5wSJEs30lZyesitksRNS8xGdR0EcbzjfpZyac722VRRt6xY2UZVpNK62BQ1/efxKio8gyZXDNWuUp0wlIEtcLOytz9GnUu/W5xLSGxMFHsXjlH8hGbJ93NfL16RN+xXIx4hMiQyMZr1iqo9ZjBKDeiM47JCnYP+EXxxRstTQCc7BzScVDZY9++ffzHf/wH69atu+C5j370o3R3d6f/feUrX0k/F4/H2bVrF5FIhD179vDDH/6QBx54gM9+9rPpY86cOcOuXbu46aabOHjwIB//+Mf5yEc+wuOPP56T7+1qJGUl35dD/STVqVwhO9uzwupKYdF7undMV6evTNF9Tme7JDMs8jtwWoyEogma+gs/N1cW8GQXo0FlaZmwks83Z9LZEBk8C8CoyTc34UhySZaUOlEUGA4lGFD8okg8ERXdqrNFMeia186IsJAf1ewUe336jWMBUuay4HcYp6zkB5tFcYWmXR1iaiwivs/z3BL2tIf511fH+AvDr3i/9oh4cPXvw42fFvsMepOIir/lubo8WF2iQz8RF9nz57C2TLzm4b4Y1G4TD7a9PHWAwSxcD67y3Ha5Ayi5YqY62wP6DuQcUnntxXYzRoOsIcl3rkuK7WfGFELROCZT9jv/pI18/vPGpX52H+/l+dP93HPTEl3GkC7KcMh5km84LEZuXVXObw518euDXazcqc8cUSeERXjcLDvb8wpzEdRfDy//O4x1c13JIA+NunM/jkQCW3RYDMlZnPvzSy6P2QE118Lx/+V69Sivc53eI7qQWIh4Mn9wxFSKzWaXtojZwlEC3gboeI3tptMQgmMdQ2xZr/fAMsv4+Djve9/7+M///E+++MUvXvC83W6nvLx8xq994oknOH78OE8++SRlZWVs2LCB++67j09/+tN87nOfw2w2853vfIeGhga+9rWvAbBy5UpefPFFvv71r7Nz584ZXzccDhMOT3XGjI4KISAajRaWxaRO1BcLcaovpOTk5xVPaOkMbr/dKH9HWaDSZcZkUAhG4rQOjFHtnb8Amfo96fH76hoOAlDqNMv5kkFWVRTx6tlhDrYOsdhX2HOkc1jEtpQVyTmSLZaVOTnWNcqxzgA3LbtyMfLc30vO358DovB00uwnqikg50hGMSpQ67XTOhTk+LDCGz21KEPNxHpPovmWXfwLtWSnGwAK0Zh+c0QZaMEIdGgllHmd8jqSYVaVF3HybC1vM+whMXyWeDwBCQ1CQbBd5GetaTDQCIkYqEYwWojaStNPF8x9fmQCIhGwWkRePdA+GuOe3w7xh+oT/JXp5wDEl95BYsufgmJKH6crkQhoRkioc79mWovBsxj6T0KCdHzmKr+QkY/0RYiu3YLpxMNoPUenO+3ENGEtb3Zd+hyJ+NQ/NDDbdb0fmQ2zHZcU2yVXzNqk2N7UP85EOIbDov80Gk5aP8uu9sJgkd9BmctC72iY/W0Bblwx8+ZepojEEoxMiouiFNvzl1Ru++ttw4yHY1hyXDeTSGgMpmzki+S1JB/5vfWV/OZQF48c7uKvb12syxhMk0nnBatXil/5hKqCb5nImRo4za28wkPclvtxhAIYEdbKdrfMFsxLzE4oWwU2L47Jc6q1YxEgTywqoyHME8JFI+4oFXbnkuxgdQm7PGB5QnTyHOudhOgkmBZOp90999zDrl272LFjx4xi+49//GN+9KMfUV5ezlvf+lb+/u//HrtdZHPu3buXtWvXUlY2Zc+5c+dO7r77bo4dO8bGjRvZu3cvO3bsmPaaO3funGZXfz5f/vKX+fznP3/B40888UT63JKLM96nAAb6J2H37t1ZP99IBGIJIyoa+198GlXGK2cFv9lA96TCTx97ltXezHW352KOnM/B0yqgMtjRxGOPNeb8/AsVe1j8XB/dcwRbz6GMva4ec+RQY3KOtDfy2GOnc37+qwFtSLxXPHewkUWTp6746+PxKYvwp59+GoPBkMHRXZotI20AdCtlPPbM3pyd92rCpYm/wV81wyKtnBqaaTzbwelQx4zHX9vyDUpHj9DpvZazvpsZdiwhfo4TS67nSH3/U6wHOjU/7R1dPPbYRTLnJXPCElI5odUAMDHcy9OvJ+dF0zHg2KxfJx4/nP4413Nk/ohC03AcvnnUwLLwSf7B8v8AOFn++5xy/j4c1s+d9aKcfCpDLzSc/AexBJgUA8Mh+MHZMj6KgjI5yNMvHyZkPqfRpPkgcHDOZ9TjfmQ2BIPBWR2nv0oqKThKi6xUuK10j4Q42jnC1kX6W7UMSrG9oFAUha31xTx8uJvXcyC2DyXnh0FV8NikOJav1Pkc1PnstA4Gebl5kDctzW1X6MhkNL1Q8MnO9rzkTctKcNtM9I2Fea1VH5tfe2QQAKOzWHa25xs2D1RtgYHTrJ18FfQQ28eF80FAc+Dz6NBZL7k8qgHsJVC1GZqenXo8FgS8eo1qOsOtGLUIEc2A3eVPV5NLsoC5CMrXAQqu+DAlBHil24MWGkVZIGL7gw8+yOuvv86+fftmfP4P//APqauro7KyksOHD/PpT3+aU6dO8ctf/hKAnp6eaUI7kP68p6fnkseMjo4yOTmJzXbhz/Jv//Zv+cQnPpH+fHR0lJqaGm677TZcLllgcjkq2gL8tPlVuoIKO3bswGzO7jr4UMcI7H+FUpeVt+x6U1bPdTXz+Nghuo/24q1byR3X18/79aLRKLt37+bWW2/NiZvcufxX+yswNMLN2zZz66rSy3+BZFbEDnXz7C+OMGb2cscdW+f9enrOkX8/sxcYY8cbtnBjsvBeklnczYP86oH9jChO7rjj+iv++kgkwpEjInbp5ptvxuHI0T2pphE/+GcAVJf6uGPnbTKeKwucMjdx5LkWTGYLlZXLYPgllseOs2TTn194cHQS44EDKGjUDr1E7dBLJCo2Mnntx0hOkdzOEUB98hXogA7Nz51v2kBtZUXOzn01oB7r5b4HRYSjM9TNHatcwko+EYearWCc4d5zchjaX4Wi5LpgrJdI+RZ9riPzYbQbug+BuxJN07j3iREiwW6+bf0WRhIkytax+OYPsdhWpPdIpxMcEsXklZsy83qRCZgMwGgnTAb4Xb+F3S1hhq3lULwYhpq4xXEabe07xfETg+L8VZsv8ZpBaH8l2bykQDwCNduIKkbd7kdmQ8qF7XLIdyrJnFhX7aZ7JMThjvwQ24eDUmwvNNZVu3j4cDdHOrOfvZyyBvc5zKiyDSOvuWGpn9bBNp473Z9zsT01T9w2E2ajjKPIR8xGldvXlPPgvnaeOt6HHn/Nzpio6rQW+UQ3tSR/sLqg/gY49BN8E40UMbvK04ySFNv7NQ8l7gJYRF6t2DxQuXG62B6Z0Gs0FzIgOo/atDJqPRaZ155NVBV8i8BVCaOdbDE08tuxa2jqGmTp8rLLf32e097ezsc+9jF2796N1TrzPPrTP/3T9Mdr166loqKCW265hebmZhYvzp6LjMViwWK5sLjRZDLl5QZLvrGuthiTQWE0Cn0TcRoc2f2ZDUwIl7AKj03+frLIsnIXjx3tpWUgmNGfsx5/V6k87mqfQ86ZDLKhTqyRT/aMoaiGjMUo6jlH6vxFco5kiTXVopD07FCQmKZiM19ZR6mmTXUt53SOBIcwaZMAOP2VmKwLowAy31hZKYrDT48oGFYtB0AdakLV4mA873fd3wJoYLKLQtX2l1G7D2AKDaYPyfV1JDTUigHo0vzUlnrldSTDbKgtpg8PA5oLvzKKqf841G4Vex6JEJhm2O8IRkAFjEnJUQVNmbIZL5j7fCUGBgUMKv/+2hiPN43zU/O38DMCjlLUN34C1ZmPzRUxsBZBpn7GJg84POBwQ+d+3lIPu1vgd80hPrVkPQw1Yew7AoZ3i+MtVkiExRy4mBNoJAZEwO4TQns4CiajsOInf+fIbMckd4klc2JdtQeAwzkQSmdDqnPZK8X2gmFdlXhTOtI5Ou0GPhv0p8R2aSGf99y4THQ9PHOqL+vz4nxS88TvlNeRfOb6pcKa+9XWYVasWIHb7UbNleidiOPRAgA4PbJDJ+8wWqBiHRRVomgJdpgOcybuxeLM3RzRxvsA6Nfc+D2yMzNvsTihfB2aYmaVdpoxzYQaDek9qikGhJVqi1ZBvdck5rYke1jd4FsKwB1Fwup49/FekTlY4Ozfv5++vj42bdqE0WjEaDTy3HPP8a1vfQuj0TjNHjbF1q2iU7KpqQmA8vJyent7px2T+jyV836xY1wu14xd7ZL5YzUZWFUh3mdebwtk/XwpUazCLYt/ssnSUtEhdbpvXOeRzI9wLE7fmFhbVXnkNSCTNPgcOC1GQtEETf2FO0/Gw7F01J+8rmQPv9OC32lB0+B079jlv+A8VFXN/ZobICAs5Ps1N2VeuabKFivKk+85QzESRVXC8SkegY4Z3JAGknEgrmq4/Z+EUAaoY52sWFST+zkCxIbEPJmw+DGZ5XUk01R7bbitRo4n6sQD/SeFS5ymQfgiXb7hcVCmt8Woiag+15H5EJ4Ag5Fnzob4yp4xPmH8Bdeop8BggevuAW+93iOcmUQ8O1FoNi8UL+aWskksBjgTiNNhXy2eGzgnBsZghlgYYpfYW4lNgoZwSViALMzvSpJ11iVz2w93BPQdSJKU2F5slyJZobCyoghV0RgYj9A1kt0N7oExKaIWCtct8WE2qHQMT9IykNuu1IFxcR2RRRn5zbX1opvjRO8EO25/Kw0NDRiNuTHqSUwMCssoTcHjL/yOxwWJ3QfVWwD4A9MLPBtZzKRnUc7mSCTQDUA/HvxFcnM5bzE7weZBrVrHu3iEmthpRsdnZwuWdRIJtMEWQIjtDV7zgsoOz0tMdigVue1bFLFZ8OSZSYjq4I6RYW655RaOHDnCwYMH0/+2bNnC+973Pg4ePDhjZuLBgwcBqKgQVpzbt2/nyJEj9PX1pY/ZvXs3LpeLVatWpY956qnp2YC7d+9m+/btWfrOJACbaj0AHGgPZP1cPWmxXV6PssmyMicATb1jOS88ziSp+WI1qdJ9MMOoqsLqSiE+Hu7Ij+aXudAVEF3LLquRImv+dbAtJFZWCEH1ZM+V3+sajUbe8Y535HTNDaANnQGgQyuhslRGDGSLBr8Ds0ElGNVonzBAVdJ6um3vhQenxHZvrfi/SBRcGicHeMctW3M+RwCMYyKjXbOXXLyLVjJnFEVhTWURx7R68cCgKMTFaIaJgZm/KBSYXiiugJGELteReREe48y4kf/zu2HWK03cZXxEPL7u3VB3g75juxyGLN13uWtw+iq5sUrcnz48ukQ8Pt4LE/1T505E4VKNDJHgBQUZCwkptkvmxLoqDwCtg0FGglF9B8M5YrtcyBUMVpOBSrv4+HCWN4hSImqJFFHzHrvZyNZFQkx9vvEiN29ZIlWUIedJflPqslLns6Np8HoONpfPZXxQLOYGKaLYm9uYA8kssbph2U4ANiaOUK300RVUIBbJyelDQ2KODOPCYZPXkrxFNYDNh6lqIwC3qq9zrHcyZ/PkksQjRIfbATijVVDrtWRvwSwRmGxQthaAsshZVBIc6EvQPzik88DmT1FREWvWrJn2z+Fw4PP5WLNmDc3Nzdx3333s37+fs2fP8vDDD/PBD36QN77xjaxbtw6A2267jVWrVvGBD3yAQ4cO8fjjj/OZz3yGe+65J20Df9ddd9HS0sKnPvUpTp48yf33389DDz3Evffeq+e3v+DZWCMK4HPR2d4lO9tzQp3PgVFVmIjE024ChUhnUkit8thQFvCGql6sTboEHs0Tp8m5kJ4jXrvOI1n4pLqXT3RfurP9wVfb+IufvM5/7z3L2QF945Um+0Thaafmp6w4zzKRFxBGg8rSZJHXiREjVG4QT/QeufDgwWT3avEi8b8j6fQXHBTiWq6JhrBGhIW92V2e+/NfJayp8k51to8IJwFMDtHZHp2cfnAsDJHJ6RFoqgmieRTXNhtiESKREP/nmSiRSJhv2b6LSkKsF9e/N//jJLO1d2Awgm8xdywSBRO/OGtBc1eL59peFv8rinA+iE1e5EW4sCBjgZHns0OSr7jtJup84qb4cGdA38EgM9sLlVqHqIY6lOWK7MGUPXjRwr2YLyRuXC5u2p893Z/T8w5OSAeEQuGaZHf7a2cDOT3v2IAQUodwY3XkYz6TBLMdSldB2VpUNN5neIquIBC5ctvEuRAbEZ3tk0a3WFhK8hd7MVSsJ4HCKrWVjo72Sy8Kc0UshDYqrjWjplKsDs+CrvzOCxQFKteDwYIaj3C7txsNhWdO9Og9sqxjNpt58sknue2221ixYgWf/OQnufPOO/nNb36TPsZgMPDII49gMBjYvn0773//+/ngBz/IF77whfQxDQ0NPProo+zevZv169fzta99je9973vs3LlTj2/rqiHV2X6yZ4yJcCyr5+oZEdfHcim2ZxWzUaXeLzJQGwvYSr5zWMyXSmkhnxXWVqci+QpYbB9OFWTIa0q2WVEunBAu1dmeSGh8/jfHeeRwN3//62Pc+NVn+cD3X9HNYWOyX4jtA6oPq0XOkWySnh8jBigRjkUE2mDynOtLPAJDZwF4fHIV//LyKE8OeMRzk0Pi+Vwz0gHAhGbB55PuB9lidbVnqrN9rAeiETBZhU14+Lw9lugkxEPThVTVKDqZC4l4mK/tC3OkP8GnrP9LbaJDFBhsu0vsN+UriZhoKsimy4PNwy0ry9NW8gGvKM6m+9DUMao6/fpxLvGomA8LuJlAiu2SOZPObc8D66rBcZnZXojUOsWNe7bjCAZkFndBcdNycaO87+ww4QujRLPGwJi4jvhlZ3vec219MUbiRF77OQcPHiQSyVHX8rAQUkcV94KuxCx4HCWw5GYiGOkzLeXa0OtEJnJzr5LKbA+b3GJhKclf7MVEbKXcp9zL55VPYOk9IKrx9SY0ijkkujQSjjKRLy/JPjYPFDcA8Hb3KQB2nx6CRELHQWWHZ599lm984xsA1NTU8NxzzzE4OEgoFKKxsZGvfOUruFzT81Hr6up47LHHCAaD9Pf389WvfvUCG8gbb7yRAwcOEA6HaW5u5kMf+lCOvqOrlzKXlWKLRkKDQ1l2++mWNvI5I2Ul3ziHfOV8IdW1XO2V8yUbpDrbj3eNEosX5vtUykZeFmRknxVpG/mLx1N0BiaZjMYxGRS2NojC9hcaB+gaGuNLX/pSTtfcAInhZBa3ybegRZl8IB0zEDCAzQVFFaAloPXFqYOGz4IWZxQHf7avhG+9Os6zgx4AJsZH+dIDv835HEl1WXdoJVT7XZc5WDJXNtZ4OKuVM6FZhJg7cCqZta3NLLYn4tP3QQwmIqFJXa4jc2VPYz/fPRJntXKWPyZZgLz2Tihbre/ALkc8KoT2LF8zncXlaSv5PdHl4sH+U1MHWFww0QeRGRwNopMQDy/o/VQptkvmzLqq/MltT3W2+6TYXlCkxPYjHSMkEtmrmE1ncTsW7sV8IdHgd1BbbCca1zg9krtuvlRRhsxsz3+uadDHwj0W6AJg3OCSmWD5jNUNNdtJWKfmydhYICenNk4KR46ExSPnSL5jtIjNpCSOYMeFVnh6MHAKBY2A5sDncU234ZNkD5MdfEsB2MwJAF5ojxEKXnm+qUSSS+qT66n9rcNZO0ciodE7Km3kc8WSUiF8NPYWbmd7WkiVxRlZod7nwGkxEo4lCtYBoeucqAFJdllS6sSgKgSCUXpHZy4sbUrOo0V+Jz/7s+3UJ51MW/r1sX82JLO4Y/YSKbZnmVRn+6nBmCgyrd4inuh4beqgAWEhfzheDyi8a5WNgNEPQFSve+XAlNhe45eug9mi2mvD77RwUqsVD/QdE/8brTDeJyzDU0SDdE/Aod5zBHWDCTT944dnSyAY4d5fNaKh8Pclzwv7+JIVsO4P9B7a5UnEhLtitq+ZNi93LBbrgR8NLBaPjXbCZPJaYHYIoX1ihmjYWChZFLBwr+tSbJfMmXXVKbFd3852TdMYnhAXbtnZXliU28FqUhkLx2jJYibUgLSRLygURUl3tx8P5F5slw4I+U+9z65L8UxiTFgKh40u2bWcz1iKwO5FXXJz+qHGntGcdKimOpKxFwsLL0l+Y/elPyyLdaGF8qCLcKAREHntDV7Tgq76zitM9nS3gnfsNFVFBkJxeOnUwreSlxQ2DUVJsb0te2L7wESYaFxDVaBErqeyztLSZGd7Xx68J82RqTxuKaRmA1VVWF0pBLJCtZLvCogCHtnZnn0sRgOLS0Q8xYmLWMmnxPYlyetP6v/mfn2KOeyT4v5LcZbKe+Esk3I+ODscIRhToHKTeKL32NRBA00AHNUa8FgU/vkWD85iUbRsCOtzDUoMtQLQqfmp8Tl0GcPVgKIobKnzcCxRLx5IrlWFoDo+vXs5NMI9uyf45EMH+fCvB2gbiYl9s3jhiO0/fqWN3vEYi1xwrTH5vdZfn9/28SlSIrYhy3uVRgs3ryzHoGjsHSkmZvMLN4zOV6eOMdthpFM4HZxLPjQ3ZBkptkvmzJoqN4oiLOX6xkK6jWM8HCOStM4qtkuRrJAwKLC6QiwSs+mQIEXUwiOV235iWMlZTljKAUEWZeQ/iqKwud6T8/OqE8IiPGbxSLE9n1EUKCqHmm3ph/raGrOfxx2PYouJzQaTQx/3BckVYp3qglisdNI9MDC9Ol8Pkp0jLVoF9V6T7GzPFSYrVG0GFJSJft5WLTaXnzw1Q0W+RJJHpMT211uHs+YU1pO0kC8psmAyyC2kbLM0ZSPfN65bXvJ8kUJq9kk1vxy5wuaXRELj7MAEYyF9xY9OaSOfU1Ldyye6Ly22L06K7Kn/delsnwxgTYjz2jzlsoA5y/idFvxOMxpweswCpSsBRdhAB0Quemp9cjxRT31RHEY7KSouB8Ci6ZPHPdnXDEA3fkrdRbqM4WphU72f41qd+CQgihwwWCB6Tm57PEZwfJTPTv5fnrR8is/33MXjP/kG//tq04WCa56iaRq/PiDm/D1rNdThM+KJig36DepKSMRyVhRQVFzGWp9ojutyrhEPdh2YOsDqhtAwBIemf2F4bMFf0+VKSTJnHBYjS0rEDdiV3uBnklRXu81kwGZe2H+wC5G1VSmxPTtzKJ7QGJoQImqJtAcvGLYt8mExqgxHFJr6sr/A0zQtXZQh50lhsLnOm/NzmkNCdNGsHmkRnu84SsFdk/7U3LM/+1W0EwMoaMQ1Bbsr9/NTMgfUqaWQU5mkva0V4jrmyCUSaMlFfXOiggavWYrtucTbAO5qAN5uPcAddfCm+gLoYpBc1VQ6wGZSGQ3FstaBKPPac0uD34FBVRgLxegbm9nyOZ9JJLSpznYppGaNNclYx6Nds9tH+eXrHbzz3/ew9nOPc+NXn2XXt14knsUov0sRiyfoSUZTVEv3g5ywJrnv9mLjzEWETf3ndbaX6NjZnrQHH9BclHqkPXguSBVjnBwziX0Ov4hWovEJIZQOCmH7qFbPIo8JiiqpdiQIaPp1lMeHxTwJWXyoJrmHl00213nTne3aSCfEE6LBwWiC4bNCdI9N0to7xAZVzJVqZYCPGh5hx6F7GQ0WRmf7ie4xTvdNYFY13uztEN3aVg8UL9J7aLMjEQWzMzfnsnnZXi0aGl9JrBCP9Z2Yel41AgqM9049pmkQGlnwbiVSbJfMi3XVHgAO6Si2DyXz2oulhXxBsja5SDzYHsjK6w8HI6TWkDJmoHCwmQ1sbRBi1Ustg1k/33g4RjgmHDJ80gGhINhcO9U5nKuNIltEbE6ojmLZ2Z7vmO3gqkp/mpM87uRCYhA3xW65MVSIjPQ062ttFg8TGxbV9Ge0Cqp9rmkFAZIsY3FAqbCSXxY8yP03wZuXyk4ZSX5jUGB9ssM1W7nt3UnhVOa15waL0UBdMi/5dG/hWckPTkSIxBKoCpTLOZM1VleKv/uT3WOXXQtpmsZnf32M11qHmYiIDsO2oSCdw/rc8/SNhYknNEwGRRa654hd6ypRFNjTPEjH8PROZE3TpmzkS6bbyOvS2Z7snO3Q/JSXSLewXLCiXNzvngwYhXC6ZId44vBP4cwLEA8TxsIZrZyGYjNUrGdxbQ1dml+3MRvHOgHQHKXZt82+ylld6eaMWkNMU1Fik+mCGBwlMNEvijEiQQa7hNA+iBdt+18yjo0iZZKu9iYdRz97fn1QzKmbaxQcw0nh2FNbGBbyIGITc5WFbjCyfYlwpP3lyDLxWKAVwufcV1hdYo8snCzaioXFP9UEkwFRzLAAkbs3knmxodYDwHOn+3Ubw9CEqPb2OmSXYSGS2hw63j1KJJb5C22qW9lrN0nbwwJjU60Q2w+1Z7+YJ2UhbzcbsJvljXohsLx8SgA53ZubivuimNjEthb5RSWvJL8pKkt/WBXvIBqc2TYxY4yLmIF+zY3PnaOKYklG0YbbIKZfNBLRSZSxLgDGLGVYnR79xnI1YrJDedIGr/+kvmORSK6Ajck1+WvZEtuTHahSOM0d6dz2HN3jZpJUV3uZyyrX31mkwe/AalKZjMY5O3hpQbR/PMx4OIaqwO8+fgPLklEFTf36FHOk5ki524qqyjVVLqjy2LhusQ+A/9nfOe25gfEII5NRFAUWJbPdUzby/eO5d9fQhoTLU4dWQmWxLHrMBSuS8Z4nBuNin2P170P5epEB/cwXAWhRatBQqS9xg8HI4kVL6NRLbI9FsIWFDmF2l+szhqsIs1FlZWUxTVqymaH3iPhfUcFZAoGzMN5DbEgUyvSaq1HW3kmraTEAY/0dOoz6ykgkNB4+JK6Nb19mgZ6j4omUy0O+M9EPFte0mLxss2VJJSYV9o6XEbN6hQtGx8tTB5gdEJmA0S7R1R4Nir2Wp78A//12+P5t8NAH4Ym/h4mFE90m73wl8+LNq8sxqgqH2gM06lR1PZS0kS92yIrYQqS22IbbZiISS2Slcn9gLJnDLSumC44NNUnXgxw4Z6SKMuQ8KRxMRgMTFj/tcTeHW3sv/wXzJTqJQ0tlx5Vk/3ySeaPaXCyuKGap1kKD0kNTa1t287iTne39mgdfUYFUP1/lqKrK4sWL8RhDqGg4Jzv17Wwf7cYYC5LQFExFJQveYi3vMNmhYiOgQHAARrv1HpFEMiuuSUbr7GkayErGdyqzvVLayOeMpaVCYGrsK0CxfVhmcecCg6qkrZ+Pd126oLR1UHQyV3psrCh3sbRMzK8mneZXl4wZ0IV3bhZROb94vZ3EOW4IqXlQ47VjNYloTpfVRJnLgoZCSVUtLpcLNUduS6H+FgA6NT9lHv1syq8m0p3t/ZNoRpsQxG79PNi8IgcaOJRoAKChVFx3yr0OehU/Khp1ltGczhFGO1HQCGkmin1llz9eMm821RdzLJXbPnB66gmjVfwb78My3g5A0CGuNWOOegCU8W4WV5fkdo5cIa+cGaJ7JIzLDDc2OKDvuHiibK2+A5sNoRHQgLJVops8R9hdxWwoNQAKHc714sGO1847yCfmy/BZcV2JTk4dk4iJhpX+E2BcOA6z+TnDJQVDSZGFm1YI24if79enUmk4mcddbJed7YWIoiisSlZRnuzJvNg+OCFF1EJlXZULBY2O4UkGs1xRPZgW2xfOG/xCx2g04lt/E09GlnKyI/tRA6mu5ZBmwu0tzf75JPPGaDTynvf+AW/mGcxKjK7Wk8K2KkskRnsA0dnud8nNw0LAaDTynve8h8WlNozEKY91oIX0i0ZiQHRTd2p+qjxWmdeea4xmcJWDN7mR1HVQ1+FIJLNlS50Xs1GlayREcxYsf1OZ7bKzPXcsTXYen+qZnSvPMyf7+Lv/PcLf/M9hPvWLQzx8qCubw7skUkjNHasqk2J79+zE9lQ8gd7OCanOdlmQkVvevLoCp8VI+9Akr54dSj9+fl57isUlTuKoFK+9iUWLFmE05sYBMDooumMDBj9mi3zfyQVLSp0YVIWRyRi9FENkHGweuPU+0b0M7I8lxfYyDyD2csP2cozEuc1yIKdzJGVj3qn5qSmV8W25YHO9n8MJ0ameFqJT2IsBjZKQ+Ns1uiogHkVLZp27Q528501rcjtHrpCUhfwddRrWiU7RhW0wQ9mai3+RpkFwCAIdons7NCrcICLjYv9wrAfikewOPBoU3eMly8GZ431K1cD2evH3tzchotjoPTr9GLNdFAD0nRDNKaNdoCWIqHbib/lXuOnvYds9oit/gSDFdsm8efeWGgB++XoH0Xju8xYGk2K7zOMuXFJ20NnobO8fS4qoRVJsLzSKrCbKkuvvg+2BrJ6rf1w6IBQiK1MV2ANZvoEFEmNTXct+ryfr55NkCIubTmMtAMHeFrEYyRKRgNjY7seD1yk3DwuKInEvW0cPA3292XVAuBT9okugRaug3mPKXeaaZApbMZSuEh9LK3lJgWAzG7i2XuTavtCY+Xi37hGZ2Z5rVidF1BOzyONOJDT+z08P8JNX2nhwXzsPvdbBJx86yGQymzvXSCE1d6SaFi7X2d6WtJmvLRZdwilRNSWy5pqU+4EsyMgtNrOBt6yrAOAX5zRLNffNLLanPs9GEdclGRFjC1v98l44R1hNBhb5xfXhxJgV4nGxHipfAzf9fwz7NvFYfCuldnA4pqz9VbewFVdCQzO+btYYER3UHVoJNX5Pbs99lbKpzstTiU0AaEMtMNY37XnN6qEmLv52PRWLYKwXh0/Mj4p4pxCh85RwLM6jR4Sj2duWmqcEY3dVspBgpi8aE9cq1QgVa8ErilGYGIRYDOx+cJbBeL+weM9GPnkiLs7nWyqy5XVg+1LhLPGz4WRu+0g7TJ53T2IpEpbyIx2QjMx7LVrPB16uoLd0Oyy+KZdDzjpSbJfMmxuXl+B3mhkYj/Dsqdxnt6c6231SbC9YliVtzE5lobM9lcUt50dhUucUm0sH2gJZPc9AsijDJ8X2gmJVqY3Vylm6h0aIhbObsxwcFJWu/bjxumV2XMGgKAxbxcLDPNaeVYvw6IhYoI2pLsxmeS0pJOJWL6M4MCoJutpO65fbPtgEpMR2o+xs1wOzA0pXio+HmvUdi0RyBdywVOSmvtCY2czDREKjd0TcJ8vO9tzR4HdiMxmYjMY5M3BpoatnNMRYOIZRVfjkrcsoshqJxjWa9RJSU53tXimkZptZd7YPTe9sT4vtveNZiZ64HF2yIEM3Ulbyjx3pZiIs7MFTNvJLSvJDbLcExZoq4SgFg1xT5YpUE9TJIU10o6bWzUtu4dnVX2QCGw0eI5im7gXsJcINyhYdzulYYwNngGRnu895maMlmcDvtGDyVnM40YCCBqd/N+35ga5WLEqUMc1KZc1iKFtNeUUNCU2hmFEiIz06jfzy/NeLZxkLxahwwNa6oqm8du9iON/2PhIUnezxCJSshJprwVsvLNxrt0PdNqi/Dqo3Q8UGqNoMJgeMdIqvzSQT/aKb3VsPipLZ154lGxdXYjbAwUk/UVuJKCo4N7c9hdUFdh+hYXF9P6HVsacjwpt/NsJTbblv3M0mUmyXzBuTQeUdm8QN20Ovtef8/ENB2dle6CwvT9nkZUNsF5tDJbKzvSCpLxKL/2x3tqfnibSRLxgikQgPff9bvMHSzVbtCGd6s1tNHRwSFZjDilsKqQVCJBLhn//5n3luYhkRjJRF20mEs7dZpCWjBsImDxhktE0hkJojh48cod0oqtHHes/ok9sej6INC+u9Zq2Seq91QWWXFQxmB5SvAxSYHILRTr1HJJHMihuWlgCwt3mQcCxzHc1DwQiReAJFgTKXFNtzhUFVWFEhhI9jXZeON2npT3Ut2/nLW5ayMtntnA3XuNkw1bUs50u2WVFehKIIN7++sYsXCqZs5OuTYnuD34GqwFg4Rt9YduPaZqIrIMYqO9tzz+Y6Lw1+B8FInF8lbZNTYvvi8zvbS5wYiVPd9jiHDx8mEsm+mxyhESwxce0yu8ovFLokWWN1pbCD3nN2VGS1R6YKts4ERGHGIt/0v9niqqVEMHJ/4gMcPnQoN3MECPYLsb1P8eMpcuTknBLYVOPmt/Gt4pPWF6c9N9zZCMBZpQZzkQ98iyhZvo1Wyolg5Bu/a8zddeQKaB2c4BtPCne5v9qooZqs0HtMPFm2aupATRMW6NEg+BZD9bVQshRM5/xNGM3ib8eY3C9UVXBVQPUWKFkB4VFhL5+JLvdIUEQ8+JboumdgtVrZUmkFFFrTue37Zz7YZCPcL4rZO4w1rC4xMRzS+PDuGPvbdIzyyzDyXUuSEd6VrI58+mRf2rY7VwylM9vlhmShsjTZ2d4zGmIkmFlrmQGZxV3QpDrbD7UHSFzGQnE+pOeJLMooKKKagahiYrV6huOd2RXbIwGxGTGuuoVVlKQgiEajJBBVvktpp62rO2vnUoPC3Sdu8YBqyNp5JJklGo2SSCQYKxJiuxJoh1juN56JhYgnbTNbtXJqS2T+oC6YHcLyL5kxSNtefccjkcySFeVF+J0WJqNxXm8NZOx1u5OiWInTgskgt49yScpK/nIW4S0DQhBZVCIEh+VlqYg2fTrbu0ZSYrtdl/NfTdjNRhpS1s/dFy+uaD3PRt5iNFDnEx/n3CIc2dmuJ4qi8P5tohv5W0810j8WpmdUXOcv1tluIEEikYBEDroPA6KBa0hz4vMsnAzfQmDXWhEx8GLTAB1xz7Ss6TMBUcTXcN4cqaldREgzEVVMJHLokhEfEgXKIasPxSj38HLF9cvK+G3iGgC0wSZhYZ4k2i/E9n5zlYjlAhSHjy6TuN7ENEVcR/IITdP4//73KOFYgjdUwDvWFkNwMG11TsX61IEw1g1WtxDOy1aJTu3ZYrRAyTLR5W5OdbnP471XS4hxehvA4Z/762SI7Yt8ALwYS7rDpYoVzkfTsIyIQhmDp4ZfvsvPn6yzsKtBZVPNwrney9WSJCMsLStiQ42HeELj1wdz2wGSspEvlp3tBYvLaqIyaUt4ui+zFfiDMou7oCm3g82kMhaOZdUKUc6TwmaN0srxzktvRM6XxKiwvZo0umXXcgESxoRDCdN6tilreWHmkFhsajZvVl5fkl00/1IAnKFOfTrbw+OoE8IdIWgpxWyTloi6oCgiY690hfi88zV9xyORzBJVVc6xks9ctJvMa9ePVJfhscuJ7UmxdFFSBFlWJv7Xo7N9IhwjkCyer5Sd7Tkhldt+MQeE0VCU4eTvpNY3VQAxldueW7F9ZDLKWNK+XM4RfXj/tlqqPDZ6R8N85ldHALEP4rZPX+OWFFlwWs8pMo/l4P44IETUDq2ESp8sPM0ltT471y32oWnw8+MTIs4qGa11ZlhcQ+pLpgti9aUuujVfzsdqGk/qDo4yWeSeQ25fV8WYtYoTiRoULQGNj6efs46IjuWQvRosU24Do0WLcz7O2fK/Bzp5sWkAiwH+8ToFxWSDHnFNxFEK7uqk0N4DFheUrxWd63PFWQpVyS736KTolJ/L3tR4r1iveuvmPpYM8oYVlQD8aDCZ2z7aCcHAhQdO9GONjxHTVFxldViMCp+93sG3bjSg6GSDnw2k2C7JGG/fIP64njjem9Pzpmzkpdhe2KTygTJtJZ/qWJZZ3IWJQYG1VWKRlc3c9ikHBDlPCpHV6llOdGfXdiglgkXNblCl2F5o9JtFbnugu1lYf2WaSBBzXGxWGpy533CQzJ+iqjUAVMU6hMVbrhlsRtXiBDULjiLvtDxESY6xuqByMyy+GZbcpvdoJJJZk43c9lTHY4VbdqDmmlRn+7GukUvmarckM91THc5L053tuRfbU3ntLquRIqu8X84Fqy7jgNCWtJD3O804LVPC6VQed24dEFJd7cUOM3azdAvTA4vRwF/vXA7A48fE/u2S0gutuBVFYfG5nczz6cScLUOi67FDK6HKv3A6HQuF91xTA8AvDvYRN7sgJN5/0jbypZ5px1tNBoYMOe6sjcewh8S8NXvKcnvuqxyrycD7NvqmrOTPviT+1zRKJsXfrtFXP91avWRFbgc5S0LROF989AQAH1uXoL5CxDHRuFv8718mOtKDA2B2CqHdmoECIJNVdLlXbwFnubCVDw4KUX82TPSD0SYKw/PE1WFDXQl+u0pTxMekrQLQoO2lCw8cEgUZzVolS8unihYM6sIR2kGK7ZIMcstK8Sa3v3WY4WBuMjhi8US6clpmthc2y8ozvymgado5HctyfhQq66uTYnsWc9sHkvPEJ+dJQeJUJpkYbEeLZK/a3jgpusQ0q1d2thcgkx7RtawGWrPTtZwsxghpJopcsrO9ECldLGzi/MoII90ts1/wZooBkRV3RiunzmsEQ34snq9KLEVQuR42vE8I7hJJgXD9ErHhfbRrhMHxzMRhpMTTctnZnnOWlRVhUBWGg1G6Ry6ex92SFEsXJcX2ZUmxvWN4kolkB3GuSM2XKq+0kM8Vqc72490zi+2pvPba4um/k5RleK5t5Kcs5OU1RU9+b31luqAHpoovzudmW/PUJ+HsF2Ykhs4CQmyv9kqXp1yzc3U5LquRzkCIlwJeUAz0Dw0zEQVVgZrzxHaASes5gncubMLHulFJENEMePyV2T+fZBrv31bLbq4FINF/EiZHITiISxslrikUVy4B09T7jbN2g04jvTQH2wMMTUQosSl8dJNDREUGB6cixJYmC66jYShuAJsnswOweaFyA1RuFM08Ix0wGZgW33ABwSGR0162BuzFmR3PPFBVhR3LxB7YMfNa8WD7qxcclxhoAuCEVssKTzQ3BVw6IMV2ScaoKbazoryIeELj+dOZq6a/FIFJIbQrCnhsUvwoZFLZcpnsbO8ZDRGJJzCoCqVFcjFXqGyoSXW2D2fl9UPROOPJjSjZ2V641ETO0Decve52a0RYhCuOYvGmIykozOWrAPCG2rMjto8LsX0AN8WuCztDJPmPw1NKJ6UA9LSeStsm5oz+kwC0aBXUe4x5U6l+VWKyiQ6GRG5FKolkvpS6rKwoL0LT4KXmwct/wSw40iHurZaWSdEj11hNBpYmBbCLWcmHovG0wJ2ykS92mNNrmlxbhHcOp/La5do7V6Q6288MTBCMXPi+1Tok5kAqoz1F6m+6qS/HcyQltku3DF1RVYW/uX2q4/T8vPYUntLqqU/GurJejBoaOAtALz78Llm0k2usJgO/v7EKgJ8dG4fSVZwZEuJflcuIxXShG4Xmqpr6JCdRA20AdGl+ako82T+fZBql/lJWLmqgKVGJqsXhte8R6RNF4y1aJQ3l3mnr2KrFqxjX8u96/9rZIQCuLUtgsic71ht3izx0V5XoPI9HwWAUhdjZQDWAuwqqr4GS5UnBf1gI7yOdIid+vE/YzY90QiIOpavAWZKd8cyDHasqAHhwbKN4oG2vKMQ4h4neZGc7NdRaQmK/ZSy37ti5QIrtkoxyy0qxSfn0qczlxF2KVF6722bCaJDTuZBJVeCf6h27pE3eldDYKypv6312zEY5PwqVVGf76d6xrHRn9I+Jzh+zQcVllVZ2hcp6tZnjHUPZeXFNwxkVxR5mZ45t0iQZwb94AwCLaadvKJD5E4yLRUK/5sEnxfbCRDXQYxa5ZxMDWXJAuBiJBAw2AnA6UU2DxyTFdj1RFJHTp8h7R0nh8cZlYgPu2VN9836teELjUNJZalOtdG3Rg1WVl87jbh0MomlQZDVOc3JL5bY39uljEV7lyb+N9YVKaZEVv9OCpsHJGRoXWgdEZ3udb7pwmbIHH5yIMDGHyNi5MuV+IOeI3tywtITb15RjUBXesGTmNa6/ZkqQV7oOZCeO6xy0QDsAQbMf1SyLdvTg3Ukr+SeO9TBkLOGMVg5AQ/HMvw+z75zc6GgOindGxBzp0PzUlORPd+9Vg6ryJ9ur+XH8FvH5iYdRn/0SAI3UUuyfbu1fW1pMo3ZO0U4sN07Il2PfWbG/d02JJtZ8mganfiuerN0OZru43pnsogg7m5jtQmyvuw5qt0HlJihbDZ56KCoH31JhY1+1CVz56ebwhuVVWI3wi+A6QkX1okP/9R9OO0YZFGL7mK0ag8Uu8uurNoGzYkG5h8odBElG2ZG0kn+ucYBYDtxjhpJie7FdWj8XOktKnagKBILRtPg5X1KW9EtLs1SFJskJZS4rFW4rCe3iXR3zYXBiKmpAkR3LBYOiKNTW1lJsiaMAa9UzHO8KZOdkk8MYEYUedk9pds4hyTipOeJwOLDUbAagQhmiuaU5810ZYz0A9GtufE65eVgonDtHFEVhvGgRAOpoB8Qycy8yK+JhtOGzAJzWqqn32RbUgrMgsRRNzxuUSAqEm5aL+5RnTvYRT8zvve507xgTkThOizFdGC3JLasrRdHxxdZA51rIn7uOSf2+mnIstqe7lqXYnlMulds+1dk+XWx3WIzpoojeHNYXptwPZGd7fvBvf7iJ1z9zK0svco1fWeXGqY1Qp7UT6zqYddtf03gXAAlnCRjkPq8erK50s6bKRTSu8fv3v8QvTybdU8o8Mx7vqVhCndZOudaDEsluMQZAqL8FgE6thBqfdN3RgzUNlRzzv5lPRT9KCDPGqLjX6DdXo1inX0uMBpUeUy11WjteQxAlnsM3nIsQT2i83irE9i2VyTV3/wkItIru8uW3i8eiQbD7RXd7LlANwq7eXSWs68tWQcV6kfHurQNH/jb+2CxGbqgvAhSe8b5LPHj6sakCrVgYe7ATAJO3GswO8b16aqFizYLa+5BiuySjrK/24HdamAjHaR7NvmiVFttlXnvBYzUZqE9am53KUG57anNB2h4WPqkNo+b+zG8YDSSLO3zSQr6gMJlMvP/972dVQwUmYqxWznKyK5Ada7tk1/Kw5sTjkZ1dhUJqjixduhRTkY9eg6jKH+8+nfmu5bFuQHS2+2VsScEwbY6YTCR8ywBwTnblxgYxRTQo7OKAJq2aap87d+eWzIzFOS1vUCIpFLbUe3FZjQwHo7w+zwim1Nevr3FjUGVBqh6svoSICtAyIISvRedZQKfWTqd7cyu2tw2JTdVqmdmeU9Yk58lvj3Zf4BLYls5sv9B5aXEypqBnMnd/3x1Jsb2mWIrt+YBBVXDbLy5y1PpdRFUDH+LnJPpOQzhzsY8XEB7DHBPXOmNRuYxu05HP7FqF126idTDIq8kO4PoS14zHNqzdxge1X/Bn/IShjlNZjxqYTFpRD6o+HHb5XqMLVjeffZOX3xluYlf4HzmeqCOuKQy6V824fppwL+ZD/Jwdhr2YtBxaqVyEUz1jjIVjOE2wojT5XnQy2dVeugq8DeLjRDzzWe0LmFtXCnetfx/aDEUVYs/t4E/Fk0NnUEkwqBVR5XOD1aPfQLOMFNslGUVVFW5eIf64jg7nQGwPCrHdK8X2BcHy8szmtjemxXbZiVHoLCoRmwNnBjJfST0wLsT2c60XJYXDuKWcmMGGTYkQ7D+bpTxuIbb3aR58bnk9KVSGHEvEB8NnMp7HnRhJiu24KS6Si/5CxV69GoDyeBcEA7k78WAzSjxCWDMRt5VgtssiQd0x2bKXzyeRZBGTQeXGZHf7kyfml4N4oC0AwMYaWWioF6mO5c7AJIHghdarLclM9kX+6UKqXjbyKWH3/C5qSXZ595YazEaVl5oGefRId/rxcCxO96i4553pd5LK6e7NodjenizIqCmWc6RQCFduBcA50QbDbdkTU5MW8sOaE3+xfN/Rk22LfLz0NzfzmV0rKS2yoCqwpX5my3anx89ZQy0AXaf3Z3ydfT7xYZHZHrb6pfuBXigKaxbV8JNb4wxbatgV+Uc2h7+DtWIlGGdoPChNrrEjbRDKvFvplbIvmde+sQSMFruYs81PiScX3wyqKmzQVZMowJbMiptX16AAhwcSBFa+Vzx4/FfiZzkkimROJGpZ4TeJzvYFihTbJRknZSV/dFjJWPb2xRiWNvILiqkK/PmL7Zqm0Zi2kZdvjoVOqlujJRud7WmxXXa2FySKSjzZjeqbbGFiIvPV9omxVB63W+ZxFzBh30oguVGU4aKM6OjUHPE65eZhoVKyaAMxTcXJJNpgY9Y7M9L0HQegWauk1mMAg3w/ygucpbmzDZRIMsiOVWI9/tSJ+eW2pzrbN9V55jskyRxxWU3UJkXJmbrbWwbE2qihZPr9aarYvGc0zGQsy4NMMhqKpuO5pNieW+r9Du5+02IAvvCb44yFROdg+9AkmgYOswHfDA0qKQfAnuw7PwMwHo6l54gU2wuHmmXr6dM8GIlC52vZKW4HSEYqdWp+qvzS5Ulv7GYjH7lhES9++mZe+bsdrKm6+O+kz70BgHjvcQhnt8jLNC6sqHGWClFUog82L2vKbfzsbU78diMBilhbZp0xhstZv4mYplKsDUPfCUjkIHf4EqTE9mtLE8I2/uj/CKc5qwcWvVEcFA2KLPVs57UvIPweJ5sqxL3GrxI3gN0n3FB+9E4S+/8fACe0Olb4DAs6rk3Xq9LnPvc5FEWZ9m/FihXp50OhEPfccw8+nw+n08mdd95Jb+/06uy2tjZ27dqF3W6ntLSUv/7rvyYWm76aePbZZ9m0aRMWi4UlS5bwwAMP5OLbu2q5fqkfs1FlKKxkvZI6daNeLDtSFwTpzvYM2N31j4UZDcVQFWjwS3Gs0El1a6S6NzLJwHgys71IihuFRCQS4etf/zpHjhwhUb4egLXKGU52zc8ydSZCQyI7rh8PXmkRXjCcO0cikQjGynUAlEfbM75JlCrICJk8GEwLJ29qoXP+HKkpL+WsJuIGAp2ns96ZkaZXiO2ntGqWelQwyvejvMBTB956vUchkVwxb1pWglFVaOobp3VwbvfOgWAkfd8tO9v1JWUlf7Rr5ILnUq5fi/zTN4PdNhPlLnHP2pOjVJRUV7vfaabIKu+Fcs3dNy6m3menbyzM13c3AtCWzmt3oMxgyZ1qSugKZr9RBqa62j12Ey45RwqCSCRC755f8p/KB4hgJNH5evZy24fOANChlVDlle5C+YLZqFJyib2ySCTC02NL+GfuojjYApEs6gCJBM6QWHdbPBXZO4/k8pidYCtmqWWIx97l5IFbFa5fUjJj/ENteSn/rNzNP3MXkY4DUzneOqBpWlps31KuQnAIDvxYPLnszWBLOjhEJsHmEznqkllz+yqRK/+lvZO8vuTPRRZ7eBR1Qvzddhhr8ThmLspYKOheArR69Wq6u7vT/1588cX0c/feey+/+c1v+PnPf85zzz1HV1cX73jHO9LPx+Nxdu3aRSQSYc+ePfzwhz/kgQce4LOf/Wz6mDNnzrBr1y5uuukmDh48yMc//nE+8pGP8Pjjj+f0+7yasJuNbF8kLk5Pn+zP6rlkZ/vCItXZ3tg7RiIxv8VeqtCjzufAapJvjoVOyka+bShINJ7ZKkjZ2V64TE5OEo/H0VJiu9pCS1/mbanCwyJLeVRxYzbJeVJIpOYIgHepsECs1npIBNoyeh4lOABA1OzJ6OtKss+5c8RkNNBhFDaII71ncye2D5wGoDFRzXKfQYrt+YKiyLxQSUHitpm4Jmn3+uQcu9sPtAcAUbQsI9v0ZVOtKHb44Z5WgpGpxpKhiQiBoOhgnqm4PNW13B3MzXWsNZ0NLjuW9cBqMvCFt60B4IE9Z3jqRC9nBy5t67+60o1BVRiNKnSPZP+eJyW2yzlSWETCIWKKKI6Y7D6Ztdx2LdAKQIfmp0rOkYIiGtcIKnbqEp2Md53K3onGezAQI6apeEqqsnceyeVRFPAvBe8iSowhbiwPo9hcMx66pNRJRLEQVOwEu07pKrZ3DE/SOxrGqMKGShvs/4EYT1EFrH/31IEyr31OvH9bA7fVKkTi8M59K/nZlp/Bzi+xv/w9fC36Tjq8W0XUwExxAwsE3cV2o9FIeXl5+p/fLyogRkZG+P73v8+//Mu/cPPNN7N582Z+8IMfsGfPHl5++WUAnnjiCY4fP86PfvQjNmzYwO233859993Ht7/9bSIRIcJ+5zvfoaGhga997WusXLmSv/iLv+Cd73wnX//613X7nq8Gbknmtj91Krti+1BycSk3ABYG9T47FqNKMBLn7By7MFKkrOiXSAv5BUG5y4rdbCCW0GgbyuyNmcxsL3y0yk0ArFTaaO0ZyPjrx5N53EGjS1RmSgqSsuolDGtOTEqcwdYTEM+Qr2oigSk0CIBmld1/hc6wvR6ARKADojkQ2+NRtGHRyXNaq2aF37ygF58SiSQ33LJS5LY/Ncfc9gOtwiloY60nU0OSzJE/3FpLlcdGZ2CSr+8+nX48Fa9V6bZiM19YXJ4qZO/JkdieWr/X+6SrnF68cVkJb1lXQUKDD//wNb7znMhIrb2I2G4zG1iZdBc80H6hc0KmaZN57QWPZbwNAtnJbY8OngVEZ3tF8cyinSS/URWN1uOvQiySnRME2gHo1nxUl8ycIS/JIVYXVKyFmmuhfJ2wYZ8Bs3FKfowPncmeO8YseK1VdLWv8SnYJvvg5KPiibXvnupqj4XBaJZ57XPA6nBx/5vdvGeZgYQGn34uxJbdi/jz/t/nX+PvYKnfIubJAi5o1z2ErrGxkcrKSqxWK9u3b+fLX/4ytbW17N+/n2g0yo4dO9LHrlixgtraWvbu3cu2bdvYu3cva9eupaysLH3Mzp07ufvuuzl27BgbN25k7969014jdczHP/7xi44pHA4TDofTn4+Oii65aDRKNBrN0He+sLlhkdhsPtQ+QvfweNY6RgfHxSao26rK302Bkfp9nf97W1VRxIH2EV5vHaLGM/d5c6pH/N0u9tvl3ChQzp8j9T47x7vHaOwZoXYec+N8+sfE9d5jNci5UkCc+7uKuhsIGZxY4+OE+5qJTk6Im+MMoY2Ljeqw2Us0DmhynhQC0+ZINIrZbKZFrWezdpRA12k8obHMZHANncWkRYloBjRnmbyOFBDnz5FoNErYswTGwRLsJhqaAFuWf5+hAMYR4Z5xWqumvtgmHFwy7OIimTsXu2fNF/J1XBJ92bGyjC8+eoJXzwwxGopesWVzqrM91VUt0Q+HxcgX376GP35gH99/8Qxv21DFmio3LSkL+ZKZ72WWJ8X27hzZyKciCy4m7Epyw1fftZ7SIis/2HOGvuQ6t6744gUQG2vcHO0a5UB7gLdvqsnq2GRne2EzoLmpZBDaX4GqjWDNbK56bLgdMzBm8mGxLFyL4YXOZOcxYSVvzLwYrgVaUYBO/NSUSrE9b7C6Z309cIW7YKQDihuyPKiZ2XdWFJNeU5KAfd8DLQElK2HF7VMHRSfBJPPa54SqYiwq5Z+2DVPmKeLb+8YZCE7ta6wrTiz4IgZdxfatW7fywAMPsHz5crq7u/n85z/PDTfcwNGjR+np6cFsNuPxeKZ9TVlZGT09PQD09PRME9pTz6eeu9Qxo6OjTE5OYrNd+Ab+5S9/mc9//vMXPP7EE09gt8ubwtlS4zDQPqHwrV88zbbS7OQ/dQ0YAIUTB/YRbMrKKSRZZvfu3dM+d0VVQOXhFw9h6jww59fdd1LMjYmuJh57rHF+g5ToSmqOWCJibvz2xf2EWzJ3TekeFnPl+IFXCGTR8UqSWVK2zwBPP/MMG011NMSPoQRaeeyJJzN6rq3Jzvb+hJvHfve7jL62JHtMmyNPP43BYCCm1rA5fpS+ni6OP/l8Rs5TOnKI7UCLVslIMMpjjz2WkdeVZJ+Z5khrSCz+iqM9PLbnCCjHsjoGZ6ibWxIxgpqFkMXPM8f74LicQ/nI+fes+UIwOHvHn3/6p3/ib//2b/nYxz7GN77xDQBCoRCf/OQnefDBBwmHw+zcuZP7779/2hq6ra2Nu+++m2eeeQan08kf/dEf8eUvfxmjcWo74dlnn+UTn/gEx44do6amhs985jN86EMfytS3KblC6v0OFpc4aO6f4JmTfbxtw+ztVhMJjYNtAUB2tucLN60o5S3rKnjkcDd/88vD/PjD22jpT4ntMwupKyqE2N4xrsw7om02pGzkZWe7vlhNBj771lW8eU05n/rFIdqGgmyq81z0+A01Hv77lXYOJP/ms0n7sKj8qPHKfdVC5NXEct5u2AOdr8NkIONiu3GsE4C4rRQMuvcGSuaIZ+y0ENvtmRfDg31ncCCiBjYWL2zBbqFiJAadB6Bma0YbZGbLvjOis/2Nrh44uA8UFTZ+YMpZTtNEVEbJSpnXPlccPpRhE5/YbOLPNpXTOBTj1GCUWEJjZ/WIKGRYwOj67nX77VNVI+vWrWPr1q3U1dXx0EMPzSiC54q//du/5ROf+ET689HRUWpqarjttttwuaSVzWyIRqP8tv1J2icMDJgruOOODVk5z9+89iSQYNetN8rq2AIjGo2ye/dubr31VkymqU6L2KFunvvFEUaMXu64Y+ucXlvTNP7h4LNAlDtvfQOrK+XfbSFy/hxptDRx4NkWLP5a7rhjdUbOEYrGCe59CoB33L6DYhlJUTBEIhGOHDkCwM0334wWfQpOHKM21sqbb/gYalF5xs4VO3wPAL5iH3fccUfGXleSXc6fIw6Hg1/3HYXO31ITbeHaN6wDd/W8z6O+cBxaoEmrYuOSCu7YuX7erynJDTPNkaLjbYR/+RlsSpg7qidg3buzanOm7P8BnIAmrZKNpQp3vGEduGUGYT5xsXvWfCHlwnY59u3bx3/8x3+wbt26aY/fe++9PProo/z85z/H7XbzF3/xF7zjHe/gpZdeAkRRyq5duygvL2fPnj10d3fzwQ9+EJPJxJe+9CUAzpw5w65du7jrrrv48Y9/zFNPPcVHPvIRKioq2LlzZ2a/Ycms2bW2gm893cTP9rVfkdje1D/OWDiG3WxId0dL9Oezb13F86f7Odo5yvovPIHZIGxZF82Q1w6wssKF3WwgGIlzum+ctTXZ7QJMie0XyweX5JZrG4rZ/Yk3EQhGKSm6uCvcpmRBzfHuMULROFZT9sSFNtnZXtC8lljB2w17iPQcxzzWDZ7azN0jRyYwR0WUgdFddpmDJflMXbyV2GArRk9txl97srcZBxAw+LBYpftBoRLtOoQpmlk3ytkwPBGhsU9E8GzSkgX17hqo3jJ1UHgELEVyPT4fbF4oqoBAGw53FRvKzWwoNwt7/pAZTAs7Mi+vSsU8Hg/Lli2jqamJW2+9lUgkQiAQmNbd3tvbS3m52EAvLy/n1VdfnfYavb296edS/6ceO/cYl8t1UUHfYrFgsVx4M2oymfJygyVfWePV+F0HvNg0SBw14zftI5NRJqPCiqLc48BkyqvpLJkl5/9dba73AXCiewxNMUzLdpktA+NhApNRFAWWV3gwZXHBKMk+qTmytFwUTZwdnMzYtbixX1TXu20mSt12lAWcG7PQ0M7JiTOZTFgbNsGJH7FMaWVwNEhlcYberyMTmOJj4jyuEnkfUECcP0dMJhNq5QboBH+kUyzwMvH7HBTuKY2JKkpddjlHCoiZ5sji6nJatEpWKm0YhppQlUR2F4T9xwE4rdWw0qdgshVlZl5KMk6+rgVnM6bx8XHe97738Z//+Z988YtfTD8+MjLC97//fX7yk59w8803A/CDH/yAlStX8vLLL7Nt2zaeeOIJjh8/zpNPPklZWRkbNmzgvvvu49Of/jSf+9znMJvNfOc736GhoYGvfe1rAKxcuZIXX3yRr3/96xcV22V02/y5XMTBOzZW8K/PNLGneZDGnsCsO45faR4AYG2VCy0RJ5qIX+YrJLnAazXw1Xeu5b5HT9I+PEkkGTeyqsJ50TmwsdrFSy3D7G0eYEV59gonQtE4PaMi4q/SZZZ/w3mE5zKRi6UOAy6TxmgUDrQOsqUuO9ERmqalbeQrXCY5RwqEc39Prd7rSIz9APNYG9HeRiheCpYMNbb0ncYEjGh23J5iOT8KiHN/V12an3qlh6bDL1BXsRFMmY2UjQ+3AhC2lBBNKCDnSUFw/t/zaHcTruAYGHPrTvBKSz8Ai9wKtkFRcB8vWU1CMYoINy0BE6NQthoUk5xf88FRDsOdEA5OuQaEQ6CYgZl/tgslui2v1Mnx8XGam5v5wAc+wObNmzGZTDz11FPceeedAJw6dYq2tja2b98OwPbt2/nHf/xH+vr6KC0tBYS9n8vlYtWqVeljzrfz3L17d/o1JNmj2gFlLgu9o2H2Ng9y04rSjL5+c7+oRipzWXBY8moqS+ZBnc+Ox24iEIxysmeUddWeK36N071CGKvx2rGZpdC+UFjkFzdiLQPjGXvNpuR1ZEmpUwrtBYaiKFRUVBAIBFAUBUP1ZgBWKm0c7h6gsn55Zk40IITUAc2Fwyur7AuJ8+cIgLt2LeFXjdiVSeg9ChVr5n+iwdMANGpVrHRmdkNBkl1mmiNVHhuPadWspI2xvjO4Y5PZE9vjsfQ15nSiik0+04Kv9Jbowz333MOuXbvYsWPHNLF9//79RKNRduzYkX5sxYoV1NbWsnfvXrZt28bevXtZu3btNFv5nTt3cvfdd3Ps2DE2btzI3r17p71G6piPf/zjFx2TjG7LHJeKOFjhVjkRUPmnh17g9+oSFz3uXH5+SkQ3+WIDMholD/mrFTARhc6gggJ0H9lD95GZj/VEFcDAI6+eomzkRNbG1B0EMGIzaOx99slsGsJIskB9kcrhIYWfPvEyfVXZiRwYiUA4ZkRB49CeZzl65T0VEh1IJBLpRjXFYOZVbQXblBOcOvAizV2Zc5BMxXJ1aiUEAmPyvaeAOHeOnJpcSr3WQ0vTKY6pT2X8XFuTYvuA4uex3/42468vyQ6pOTIRCqFooI608diLB4C5R8fOhYdbxf1tpTlMouswKvC8up3R1zumH9hyDMhulNzVw8CFDx1//JJfUejRbboqlH/1V3/FW9/6Vurq6ujq6uIf/uEfMBgMvPe978XtdvPhD3+YT3ziExQXF+NyufjLv/xLtm/fzrZt2wC47bbbWLVqFR/4wAf4yle+Qk9PD5/5zGe455570p3pd911F//2b//Gpz71Kf7kT/6Ep59+moceeohHH31Uz2/9qkBR4OblJfx0XwdPnujNuNje1DclkkkWDoqisL7aw3On+znUHpiT2J6aG0vl3FhQNCTzCAfGI4xMRnHb5t9dlr6OlMi5UmiYTCb++I//mMcee0x09ZWtIYqJImWSvvZG2Lod1Azs4vSJrtMWrQKfS86TQuKCOQLUlXlp1KpZo5xF6z2OEguDcR4CuabBYDMAjVo1PpfMKS0kZpojRoNKn7UOonsID3VAJCis0LJBdAJt+CwKcFqr5r0lFjBKS0RJZnnwwQd5/fXX2bdv3wXP9fT0YDabpznJAZSVldHT05M+5lyhPfV86rlLHTM6Osrk5OSMjnIyum3+zCbiwNzQx90/OciBEQvfuu1Nl3UNi8YT/H+vPwvE+PAd17GuOrOZvJLc4mvu59EHDtAesnD77Tdmrbj4yRN9cOggi8vc7Nq1LSvnkGSHaDTKU51PcngIJh3Zi4Dc3zoM+/dR5bHx1re8MSvnkGSH1HvN4sr1/OZX29mmnmD12PMsv/WPoGpLRqzk1RdPQYvI4r5181Ju2bAkAyOX5IrUHFGGw3D2BZaGDlK9+ZNQtjJzJ9E0OPgRACorK2W8X4ERjUb5zH89gtofx5sY4o7aEKx6W05z0X/4n68CAT5cdhpjUxjN5OD6zavEWj8Rh9EeqJCRbhkjOASd+8FaBIoBJgagZCUU1894+EKJbtNVbO/o6OC9730vg4ODlJSUcP311/Pyyy9TUlICwNe//nVUVeXOO+8kHA6zc+dO7r///vTXGwwGHnnkEe6++262b9+Ow+Hgj/7oj/jCF76QPqahoYFHH32Ue++9l29+85tUV1fzve99T2bH5YibVwix/akTfXzx7VpGF3epznYpki081tcIsf1g+wgfmIMJRWNvcm6UybmxkHBajJQWWegbC3NmYIINNZ55v2ZTn3BBkEU7CwCDkX5rHZWhJmL9zRCbBHMGhM/+UwA0JyqpKZIiWKFT7bXzm0Qta9SzTPY2Yo8G5ye2j3ZBNEhMUzmrlVMsxfYFQdC9BAbAON4NkYnsnWhyGMa6AWhVqqkt8WSmSEgiSdLe3s7HPvYxdu/ejdWaX64JMrotc1zqZ3br6grKXCfoHQ3zTOMgb1lXecnXOtAxxHg4RrHDzMY6H6oqW5QLmY11PkyqxnAwytnhMMvKsmMl3xEQkRB1fof8+y1AGopEN/uB9hGMRmNWijK6RiMA1PrkHClUblpRyv/VtvJ57QGMY12Y+o9DxWqwZqAoa0g4hbVppbyhxC3nSIFStuX3iZ/5Gg3xs4x1HKSobFnmXLvG+0CLkNAUvGXVco4UIJUuKyf7almttGLqeR2W3wqW4pycOxSNc6RzBIBNiIYapXgxJqeIsmVyAFwl4K0Cg5xbGcFVChMVMNYjxHZrEdhdl43My9e14GzHpOtuzoMPPkhXVxfhcJiOjg4efPBBFi9enH7earXy7W9/m6GhISYmJvjlL3+ZzmJPUVdXx2OPPUYwGKS/v5+vfvWrGI3TawhuvPFGDhw4QDgcprm5mQ996EO5+PYkwPaGYuxmAz2jIU72jGX0tZuTHamLpUi24NhQI27WD3UE5vT1jUkBdVlp9nLpJPqwKNnd3tKfGSt56ZCxsAh6hHW8dewMREOZedGkxXOLVkGxM7+ECsmVYzUZ6LQsAiAy1DZ/ITXpfNCqlRHFSLlbFmQsBJQS0YXhivSmxfCs0HsCRYszptkocnkx2D3ZO5fkqmT//v309fWxadMmjEYjRqOR5557jm9961sYjUbKysqIRCIEAoFpX9fb25ted5eXl9Pb23vB86nnLnWMy+WasatdkjuMBpX3bKkB4Kevtl32+OdO9wFww1K/FNoXAGajSr1TCKmvtAxm7TytQ+J+qt4nIyAKkRonmAwKA+NhOoYns3KO9iHxujVeOUcKlSKriRWLG3gxsVY80PSEKBydL5pGol247+xPLKOyWO7jFSorly3jgGEdAC37n4RQIHMvHhD3MD14qS4rydzrSnJGjVPj9cRSAIKdxyE0krNzH2oPEI1rlNrANZTM3qkQc5VYCBIJ8C2WQnsmURQoXQV114l/tVvBmVnX63xEtk5IsorFZGB90gY8VUGUKaT988IlZR3f3D/OaCh6RV+raVq6sCNblfsS/ViU/Htv6Z9/p2EsnuDMgHgdKbYXHtFolG9/+9scO3aMaFRcJ9SK1QD4Qu0QnV2ezuXQkhbhLVoFfpcUDAqJmeYIwKh7BQCm8U6YDMzvJL1CbG/Uqqlxm3BYdDWNklwhF5sjnuoVBDULRpKZ6rFw5k+uadAhNhZPaTUsL1alhbwk49xyyy0cOXKEgwcPpv9t2bKF973vfemPTSYTTz01lat56tQp2tra2L5d2Ett376dI0eO0NfXlz5m9+7duFwuVq1alT7m3NdIHZN6DYm+vPuaGhQFXmoapLH30gXwz57qB+BNy+RG9kJhiUuI7S+fGcraOVoHxX13XbF0+ClETCqsrBB7J/tbMyCezkDbkJgjtbIgo6A4/15555pyHo4n39s79sNotxCp5sNIO2rgLADHDCtxO+V1pJA4d47EYjEmFt8OgG/gFRjru8xXz57EQBMAnZqfmlJfxl5Xkn1Sc6T55DE6HWLPLjrQDGO9Yk2cA15Lvrfd7AugDIk9PuquE+cf7wdPDTjkvW/GMVqE+4nFCWZ7RmJH8h0ptkuyzsoKkbl3ont22QazIRyLp2/WpUi28PA7LVR7bWgaHO24siKNs4NBAsEoZqPK8nIpti80FvmTne0D8+9sbxsKEo1rWE0qVR4pcBQamqYxMjJCNBpFS96gu+s3A9CgtRGdzID7QSKBNnwGgC61khKPzJAtJGaaIwCJUrHAc8SGYbBF5HPNlf4TADRqVSwvldeRQuNic6S+1MUpTXSC0n8yY8U704iFoOcQAK8llrPCZwCTnEOSzFJUVMSaNWum/XM4HPh8PtasWYPb7ebDH/4wn/jEJ3jmmWfYv38/f/zHf8z27dvZtk3kLt92222sWrWKD3zgAxw6dIjHH3+cz3zmM9xzzz1pG/i77rqLlpYWPvWpT3Hy5Enuv/9+HnroIe699149v31JkmqvndtWlQHwhUeOT7venUvfWIhjXWLNfsNSueG4UEiJ7a+0DF30dz9fzg6KAuY6KaQWLBuTEW2vt2VHbG9P7t/VFMs5Ukicf69866oydie2ENZMEBwQebzhee71tjwHwMlEDfYij4xUKjDOnyNrbn4vIc1EtdbD2aMvQjQzbhljXSLer0srodwrdYBC4tw5Eq7aTlxTcIc6RVF7OLMuyBdj31lRcHi7/ah4wFkuOtlDAbC4oLjhqhCCJdlHvoNJss6KZIVsJsX2swNBEhoUWYyUFM0ja1WSt6xPLvYOXqGV/MF2sThcXenCbJSXuIXGlI38/DvbU+4Yi0uc0iZzgVC89FoAKpQhOs+env8LjnaixsNENAMWdxmK0Tz/15ToTll5FWcTQnSg+yBE5lGYMSDmWWOiSkaXLCDqfQ6OJeoASAw2ZWyTaBqRiXQMwb7EMlb4LVJsl+jC17/+dd7ylrdw55138sY3vpHy8nJ++ctfpp83GAw88sgjGAwGtm/fzvvf/34++MEP8oUvfCF9TENDA48++ii7d+9m/fr1fO1rX+N73/seO3fu1ONbkszA392xErNR5YXGAX57tGfGY144PQDAmiqXXGMvIOqKhJ38wHiYloH5r6HOJxJL0Jm0Hq/3y47UQmVLnReAFxoHslKUkWqWqfHKe51CprTIyrLaSp5ObBAPND01f6vwsy8C8HJiJZVuGdtW6PjK6zhq2wJAz5Fn5u8klyTe/hoAvaZKDJnKgZfknMUNi3hNE/GPWvPTObGSjye0tGvL+tgx8WDJclANEAkK0d0s718kmUEqUZKssyrZ2X6yZyxjN+1N5+S1K7LyaEGyIWklf6g9cEVfd7BNHL8hKdZLFhaL/KKC9ezgBInE/K4nTf0yr32hodiL6VZEBtBwx3GIReb3gv2ierpVK08XekgKnzqfnQPaEvFJ37G557ZrmqjGBpq0KpaVS+eDhUKlx8YppR6A8GAHhDPglHE+Ix0w2gWIfMrlZQ6ZESfJCc8++yzf+MY30p9brVa+/e1vMzQ0xMTEBL/85S/TWewp6urqeOyxxwgGg/T39/PVr34Vo3F6bMaNN97IgQMHCIfDNDc386EPfSgH341kttT5HNz9psUA3PfIcSbCsQuOee60sJC/cdnCz1O8mjCpsL7aDYju9kzTGZgkoYHVpFIqizQKluuX+DAbVc4MTNDYl9n7nlA0Tu9YCIBa2dle8OxcU85vUlbyXa/DyDyt5NtfAeDlxCqqZDFG4aMomNa8DYBFo/uIDHfN/zXjMYr6hNje7lgr10wFzJtWlPK0Jhwpx9sOwnjmogYuxqmeMcZCMYqMCdxDwlmOqk2i6cLqhqKKrI9BcvUgxXZJ1llS6sSgKgSCUXpGQxl5zXReuxTJFizr0zZmgSsq0jiYFOc31nqzMCqJ3lR7bZgNKqFoIl0dP1fS15ESeR1ZSPTYxEZyuP8MROfZvZO0CG/RKlhSJruWFwp1PjuvJ5aKTwabIDRH552JfgiPktAUmrVKllUWZ26QEl0xqApBj6i4V8e7ITiY+ZO0vgTA6UQVRqsTf7HMHpRIJNnl7hsXU1Nso3skxL8+3TTtuXhC4/nGZF77cmkhv9C4tl6sjV9qGsj4a6ct5IsdshGigHFajNywxA/A7y7ifjFXOgOTaBo4zAaKHdIprNDZubqcpxMbGdesoqu97aW5W8mPdkMytu2VxAq5j7dAWP2mdzOCg1JlmIOvPDX34vYU7a9gigcJaA7ixcuk3XcBU2w3oy3eAYBj/AwEWuc/Py7Da62i0PBd3kaUUAAMFqi+VnS1O/xgMF76BSSSK0CK7ZKsYzUZWJzsCMyUlXyz7Ehd8KyrdmM3G+gfC3N4lrntoWic48k5tlF2ti9IjAaVlZWie/Rw5/zshppl0c6CZMItBDLLaAZu2pMW4S1aBUulRfiCoc7nSIvtWqAdxnvn1o3RJ4ox2rRSYoqZRXKOLCgcNeuIawqW+DgMt0IsnLkXj8egYx8g8to3lwJm2eklkUiyi9Vk4B/eshqA7z7fzL88cYpYPEEoGucLvzlGIBilyGqU66gFyI4Vwq3g8WM99GaoASJF26AogJZ57YXPztXC1eTxY5kV29vOyWuXBRmFT53PQX2Zj90J0Z1KyzNzt5JvfhoQee3DuHjD8srMDFKiK0ZnMWc81wEQbnoeJofn94JNuwHYk1jN2hpZoFzo/N4N13A8UYeKxsiJZ+be/DBL9p0V8+8O8+viAf9ScJYJp0KrJ6vnllx9SLFdkhNWJq3kT3SPZeT1zs1alixMrCYDNy0XmwIXyxU8n2Ndo0TjGj6HmWppP7VgSdkgHr7CiIFz0TSN5mTuuxTbFxZq+VoAfOF2mJxfQYaWtAhv1ipZKi3CFwxOi5FhxxKCmgUlHhaieXQOThk9RwBo1Kqo95qxmgwZHqlET5bXVtKiJTf8+o7NbY5cjOgEWs9RAPYllnNHvSLz2iUSSU7YsaqMD2yrI6HBt55u4p3f2cuub73AD/e2AvDnNy7BaJDbRAuNNVUuttR5iSU0/t/esxl97cY+sccj89oLnx2rylAVsa/SPk8XuXPpOEdslywMdq6p4OG4EFPpOgiBjrkVL595HoC9iVUs9hop88ri5QWBouC/9l0ArA/vZ7i7RQibcyTeIubJi4m1bF8i3XcKnTUNlRyxXwNAf8sBmMi8606KYCTGsyd7AVgdSortVVsgEQGjBSzymiPJLHIVJckJKbH9eAY62xMJjZYB2ZF6NfDmNaKy+ndHu2dlJZ+ykN9Q45EV0wuYddUeAA51BOb8Gj2jIcbDMQyqQp1PbgwVIoqi4Pf7sVqt0/7eixaJCvtKrRvGuiARn/M54gMtALRRQY3fM6/xSnLPxeYIwLJKL4e1ReKT7kNzc0HoPQaIvPblpVIoLUQuNUfWVLk5rtUBoA02QXQycyce70MbEteXw8oybqm3SLFdIpHkjPvevoZvvXcjRVYjB9sDNPdPUFJk+f/bu/P4qKrzj+OfmclM9g1C9gAhEAj7jiAIooJAEeuGSl2wFheoorUudUGtW20tWrdaS6H9uYCKVisooALKJvu+hy1AIGxJSEKSycz9/XFhNAKZDJJMMvm+X6+8au42Z6YPk3vuc85zmDy6B3cNyPB386SG/LpvOgDvfr+HE+Xnfn/8Uyt35wPQ6WQfTeqvRuEOeqabyyKdz9ntpybLNFOyvd45273ylV1SWODuQL4Rbq57vOtb30vJu92wZxEAS9xZXJiuwe310dliJLXrEA5ZGhFlKWHtkq/OfamBsiIsueY621sdbWmWqJnt9c1pMWK1kdhpMACpJZspO7Lr/Pa1f2TG2lyOl7noG76P0OJ9YLFCi37m8x9HBDj0PFjOLyXbpVb8MLP95yfb9+WfoNTpxmGzkqbZywHt4jbxOIKs7DpSwpaD3qsi/DjZLoHr1Mz29fsKqXCdw+hpftThbxyGI0h/Cusju93OmDFjaNOmDXa73bM9tXkWR4xIgnBTmnOOSVSAsiKCSswRsO7IZGz2kPPRbKlFZ4sRgLbJ0T+s2354K5SdQ+Wdw1sA2OZOIVMl5OulqmKkdWIkm04m208czoGy87iW3M7vsBouDhoxtExuQkRYKATpO0ZEas8VnZL54t5+DG6XwHXdU5k9/iJPVTEJTIPaJZIaG0p+iZOPV+09L9csKqtg8wHzGU/XZjHn5ZriX5fXQCn5lXvyAeio5zT1ztnuldPjwunYNI6Zrp7mhuy5UJTn28WPbIf8PQB8786iT8u489VsqUVn7U+FRJEbPwAAR85CKDl6bi+w81usRgV73E1IS0nF4tCku/rmTDHSp09/9tGEEIuTlcsW/fylBs5i2rIcAO6OOzmrPTYdYpqBsxQi4kET9eQ8U4ZBakVWkvkQetfh4p89inr7yfXam8eFqcRdgIsIDuKiVmaJoC/Wee/src4x/zh3bhpTk80SP2vRJIJwh40TTpfn+8BX2w6a57VSdYyAExsZwgZLJgCHd28492T7ke3mNYwoEhvHgFV/bwJJ2+SoH5Lt+buh5LBvpe0qyj1rtm8xmpKpZQYCTojdRmFUawCM4/uh9BwfEP1UeQnGyfXal7lbMyzdAmGN1dEXkVqXGhvGWzd158VrOhEb7vB3c6SG2awWbu3THIB/LdiJ233uJX1PWZOTj9uAlJhQkqI1ESIQDDqZbF+++xiHjpf97OsVl1V4Klx2bxb7s68ndcc13dP4n9ssJW8cWGv2n0/kV/8CW78AYJO7KYWWSHq3SqqBVoo/pfQZCUDXijXk7Nh0blUHt38FwAJ3e3q3aKznMgHCHtGIvMYXAODcu8r3wTrVsO3gcZbvPobNAj0qVpkbkzqZs9sBgvUMR84/fUNJrWgSEUzjcAdug2rNUK5Kdp5KyDckQzyl5KtOth8pKiPn6AksFuikEdMBzWa10MGzbvu5rcl9Kkmv75HAdDCyPQCuw9nnNmMZPInUbCOZVgnR56tpUkdkJUWxyt3S/KX4EBTs821gxt5l4CyhyAhhk9GUzGQ9PAxEwSkdAQgvPwLHD0LFz3/oTFkhRXvNJQhWG5lckmpAiL5jRESk5o3skUZEcBDZh4qZv+3Qz77eit3mYPeuSqIGjOSYUDqlRmMYMHvjz5/dvmZvPi63QXJ0CMkxGpARSIZ1TGK1tR0HjRgsFaWQ/Q0c2129AczOUtj4PwC+cXemfZMgoiP1bCbQNM66iL22FIItTrYs+wpKfX9+594xH4CF7g70bqUKPAHDZiet+xAAOlWsISc399wnypzF1JOz2n+ZdBT7EbMqIS36g7ME7GFar11qhJLtUissFst5KyV/qvxzyya6EWsILs1KIMhqYcvB4+yoYhbzqRLyGU0iiAqxn/U4CQyn1gRcfY7rtm/XoJ16z+l08o9//IPNmzfjdDor7XMndwUgtmSnOULWlxnLp+RtBGCHO4lWCboJr4+qipHmjcMpsTdit/tkh33fCijNr/7Fd34LwHJ3a4KsVpo30ajo+qiqGAFIb96CXMNcu5QD66D05y+HRNEh7Me2AWCJa0l4SLDWihMRkVoRGWLnuu5pALy7ZM/Pvt7KPWayvZsqywWUwdWc8FAdK3ZpQEZ9VtW9clSIncvaJfGJq6+5YctMKMiB4sPeL3xkG+xfAcDHrn70SY85zy2X2lJlf8oRRn7aZQDE5i3GKNjn27OZ/D1Yj27HbVjYGdKWtHit114fnS1G4toNpMASSbSlhBUrl/lWGcOLsgoXH680l8y5vfFawIDIZGjSGspLzMHuWipSaoCS7VJrTpWS/7nJ9uyTCdcMJckahOgwO70zzBuqL6tYN0zrtTcsHU8m29eeQ7Ld7TbYfPJ7qJXWWa63DMPg8OHDlJaWYvykwxbfuhflho0oo9AsZ+cs8f36h8yRrzuMJFrGK5FaH1UVIzarhTZJkaw0TpaSP7TFnLlc3c5/zhIAlrpbk9HYoWVt6qmqYgSgfVoMG93muu3Goa0/fy05lxNjz2JC3CUcN0LpmJEG9lDQ2oMiIlJLbuhpJtvnbsn7WWXC3W6DlSdntndr1ui8tE3qhqHtzXLei7KPcLS4/Gdda/nJGFEJ+frJ273y1d3S+EfFLygyQqBwH+yYD8d2gKuiqovC6vfBcLPBSCfbSOFCrddeb3mLkfSLfgVAF/cmNm5c69va7es/BmCD0Yy2qY00QLmeOmuMhDXiSFxP87/3r8YozD1vrzl7w0GOlThJDIPMkpMl5BM7QlCIuSRguL5zpGboyaDUmvMxs90wDLYcMEsCa0ZqwzHkZGdv+oq9uM6wtpzbbTBn40FAyfaGouPJMvKbc49T6vRt3acdh4spLK0gxG6ldaKS7YGoXXoaG4x0AEpzVp5TOaqKQ+bM050k0TxeJZ4DUVbSj9dt3wUnjkL52SuoeDjLYP9qAJa529A6PqzG2ij+lZUYxUbDTLaXHtsLRQfPba3BU0oLObblOwC+c3dkYLNgc712rT0oIiK1pFVCJJ3SYnC5DT5dve+cr5N9qIjC0gpC7TbaJKlPFUiax4XTLjkKl9tgVhUTHrxxu40fqh9oQEZA6tsyDntkHG9V/MLcsGE6FOyHQ5vBeeLMJ5Uc9azXPq3iIhxW6N4ioZZaLLUtvFkXdjoysVoMdi3/Eo7tArfb+4kuJ6x+D4AZrgvo3SIOLJaabazUrqBgEjsNBuAC90rW7jkEZdV4HlMN/11l3t/8ulke1n3LzY3p/c24sgaphLzUGD3ZkVpzKtm+Off4GUe7VceuIyUUllbgCLKSqbK+DcYvOiURHWon+1Axn605/YHAJ6v2sfnAcSKDgxjaIckPLZTalhobSuNwBxVuw+cBPKtOdvg7psRg12zUgBQfG8FmWyYABfu3+b5ue8lRgvJ3AVAY1gxHiJKpgahtpWR7jrmGXHVKlx1YC6X5OAlirdGCTA3+C1ihDhv5EeZ3iTM/1+z8n8Nagx6lBVj3mSUzd4R3JjxI67WLiEjtu7ZbKgAfLt97zs9mTq3X3iktWn2qAHTqucrMdec+03BbXhHHSysIc9g8lS4lsNisFkb2bMYk11COEg0lR8zZ7Ud3mMt0FR2qfEJpIWz5Ao7uwIWV/7l60zXJTmi4+lMByxaEvfvNAFxcMovNm9dD8SEvJ2Eu23Z4C+WGjQ9d/bmgZZMabqj4Q1jbyzlhCSHRcoyla9ab3yE/U0l5BQu2m8tZXGX7DjAgNh1Su0JZIYTGqA8uNUZ3xFJrMppE4AiycryswrNesq9OlYxumxSlDl0DEhViZ8xFLQCYOGcbTtcPoyBLnS7+Mtss9zx2YEsahTv80kapXRaLxTO7fc3JJQSqa9XJ47tobcGAVtCoEwBBBbugpBrrxv3Yru+w4GavEUds4wSwBZ3/BorftU2OYpPRjDxiwVUGOUvh+AHvpeR3mTOTN1taUIaDzCSVxQxklqT2AISXHjAT7WXnWKHJ7YZ9K4kp24fTsGFP7QI2OwTr4aKIiNSu4R2TcQRZ2XLwOOv3ndvftRWeEvK6DwpEwzr8/FLyy3eb5aI7p8VoyaUAdtuFzbEFh/NX51XmhvUfQmi0WV1u3wo4sN4c0FxeDAc3wNYvAfjO6MwxohjZJdF/jZdakdrvJrLtmYRZyjjy/ftmVbmqlhpwu2D5JABmuXsQHhlNitZrD0wRTSiM6wpA0MHVOI/sNsu8/wzfbTtMWYWbFuHlNNr7tbmxxcUQFGyu1x6VDFbbz225yBnpbkdqjSPISq90s3TUd9t8THyctHavOZuoU6pGIDU0oy9sTlyEgz1HS/hw+V7P9kkLdpJbUEpKTCi39mnuvwZKrfth3XbfZhmu2pMPKNke6BzNzbWfYstzoTDXt1LyuxcDsMzdWkuWBLA2iZEYFisfV/Q1N+xaAKX5VVdCqCg3jwPmO7MAaN9Uo+wDWUp6O/YacdhwwYE1Zin5c5kFWJqPseMbAJa4s+jarBE4wrReu4iI1LroMDuD25kJrg9X5JzTNVbsUbI9kP24lPzscywlrwEZDUNMmIPb+qYz1XUx+0gwl+Wa8XsIiTJ/ju2CvcvMZbiO50HOEgCmOi8iOdzCL7q39Gv7pRaERBPa7x4ALnQuZuOK7yB3DRTlnV5S3jAgbxPGdjNJ+p7rEoZkhIBd1QYDkiOMuHYDAbjIWMG3O/Kh6NyXLwH46uTfrLGNV2ApzTdLxre+3Ey0O8LMZdxEaoiS7VKr+raMA+C7bdUoGXMGp2a2n0qyScMR5ghi7MXmTfir32zjeKmTjfsLeXNeNgC/H9yaELtGpjUkndLMQTerT34vVEdRWQVbDpizN7o0Vac/kGW0ymKPuwlWDNi3HE4cq/7Je5cCsMKdSat4lTwMVGGOINIbhzPd1c/ccGiTObO9NP/sJxXuM8vIA0vdbUiMCCIxJrTmGyt+075pI+a6OgNg7F9tlr8sP4cKTcWHObHLXC9uHl3p2LgCQhtrVL2IiPjFNSdLyX+6ej+lTpdP5x4tLmfHIXMga5c09akC1alS8jPOsZS8ku0Nx6/7tSAsJJgxZffgtIbA4S0w80HzPjc6xUyUlh2HrTOg+BDHCeMbdxdu69EYu5ZsC3wWC8ldBrEizOx329a+j3H8gDkI49RyA6cGMxfsheWTsVSUku1OYr0tizv7p2u99gBmazucCoLIsOYyb2MuHDv32e0ut8E3mw8CcFn5yVntqT0hIt6sUhfWRJXlpEYp2S61ql8rc/bXkh1HKavwrUNX4XJ7SpydSrJJw3Jjr6YkR4eQW1BKhydnM/Rv31FUVkGHlGiu6JTs7+ZJLeuSFovNamHHoWJ2Hq7erOW1e/NxG5ASE0pCVEgNt1BqksViITo6GrvdjuUMHa+OaY1ZYZhrLZfmbq5eeXAA5wmMgxsAc2Z7ZmLUeW231B5vMQKQlRTFNiOVQyHpYLhhx1zI3wslR08/uLQQcpZB8SHcWFjhzqRzqgZj1GfViZF2yVF8RxcAKg5shPITZiz4oqIMDm4kJH8rAIdiuxGM2yyxKSIi4gd9W8aRFB1CwQkns3ycubzyZBI1o0k4sVrGLWD9nFLyh46XsftICRaLBrnXZ9W5V4aTSz/2a8EGI537rQ9j2BzmAOWZD8LRneZs0m2zYO00AJ52/opgh53rL8iorbciNaS6MUJ4E1Ivu5tiI5jW7u3smDsZQhuZS/7tW24uMZC/B/I24j5ZDew910Du7BxCbJye99ZnXmMkOoWyJubSbXGHF5Nz6Ng5z25ftfsoR0oq6ObYQ1T+RrBYIesKc2kCDIjUshVSs5Rsl1rVJjGSuIhgTjhdnlGu1ZV9qJgTThfhDhvpcRqF1BAFB9n4/eWtPb+HOWx0bRrDX67thNWqUY4NTWy4gwtPVsv4dPW+ap1zqoR8Z5WQr/fsdjtjx46lXbt22O320/bHhjvYEWyW+T5xeLeZPK2qPPgpOUuxVJRSaISxz5ZC6+RG57vpUku8xQiY67YDfBcywNyw53tzZvu+FXAk20ySgjlQ49huyF0NQI4tjWJC6dxUJcjqs+rESJgjCFdqL8oMO/byfMjfCcWHfCslX3IEdi3AisE6d3Mym6WALQgc4efnjYiIiPjIZrVwfY+mALw5Lxu3u/p/177enAdArxa6DwpkzePCaZtklpKf6ePs9hUn12vPjI8kOvTM91hS91XnXvmUW/u2ICbUxv+KMnnQ9hBuS5DZd/poNEz/NSz/FwCT7dfzoWsAN3UIJyJa3yH1XbVjxGoloVUPFiXdituwkJE3h4LPH4fQGAiJNgdlHFgHW7/AenQHZYadb+0XctuF6WDXRJn6zGuMOCIIbzcEgOts83l/u+2cZ7fPWbcHgN9FfmVuaNIG4rPMZ4HBURCm53tSs5Rsl1pltVro1+pUKXnf1m1fc7JUdPuUaGxKrDZYv+ySyvzfD2DZo5ey4anBfHz3hbRO1MzChurKzuYI109X78eoRuLDs157WkwNtkrqirKErgBEFO0yE6gnzjBb+ad2LwJghbsVnRIc2EI0uCuQtU0yk+3vO/uao54L95rlxYKCzdH1uxfD3uVwaIu574g5M3mRyxzI0bl5nN/aLrWnX1Yqi91tzV/2rYLjueYag9VhGFCYi5G7CoA5ru5ckGQxy2k6dP8iIiL+c2uf5kQEB7H5wHHmbDpYrXMqXG7PTPgh7TVDLNBd1TUFgP9bvLta/e1T5m0xl468oIUSGw1FRHAQL4/sQuNQKx8WZjGi9ElWB3fDwGIOYgbesf2Sp45fgcNqcOuFGSoN3tCENeaSX4zkzah7KDPsRB9ahnP6XeagjOhkXFtnw/LJALxeMYLbusUR2ijJv22WmmexQLurKQ+KIMlylP1bV1BWdMxMuPsywN0wmLMpj3iO0atknrktczAEOaCsCCKTwKbBX1KzlGyXWncq2b7Ax2T7qfXaOylJ1uA1axxOk8jgqksUSYMwqF0iIXYrOw8Xs25fQZXHGobB6hyzooZK2TUMjVt2I8fdBLtRbibRC3PB7T77CYYBOUsAWO5uTdeUCK2nHODanZzZvuJoKGXxnc2NG/8LwZEQlQxWK5w4Bke2Q94m2GaOkP6svDtWC3RIURnwhmBg2xS+cXcGoGL/GnNgxuGt4Dzh/eTSAji2EyN3LQDf0pWOUaUnO/tBNdhqERGRqkWH2bm5dzMAXv1mW7WSqd/vPMrR4nJiw+z01sz2gHdt9zRC7Ta2HDzOkh3VGLgMuN0G35ysfnBJVkJNNk/qmAFtEvh6/IXc0DaUdUYLriz4HReWvsJLzmt42Hk7jxVfS4gNHr8wnPgEJVEbHKsVa2xzbr7sAh4MfpR8Ixx7wU748hHy/3UNtpVTAJjovJrZ4Vdwbc9mqgTWUETEEdTyYgBGuL7ii9xI8xnM8eqXk8/es5cdxyoYY5+JzaiA6DRoeYnZZw8KhnBNlJCap2S71Lq+J8s+r99fwJGismqft3avmUjrmKoH2yJiiggO4rK25oyK/67aX+Wxe4+d4HBROXabxZNgk/rL6XQyefJktmzZgtPpPOMxHZrGMd3dz/xl10Jzdntp/tkvWnIUctcAsMLIpGtzzcSoz6oTI/FRIXRKjcZtwJKIS8yN2+ZA9jxzoEVwpNkpc4TBolcBgz2xF7DY3Y7MJmGEBytZWp9VJ0YAmjaJYmdkDwCs+bvA6jg5CCPb+2j7okOwbQ5Wdzkb3M2ITEjHEWSBMCUoRETE/27v14Iwh431+wo9s5GrMuNkOfHB7RIJsumRYqCLDrXzy5Oz2/+zeFe1zlm/v4C842WEO2z00sz2eq2698o/FhMdxfM39OG/1zbiN23dxMYl8JrrKmbYLmFcJysLbwjjpoGdNeg0QPgcI2GNiIxvyr0DW3KN8QL/qBhGoRFKjMsczPM3YySWjtfy/vBQgqI1ICMQVCtGgqOxth0BQH/rWr5ctxeC7HBoszl43Ru3i/8s2EYMx/mV7WtzW+YQcETAiXyISDCXLBCpYbozlloXHxVCm8RIDAMWZh+p1jllFS425RYC0DElpgZbJyL1zYhOZin5/63dj6uKtQZX7jFntbdNjibErtnK9Z1hGOTm5nLixImzzsLpmBrDZ+6LzOOPbIP8vVB8lr87hgE5S+HEMZyGjdXuDLo2j6+p5kstqE6MAAzraHbi3yq4AFK6g+GCb56GrbNPXsgN8543lyEIb8IH0bcDqpARCKobIwBtW7ch252EFTfsnAcR8ZC/GwqrGOh1/CAczYZss8M/xTWYC5KDzPXiQjR4VERE/K9RuINfXWDObv+bl9ntFS43s9abs8yGdlASpKG4pXdzAGZvPMj+fO9Vfb7aZM5q79eqCcFB6nfXZ77cK1diD6Fzxy48emkaM37hZu2vrCwfZeeB7kE0bt4OQtWPChQ+x4jFAk3a0CIjk/9c0YjoPrfxabcpfJP8G5Y1v4s7f3Uj4ztUEJuYrv5SgKhWjFitkNKN8sZtsVoM2h+bw+rCSCgvNhPuztIqX+O/Szbx73UnuMU2mxBKzeR6m2HgOpncj045z+9K5MyUbBe/uCizCQDfbfU+chpgy4HjOF0GsWF20hqF1mTTRKSeuSizCTFhdg4dL2NR9tmXp9B67Q1PRHAQaS0yWeLOwoJhJsgKdp+5FNWJY7BrAQAbjOakxoQQHR1Tq+0V/xjS3nxYvGRfGYe63gvp/X9IsL83EiYNNgdiWIOg5x0szzdL2XVuppk6DcnArAS+cXcBwL17sVmKzh4Khzadef324wfh4DrYtxyKD3HMiOAzVx8uaFIGEYmazSMiInXG7f3SCQ6ysmpPvmc99jNZuusoR4rLiQmz0ztDFVoaitaJkfRu0RiX2+CdJbu9Hv/N5oMAXJKlgcsNmj0EEjtCancioxsTbLNAfJY5YFUaNlsQxGWSnNGekS3d3NTazcAh19Hj0qtxOAuhUTo0buHvVkptC4vFkTUYgOts83ly7hFc4QlwPA8OrDXXXT+D9XsO8dCM3YRzgjtDZpkbMwdDaLT5nC+8MYTq2Y3UDiXbxS9Ords+d0sepU6X1+PXnCwh3yE1Rut0i0gljiArw07OrPh45b4zHnO4qIzpK/cC0CtdN1kNyaB2yXzkMme3s2exOYM9dw3k7/mh/LNhQH4O5G0AYJm7NV2TQ5UMayDSGoXRKS0GtwFf5obBheOh5aWAAUUHwe0Eqx3aXYWreX/WHXYD0DlNMzIakm4ZSXxvNZPtxv5VUHzMnJVjGHBgvVkuHqCiDAr2mol2gO1fAfC+ayCNIkLo2MRqdvhFRETqiPjIEG7vlw7AY//dwLHi8jMeN/NkCflBbROwq4R8g3JLn+YATF2WU+UzvAMFpazfV4jFAhe3UVK1wbNazeR6SjezgliUZpfKSRYLxDY3YyM8wRy8XLDPjJG4THNJN2lYgqMh4xJcwdHEW/Jpf3Q2/1lfClFJZnzsX2Uu/ehygtsNzlIO79vJHf9ZTpkLnor7hlBXkZlYb3MFuF1QUW6u3W7VPYvUDkWa+EWv9MakxIRyuKicSQt2ej1+TU4+AJ20XruInME13VIB+O/qfSzbdfS0/c/P3Mzx0graJUcxqF1ibTdP/OjSdknMdPWi2AiGkiNwZLs5I/XAesjbaCbZC/aaN+67vgPgW3dHujVTIrUh+cXJATszdlmgohQufhSG/hUGPg5X/A1Gvgu9xrDtwDGKKyyEO2y0jI/wc6ulNgXZ7YQ17856d3Ns7nJY+pa5IzzOHJBxYD3kroPdi2D/anNfaQHsX4ULC+9UXMrt7W3YQ1VCXkRE6p7fDmxFq/gIDheV8dT/Npy23+U2+HK9OWNZJeQbnkuz4kmJCeVocTlTFu0663Ffn5zV3jkthriI4FpqndR5Vpu5XrImT8lPhTeG5M5m0r1RBsS3gSCHv1sl/mC1QnQqtpNrtz8c9D7vLt5JbjEQlQzlx2HfCti1EHZ9R3H2Im57bwP7itz0iMrn6tLp5nUyB0FEHJQVmt87YXF+e0vS8CjZLn7hCLLywOBMAN6cl82RorKzHnvoeJlnBHWP5pqRKiKn69I0lmu7pWIY8PsP11BSXuHZt2zXUc+s9j9e2R6bVR28hiQhKoTMlDhmunqZG9Z/Yia6QqLh2G5zlnvuGlj1H3BXsMjdnu/cHejaXDfkDcmQDuYgnO/3lpJXYkDFCUjtCi0vMcsfRsaD28Wag+Z3S4fUaH2XNEAD2yTwYsVIAIztc6Dg5HrtEfGACwpyzIeJUUkQ1hjWfwzALFcPSoKbcH16mfmgQDM1RESkjgmx2/jztZ2wWuC/q/cz+0fl5IvLKrj/g9UcLiojOtTOhS11n9zQBNms3HeZ+Qzvb19vO+va7d+cXK/90qyEWmubiNRzp/pPSR3AEe7v1og/hTWCNsMw4loTYSnlMf7JA3OO8tm2UpYUNCLPGQo2G063wd1fl7P2iIVGIVYmJ3yApaLUnMXe8Qaz+lxZEUSnavCG1Col28VvRnRKoX1KFEVlFbz6zfazHvfaN9soKXfRKTXaU35eROSnHh/elqToEHYdKeHFL7cAUOFy8/h/1wNwfY80ujbVbOWGaFD7ZKa6LjZ/2fs9rHoPHGFmhy46xZzZvm8FBlYed95ClMNCRpL+3jQkqbFhdGkagwF8eTgOio+aM9x/rPgwS4+dXK9dJeQbpMvap7DR0ZHFrrZYDBd8/+YPO8Mam98pwZFgscL+VRibZwAwueJybu0YSnhwkFl6XqSWvPnmm3Ts2JGoqCiioqLo3bs3X3zxhWf/gAEDsFgslX7uvPPOStfYs2cPw4YNIywsjPj4eH7/+99TUVFR6Zh58+bRtWtXgoODadmyJVOmTKmNtyci51nntBjGXJQBwP0frOH+aav5YFkOV76+kE9X78dmtfDo0CyVkG+gruqSQvdmsZSUu3hmxsbT9heccLJg+2EABqqEvIiI+Co4CsIaY7ngLtzWIAbY1pC4/2vu+fIY1398hJ7/KWDg1GJGfl7O/BwXIUEWpvXZQ0TOPMACXX5lrtVemm9OsIlUJR6pXbpDFr+xWi38YUgWAO8s2c3Ow8WnHbPnSAnvLd0DwEOXt9F67SJyVlEhdv50dUcApizaxXVvLabPC9+w+cBxokPtPHh5Gz+3UM630NBQbDbvM0QHtUtihdGav1Zca25Y9g/Y+F/zv11OWPwaABujLyLbSKFLogOrI6SGWi21qboxAjDsZEnUqZvKcMZmmGtwlxdB2XEoOsj24lA+3VICQP/MJjXWZqldvsRIeEQU93YP+2F2+64FcPgMA0ZLC2Dus1gwmFoxgA22NtySWQ5hsRAScx5bL1K11NRUXnjhBVasWMHy5csZOHAgI0aMYMOGH0pE/+Y3vyE3N9fz8+KLL3r2uVwuhg0bRnl5OYsWLeLf//43U6ZM4YknnvAcs3PnToYNG8bFF1/M6tWrGT9+PLfffjuzZs2q1fcqIufH+Etb0Sk1mqKyCj5etY8Hp69lW14R8ZHBvP+bC7iuR5q/myh+YrVaPJXiZq47wLdbD3n2zd2cx5CXv6Wswk1ao1DaJEb6saVyPvlyrywNk2JEvKl2jFgsEJkIYY2xdhsNwB+D/8P18XtoHm3DAuzId7HygBOrBd4YHEmrTW+Y56Z0hxYXg+E2Z7XHNgO7nutJ7VKyXfyqT8s4BrRuQoXb4IlP11Phclfa/9KcLThdBv1axdFHpcpExIuLMptwQ8+mACzdeZS842U4bFb+eGV7GoWrdFAgcTgc3HfffXTo0AGHo+r/b1vGR9CicQh/q7iS7clXmBsXvALvXQ+Th0BBDoY9nFetowDo1lTrKQcCX2IE4IrOyUSGBLEx9zgvr7ZAoxbgLAXDwAiOZsJyOxVug0uz4umd0bgW3oHUNF9jBKuV67unkB/ZitmublgwYO4z5oCMUwwDvv0LFB9mnyWBpypu5ob24cTayswydlZ1v6T2DB8+nKFDh9KqVSsyMzN59tlniYiIYMmSJZ5jwsLCSExM9PxERUV59s2ePZuNGzfyzjvv0LlzZ4YMGcIf//hHXn/9dcrLywH4+9//Tnp6Oi+99BJZWVmMGzeOa665hokTJ9b6+xWRny/EbuOju/owdcwF3NG/BVlJUVzSJp4Z9/SjZ7qW9WvospKiuKV3cwB++/4qrnx9ISNeX8joKcvYX1BK00ZhvHpDV02UCRA+3ytLg6MYEW98jpHQWLMSZdYV0KQ1YUYJLxQ/wbwBu1g9JpG3f9GIO7tF8M6AEgau/T0c2wX2MOh+m1ky/sQx8xqRyTX+3kR+KsjfDRD5w9AsFmcf4btth5nw2QaeubI9FouF1Tn5fLraXAvzIc1IFZFqmjC8LW2TIgkPDiI9LpwWTSKIDrX7u1niZ4PaJfP3b3fwF9cN/L1VGWybBUUn16K0OViaNIovt4YTZIHLO6b6t7HiF/GRIbxwVUfGvreSN77dQe+MHvRt3gJsDr7YeIiFu1biCLLyxC/a+bup4kf28Gge6mrw/Pwb6WndQsyxXfDpOLjib+B2wap3YNd3uLAxpvQebPZQftPOMMvLh2ngqPiPy+Xiww8/pLi4mN69e3u2v/vuu7zzzjskJiYyfPhwHn/8ccLCwgBYvHgxHTp0ICHhh7V3Bw8ezF133cWGDRvo0qULixcv5tJLL630WoMHD2b8+PFVtqesrIyysjLP74WFhQA4nU6cTufPfbsNwqnPSZ+XnM3PiZFuaVF0S4vigUtbnnY9CRznEiPjBqQzc91+DhSWsTonHwCrBW7t3Yzxl7Qk1GFTrAQQ/a0RbxQj4o1PMWINhpA4KMiBy/+CbdYjWPPWY8x+hIi2VzMwvDED7cVYl34ArjKMoBBcHW/EiGsNTiecKIHkVmBYzN+lXqjr3yPVbZeS7eJ3mQmRvHJ9Z+56dyXvfr+H+MgQnC43//h2BwC/6JhE+xTNMhSR6gmx27jp5Gh7kVN+0clMtn+52+CzwfdyRctL4cRRiEwmz5bAHZ+4AINxFzQis6lGwDZUwzomsWB7U95fuofxH6zl5ZGdCbaX8szn5rqUd/bPoGnjMD+3UvwqtBGDM6P4x0Yr1x96jGkhzxKdvxs+HA3lx82lKYDnnTewwUjn5YujSQw6BpEtVcZO/GLdunX07t2b0tJSIiIi+OSTT2jbti0AN954I82aNSM5OZm1a9fy0EMPsWXLFj7++GMADhw4UCnRDnh+P3DgQJXHFBYWcuLECUJDQ8/Yrueff56nnnrqtO2zZ8/2JPuleubMmePvJkgdpxgRb3yNkXGZsK/YgtMNTjckhxkkGtnM/Sq7hloo/qbvEfFGMSLe+B4jx7Ak3U8X5z9JO7YI24YPK+09FNGWlc3GUOpuBCv3/rBjx2pg9c9srfhDXf0eKSkpqdZxSrZLnXB5+yQeH9aWpz/fyMSvtnq292sVx5NXaAaZiIhU5nQ6eeeddzhy5AhOpxO7verqBe1Torm7bxpvLMjhoa/zaT2yM63T7BiGwR8+P0Z+WQXtGlsZO6hDLb0DqWm+xsgpT/yiLSt2H2XrwSJ+Nel7z/bU2FDuHpBRU80VPzinGAlyYGncgie7r2Lk7KZcVfoE7zueJf7EUQBKw1N5ouRaPnD14JetQ7mypQ1OBEFEfA2/G5Eza926NatXr6agoICPPvqIW265hfnz59O2bVvGjBnjOa5Dhw4kJSVxySWXkJ2dTUZGzX7fPfLII9x///2e3wsLC0lLS2PQoEGVStnL2TmdTubMmcNll11W7b9x0rAoRsQbxYhUxel0MnXqVI4ePcpvfvMbDYaT0yhGxJtzjpHD2+HwVohuBt3+iGv9R1j2rQC3E9wu3I1bEdP5RgaGxprHu5xQfBiSOkNkQpWXlrqnrt+PnKrC5o2S7VJn3NY3nX35J5i0YCcpMaE8Mbwtg9omaK0nERE5jWEY7Nmzx/Pf1fG7oR1Yu6+ABTsLufN/h7m2fQQLc8pYmFOO3Wrw0ogM7KERNdlsqUXnEiMAoQ4bb/6qGxM+3UBuwQlcbgOb1cIfR7QnxG6rqeaKH5xrjBCRQMemjfl8RBH3ftucqw4/ySjb18x1dWZpaRvAQlqUjacHREPpYQiPM9eNE/EDh8NBy5ZmGehu3bqxbNkyXnnlFd56663Tju3VqxcA27dvJyMjg8TERJYuXVrpmIMHDwKQmJjo+d9T2358TFRU1FlntQMEBwcTHBx82na73V4nH7DUZfrMxBvFiHijGJEzMQyDnJwcAIKCghQjchrFiHhzzjES1xxO5IGz0OxLdx5p/pxk/enxJUchOhFiksCq5zb1VV29H6lum5RslzrlsWFZXNMtlfS4cD3QFhGR88pmtfC3X/Vi+Cvz2Fno5MVFxz37HrognDYZ6X5sndQlGU0ieOf2Xv5uhtRVNjs0ak7GiZV8fE08L33vYOKaG6gAwuwWUiJtvHRZDJFBFVDqhKhk0OBRqSPcbneltdJ/bPXq1QAkJSUB0Lt3b5599lny8vKIjzerM8yZM4eoqChPKfrevXszc+bMSteZM2dOpXXhRUREREREqs0eCo1awP7VEBwJ1irSmBWlgAVimyvRLn6lZLvUKRaLhawklQ0UEZGa0SjcwVu39OQPH64gMaSC3k3D6NsihlbNUs0EmohIdYTHQ3gcjtJjPNI3nocujMICP1RkcrugcL/Z4Y9QGTvxj0ceeYQhQ4bQtGlTjh8/znvvvce8efOYNWsW2dnZvPfeewwdOpTGjRuzdu1a7rvvPi666CI6duwIwKBBg2jbti033XQTL774IgcOHOCxxx5j7Nixnlnpd955J6+99hoPPvggt912G9988w0ffPABM2bM8OdbFxERERGR+iwyCaLyoHAfRDSBoJAzH1d8BGKaQVjj2m2fyE8o2S4iIiINSvuUGD4bfwkYhmabisi5sQWZI+0PrIWig1jD43/4PjEMKDpgrhUXl6nR9eI3eXl53HzzzeTm5hIdHU3Hjh2ZNWsWl112GTk5OXz11Ve8/PLLFBcXk5aWxtVXX81jjz3mOd9ms/H5559z11130bt3b8LDw7nlllt4+umnPcekp6czY8YM7rvvPl555RVSU1P55z//yeDBg/3xlkVEREREJBDYgiCxg5lkP7YTQqLMWe6nGAaU5kNQMMQ21fM98Tsl20VERKRh0o24iPwcEfGQ1AXyNpqz2EOjweWE8mIIiYH4tmA/y+h7kVowadKks+5LS0tj/vz5Xq/RrFmz08rE/9SAAQNYtWqVz+0TERERERE5qyAHxGeBIwyObIfSAjP5bguC0uNm8j2uFYRE+7ulIkq2i4iIiIiInJPwxpDSFfI2Q2mhOao+Jg6iUyuPuhcRERERERER31it0CgdwhrBiXw4ngsVZZDQHiITzUS8SB2gZLuIiIjUS3a7HZfL5e9mSB2mGBFvzkuMOMIhuQu4nWBzqGqGiIiIiAQE9afEG8WIeHPeYiQk2vyJaWpWlAty/PxripxHVn83QERERMRXDoeD3//+93Ts2BGHQzfYcjrFiHhzXmPEajVntSvRLiIiIiIBQP0p8UYxIt7USIxYLEq0S52kZLuIiIiIiIiIiIiIiIiIiIiPlGwXERERERERERERERERERHxkdZsFxERkXqnoqKCadOmcejQISoqKrDb7f5uktQxihHxRjEiIiIiInJmulcWbxQj4o1iRBoSJdtFRESk3nG73WRnZ3v+W+SnFCPijWJEREREROTMdK8s3ihGxBvFiDQkKiMvIiIiIiIiIiIiIiIiIiLiIyXbRUREREREREREREREREREfKRku4iIiIiIiIiIiIiIiIiIiI+UbBcREREREREREREREREREfGRku0iIiIiIiIiIiIiIiIiIiI+CvJ3A+oDwzAAKCws9HNL6g+n00lJSQmFhYXY7XZ/N0fqIMWIeKMYkaqUl5dTWloKmH+fXS6Xn1skdY1iRLxRjEh11PX7kVN91FN91vpKfW7f1fXYFP9TjIg3ihGpiu6VxRvFiHijGJHqqOv3I9Xtc1uM+t4rrwV79+4lLS3N380QEREREREROU1OTg6pqan+bsY5U59bRERERERE6ipvfW4l26vB7Xazf/9+IiMjsVgs/m5OvVBYWEhaWho5OTlERUX5uzlSBylGxBvFiHijGBFvFCPijWJEvKnrMWIYBsePHyc5ORmrtf6uEqc+t+/qemyK/ylGxBvFiHijGBFvFCPijWJEvKnrMVLdPrfKyFeD1Wqt17ME/CkqKqpO/gORukMxIt4oRsQbxYh4oxgRbxQj4k1djpHo6Gh/N+FnU5/73NXl2JS6QTEi3ihGxBvFiHijGBFvFCPiTV2Oker0uevv0HcRERERERERERERERERERE/UbJdRERERERERERERERERETER0q2S40IDg5mwoQJBAcH+7spUkcpRsQbxYh4oxgRbxQj4o1iRLxRjEhdpdgUbxQj4o1iRLxRjIg3ihHxRjEi3gRKjFgMwzD83QgREREREREREREREREREZH6RDPbRUREREREREREREREREREfKRku4iIiIiIiIiIiIiIiIiIiI+UbBcREREREREREREREREREfGRku0iIiIiIiIiIiIiIiIiIiI+UrJdRERERERERERERERERETER0q213PPP/88PXr0IDIykvj4eK688kq2bNlS6ZjS0lLGjh1L48aNiYiI4Oqrr+bgwYOVjrnnnnvo1q0bwcHBdO7c+bTX2bVrFxaL5bSfJUuWeG3j66+/TvPmzQkJCaFXr14sXbrUs+/o0aP89re/pXXr1oSGhtK0aVPuueceCgoKqrzmvHnzGDFiBElJSYSHh9O5c2fefffdSsd8/PHHdO/enZiYGM8x//d//+e1vYFGMXL2GPmxqVOnYrFYuPLKK722N9AoRs4eI1OmTDmtvSEhIV7bG2gUI1V/j+Tn5zN27FiSkpIIDg4mMzOTmTNnem1zIFGMnD1GBgwYcMY2Dxs2zGubA4lipOrvkZdfftlz7bS0NO677z5KS0u9tjmQ1PcYAbjjjjvIyMggNDSUJk2aMGLECDZv3uz1umvXrqVfv36EhISQlpbGiy++WGn/hg0buPrqq2nevDkWi4WXX37Z6zXl/Knvsak+d81TjKjP7Y1iRH1ubxQj6nN7oxhRn9sbxYj63N7U9xiBut3nVrK9nps/fz5jx45lyZIlzJkzB6fTyaBBgyguLvYcc9999/G///2PDz/8kPnz57N//36uuuqq06512223MXLkyCpf76uvviI3N9fz061btyqPnzZtGvfffz8TJkxg5cqVdOrUicGDB5OXlwfA/v372b9/P3/5y19Yv349U6ZM4csvv+TXv/51ldddtGgRHTt2ZPr06axdu5bRo0dz88038/nnn3uOadSoEY8++iiLFy/2HDN69GhmzZpV5bUDjWLk7DFyyq5du3jggQfo169fldcMVIqRqmMkKiqqUnt3795d5XUDkWLk7DFSXl7OZZddxq5du/joo4/YsmULb7/9NikpKVVeO9AoRs4eIx9//HGltq5fvx6bzca1115b5bUDjWLk7DHy3nvv8fDDDzNhwgQ2bdrEpEmTmDZtGn/4wx+qvHagqe8xAtCtWzcmT57Mpk2bmDVrFoZhMGjQIFwu11mvW1hYyKBBg2jWrBkrVqzgz3/+M08++ST/+Mc/PMeUlJTQokULXnjhBRITE6tsp5x/9T021eeueYoR9bm9UYyoz+2NYkR9bm8UI+pze6MYUZ/bm/oeI1DH+9yGBJS8vDwDMObPn28YhmHk5+cbdrvd+PDDDz3HbNq0yQCMxYsXn3b+hAkTjE6dOp22fefOnQZgrFq1yqf29OzZ0xg7dqznd5fLZSQnJxvPP//8Wc/54IMPDIfDYTidTp9ea+jQocbo0aOrPKZLly7GY4895tN1A41ipHKMVFRUGH369DH++c9/GrfccosxYsQIn64ZiBQjP8TI5MmTjejoaJ+u0RAoRn6IkTfffNNo0aKFUV5e7tN1Ap1i5Oz3IxMnTjQiIyONoqIin64baBQjP8TI2LFjjYEDB1Y65v777zcuvPBCn64baAIhRtasWWMAxvbt2896zBtvvGHExsYaZWVlnm0PPfSQ0bp16zMe36xZM2PixIk+tV3Or0CITfW5a5ZiRH1ubxQj6nN7oxhRn9sbxYj63N4oRtTn9iYQYqQu9bk1sz3AnCqr0ahRIwBWrFiB0+nk0ksv9RzTpk0bmjZtyuLFi32+/hVXXEF8fDx9+/bls88+q/LY8vJyVqxYUem1rVYrl156aZWvXVBQQFRUFEFBQT61raCgwPO+f8owDL7++mu2bNnCRRdd5NN1A41ipHKMPP3008THx3sdJdeQKEYqx0hRURHNmjUjLS2NESNGsGHDBp+uGYgUIz/EyGeffUbv3r0ZO3YsCQkJtG/fnueee67KEZUNgWLkzPcjAJMmTeL6668nPDzcp+sGGsXIDzHSp08fVqxY4SmPtmPHDmbOnMnQoUN9um6gqe8xUlxczOTJk0lPTyctLe2s1168eDEXXXQRDofDs23w4MFs2bKFY8eO+fiupDbU99g89R7U5645ihH1ub1RjKjP7Y1iRH1ubxQj6nN7oxhRn9ub+h4jda3PrWR7AHG73YwfP54LL7yQ9u3bA3DgwAEcDgcxMTGVjk1ISODAgQPVvnZERAQvvfQSH374ITNmzKBv375ceeWVVf4jOXz4MC6Xi4SEhGq/9uHDh/njH//ImDFjqt02gA8++IBly5YxevToStsLCgqIiIjA4XAwbNgwXn31VS677DKfrh1IFCOVY2TBggVMmjSJt99+26drBTLFSOUYad26Nf/617/49NNPeeedd3C73fTp04e9e/f6dO1AohipHCM7duzgo48+wuVyMXPmTB5//HFeeuklnnnmGZ+uHUgUI6ffj5yydOlS1q9fz+233+7TdQONYqRyjNx44408/fTT9O3bF7vdTkZGBgMGDGhwJe1+rD7HyBtvvEFERAQRERF88cUXzJkzp1Kn/qcOHDhwxuue2id1S32OzR+foz53zVGMqM/tjWJEfW5vFCPqc3ujGFGf2xvFiPrc3tTnGKmrfW4l2wPI2LFjWb9+PVOnTj3v146Li+P++++nV69e9OjRgxdeeIFf/epX/PnPfwbgu+++8wR4REQE7777rs+vUVhYyLBhw2jbti1PPvmkZ3u7du081x0yZMhp582dO5fRo0fz9ttv065du0r7IiMjWb16NcuWLePZZ5/l/vvvZ968eT63LVAoRn6IkePHj3PTTTfx9ttvExcXd25vOgApRip/j/Tu3Zubb76Zzp07079/fz7++GOaNGnCW2+95fsHECAUI5VjxO12Ex8fzz/+8Q+6devGyJEjefTRR/n73//u+wcQIBQjp9+PnDJp0iQ6dOhAz549fW5XIFGMVI6RefPm8dxzz/HGG2+wcuVKPv74Y2bMmMEf//hH3z+AAFGfY2TUqFGsWrWK+fPnk5mZyXXXXUdpaSngPUak7qvPsQnqc9cGxYj63N4oRtTn9kYxoj63N4oR9bm9UYyoz+1NfY6Rutrn9q3+gtRZ48aN4/PPP+fbb78lNTXVsz0xMZHy8nLy8/MrjUg5ePAgiYmJP+s1e/XqxZw5cwDo3r07q1ev9uxLSEggODgYm83GwYMHK513ptc+fvw4l19+OZGRkXzyySfY7XbPvpkzZ+J0OgEIDQ2tdN78+fMZPnw4EydO5Oabbz6tjVarlZYtWwLQuXNnNm3axPPPP8+AAQPO+X3XV4qRyjGSnZ3Nrl27GD58uGeb2+0GICgoiC1btpCRkfEz3n39oxg58/fIj9ntdrp06cL27dt9fq+BQDFyeowkJSVht9ux2WyebVlZWRw4cIDy8vIqR1YGIsXI2b9HiouLmTp1Kk8//fQ5v9dAoBg5PUYef/xxbrrpJs/siw4dOlBcXMyYMWN49NFHsVob1vjo+h4j0dHRREdH06pVKy644AJiY2P55JNPuOGGG84YI4mJiWe87ql9UnfU99hUn7vmKUbU5/ZGMaI+tzeKEfW5vVGMqM/tjWJEfW5v6nuM1Nk+t8+rvEud4na7jbFjxxrJycnG1q1bT9ufn59v2O1246OPPvJs27x5swEYixcvPu34CRMmGJ06darWa99+++1Gly5dqjymZ8+exrhx4zy/u1wuIyUlxXj++ec92woKCowLLrjA6N+/v1FcXFyt1zYMw5g7d64RHh5uvPbaa9U+Z/To0Ub//v2rfXwgUIycOUZOnDhhrFu3rtLPiBEjjIEDBxrr1q0zysrKqv069Z1ipPrfIxUVFUbr1q2N++67r9qvEQgUI2ePkUceecRo1qyZ4XK5PNtefvllIykpqdqvEQgUI96/RyZPnmwEBwcbhw8frva1A4li5Owx0rVrV+PBBx+stO29994zQkNDjYqKimq/Tn0XCDHyU6WlpUZoaKgxefLksx7zxhtvGLGxsUZ5ebln2yOPPGK0bt36jMc3a9bMmDhxYpVtlfMrEGJTfe6apRhRn9sbxYj63N4oRtTn9kYxoj63N4oR9bm9CYQY+am61OdWsr2eu+uuu4zo6Ghj3rx5Rm5uruenpKTEc8ydd95pNG3a1Pjmm2+M5cuXG7179zZ69+5d6Trbtm0zVq1aZdxxxx1GZmamsWrVKmPVqlWezs+UKVOM9957z9i0aZOxadMm49lnnzWsVqvxr3/9q8r2TZ061QgODjamTJlibNy40RgzZowRExNjHDhwwDAM8wu0V69eRocOHYzt27dXeg9VfdF98803RlhYmPHII49UOufIkSOeY5577jlj9uzZRnZ2trFx40bjL3/5ixEUFGS8/fbbPn/O9Zli5Owx8lO33HKLMWLECG8facBRjJw9Rp566ilj1qxZRnZ2trFixQrj+uuvN0JCQowNGzb4/DnXZ4qRs8fInj17jMjISGPcuHHGli1bjM8//9yIj483nnnmGZ8/5/pMMeL9b03fvn2NkSNHVvszDTSKkbPHyIQJE4zIyEjj/fffN3bs2GHMnj3byMjIMK677jqfP+f6rL7HSHZ2tvHcc88Zy5cvN3bv3m0sXLjQGD58uNGoUSPj4MGDZ71ufn6+kZCQYNx0003G+vXrjalTpxphYWHGW2+95TmmrKzM8z6SkpKMBx54wFi1apWxbds2nz9n8V19j031uWueYkR9bm8UI+pze6MYUZ/bG8WI+tzeKEbU5/amvsdIXe9zK9lezwFn/PnxSI4TJ04Yd999txEbG2uEhYUZv/zlL43c3NxK1+nfv/8Zr7Nz507DMMx/IFlZWUZYWJgRFRVl9OzZ0/jwww+r1cZXX33VaNq0qeFwOIyePXsaS5Ys8eybO3fuWd/Dqdc+k1tuueWM5/x4BP2jjz5qtGzZ0ggJCTFiY2ON3r17G1OnTq1WmwOJYuTsMXKmcxpix18xcvYYGT9+vOd1ExISjKFDhxorV66sVpsDiWKk6u+RRYsWGb169TKCg4ONFi1aGM8++2yDGhlrGIoRbzFyaiTw7Nmzq9XWQKQYOXuMOJ1O48knnzQyMjKMkJAQIy0tzbj77ruNY8eOVavdgaK+x8i+ffuMIUOGGPHx8YbdbjdSU1ONG2+80di8ebPX665Zs8bo27evERwcbKSkpBgvvPBCpf07d+70+b5Wzp/6Hpvqc9c8xYj63N4oRtTn9kYxoj63N4oR9bm9UYyoz+1NfY+Rut7nthiGYSAiIiIiIiIiIiIiIiIiIiLVZvV3A0REREREREREREREREREROobJdtFRERERERERERERERERER8pGS7iIiIiIiIiIiIiIiIiIiIj5RsFxERERERERERERERERER8ZGS7SIiIiIiIiIiIiIiIiIiIj5Ssl1ERERERERERERERERERMRHSraLiIiIiIiIiIiIiIiIiIj4SMl2EREREREREREREZF64NZbb+XKK6/0dzNERETkpCB/N0BEREREREREREREpKGzWCxV7p8wYQKvvPIKhmHUUotERETEGyXbRURERERERERERET8LDc31/Pf06ZN44knnmDLli2ebREREURERPijaSIiInIWKiMvIiIiIiIiIiIiIuJniYmJnp/o6GgsFkulbREREaeVkR8wYAC//e1vGT9+PLGxsSQkJPD2229TXFzM6NGjiYyMpGXLlnzxxReVXmv9+vUMGTKEiIgIEhISuOmmmzh8+HAtv2MREZH6T8l2EREREREREREREZF66t///jdxcXEsXbqU3/72t9x1111ce+219OnTh5UrVzJo0CBuuukmSkpKAMjPz2fgwIF06dKF5cuX8+WXX3Lw4EGuu+46P78TERGR+kfJdhERERERERERERGReqpTp0489thjtGrVikceeYSQkBDi4uL4zW9+Q6tWrXjiiSc4cuQIa9euBeC1116jS5cuPPfcc7Rp04YuXbrwr3/9i7lz57J161Y/vxsREZH6RWu2i4iIiIiIiIiIiIjUUx07dvT8t81mo3HjxnTo0MGzLSEhAYC8vDwA1qxZw9y5c8+4/nt2djaZmZk13GIREZHAoWS7iIiIiIiIiIiIiEg9ZbfbK/1usVgqbbNYLAC43W4AioqKGD58OH/6059Ou1ZSUlINtlRERCTwKNkuIiIiIiIiIiIiItJAdO3alenTp9O8eXOCgpQiEBER+Tm0ZruIiIiIiIiIiIiISAMxduxYjh49yg033MCyZcvIzs5m1qxZjB49GpfL5e/miYiI1CtKtouIiIiIiIiIiIiINBDJycksXLgQl8vFoEGD6NChA+PHjycmJgarVSkDERERX1gMwzD83QgREREREREREREREREREZH6RMPUREREREREREREREREREREfKRku4iIiIiIiIiIiIiIiIiIiI+UbBcREREREREREREREREREfFRkL8bICIiIiIiInWT2+2mvLzc380QER/Z7XZsNpu/myEiIiIiIhLwlGwXERERERGR05SXl7Nz507cbre/myIi5yAmJobExEQsFou/myIiIiIiIhKwlGwXERERERGRSgzDIDc3F5vNRlpaGlarViATqS8Mw6CkpIS8vDwAkpKS/NwiERERERGRwKVku4iIiIiIiFRSUVFBSUkJycnJhIWF+bs5IuKj0NBQAPLy8oiPj1dJeRERERERkRqi6QkiIiIiIiJSicvlAsDhcPi5JSJyrk4NlHE6nX5uiYiIiIiISOBSsl1ERERERETOSGs9i9Rf+vcrIiIiIiJS85RsFxERERERERERERERERER8ZGS7SIiIiIiItLgNG/enJdfftnfzThvAu39iIiIiIiIiNQHSraLiIiIiIhIQMnJyeG2224jOTkZh8NBs2bNuPfeezly5Ii/myZ+cOutt2KxWE772b59u7+bdk6mTJlCTEyMv5shIiIiIiIiKNkuIiIiIiIiAWTHjh10796dbdu28f7777N9+3b+/ve/8/XXX9O7d2+OHj3ql3a5XC7cbrdfXlvg8ssvJzc3t9JPenq6z9cpLy+vgdaJiIiIiIhIfaVku4iIiIiIiASMsWPH4nA4mD17Nv3796dp06YMGTKEr776in379vHoo496jj1+/Dg33HAD4eHhpKSk8Prrr3v2GYbBk08+SdOmTQkODiY5OZl77rnHs7+srIwHHniAlJQUwsPD6dWrF/PmzfPsPzX7+LPPPqNt27YEBwfzz3/+k5CQEPLz8yu1+d5772XgwIGe3xcsWEC/fv0IDQ0lLS2Ne+65h+LiYs/+vLw8hg8fTmhoKOnp6bz77rvn8RMMTMHBwSQmJlb6sdlszJ8/n549exIcHExSUhIPP/wwFRUVnvMGDBjAuHHjGD9+PHFxcQwePBiA9evXM2TIECIiIkhISOCmm27i8OHDnvPcbjcvvvgiLVu2JDg4mKZNm/Lss8969j/00ENkZmYSFhZGixYtePzxx3E6nZ79a9as4eKLLyYyMpKoqCi6devG8uXLmTdvHqNHj6agoMAzQ//JJ5+s+Q9QREREREREzkjJdhEREREREamSYRiUlFf45ccwjGq38+jRo8yaNYu7776b0NDQSvsSExMZNWoU06ZN81zzz3/+M506dWLVqlU8/PDD3HvvvcyZMweA6dOnM3HiRN566y22bdvGf//7Xzp06OC53rhx41i8eDFTp05l7dq1XHvttVx++eVs27bNc0xJSQl/+tOf+Oc//8mGDRsYNWoUMTExTJ8+3XOMy+Vi2rRpjBo1CoDs7Gwuv/xyrr76atauXcu0adNYsGAB48aN85xz6623kpOTw9y5c/noo4944403yMvL8+H/0fPEMKC8uPZ/fIiJquzbt4+hQ4fSo0cP1qxZw5tvvsmkSZN45plnKh3373//G4fDwcKFC/n73/9Ofn4+AwcOpEuXLixfvpwvv/ySgwcPct1113nOeeSRR3jhhRd4/PHH2bhxI++99x4JCQme/ZGRkUyZMoWNGzfyyiuv8PbbbzNx4kTP/lGjRpGamsqyZctYsWIFDz/8MHa7nT59+vDyyy8TFRXlmaH/wAMPnJfPQ0RERERERHxnMXx5ciEiIiIiIiIBr7S0lJ07d5Kenk5ISAgl5RW0fWKWX9qy8enBhDmCqnXs999/zwUXXMAnn3zClVdeedr+iRMncv/993Pw4EF69uxJVlYWX3zxhWf/9ddfT2FhITNnzuSvf/0rb731FuvXr8dut1e6zp49e2jRogV79uwhOTnZs/3SSy+lZ8+ePPfcc0yZMoXRo0ezevVqOnXq5Dlm/PjxrFu3jq+//hqA2bNnc8UVV3DgwAFiYmK4/fbbsdlsvPXWW55zFixYQP/+/SkuLmbPnj20bt2apUuX0qNHDwA2b95MVlYWEydOZPz48dX6rM6L8mJ4Ltn7cefbH/aDI7zah99666288847hISEeLYNGTKEzMxMpk+fzqZNm7BYLAC88cYbPPTQQxQUFGC1WhkwYACFhYWsXLnSc+4zzzzDd999x6xZP/yb2Lt3L2lpaWzZsoWkpCSaNGnCa6+9xu23316tNv7lL39h6tSpLF++HICoqCheffVVbrnlltOOnTJlCuPHjz+tQsJP/fTfsYiIiIiIiJx/1XtiISIiIiIiIlJPVHdMee/evU/7/eWXXwbg2muv5eWXX6ZFixZcfvnlDB06lOHDhxMUFMS6detwuVxkZmZWOr+srIzGjRt7fnc4HHTs2LHSMaNGjeKCCy5g//79JCcn8+677zJs2DBiYmIAs3z42rVrK5WGNwwDt9vNzp072bp1K0FBQXTr1s2zv02bNp7z5cwuvvhi3nzzTc/v4eHhjB07lt69e3sS7QAXXnghRUVF7N27l6ZNmwJU+qzB/P9o7ty5REREnPY62dnZ5OfnU1ZWxiWXXHLW9kybNo2//e1vZGdnU1RUREVFBVFRUZ79999/P7fffjv/93//x6WXXsq1115LRkbGOb9/ERERERERqRlKtouIiIiIiEiVQu02Nj492G+vXV0tW7bEYrGwadMmfvnLX562f9OmTcTGxtKkSROv1zo1S/mrr75izpw53H333fz5z39m/vz5FBUVYbPZWLFiBTZb5fb9OAEbGhpaKZEL0KNHDzIyMpg6dSp33XUXn3zyCVOmTPHsLyoq4o477qi0PvwpTZs2ZevWrV7bXmvsYeYsc3+8ro/Cw8Np2bLlOb1ceHjlWfRFRUUMHz6cP/3pT6cdm5SUxI4dO6q83uLFixk1ahRPPfUUgwcPJjo6mqlTp/LSSy95jnnyySe58cYbmTFjBl988QUTJkxg6tSpZ4xrERERERER8R8l20VERERERKRKFoul2qXc/alx48ZcdtllvPHGG9x3332V1m0/cOAA7777LjfffLMnAb5kyZJK5y9ZsoSsrCzP76GhoQwfPpzhw4czduxY2rRpw7p16+jSpQsul4u8vDz69evncztHjRrFu+++S2pqKlarlWHDhnn2de3alY0bN541MdymTRsqKipYsWKFp4z8li1bvJYUrxEWi0/l3OuarKwspk+fjmEYnphYuHAhkZGRpKamnvW8rl27Mn36dJo3b05Q0On/Llq1akVoaChff/31GcvIL1q0iGbNmvHoo496tu3evfu04zIzM8nMzOS+++7jhhtuYPLkyfzyl7/E4XDgcrnO5S2LiIiIiIjIeWb1dwNEREREREREzpfXXnuNsrIyBg8ezLfffktOTg5ffvkll112GSkpKTz77LOeYxcuXMiLL77I1q1bef311/nwww+59957AXNd7EmTJrF+/Xp27NjBO++8Q2hoKM2aNSMzM5NRo0Zx88038/HHH7Nz506WLl3K888/z4wZM7y2cdSoUaxcuZJnn32Wa665huDgYM++hx56iEWLFjFu3DhWr17Ntm3b+PTTTxk3bhwArVu35vLLL+eOO+7g+++/Z8WKFdx+++2VBhZI9dx9993k5OTw29/+ls2bN/Ppp58yYcIE7r//fqzWsz8uGTt2LEePHuWGG25g2bJlZGdnM2vWLEaPHo3L5SIkJISHHnqIBx98kP/85z9kZ2ezZMkSJk2aBJjJ+D179jB16lSys7P529/+xieffOK5/okTJxg3bhzz5s1j9+7dLFy4kGXLlnkGgjRv3pyioiK+/vprDh8+TElJSc1+UCIiIiIiInJWSraLiIiIiIhIwGjVqhXLly+nRYsWXHfddWRkZDBmzBguvvhiFi9eTKNGjTzH/u53v2P58uV06dKFZ555hr/+9a8MHmyWy4+JieHtt9/mwgsvpGPHjnz11Vf873//86zJPnnyZG6++WZ+97vf0bp1a6688kqWLVvmWee7Ki1btqRnz56sXbuWUaNGVdrXsWNH5s+fz9atW+nXrx9dunThiSeeIDk52XPM5MmTSU5Opn///lx11VWMGTOG+Pj48/HxNSgpKSnMnDmTpUuX0qlTJ+68805+/etf89hjj1V5XnJyMgsXLsTlcjFo0CA6dOjA+PHjiYmJ8STpH3/8cX73u9/xxBNPkJWVxciRI8nLywPgiiuu4L777mPcuHF07tyZRYsW8fjjj3uub7PZOHLkCDfffDOZmZlcd911DBkyhKeeegqAPn36cOeddzJy5EiaNGnCiy++WEOfkIiIiIiIiHhjMQzD8HcjREREREREpO4oLS1l586dpKenExIS4u/miMg50L9jERERERGRmqeZ7SIiIiIiIiIiIiIiIiIiIj5Ssl1ERERERERERERERERERMRHSraLiIiIiIiIiIiIiIiIiIj4SMl2ERERERERERERERERERERHynZLiIiIiIiImdkGIa/myAi50j/fkVERERERGqeku0iIiIiIiJSic1mA6C8vNzPLRGRc1VSUgKA3W73c0tEREREREQCV5C/GyAiIiIiIiJ1S1BQEGFhYRw6dAi73Y7VqnHaIvWFYRiUlJSQl5dHTEyMZ/CMiIiIiIiInH8WQ3XFRERERERE5CfKy8vZuXMnbrfb300RkXMQExNDYmIiFovF300REREREREJWEq2i4iIiIiIyBm53W6Vkheph+x2u2a0i4iIiIiI1AIl20VERERERERERERERERERHykhfdERERERERERERERERERER8pGS7iIiIiIiIiIiIiIiIiIiIj5RsFxERERERERERERERERER8ZGS7SIiIiIiIiIiIiIiIiIiIj5Ssl1ERERERERERERERERERMRHSraLiIiIiIiIiIiIiIiIiIj4SMl2ERERERERERERERERERERH/0/I6213juEBW8AAAAASUVORK5CYII=\",\n \"text/plain\": [\n \"| \\n\",\n \" | \\n\",\n \" | mean | \\n\",\n \"0.1 | \\n\",\n \"0.2 | \\n\",\n \"0.3 | \\n\",\n \"0.4 | \\n\",\n \"0.5 | \\n\",\n \"0.6 | \\n\",\n \"0.7 | \\n\",\n \"0.8 | \\n\",\n \"0.9 | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|
| item_id | \\n\",\n \"timestamp | \\n\",\n \"\\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" |
| T000000 | \\n\",\n \"2015-02-26 00:00:00 | \\n\",\n \"5223.812012 | \\n\",\n \"5153.143066 | \\n\",\n \"5178.589355 | \\n\",\n \"5193.954102 | \\n\",\n \"5210.103027 | \\n\",\n \"5223.812012 | \\n\",\n \"5234.564453 | \\n\",\n \"5248.638672 | \\n\",\n \"5265.144531 | \\n\",\n \"5295.290527 | \\n\",\n \"
| 2015-02-26 00:30:00 | \\n\",\n \"5001.890625 | \\n\",\n \"4940.849609 | \\n\",\n \"4967.337891 | \\n\",\n \"4982.128906 | \\n\",\n \"4991.323242 | \\n\",\n \"5001.890625 | \\n\",\n \"5012.041504 | \\n\",\n \"5026.311523 | \\n\",\n \"5047.328125 | \\n\",\n \"5078.305664 | \\n\",\n \"|
| 2015-02-26 01:00:00 | \\n\",\n \"4759.131348 | \\n\",\n \"4684.923340 | \\n\",\n \"4712.408691 | \\n\",\n \"4729.202637 | \\n\",\n \"4743.586914 | \\n\",\n \"4759.131348 | \\n\",\n \"4770.625977 | \\n\",\n \"4784.588379 | \\n\",\n \"4803.948242 | \\n\",\n \"4828.080078 | \\n\",\n \"|
| 2015-02-26 01:30:00 | \\n\",\n \"4560.188477 | \\n\",\n \"4505.166016 | \\n\",\n \"4523.577637 | \\n\",\n \"4535.823730 | \\n\",\n \"4550.487793 | \\n\",\n \"4560.188477 | \\n\",\n \"4580.130859 | \\n\",\n \"4591.911133 | \\n\",\n \"4615.944336 | \\n\",\n \"4636.415039 | \\n\",\n \"|
| 2015-02-26 02:00:00 | \\n\",\n \"4439.416992 | \\n\",\n \"4369.610352 | \\n\",\n \"4390.421875 | \\n\",\n \"4412.242676 | \\n\",\n \"4428.110352 | \\n\",\n \"4439.416992 | \\n\",\n \"4456.724121 | \\n\",\n \"4474.257324 | \\n\",\n \"4496.027832 | \\n\",\n \"4509.391602 | \\n\",\n \"
\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"import matplotlib.pyplot as plt\\n\",\n \"\\n\",\n \"# Generate predictions for multiple windows\\n\",\n \"predictions_per_window = predictor.backtest_predictions(test_data, num_val_windows=num_test_windows)\\n\",\n \"\\n\",\n \"# Plot predictions for the first two time series\\n\",\n \"item_ids = test_data.item_ids[:2].tolist()\\n\",\n \"all_predictions = pd.concat(predictions_per_window)\\n\",\n \"predictor.plot(test_data, all_predictions, max_history_length=300, item_ids=item_ids)\\n\",\n \"\\n\",\n \"# Optional: Plot the cutoff dates with dashed vertical lines\\n\",\n \"for cutoff in range(-num_test_windows * prediction_length, 0, prediction_length):\\n\",\n \" for i, ax in enumerate(plt.gcf().axes):\\n\",\n \" cutoff_timestamp = test_data.loc[item_ids[i]].index[cutoff]\\n\",\n \" ax.axvline(cutoff_timestamp, color='gray', linestyle='--')\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Forecasting with covariates\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The previous example showed Chronos-2 in action on a univariate forecasting task, i.e., only the historical data of the target time series for making predictions. However, in real-world scenarios, additional exogenous information related to the target series (e.g., weather forecasts, holidays, promotions) is often available. These exogenous time series, often referred to as covariates, may either be observed only in the past (past-only) or also in the forecast horizon (known future). Leveraging this information when making predictions can improve forecast accuracy. \\n\",\n \"\\n\",\n \"Chronos-2 natively supports (dynamic) covariates, past-only and known-future, real-valued or categorical. Let's see how we can use Chronos-2 to forecast with covariates on a **Electrical Load Forecasting** task.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"\nIn Depth \nForecasting with Chronos-2 \nMetrics \nModel Zoo \nAdvanced \n```\n"
},
{
"path": "docs/tutorials/timeseries/model_zoo/index.md",
"content": "# Model Zoo\n\n::::{grid} 2\n :gutter: 3\n\n:::{grid-item-card} Forecasting Models\n :link: ../forecasting-model-zoo.html\n\n List of available forecasting models in AutoGluon-TimeSeries.\n:::\n\n:::{grid-item-card} Ensemble Models\n :link: ../forecasting-ensembles.html\n\n Ensemble models in AutoGluon-TimeSeries.\n:::\n\n::::\n\n\n```{toctree}\n---\nmaxdepth: 1\nhidden: true\n---\n\nForecasting Models <../forecasting-model-zoo>\nEnsemble Models <../forecasting-ensembles>\n```\n"
},
{
"path": "docs/versions.rst",
"content": "Available Documentation for AutoGluon\n-------------------------------------\n\nWeb-based documentation is available for versions listed below:\n\n- `AutoGluon 1.4.1 (dev) documentation `_\n- `AutoGluon 1.4.0 (stable) documentation `_\n- `AutoGluon 1.3.1 documentation `_\n- `AutoGluon 1.3.0 documentation `_\n- `AutoGluon 1.2.0 documentation `_\n- `AutoGluon 1.1.1 documentation `_\n- `AutoGluon 1.1.0 documentation `_\n- `AutoGluon 1.0.0 documentation `_\n- `AutoGluon 0.8.2 documentation `_\n- `AutoGluon 0.8.1 documentation `_\n- `AutoGluon 0.8.0 documentation `_\n- `AutoGluon 0.7.0 documentation `_\n- `AutoGluon 0.6.2 documentation `_\n- `AutoGluon 0.6.1 documentation `_\n- `AutoGluon 0.6.0 documentation `_\n- `AutoGluon 0.5.2 documentation `_\n- `AutoGluon 0.5.1 documentation `_\n- `AutoGluon 0.4.2 documentation `_\n- `AutoGluon 0.4.1 documentation `_\n- `AutoGluon 0.4.0 documentation `_\n- `AutoGluon 0.3.1 documentation `_\n- `AutoGluon 0.3.0 documentation `_\n- `AutoGluon 0.2.0 documentation `_\n- `AutoGluon 0.1.0 documentation `_\n- `AutoGluon 0.0.15 (legacy) documentation `_\n"
},
{
"path": "docs/whats_new/index.md",
"content": "# What's New\n\nHere you can find the release notes for current and past releases of AutoGluon.\n\n```{toctree}\n:hidden: true\n:maxdepth: 1\n\nv1.5.0\nv1.4.0\nv1.3.1\nv1.3.0\nv1.2.0\nv1.1.1\nv1.1.0\nv1.0.0\nv0.8.3\nv0.8.2\nv0.8.1\nv0.8.0\nv0.7.0\nv0.6.2\nv0.6.1\nv0.6.0\nv0.5.2\nv0.5.1\nv0.4.3\nv0.4.2\nv0.4.1\nv0.4.0\n```\n\n:::{dropdown} v1.5.0\n:open:\n\n```{include} v1.5.0.md\n```\n\n:::\n\n:::{dropdown} v1.4.0\n\n```{include} v1.4.0.md\n```\n\n:::\n\n:::{dropdown} v1.3.1\n\n```{include} v1.3.1.md\n```\n\n:::\n\n:::{dropdown} v1.3.0\n\n```{include} v1.3.0.md\n```\n\n:::\n\n:::{dropdown} v1.2.0\n\n```{include} v1.2.0.md\n```\n\n:::\n\n:::{dropdown} v1.1.1\n\n```{include} v1.1.1.md\n```\n\n:::\n\n:::{dropdown} v1.1.0\n\n```{include} v1.1.0.md\n```\n\n:::\n\n:::{dropdown} v1.0.0\n\n```{include} v1.0.0.md\n```\n\n:::\n\n:::{dropdown} v0.8.3\n\n```{include} v0.8.3.md\n```\n\n:::\n\n:::{dropdown} v0.8.2\n\n```{include} v0.8.2.md\n```\n\n:::\n\n:::{dropdown} v0.8.1\n\n```{include} v0.8.1.md\n```\n\n:::\n\n:::{dropdown} v0.8.0\n\n```{include} v0.8.0.md\n```\n\n:::\n"
},
{
"path": "docs/whats_new/v0.4.0.md",
"content": "# Version 0.4.0\n\nWe're happy to announce the AutoGluon 0.4 release. 0.4 contains major enhancements to Tabular and Text modules, along with many quality of life improvements and fixes.\n\nThis release is **non-breaking** when upgrading from v0.3.1. As always, only load previously trained models using the same version of AutoGluon that they were originally trained on. Loading models trained in different versions of AutoGluon is not supported.\n\nThis release contains [**151** commits from **14** contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=2021-09-01&to=2022-03-09&type=c)!\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.3.1...v0.4.0\n\nSpecial thanks to [@zhiqiangdon](https://github.com/zhiqiangdon), [@willsmithorg](https://github.com/willsmithorg), [@DolanTheMFWizard](https://github.com/DolanTheMFWizard), [@truebluejason](https://github.com/truebluejason), [@killerSwitch](https://github.com/killerSwitch), and [@Xilorole](https://github.com/Xilorole) who were first time contributors to AutoGluon this release!\n\nFull Contributor List (ordered by # of commits):\n- [@Innixma](https://github.com/Innixma), [@yinweisu](https://github.com/yinweisu), [@gradientsky](https://github.com/gradientsky), [@zhiqiangdon](https://github.com/zhiqiangdon), [@jwmueller](https://github.com/jwmueller), [@willsmithorg](https://github.com/willsmithorg), [@sxjscience](https://github.com/sxjscience), [@DolanTheMFWizard](https://github.com/DolanTheMFWizard), [@truebluejason](https://github.com/truebluejason), [@taesup-aws](https://github.com/taesup-aws), [@Xilorole](https://github.com/Xilorole), [@mseeger](https://github.com/mseeger), [@killerSwitch](https://github.com/killerSwitch), [@rschmucker](https://github.com/rschmucker)\n\nThis version supports Python versions 3.7 to 3.9.\n\n## Changes\n\n### General\n\n- [AutoGluon now supports Windows OS!](https://auto.gluon.ai/0.4.0/index.html) Both CPU and GPU are supported on Windows.\n- AutoGluon now supports Python 3.9. Python 3.6 is no longer supported.\n- AutoGluon has migrated from MXNet to PyTorch for all deep learning models resulting in major speedups.\n- [New tutorials](https://auto.gluon.ai/0.4.0/tutorials/cloud_fit_deploy/index.html) showcasing cloud training and deployment with AWS SageMaker and Lambda.\n\n### Text\n\nAutoGluon-Text is refactored with [PyTorch Lightning](https://www.pytorchlightning.ai/). It now supports backbones in [huggingface/transformers](https://huggingface.co/docs/transformers/index). The new version has better performance, faster training time, and faster inference speed. In addition, AutoGluon-Text now supports solving multilingual problems and a new `AutoMMPredictor` has been implemented for automatically building multimodal DL models.\n\n- **Better Performance**\n - Compared with TextPredictor in AutoGluon 0.3, TextPredictor in AutoGluon 0.4 has **72.22%** win-rate in the [multimodal text-tabular benchmark published in NeurIPS 2021](https://arxiv.org/abs/2111.02705). If we use `presets=\"high_quality\"`, the win-rate increased to **77.8%** thanks to the [DeBERTa-v3 backbone](https://arxiv.org/abs/2111.09543).\n - In addition, we resubmitted our results to [MachineHack: Product Sentiment Analysis](https://machinehack.com/hackathon/product_sentiment_classification_weekend_hackathon_19/overview\n), [\"MachineHack: Predict the Price of Books\"](https://machinehack.com/hackathon/predict_the_price_of_books/overview\n), and [\"Kaggle: Mercari Price Suggestion\"](https://www.kaggle.com/c/mercari-price-suggestion-challenge). With three lines of code, AutoGluon 0.4 is able to achieve top places in these competitions (1st, 2nd, 2nd correspondingly). The results obtained by AutoGluon 0.4 also consistently outperform the results obtained by AutoGluon 0.3.\n- **Faster Speed**\n - The new version has **~2.88x** speedup in training and **~1.40x** speedup in inference. With g4dn.12x instance, the model can achieve an additional 2.26x speedup with 4 GPUs.\n- **Multilingual Support**\n - AutoGluon-Text now supports solving multilingual problems via cross-lingual transfer ([Tutorial](https://auto.gluon.ai/0.4.0/tutorials/text_prediction/multimodal_text.html)). This is triggered by setting `presets=\"multilingual\"`. You can now train a model on the English dataset and directly apply the model on datasets in other languages such as German, Japanese, Italian, etc.\n- **AutoMMPredictor for Multimodal Problems**\n - Support an experimental AutoMMPredictor that supports fusion image backbones in [timm](https://github.com/rwightman/pytorch-image-models/tree/master/timm), text backbone in [huggingface/transformers](https://huggingface.co/docs/transformers/index), and multimodal backbones like [CLIP](https://openai.com/blog/clip/) ([Tutorial](https://auto.gluon.ai/0.4.0/tutorials/text_prediction/automm.html)). It may perform better than ensembling ImagePredictor + TextPredictor.\n- **Other Features**\n - Support continuous training from an existing checkpoint. You may just call `.fit()` again after a previous trained model has been loaded.\n\nThanks to [@zhiqiangdon](https://github.com/zhiqiangdon) and [@sxjscience](https://github.com/sxjscience) for contributing the AutoGluon-Text refactors! ([#1537](https://github.com/autogluon/autogluon/pull/1537), [#1547](https://github.com/autogluon/autogluon/pull/1547), [#1557](https://github.com/autogluon/autogluon/pull/1557), [#1565](https://github.com/autogluon/autogluon/pull/1565), [#1571](https://github.com/autogluon/autogluon/pull/1571), [#1574](https://github.com/autogluon/autogluon/pull/1574), [#1578](https://github.com/autogluon/autogluon/pull/1578), [#1579](https://github.com/autogluon/autogluon/pull/1579), [#1581](https://github.com/autogluon/autogluon/pull/1581), [#1585](https://github.com/autogluon/autogluon/pull/1585), [#1586](https://github.com/autogluon/autogluon/pull/1586))\n\n### Tabular\n\nAutoGluon-Tabular has been majorly enhanced by numerous optimizations in 0.4. In summation, these improvements have led to a:\n\n- **~2x** training speedup in Good, High, and Best quality presets.\n- **~1.3x** inference speedup.\n- **63%** win-rate vs AutoGluon 0.3.1 (Results from [AutoMLBenchmark](https://github.com/openml/automlbenchmark))\n - **93%** win-rate vs AutoGluon 0.3.1 on datasets with >=100,000 rows of data (!!!)\n\nSpecific updates:\n\n- Added `infer_limit` and `infer_limit_batch_size` as new fit-time constraints ([Tutorial](https://auto.gluon.ai/0.4.0/tutorials/tabular_prediction/tabular-indepth.html#inference-speed-as-a-fit-constraint)). This allows users to specify\nthe desired end-to-end inference latency of the final model and AutoGluon will automatically train models\nto satisfy the constraint. This is extremely useful for online-inference scenarios where you need to satisfy an\nend-to-end latency constraint (for example 50ms). [@Innixma](https://github.com/Innixma) ([#1541](https://github.com/autogluon/autogluon/pull/1541), [#1584](https://github.com/autogluon/autogluon/pull/1584))\n- Implemented automated semi-supervised and transductive learning in TabularPredictor.\n[Try it out](https://auto.gluon.ai/0.4.0/api/autogluon.predictor.html#autogluon.tabular.TabularPredictor.fit_pseudolabel) via `TabularPredictor.fit_pseudolabel(...)`! [@DolanTheMFWizard](https://github.com/DolanTheMFWizard) ([#1323](https://github.com/autogluon/autogluon/pull/1323), [#1382](https://github.com/autogluon/autogluon/pull/1382))\n- Implemented automated feature pruning (i.e. feature selection) in TabularPredictor.\nTry it out via `TabularPredictor.fit(..., feature_prune_kwargs={})`! [@truebluejason](https://github.com/truebluejason) ([#1274](https://github.com/autogluon/autogluon/pull/1274), [#1305](https://github.com/autogluon/autogluon/pull/1305))\n- Implemented automated model calibration to improve AutoGluon's predicted probabilities for classification problems.\nThis is enabled by default, and can be toggled via the `calibrate` fit argument. [@DolanTheMFWizard](https://github.com/DolanTheMFWizard) ([#1336](https://github.com/autogluon/autogluon/pull/1336), [#1374](https://github.com/autogluon/autogluon/pull/1374), [#1502](https://github.com/autogluon/autogluon/pull/1502))\n- Implemented parallel bag training via Ray. This results in a ~2x training speedup when bagging is enabled\ncompared to v0.3.1 with the same hardware due to more efficient usage of resources\nfor models that cannot effectively use all cores. [@yinweisu](https://github.com/yinweisu) ([#1329](https://github.com/autogluon/autogluon/pull/1329), [#1415](https://github.com/autogluon/autogluon/pull/1415), [#1417](https://github.com/autogluon/autogluon/pull/1417), [#1423](https://github.com/autogluon/autogluon/pull/1423))\n- Added adaptive early stopping logic which greatly improves the quality of models within a time budget. [@Innixma](https://github.com/Innixma) ([#1380](https://github.com/autogluon/autogluon/pull/1380))\n- Added automated model calibration in quantile regression. [@taesup-aws](https://github.com/taesup-aws) ([#1388](https://github.com/autogluon/autogluon/pull/1388))\n- Enhanced datetime feature handling. [@willsmithorg](https://github.com/willsmithorg) ([#1446](https://github.com/autogluon/autogluon/pull/1446))\n- Added support for custom confidence levels in feature importance. [@jwmueller](https://github.com/jwmueller) ([#1328](https://github.com/autogluon/autogluon/pull/1328))\n- Improved neural network HPO search spaces. [@jwmueller](https://github.com/jwmueller) ([#1346](https://github.com/autogluon/autogluon/pull/1346))\n- Optimized one-hot encoding preprocessing. [@Innixma](https://github.com/Innixma) ([#1376](https://github.com/autogluon/autogluon/pull/1376))\n- Refactored `refit_full` logic to majorly simplify user model contributions and improve multimodal support with advanced presets. [@Innixma](https://github.com/Innixma) ([#1567](https://github.com/autogluon/autogluon/pull/1567))\n- Added experimental TabularPredictor config helper. [@gradientsky](https://github.com/gradientsky) ([#1491](https://github.com/autogluon/autogluon/pull/1491))\n- New Tutorials\n - [GPU training tutorial for tabular models](https://auto.gluon.ai/0.4.0/tutorials/tabular_prediction/tabular-gpu.html). [@gradientsky](https://github.com/gradientsky) ([#1527](https://github.com/autogluon/autogluon/pull/1527))\n - [Feature preprocessing tutorial](https://auto.gluon.ai/0.4.0/tutorials/tabular_prediction/tabular-feature-engineering.html). [@willsmithorg](https://github.com/willsmithorg) ([#1478](https://github.com/autogluon/autogluon/pull/1478))\n\n### Tabular Models\n\n#### NEW: TabularNeuralNetTorchModel (alias: 'NN_TORCH')\n\nAs part of the migration from MXNet to Torch, we have created a Torch based counterpart\nto the prior MXNet tabular neural network model. This model has several major advantages, such as:\n\n- **1.9x** faster training speed\n- **4.7x** faster inference speed\n- **51%** win-rate vs MXNet Tabular NN\n\nThis model has replaced the MXNet tabular neural network model in the default hyperparameters configuration,\nand is enabled by default.\n\nThanks to [@jwmueller](https://github.com/jwmueller) and [@Innixma](https://github.com/Innixma) for contributing TabularNeuralNetTorchModel to AutoGluon! ([#1489](https://github.com/autogluon/autogluon/pull/1489))\n\n#### NEW: VowpalWabbitModel (alias: 'VW')\n\nVowpalWabbit has been added as a new model in AutoGluon. VowpalWabbit is not installed by default, and must be installed separately.\nVowpalWabbit is used in the `hyperparameters='multimodal'` preset, and the model is a great option to use for datasets containing text features.\n\nTo install VowpalWabbit, specify it via `pip install autogluon.tabular[all, vowpalwabbit]` or `pip install \"vowpalwabbit>=8.10,<8.11\"`\n\nThanks to [@killerSwitch](https://github.com/killerSwitch) for contributing VowpalWabbitModel to AutoGluon! ([#1422](https://github.com/autogluon/autogluon/pull/1422))\n\n#### XGBoostModel (alias: 'XGB')\n\n- Optimized model serialization method, which results in 5.5x faster inference speed and halved disk usage. [@Innixma](https://github.com/Innixma) ([#1509](https://github.com/autogluon/autogluon/pull/1509))\n- Adaptive early stopping logic leading to 54.7% win-rate vs prior implementation. [@Innixma](https://github.com/Innixma) ([#1380](https://github.com/autogluon/autogluon/pull/1380))\n- Optimized training speed with expensive metrics such as F1 by ~10x. [@Innixma](https://github.com/Innixma) ([#1344](https://github.com/autogluon/autogluon/pull/1344))\n- Optimized num_cpus default to equal physical cores rather than virtual cores. [@Innixma](https://github.com/Innixma) ([#1467](https://github.com/autogluon/autogluon/pull/1467))\n\n#### CatBoostModel (alias: 'CAT')\n\n- CatBoost now incorporates callbacks which make it more stable and resilient to memory errors,\nalong with more advanced adaptive early stopping logic that leads to 63.2% win-rate vs prior implementation. [@Innixma](https://github.com/Innixma) ([#1352](https://github.com/autogluon/autogluon/pull/1352), [#1380](https://github.com/autogluon/autogluon/pull/1380))\n\n#### LightGBMModel (alias: 'GBM')\n\n- Optimized training speed with expensive metrics such as F1 by ~10x. [@Innixma](https://github.com/Innixma) ([#1344](https://github.com/autogluon/autogluon/pull/1344))\n- Adaptive early stopping logic leading to 51.1% win-rate vs prior implementation. [@Innixma](https://github.com/Innixma) ([#1380](https://github.com/autogluon/autogluon/pull/1380))\n- Optimized num_cpus default to equal physical cores rather than virtual cores. [@Innixma](https://github.com/Innixma) ([#1467](https://github.com/autogluon/autogluon/pull/1467))\n\n#### FastAIModel (alias: 'FASTAI')\n\n- Added adaptive batch size selection and epoch selection. [@gradientsky](https://github.com/gradientsky) ([#1409](https://github.com/autogluon/autogluon/pull/1409))\n- Enabled HPO support in FastAI (previously HPO was not supported for FastAI). [@Innixma](https://github.com/Innixma) ([#1408](https://github.com/autogluon/autogluon/pull/1408))\n- Made FastAI training deterministic (it is now consistently seeded). [@Innixma](https://github.com/Innixma) ([#1419](https://github.com/autogluon/autogluon/pull/1419))\n- Fixed GPU specification in FastAI to respect the num_gpus parameter. [@Innixma](https://github.com/Innixma) ([#1421](https://github.com/autogluon/autogluon/pull/1421))\n- Forced correct number of threads during fit and inference to avoid issues with global thread updates. [@yinweisu](https://github.com/yinweisu) ([#1535](https://github.com/autogluon/autogluon/pull/1535))\n\n#### LinearModel (alias: 'LR')\n\nLinear models have been accelerated by **20x** in training and **20x** in inference thanks to a variety of optimizations.\nTo get the accelerated training speeds, please install [scikit-learn-intelex](https://github.com/intel/scikit-learn-intelex) via `pip install \"scikit-learn-intelex>=2021.5,<2021.6\"`\n\nNote that currently LinearModel is not enabled by default in AutoGluon,\nand must be specified in `hyperparameters` via the key `'LR'`.\nFurther testing is planned to incorporate LinearModel as a default model in future releases.\n\nThanks to the `scikit-learn-intelex` team and [@Innixma](https://github.com/Innixma) for the LinearModel optimizations! ([#1378](https://github.com/autogluon/autogluon/pull/1378))\n\n### Vision\n\n- Refactored backend logic to be more robust. [@yinweisu](https://github.com/yinweisu) ([#1427](https://github.com/autogluon/autogluon/pull/1427))\n- Added support for inference via CPU. Previously, inferring without GPU would error. [@yinweisu](https://github.com/yinweisu) ([#1533](https://github.com/autogluon/autogluon/pull/1533))\n- Refactored HPO logic. [@Innixma](https://github.com/Innixma) ([#1511](https://github.com/autogluon/autogluon/pull/1511))\n\n### Miscellaneous\n\n- AutoGluon no longer depends on ConfigSpace, cython, dill, paramiko, autograd, openml, d8, and graphviz.\nThis greatly simplifies installation of AutoGluon, particularly on Windows.\n- Entirely refactored HPO logic to break dependencies on ConfigSpace and improve stability and ease of development. [@Innixma](https://github.com/Innixma)\nHPO has been simplified to use random search in this release while we work on\nre-introducing the more advanced HPO methods such as bayesopt in a future release.\nAdditionally, removed 40,000 lines of out-dated code to streamline future development.\n[@Innixma](https://github.com/Innixma) ([#1397](https://github.com/autogluon/autogluon/pull/1397), [#1411](https://github.com/autogluon/autogluon/pull/1411), [#1414](https://github.com/autogluon/autogluon/pull/1414), [#1431](https://github.com/autogluon/autogluon/pull/1431), [#1443](https://github.com/autogluon/autogluon/pull/1443), [#1511](https://github.com/autogluon/autogluon/pull/1511))\n- Added `autogluon.common` to simplify dependency management for future submodules. [@Innixma](https://github.com/Innixma) ([#1386](https://github.com/autogluon/autogluon/pull/1386))\n- Removed `autogluon.mxnet` and `autogluon.extra` submodules as part of code cleanup. [@Innixma](https://github.com/Innixma) ([#1397](https://github.com/autogluon/autogluon/pull/1397), [#1411](https://github.com/autogluon/autogluon/pull/1411), [#1414](https://github.com/autogluon/autogluon/pull/1414))\n- Refactored logging to avoid interfering with other packages. [@yinweisu](https://github.com/yinweisu) ([#1403](https://github.com/autogluon/autogluon/pull/1403))\n- Fixed logging output on Kaggle, previously no logs would be displayed while fitting AutoGluon in a Kaggle kernel. [@Innixma](https://github.com/Innixma) ([#1468](https://github.com/autogluon/autogluon/pull/1468))\n- Added platform tests for Linux, MacOS, and Windows. [@yinweisu](https://github.com/yinweisu) ([#1464](https://github.com/autogluon/autogluon/pull/1464), [#1506](https://github.com/autogluon/autogluon/pull/1506), [#1513](https://github.com/autogluon/autogluon/pull/1513))\n- Added [ROADMAP.md](https://github.com/autogluon/autogluon/blob/master/ROADMAP.md) to highlight past, present, and future feature prioritization and progress to the community. [@Innixma](https://github.com/Innixma) ([#1420](https://github.com/autogluon/autogluon/pull/1420))\n- Various documentation and CI improvements\n - [@jwmueller](https://github.com/jwmueller) ([#1379](https://github.com/autogluon/autogluon/pull/1379), [#1408](https://github.com/autogluon/autogluon/pull/1408), [#1429](https://github.com/autogluon/autogluon/pull/1429))\n - [@gradientsky](https://github.com/gradientsky) ([#1383](https://github.com/autogluon/autogluon/pull/1383), [#1387](https://github.com/autogluon/autogluon/pull/1387), [#1471](https://github.com/autogluon/autogluon/pull/1471), [#1500](https://github.com/autogluon/autogluon/pull/1500))\n - [@yinweisu](https://github.com/yinweisu) ([#1441](https://github.com/autogluon/autogluon/pull/1441), [#1482](https://github.com/autogluon/autogluon/pull/1482), [#1566](https://github.com/autogluon/autogluon/pull/1566), [#1580](https://github.com/autogluon/autogluon/pull/1580))\n - [@willsmithorg](https://github.com/willsmithorg) ([#1476](https://github.com/autogluon/autogluon/pull/1476), [#1483](https://github.com/autogluon/autogluon/pull/1483))\n - [@Xilorole](https://github.com/Xilorole) ([#1526](https://github.com/autogluon/autogluon/pull/1526))\n - [@Innixma](https://github.com/Innixma) ([#1452](https://github.com/autogluon/autogluon/pull/1452), [#1453](https://github.com/autogluon/autogluon/pull/1453), [#1528](https://github.com/autogluon/autogluon/pull/1528), [#1577](https://github.com/autogluon/autogluon/pull/1577), [#1584](https://github.com/autogluon/autogluon/pull/1584), [#1588](https://github.com/autogluon/autogluon/pull/1588), [#1593](https://github.com/autogluon/autogluon/pull/1593))\n- Various backend enhancements / refactoring / cleanup\n - [@DolanTheMFWizard](https://github.com/DolanTheMFWizard) ([#1319](https://github.com/autogluon/autogluon/pull/1319))\n - [@gradientsky](https://github.com/gradientsky) ([#1320](https://github.com/autogluon/autogluon/pull/1320), [#1366](https://github.com/autogluon/autogluon/pull/1366), [#1385](https://github.com/autogluon/autogluon/pull/1385), [#1448](https://github.com/autogluon/autogluon/pull/1448), [#1488](https://github.com/autogluon/autogluon/pull/1488), [#1490](https://github.com/autogluon/autogluon/pull/1490), [#1570](https://github.com/autogluon/autogluon/pull/1570), [#1576](https://github.com/autogluon/autogluon/pull/1576))\n - [@mseeger](https://github.com/mseeger) ([#1349](https://github.com/autogluon/autogluon/pull/1349))\n - [@yinweisu](https://github.com/yinweisu) ([#1497](https://github.com/autogluon/autogluon/pull/1497), [#1503](https://github.com/autogluon/autogluon/pull/1503), [#1512](https://github.com/autogluon/autogluon/pull/1512), [#1563](https://github.com/autogluon/autogluon/pull/1563), [#1573](https://github.com/autogluon/autogluon/pull/1573))\n - [@willsmithorg](https://github.com/willsmithorg) ([#1525](https://github.com/autogluon/autogluon/pull/1525), [#1543](https://github.com/autogluon/autogluon/pull/1543))\n - [@Innixma](https://github.com/Innixma) ([#1311](https://github.com/autogluon/autogluon/pull/1311), [#1313](https://github.com/autogluon/autogluon/pull/1313), [#1327](https://github.com/autogluon/autogluon/pull/1327), [#1331](https://github.com/autogluon/autogluon/pull/1331), [#1338](https://github.com/autogluon/autogluon/pull/1338), [#1345](https://github.com/autogluon/autogluon/pull/1345), [#1369](https://github.com/autogluon/autogluon/pull/1369), [#1377](https://github.com/autogluon/autogluon/pull/1377), [#1380](https://github.com/autogluon/autogluon/pull/1380), [#1408](https://github.com/autogluon/autogluon/pull/1408), [#1410](https://github.com/autogluon/autogluon/pull/1410), [#1412](https://github.com/autogluon/autogluon/pull/1412), [#1419](https://github.com/autogluon/autogluon/pull/1419), [#1425](https://github.com/autogluon/autogluon/pull/1425), [#1428](https://github.com/autogluon/autogluon/pull/1428), [#1462](https://github.com/autogluon/autogluon/pull/1462), [#1465](https://github.com/autogluon/autogluon/pull/1465), [#1562](https://github.com/autogluon/autogluon/pull/1562), [#1569](https://github.com/autogluon/autogluon/pull/1569), [#1591](https://github.com/autogluon/autogluon/pull/1591), [#1593](https://github.com/autogluon/autogluon/pull/1593))\n- Various bug fixes\n - [@jwmueller](https://github.com/jwmueller) ([#1314](https://github.com/autogluon/autogluon/pull/1314), [#1356](https://github.com/autogluon/autogluon/pull/1356))\n - [@yinweisu](https://github.com/yinweisu) ([#1472](https://github.com/autogluon/autogluon/pull/1472), [#1499](https://github.com/autogluon/autogluon/pull/1499), [#1504](https://github.com/autogluon/autogluon/pull/1504), [#1508](https://github.com/autogluon/autogluon/pull/1508), [#1516](https://github.com/autogluon/autogluon/pull/1516))\n - [@gradientsky](https://github.com/gradientsky) ([#1514](https://github.com/autogluon/autogluon/pull/1514))\n - [@Innixma](https://github.com/Innixma) ([#1304](https://github.com/autogluon/autogluon/pull/1304), [#1325](https://github.com/autogluon/autogluon/pull/1325), [#1326](https://github.com/autogluon/autogluon/pull/1326), [#1337](https://github.com/autogluon/autogluon/pull/1337), [#1365](https://github.com/autogluon/autogluon/pull/1365), [#1395](https://github.com/autogluon/autogluon/pull/1395), [#1405](https://github.com/autogluon/autogluon/pull/1405), [#1587](https://github.com/autogluon/autogluon/pull/1587), [#1599](https://github.com/autogluon/autogluon/pull/1599))\n"
},
{
"path": "docs/whats_new/v0.4.1.md",
"content": "# Version 0.4.1\n\nWe're happy to announce the AutoGluon 0.4.1 release. 0.4.1 contains minor enhancements to Tabular, Text, Image, and Multimodal modules, along with many quality of life improvements and fixes.\n\nThis release is **non-breaking** when upgrading from v0.4.0. As always, only load previously trained models using the same version of AutoGluon that they were originally trained on. Loading models trained in different versions of AutoGluon is not supported.\n\nThis release contains [**55** commits from **10** contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=2022-03-10&to=2022-05-23&type=c)!\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.4.0...v0.4.1\n\nSpecial thanks to [@yiqings](https://github.com/yiqings), [@leandroimail](https://github.com/leandroimail), [@huibinshen](https://github.com/huibinshen) who were first time contributors to AutoGluon this release!\n\nFull Contributor List (ordered by # of commits):\n\n- [@Innixma](https://github.com/Innixma), [@zhiqiangdon](https://github.com/zhiqiangdon), [@yinweisu](https://github.com/yinweisu), [@sxjscience](https://github.com/sxjscience), [@yiqings](https://github.com/yiqings), [@gradientsky](https://github.com/gradientsky), [@willsmithorg](https://github.com/willsmithorg), [@canerturkmen](https://github.com/canerturkmen), [@leandroimail](https://github.com/leandroimail), [@huibinshen](https://github.com/huibinshen).\n\nThis version supports Python versions 3.7 to 3.9.\n\n## Changes\n\n### AutoMM\n\n#### New features\n\n- Added `optimization.efficient_finetune` flag to support multiple efficient finetuning algorithms. ([#1666](https://github.com/autogluon/autogluon/pull/1666)) [@sxjscience](https://github.com/sxjscience)\n\n - Supported options:\n - `bit_fit`: [\"BitFit: Simple Parameter-efficient Fine-tuning for Transformer-based Masked Language-models\"](https://arxiv.org/abs/2106.10199)\n - `norm_fit`: An extension of the algorithm in [\"Training BatchNorm and Only BatchNorm: On the Expressive Power of Random Features in CNNs\"](https://arxiv.org/abs/2003.00152) and BitFit. We finetune both the parameters in the norm layers as long as the biases.\n\n- Enabled knowledge distillation for AutoMM ([#1670](https://github.com/autogluon/autogluon/pull/1670)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Distillation API for `AutoMMPredictor` reuses the `.fit()` function:\n\n ```python\n from autogluon.text.automm import AutoMMPredictor\n teacher_predictor = AutoMMPredictor(label=\"label_column\").fit(train_data)\n student_predictor = AutoMMPredictor(label=\"label_column\").fit(\n train_data,\n hyperparameters=student_and_distiller_hparams,\n teacher_predictor=teacher_predictor,\n )\n ```\n\n- Option to turn on returning feature column information ([#1711](https://github.com/autogluon/autogluon/pull/1711)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - The feature column information is turned on for feature column distillation; for other cases it is turned off by default to reduce dataloaderâs latency.\n - Added a `requires_column_info` flag in data processors and a utility function to turn this flag on or off.\n\n- FT-Transformer implementation for tabular data in AutoMM ([#1646](https://github.com/autogluon/autogluon/pull/1646)) [@yiqings](https://github.com/yiqings)\n\n - Yury Gorishniy, Ivan Rubachev, Valentin Khrulkov, Artem Babenko, \"Revisiting Deep Learning Models for Tabular Data\" 2022. ([arxiv](https://arxiv.org/abs/2106.11959), [official implementation](https://github.com/Yura52/tabular-dl-revisiting-models))\n\n- Make CLIP support multiple images per sample ([#1606](https://github.com/autogluon/autogluon/pull/1606)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Added multiple images support for CLIP. Improved data loader robustness: added missing images handling to prevent training crashes.\n - Added the choice of using a zero image if an image is missing.\n\n- Avoid using `eos` as the sep token for CLIP. ([#1710](https://github.com/autogluon/autogluon/pull/1710)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n- Update fusion transformer in AutoMM ([#1712](https://github.com/autogluon/autogluon/pull/1712)) [@yiqings](https://github.com/yiqings)\n\n - Support constant learning rate in `polynomial_decay` scheduler.\n - Update `[CLS]` token in numerical/categorical transformer.\n\n- Added more image augmentations: `verticalflip`, `colorjitter`, `randomaffine` ([#1719](https://github.com/autogluon/autogluon/pull/1719)) [@Linuxdex](https://github.com/Linuxdex), [@sxjscience](https://github.com/sxjscience)\n\n- Added prompts for the percentage of missing images during image column detection. ([#1623](https://github.com/autogluon/autogluon/pull/1623)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n- Support `average_precision` in AutoMM ([#1697](https://github.com/autogluon/autogluon/pull/1697)) [@sxjscience](https://github.com/sxjscience)\n\n- Convert `roc_auc` / `average_precision` to `log_loss` for torchmetrics ([#1715](https://github.com/autogluon/autogluon/pull/1715)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - `torchmetrics.AUROC` requires both positive and negative examples are available in a mini-batch. When training a large model, the per gpu batch size is probably small, leading to an incorrect `roc_auc` score. Conversion from `roc_auc` to `log_loss` improves training stability.\n\n- Added `pytorch-lightning` 1.6 support ([#1716](https://github.com/autogluon/autogluon/pull/1716)) [@sxjscience](https://github.com/sxjscience)\n\n#### Checkpointing and Model Outputs Changes\n\n- Updated the names of top-k checkpoint average methods and support customizing model names for terminal input ([#1668](https://github.com/autogluon/autogluon/pull/1668)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Following paper: https://arxiv.org/pdf/2203.05482.pdf to update top-k checkpoint average names: `union_soup` -> `uniform_soup` and `best_soup` -> `best`.\n - Update function names (`customize_config_names` -> `customize_model_names` and `verify_config_names` -> `verify_model_names`) to make it easier to understand them.\n - Support customizing model names for the terminal input.\n\n- Implemented the GreedySoup algorithm proposed in [paper](https://arxiv.org/pdf/2203.05482.pdf). Added `union_soup`, `greedy_soup`, `best_soup` flags and changed the default value correspondingly. ([#1613](https://github.com/autogluon/autogluon/pull/1613)) [@sxjscience](https://github.com/sxjscience)\n\n- Updated the `standalone` flag in `automm.predictor.save()` to save the pertained model for offline deployment ([#1575](https://github.com/autogluon/autogluon/pull/1575)) [@yiqings](https://github.com/yiqings)\n\n - An efficient implementation to save the downloaded models from transformers for the offline deployment. Revised logic is in [#1572](https://github.com/autogluon/autogluon/pull/1572), and discussed in [#1572](https://github.com/autogluon/autogluon/pull/1572) (comment).\n\n- Simplified checkpoint template ([#1636](https://github.com/autogluon/autogluon/pull/1636)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Stopped using pytorch lightning's model checkpoint template in saving `AutoMMPredictor`'s final model checkpoint.\n - Improved the logic of continuous training. We pass the `ckpt_path` argument to pytorch lightning's trainer only when `resume=True`.\n\n- Unified AutoMM's model output format and support customizing model names ([#1643](https://github.com/autogluon/autogluon/pull/1643)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Now each model's output is dictionary with the model prefix as the first level key. The format is uniform between single model and fusion model.\n - Now users can customize model names by using the internal registered names (`timm_image`, `hf_text`, `clip`, `numerical_mlp`, `categorical_mlp`, and `fusion_mlp`) as prefixes. This is helpful when users want to simultaneously use two models of the same type, e.g., `hf_text`. They can just use names `hf_text_0` and `hf_text_1`.\n\n- Support `standalone` feature in `TextPredictor` ([#1651](https://github.com/autogluon/autogluon/pull/1651)) [@yiqings](https://github.com/yiqings)\n\n- Fixed saving and loading tokenizers and text processors ([#1656](https://github.com/autogluon/autogluon/pull/1656)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Saved pre-trained huggingface tokenizers separately from the data processors.\n - This change is backwards-compatible with checkpoints saved by version `0.4.0`.\n\n- Change load from a classmethod to staticmethod to avoid incorrect usage. ([#1697](https://github.com/autogluon/autogluon/pull/1697)) [@sxjscience](https://github.com/sxjscience)\n\n- Added `AutoMMModelCheckpoint` to avoid evaluating the models to obtain the scores ([#1716](https://github.com/autogluon/autogluon/pull/1716)) [@sxjscience](https://github.com/sxjscience)\n\n - checkpoint will save the best_k_models into a yaml file so that it can be loaded later to determine the path to model checkpoints.\n\n- Extract column features from AutoMM's model outputs ([#1718](https://github.com/autogluon/autogluon/pull/1718)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Add one util function to extract column features for both image and text.\n - Support extracting column features for models `timm_image`, `hf_text`, and `clip`.\n\n- Make AutoMM dataloader return feature column information ([#1710](https://github.com/autogluon/autogluon/pull/1710)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n#### Bug fixes\n\n- Fixed calling `save_pretrained_configs` in `AutoMMPrediction.save(standalone=True)` when no fusion model exists ([here](https://github.com/autogluon/autogluon/blob/5a323641072431091d2be5e6dbef5a87b646a408/text/src/autogluon/text/automm/utils.py#L644)) ([#1651](https://github.com/autogluon/autogluon/pull/1651)) [@yiqings](https://github.com/yiqings)\n\n- Fixed error raising for setting key that does not exist in the configuration ([#1613](https://github.com/autogluon/autogluon/pull/1613)) [@sxjscience](https://github.com/sxjscience)\n\n- Fixed warning message about bf16. ([#1625](https://github.com/autogluon/autogluon/pull/1625)) [@sxjscience](https://github.com/sxjscience)\n\n- Fixed the corner case of calculating the gradient accumulation step ([#1633](https://github.com/autogluon/autogluon/pull/1633)) [@sxjscience](https://github.com/sxjscience)\n\n- Fixes for top-k averaging in the multi-gpu setting ([#1707](https://github.com/autogluon/autogluon/pull/1707)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n### Tabular\n\n- Limited RF `max_leaf_nodes` to 15000 (previously uncapped) ([#1717](https://github.com/autogluon/autogluon/pull/1717)) [@Innixma](https://github.com/Innixma)\n\n - Previously, for very large datasets RF/XT memory and disk usage would quickly become unreasonable. This ensures that at a certain point RF and XT will no longer become larger given more rows of training data. Benchmark results showed that the change is an improvement, particularly for the `high_quality` preset.\n\n- Limit KNN to 32 CPUs to avoid OpenBLAS error ([#1722](https://github.com/autogluon/autogluon/pull/1722)) [@Innixma](https://github.com/Innixma)\n\n - Issue [#1020](https://github.com/autogluon/autogluon/pull/1020). When training K-nearest-neighbors (KNN) models, sometimes a rare error can occur that crashes the entire process:\n\n ```\n BLAS : Program is Terminated. Because you tried to allocate too many memory regions.\n Segmentation fault: 11\n ```\n\n This error occurred when the machine had many CPU cores (>64 vCPUs) due to too many threads being created at once. By limiting to 32 cores used, the error is avoided.\n\n- Improved memory warning thresholds ([#1626](https://github.com/autogluon/autogluon/pull/1626)) [@Innixma](https://github.com/Innixma)\n\n- Added `get_results` and `model_base_kwargs` ([#1618](https://github.com/autogluon/autogluon/pull/1618)) [@Innixma](https://github.com/Innixma)\n\n - Added `get_results` to searchers, useful for debugging and for future extensions to HPO functionality.\n Added new way to init a `BaggedEnsembleModel` that avoids having to init the base model prior to initing the bagged ensemble model.\n\n- Update resource logic in models ([#1689](https://github.com/autogluon/autogluon/pull/1689)) [@Innixma](https://github.com/Innixma)\n\n - Previous implementation would crash if user specified `auto` for resources, fixed in this PR.\n - Added `get_minimum_resources` to explicitly define minimum resource requirements within a method.\n\n- Updated feature importance default `subsample_size` 1000 -> 5000, `num_shuffle_sets 3` -> 5 ([#1708](https://github.com/autogluon/autogluon/pull/1708)) [@Innixma](https://github.com/Innixma)\n\n - This will improve the quality of the feature importance values by default, especially the 99% confidence bounds. The change increases the time taken by ~8x, but this is acceptable because of the numerous inference speed optimizations done since these defaults were first introduced.\n\n- Added notice to ensure serializable custom metrics ([#1705](https://github.com/autogluon/autogluon/pull/1705)) [@Innixma](https://github.com/Innixma)\n\n#### Bug fixes\n\n- Fixed `evaluate` when `weight_evaluation=True` ([#1612](https://github.com/autogluon/autogluon/pull/1612)) [@Innixma](https://github.com/Innixma)\n\n - Previously, AutoGluon would crash if the user specified `predictor.evaluate(...)` or `predictor.evaluate_predictions(...)` when `self.weight_evaluation==True`.\n\n- Fixed RuntimeError: dictionary changed size during iteration ([#1684](https://github.com/autogluon/autogluon/pull/1684), [#1685](https://github.com/autogluon/autogluon/pull/1685)) [@leandroimail](https://github.com/leandroimail)\n\n- Fixed CatBoost custom metric & F1 support ([#1690](https://github.com/autogluon/autogluon/pull/1690)) [@Innixma](https://github.com/Innixma)\n\n- Fixed HPO not working for bagged models if the bagged model is loaded from disk ([#1702](https://github.com/autogluon/autogluon/pull/1702)) [@Innixma](https://github.com/Innixma)\n\n- Fixed Feature importance erroring if `self.model_best` is `None` (can happen if no Weighted Ensemble is fit) ([#1702](https://github.com/autogluon/autogluon/pull/1702)) [@Innixma](https://github.com/Innixma)\n\n### Documentation\n\n- updated the text tutorial of customizing hyperparameters ([#1620](https://github.com/autogluon/autogluon/pull/1620)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n - Added customizable backbones from the Huggingface model zoo and how to use local backbones.\n\n- Improved implementations and docstrings of `save_pretrained_models` and `convert_checkpoint_name`. ([#1656](https://github.com/autogluon/autogluon/pull/1656)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n- Added cheat sheet to website ([#1605](https://github.com/autogluon/autogluon/pull/1605)) [@yinweisu](https://github.com/yinweisu)\n\n- Doc fix to use correct predictor when calling leaderboard ([#1652](https://github.com/autogluon/autogluon/pull/1652)) [@Innixma](https://github.com/Innixma)\n\n### Miscellaneous changes\n\n- [security] updated `pillow` to `9.0.1`+ ([#1615](https://github.com/autogluon/autogluon/pull/1615)) [@gradientsky](https://github.com/gradientsky)\n\n- [security] updated `ray` to `1.10.0`+ ([#1616](https://github.com/autogluon/autogluon/pull/1616)) [@yinweisu](https://github.com/yinweisu)\n\n- Tabular regression tests improvements ([#1555](https://github.com/autogluon/autogluon/pull/1555)) [@willsmithorg](https://github.com/willsmithorg)\n\n - Regression testing of model list and scores in tabular on small synthetic datasets (for speed).\n - Tests about 20 different calls to `TabularPredictor` on both regression and classification tasks, multiple presets etc.\n - When a test fails it dumps out the config change required to make it pass, for ease of updating.\n\n- Disabled image/text predictor when gpu is not available in `TabularPredictor` ([#1676](https://github.com/autogluon/autogluon/pull/1676)) [@yinweisu](https://github.com/yinweisu)\n\n - Resources are validated before bagging is started. Image/text predictor model would require minimum of 1 gpu.\n\n- Use class property to set keys in model classes. In this way, if we customize the prefix key, other keys are automatically updated. ([#1669](https://github.com/autogluon/autogluon/pull/1669)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n### Various bugfixes, documentation and CI improvements\n\n- [@yinweisu](https://github.com/yinweisu) ([#1605](https://github.com/autogluon/autogluon/pull/1605), [#1611](https://github.com/autogluon/autogluon/pull/1611), [#1631](https://github.com/autogluon/autogluon/pull/1631), [#1638](https://github.com/autogluon/autogluon/pull/1638), [#1691](https://github.com/autogluon/autogluon/pull/1691))\n- [@zhiqiangdon](https://github.com/zhiqiangdon) ([#1721](https://github.com/autogluon/autogluon/pull/1721))\n- [@Innixma](https://github.com/Innixma) ([#1608](https://github.com/autogluon/autogluon/pull/1608), [#1701](https://github.com/autogluon/autogluon/pull/1701))\n- [@sxjscience](https://github.com/sxjscience) ([#1714](https://github.com/autogluon/autogluon/pull/1714))\n"
},
{
"path": "docs/whats_new/v0.4.2.md",
"content": "# Version 0.4.2\n\nv0.4.2 is a hotfix release which fixes [breaking change](https://github.com/protocolbuffers/protobuf/issues/10051) made by protobuf.\n\nThis release is **non-breaking** when upgrading from v0.4.0. As always, only load previously trained models using the same version of AutoGluon that they were originally trained on. Loading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.4.1...v0.4.2\n\nThis version supports Python versions 3.7 to 3.9.\n"
},
{
"path": "docs/whats_new/v0.4.3.md",
"content": "# Version 0.4.3\n\nv0.4.3 is a security hotfix release.\n\nThis release is **non-breaking** when upgrading from v0.4.0. As always, only load previously trained models using the same version of AutoGluon that they were originally trained on. Loading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.4.2...v0.4.3\n\nThis version supports Python versions 3.7 to 3.9.\n"
},
{
"path": "docs/whats_new/v0.5.1.md",
"content": "# Version 0.5.1\n\n## Changes\n\n### AutoMM\n\nChanged to a new namespace `autogluon.multimodal` (AutoMM), which is a deep learning \"model zoo\" of model zoos. On one hand, AutoMM can automatically train deep models for unimodal (image-only, text-only or tabular-only) problems. On the other hand, AutoMM can automatically solve multimodal (any combinations of image, text, and tabular) problems by fusing multiple deep learning models. In addition, AutoMM can be used as a base model in AutoGluon Tabular and participate in the model ensemble.\n\n#### New features\n\n- Supported zero-shot learning with CLIP ([#1922](https://github.com/autogluon/autogluon/pull/1922)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n - Users can directly perform zero-shot image classification with the [CLIP model](https://arxiv.org/abs/2103.00020). Moreover, users can extract image and text embeddings with CLIP to do image-to-text or text-to-image retrieval. \n\n- Improved efficient finetuning\n - Support âbit_fitâ, ânorm_fitâ, âloraâ, âlora_biasâ, âlora_normâ. In four multilingual datasets ([xnli](https://huggingface.co/datasets/xnli), [stsb_multi_mt](https://huggingface.co/datasets/stsb_multi_mt), [paws-x](https://huggingface.co/datasets/paws-x), [amazon_reviews_multi](https://huggingface.co/datasets/amazon_reviews_multi)), âlora_biasâ, which is a combination of [LoRA](https://arxiv.org/abs/2106.09685) and [BitFit](https://arxiv.org/abs/2106.10199), achieved the best overall performance. Compared to finetuning the whole network, âlora_biasâ will only finetune **<0.5%** of the network parameters and can achieve comparable performance on âstsb_multi_mtâ ([#1780](https://github.com/autogluon/autogluon/pull/1780), [#1809](https://github.com/autogluon/autogluon/pull/1809)). [@Raldir](https://github.com/Raldir) [@zhiqiangdon](https://github.com/zhiqiangdon)\n - Support finetuning the [mT5-XL](https://huggingface.co/google/mt5-xl) model that has 1.7B parameters on a single NVIDIA G4 GPU. In AutoMM, we only use the T5-encoder (1.7B parameters) like [Sentence-T5](https://aclanthology.org/2022.findings-acl.146.pdf). ([#1933](https://github.com/autogluon/autogluon/pull/1933)) [@sxjscience](https://github.com/sxjscience)\n\n- Added more data augmentation techniques\n - [Mixup](https://arxiv.org/pdf/1710.09412.pdf) for image data. ([#1730](https://github.com/autogluon/autogluon/pull/1730)) [@Linuxdex](https://github.com/Linuxdex)\n - [TrivialAugment](https://arxiv.org/pdf/2103.10158.pdf) for both image and text data. ([#1792](https://github.com/autogluon/autogluon/pull/1792)) [@lzcemma](https://github.com/lzcemma)\n - [Easy text augmentations](https://arxiv.org/pdf/1901.11196.pdf). ([#1756](https://github.com/autogluon/autogluon/pull/1756)) [@lzcemma](https://github.com/lzcemma)\n\n- Enhanced teacher-student model distillation\n - Support distilling the knowledge from a unimodal/multimodal teacher model to a student model. ([#1670](https://github.com/autogluon/autogluon/pull/1670), [#1895](https://github.com/autogluon/autogluon/pull/1895)) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n#### More tutorials and examples\n\n- [Beginner tutorials](https://auto.gluon.ai/stable/tutorials/multimodal/index.html) of applying AutoMM to image, text, or multimodal (including tabular) data. ([#1861](https://github.com/autogluon/autogluon/pull/1861), [#1908](https://github.com/autogluon/autogluon/pull/1908), [#1858](https://github.com/autogluon/autogluon/pull/1858), [#1869](https://github.com/autogluon/autogluon/pull/1869)) [@bryanyzhu](https://github.com/bryanyzhu) [@sxjscience](https://github.com/sxjscience) [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n- [A zero-shot image classification tutorial](https://auto.gluon.ai/0.5.1/tutorials/multimodal/clip_zeroshot.html) with the CLIP model. ([#1942](https://github.com/autogluon/autogluon/pull/1942)) [@bryanyzhu](https://github.com/bryanyzhu)\n\n- A tutorial of using [CLIP model to extract embeddings](https://auto.gluon.ai/0.5.1/tutorials/multimodal/clip_embedding.html) for image-text retrieval. ([#1957](https://github.com/autogluon/autogluon/pull/1957)) [@bryanyzhu](https://github.com/bryanyzhu)\n\n- [A tutorial](https://auto.gluon.ai/0.5.1/tutorials/multimodal/customization.html) to introduce comprehensive AutoMM configurations ([#1861](https://github.com/autogluon/autogluon/pull/1861)). [@zhiqiangdon](https://github.com/zhiqiangdon)\n\n- [AutoMM for tabular data examples](https://github.com/autogluon/autogluon/tree/master/examples/automm/tabular_dl) ([#1752](https://github.com/autogluon/autogluon/pull/1752), [#1893](https://github.com/autogluon/autogluon/pull/1893), [#1903](https://github.com/autogluon/autogluon/pull/1903)). [@yiqings](https://github.com/yiqings)\n\n- [AutoMM distillation example](https://github.com/autogluon/autogluon/tree/master/examples/automm/distillation) ([#1846](https://github.com/autogluon/autogluon/pull/1846)). [@FANGAreNotGnu](https://github.com/FANGAreNotGnu)\n\n- A Kaggle notebook about how to use AutoMM to predict pet adoption: https://www.kaggle.com/code/linuxdex/use-autogluon-to-predict-pet-adoption. The model achieves the score equivalent to **top 1% (20th/3537) in this kernel-only competition (test data is only available in the kernel without internet access)** ([#1796](https://github.com/autogluon/autogluon/pull/1796), [#1847](https://github.com/autogluon/autogluon/pull/1847), [#1894](https://github.com/autogluon/autogluon/pull/1894), [#1943](https://github.com/autogluon/autogluon/pull/1943)). [@Linuxdex](https://github.com/Linuxdex)\n\n\n### TimeSeries\n\nWe are happy to announce AutoGluon-TimeSeries! Starting with v0.5, AutoGluon now supports AutoML for time series forecasting, \nleveraging both statistical forecasting methods such as ETS and ARIMA, as well as modern deep learning architectures\nthrough [GluonTS](https://ts.gluon.ai/stable/). The new module also features a weighted ensemble of time series models, \nand is geared towards probabilistic (quantile) forecasting to enable many use cases from demand and supply chain forecasting \nto financial applications.\n\nAll time series forecasting tasks are supported via the familiar AutoGluon interface, through the\n`TimeSeriesPredictor` class. Start forecasting today with the AutoGluon-TimeSeries \n[quick start guide](https://auto.gluon.ai/0.5.1/tutorials/timeseries/forecasting-quickstart.html).\n\nContributor List: [@canerturkmen](https://github.com/canerturkmen), [@huibinshen](https://github.com/huibinshen), [@Innixma](https://github.com/Innixma), [@yinweisu](https://github.com/yinweisu), [@shchur](https://github.com/shchur), [@gradientsky](https://github.com/gradientsky) \n\n#### Fixes and enhancements in v0.5.1\n\n- Add early stopping for AutoGluon-TimeSeries models ([#1917](https://github.com/autogluon/autogluon/pull/1917)) [@huibinshen](https://github.com/huibinshen)\n- Allow for automatically inferring seasonality period from `TimeSeriesDataFrame` index in `AutoETS`, intelligently setting seasonality to be used in ETS models by default. ([#1914](https://github.com/autogluon/autogluon/pull/1914)) [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur)\n- Changes in model presets, enabling `ARIMA` and GluonTS's `Transformer` models to be enabled by default and removing `MQCNN` models ([#1914](https://github.com/autogluon/autogluon/pull/1914)). [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur)\n- Fix for an issue that affected data sets with custom target column names when using `TimeSeriesPredictor` ([#1901](https://github.com/autogluon/autogluon/pull/1901)) [@canerturkmen](https://github.com/canerturkmen)\n- Capping `gluonts`, `sktime` versions ([#1914](https://github.com/autogluon/autogluon/pull/1914), [#1916](https://github.com/autogluon/autogluon/pull/1916)) [@yinweisu](https://github.com/yinweisu), [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur)\n"
},
{
"path": "docs/whats_new/v0.5.2.md",
"content": "# Version 0.5.2\n\nv0.5.2 is a security hotfix release.\n\nThis release is **non-breaking** when upgrading from v0.5.0. As always, only load previously trained models using the same version of AutoGluon that they were originally trained on. Loading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.5.1...v0.5.2\n\nThis version supports Python versions 3.7 to 3.9.\n"
},
{
"path": "docs/whats_new/v0.6.0.md",
"content": "# Version 0.6.0\n\nWe're happy to announce the AutoGluon 0.6 release. 0.6 contains major enhancements to Tabular, Multimodal, and Time Series\nmodules, along with many quality of life improvements and fixes.\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on.\nLoading models trained in different versions of AutoGluon is not supported.\n\nThis release contains [**263** commits from **25** contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=2022-07-18&to=2022-11-15&type=c)!\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.5.2...v0.6.0\n\nSpecial thanks to [@cheungdaven](https://github.com/cheungdaven), [@suzhoum](https://github.com/suzhoum), [@BingzhaoZhu](https://github.com/BingzhaoZhu), [@liangfu](https://github.com/liangfu), [@Harry-zzh](https://github.com/Harry-zzh), [@gidler](https://github.com/gidler), [@yongxinw](https://github.com/yongxinw), [@martinschaef](https://github.com/martinschaef),\n[@giswqs](https://github.com/giswqs), [@Jalagarto](https://github.com/Jalagarto), [@geoalgo](https://github.com/geoalgo), [@lujiaying](https://github.com/lujiaying) and [@leloykun](https://github.com/leloykun) who were first time contributors to AutoGluon this release!\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur), [@yinweisu](https://github.com/yinweisu), [@zhiqiangdon](https://github.com/zhiqiangdon), [@Innixma](https://github.com/Innixma), [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@canerturkmen](https://github.com/canerturkmen), [@sxjscience](https://github.com/sxjscience), [@gradientsky](https://github.com/gradientsky), [@cheungdaven](https://github.com/cheungdaven),\n[@bryanyzhu](https://github.com/bryanyzhu), [@suzhoum](https://github.com/suzhoum), [@BingzhaoZhu](https://github.com/BingzhaoZhu), [@yongxinw](https://github.com/yongxinw), [@tonyhoo](https://github.com/tonyhoo), [@liangfu](https://github.com/liangfu), [@Harry-zzh](https://github.com/Harry-zzh), [@Raldir](https://github.com/Raldir), [@gidler](https://github.com/gidler), [@martinschaef](https://github.com/martinschaef), \n[@giswqs](https://github.com/giswqs), [@Jalagarto](https://github.com/Jalagarto), [@geoalgo](https://github.com/geoalgo), [@lujiaying](https://github.com/lujiaying), [@leloykun](https://github.com/leloykun), [@yiqings](https://github.com/yiqings)\n\nThis version supports Python versions 3.7 to 3.9. This is the last release that will support Python 3.7.\n\n## Changes\n\n### AutoMM\n\nAutoGluon Multimodal (a.k.a AutoMM) supports three new features: 1) object detection, 2) named entity recognition, and 3) multimodal matching. In addition, the HPO backend of AutoGluon Multimodal has been upgraded to ray 2.0. It also supports fine-tuning billion-scale FLAN-T5-XL model on a single AWS g4.2x-large instance with improved parameter-efficient finetuning. Starting from 0.6, we recommend using autogluon.multimodal rather than autogluon.text or autogluon.vision and added deprecation warnings.\n\n#### New features\n\n- Object Detection\n - Add new problem_type `\"object_detection\"`.\n - Customers can run inference with pretrained object detection models and train their own model with three lines of code.\n - Integrate with [open-mmlab/mmdetection](https://github.com/open-mmlab/mmdetection), which supports classic detection architectures like Faster RCNN, and more efficient and performant architectures like YOLOV3 and VFNet.\n - See [tutorials](https://auto.gluon.ai/stable/tutorials/multimodal/object_detection/index.html) and [examples](https://github.com/autogluon/autogluon/tree/master/examples/automm/object_detection) for more detail.\n - Contributors and commits: [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@bryanyzhu](https://github.com/bryanyzhu), [@zhiqiangdon](https://github.com/zhiqiangdon), [@yongxinw](https://github.com/yongxinw), [@sxjscience](https://github.com/sxjscience), [@Harry-zzh](https://github.com/Harry-zzh) ([#2025](https://github.com/autogluon/autogluon/pull/2025), [#2061](https://github.com/autogluon/autogluon/pull/2061), [#2131](https://github.com/autogluon/autogluon/pull/2131), [#2181](https://github.com/autogluon/autogluon/pull/2181), [#2196](https://github.com/autogluon/autogluon/pull/2196), [#2215](https://github.com/autogluon/autogluon/pull/2215), [#2244](https://github.com/autogluon/autogluon/pull/2244), [#2265](https://github.com/autogluon/autogluon/pull/2265), [#2290](https://github.com/autogluon/autogluon/pull/2290), [#2311](https://github.com/autogluon/autogluon/pull/2311), [#2312](https://github.com/autogluon/autogluon/pull/2312), [#2337](https://github.com/autogluon/autogluon/pull/2337), [#2349](https://github.com/autogluon/autogluon/pull/2349), [#2353](https://github.com/autogluon/autogluon/pull/2353), [#2360](https://github.com/autogluon/autogluon/pull/2360), [#2362](https://github.com/autogluon/autogluon/pull/2362), [#2365](https://github.com/autogluon/autogluon/pull/2365), [#2380](https://github.com/autogluon/autogluon/pull/2380), [#2381](https://github.com/autogluon/autogluon/pull/2381), [#2391](https://github.com/autogluon/autogluon/pull/2391), [#2393](https://github.com/autogluon/autogluon/pull/2393), [#2400](https://github.com/autogluon/autogluon/pull/2400), [#2419](https://github.com/autogluon/autogluon/pull/2419), [#2421](https://github.com/autogluon/autogluon/pull/2421), [#2063](https://github.com/autogluon/autogluon/pull/2063), [#2104](https://github.com/autogluon/autogluon/pull/2104), [#2411](https://github.com/autogluon/autogluon/pull/2411))\n\n- Named Entity Recognition\n - Add new problem_type `\"ner\"`.\n - Customers can train models to extract named entities with three lines of code.\n - The implementation supports any backbones in huggingface/transformer, including the recently [FLAN-T5 series](https://arxiv.org/abs/2210.11416) released by Google.\n - See [tutorials](https://auto.gluon.ai/stable/tutorials/multimodal/text_prediction/ner.html) for more detail.\n - Contributors and commits: [@cheungdaven](https://github.com/cheungdaven) ([#2183](https://github.com/autogluon/autogluon/pull/2183), [#2232](https://github.com/autogluon/autogluon/pull/2232), [#2220](https://github.com/autogluon/autogluon/pull/2220), [#2282](https://github.com/autogluon/autogluon/pull/2282), [#2295](https://github.com/autogluon/autogluon/pull/2295), [#2301](https://github.com/autogluon/autogluon/pull/2301), [#2337](https://github.com/autogluon/autogluon/pull/2337), [#2346](https://github.com/autogluon/autogluon/pull/2346), [#2361](https://github.com/autogluon/autogluon/pull/2361), [#2372](https://github.com/autogluon/autogluon/pull/2372), [#2394](https://github.com/autogluon/autogluon/pull/2394), [#2412](https://github.com/autogluon/autogluon/pull/2412))\n\n- Multimodal Matching\n - Add new problem_type `\"text_similarity\"`, `\"image_similarity\"`, `\"image_text_similarity\"`.\n - Users can now extract semantic embeddings with pretrained models for text-text, image-image, and text-image matching problems.\n - Moreover, users can further finetune these models with relevance data.\n - The semantic text embedding model can also be combined with BM25 to form a hybrid indexing solution.\n - Internally, AutoGluon Multimodal implements a twin-tower architecture that is flexible in the choice of backbones for each tower. It supports image backbones in TIMM, text backbones in huggingface/transformers, and also the CLIP backbone.\n - See [tutorials](https://auto.gluon.ai/0.6.0/tutorials/multimodal/matching/index.html) for more detail.\n - Contributors and commits: [@zhiqiangdon](https://github.com/zhiqiangdon) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@cheungdaven](https://github.com/cheungdaven) [@suzhoum](https://github.com/suzhoum) [@sxjscience](https://github.com/sxjscience) [@bryanyzhu](https://github.com/bryanyzhu) ([#1975](https://github.com/autogluon/autogluon/pull/1975), [#1994](https://github.com/autogluon/autogluon/pull/1994), [#2142](https://github.com/autogluon/autogluon/pull/2142), [#2179](https://github.com/autogluon/autogluon/pull/2179), [#2186](https://github.com/autogluon/autogluon/pull/2186), [#2217](https://github.com/autogluon/autogluon/pull/2217), [#2235](https://github.com/autogluon/autogluon/pull/2235), [#2284](https://github.com/autogluon/autogluon/pull/2284), [#2297](https://github.com/autogluon/autogluon/pull/2297), [#2313](https://github.com/autogluon/autogluon/pull/2313), [#2326](https://github.com/autogluon/autogluon/pull/2326), [#2337](https://github.com/autogluon/autogluon/pull/2337), [#2347](https://github.com/autogluon/autogluon/pull/2347), [#2357](https://github.com/autogluon/autogluon/pull/2357), [#2358](https://github.com/autogluon/autogluon/pull/2358), [#2362](https://github.com/autogluon/autogluon/pull/2362), [#2363](https://github.com/autogluon/autogluon/pull/2363), [#2375](https://github.com/autogluon/autogluon/pull/2375), [#2378](https://github.com/autogluon/autogluon/pull/2378), [#2404](https://github.com/autogluon/autogluon/pull/2404), [#2416](https://github.com/autogluon/autogluon/pull/2416), [#2407](https://github.com/autogluon/autogluon/pull/2407), [#2417](https://github.com/autogluon/autogluon/pull/2417))\n\n- Miscellaneous minor fixes. [@cheungdaven](https://github.com/cheungdaven) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@geoalgo](https://github.com/geoalgo) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#2402](https://github.com/autogluon/autogluon/pull/2402), [#2409](https://github.com/autogluon/autogluon/pull/2409), [#2026](https://github.com/autogluon/autogluon/pull/2026), [#2401](https://github.com/autogluon/autogluon/pull/2401), [#2418](https://github.com/autogluon/autogluon/pull/2418))\n\n#### Other Enhancements\n\n- Fix the FT-Transformer implementation and support Fastformer. [@BingzhaoZhu](https://github.com/BingzhaoZhu) [@yiqings](https://github.com/yiqings) ([#1958](https://github.com/autogluon/autogluon/pull/1958), [#2194](https://github.com/autogluon/autogluon/pull/2194), [#2251](https://github.com/autogluon/autogluon/pull/2251), [#2344](https://github.com/autogluon/autogluon/pull/2344), [#2379](https://github.com/autogluon/autogluon/pull/2379), [#2386](https://github.com/autogluon/autogluon/pull/2386))\n- Support finetuning billion-scale FLAN-T5-XL in a single AWS g4.2x-large instance via improved parameter-efficient finetuning. See [tutorial](https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/efficient_finetuning_basic.html). [@Raldir](https://github.com/Raldir) [@sxjscience](https://github.com/sxjscience) ([#2032](https://github.com/autogluon/autogluon/pull/2032), [#2108](https://github.com/autogluon/autogluon/pull/2108), [#2285](https://github.com/autogluon/autogluon/pull/2285), [#2336](https://github.com/autogluon/autogluon/pull/2336), [#2352](https://github.com/autogluon/autogluon/pull/2352))\n- Upgrade multimodal HPO to use ray 2.0 and also add [new tutorial](https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/hyperparameter_optimization.html). [@yinweisu](https://github.com/yinweisu) [@suzhoum](https://github.com/suzhoum) [@bryanyzhu](https://github.com/bryanyzhu) ([#2206](https://github.com/autogluon/autogluon/pull/2206), [#2341](https://github.com/autogluon/autogluon/pull/2341))\n- Further improvement on model distillation. Add [example](https://github.com/autogluon/autogluon/tree/master/examples/automm/distillation) and [tutorial](https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/model_distillation.html). [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@sxjscience](https://github.com/sxjscience) ([#1983](https://github.com/autogluon/autogluon/pull/1983), [#2064](https://github.com/autogluon/autogluon/pull/2064), [#2397](https://github.com/autogluon/autogluon/pull/2397))\n- Revise the default presets of AutoMM for image classification problems. [@bryanyzhu](https://github.com/bryanyzhu) ([#2351](https://github.com/autogluon/autogluon/pull/2351))\n- Support backend=âautommâ in autogluon.vision. [@bryanyzhu](https://github.com/bryanyzhu) ([#2316](https://github.com/autogluon/autogluon/pull/2316))\n- Add deprecated warning to autogluon.vision and autogluon.text and point the usage to autogluon.multimodal. [@bryanyzhu](https://github.com/bryanyzhu) [@sxjscience](https://github.com/sxjscience) ([#2268](https://github.com/autogluon/autogluon/pull/2268), [#2315](https://github.com/autogluon/autogluon/pull/2315))\n- Examples about [Kaggle: Feedback Prize prediction competition](https://www.kaggle.com/competitions/feedback-prize-effectiveness). We created [a solution](https://www.kaggle.com/code/mountpotatoq/autogluon-finetune-solutions) with AutoGluon Multimodal that obtained 152/1557 in the public leaderboard and 170/1557 in the private leaderboard, which is among the top 12% participants. The solution is public days before the DDL of the competition and obtained more than 3000 views. [@suzhoum](https://github.com/suzhoum) [@MountPOTATO](https://github.com/MountPOTATO) ([#2129](https://github.com/autogluon/autogluon/pull/2129), [#2168](https://github.com/autogluon/autogluon/pull/2168), [#2333](https://github.com/autogluon/autogluon/pull/2333))\n* Improve native inference speed. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#2051](https://github.com/autogluon/autogluon/pull/2051), [#2157](https://github.com/autogluon/autogluon/pull/2157), [#2161](https://github.com/autogluon/autogluon/pull/2161), [#2171](https://github.com/autogluon/autogluon/pull/2171))\n* Other improvements, security/bug fixes. [@zhiqiangdon](https://github.com/zhiqiangdon) [@sxjscience](https://github.com/sxjscience) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@yinweisu](https://github.com/yinweisu) [@Innixma](https://github.com/Innixma) [@tonyhoo](https://github.com/tonyhoo) [@martinschaef](https://github.com/martinschaef) [@giswqs](https://github.com/giswqs) [@tonyhoo](https://github.com/tonyhoo) ([#1980](https://github.com/autogluon/autogluon/pull/1980), [#1987](https://github.com/autogluon/autogluon/pull/1987), [#1989](https://github.com/autogluon/autogluon/pull/1989), [#2003](https://github.com/autogluon/autogluon/pull/2003), [#2080](https://github.com/autogluon/autogluon/pull/2080), [#2018](https://github.com/autogluon/autogluon/pull/2018), [#2039](https://github.com/autogluon/autogluon/pull/2039), [#2058](https://github.com/autogluon/autogluon/pull/2058), [#2101](https://github.com/autogluon/autogluon/pull/2101), [#2102](https://github.com/autogluon/autogluon/pull/2102), [#2125](https://github.com/autogluon/autogluon/pull/2125), [#2135](https://github.com/autogluon/autogluon/pull/2135), [#2136](https://github.com/autogluon/autogluon/pull/2136), [#2140](https://github.com/autogluon/autogluon/pull/2140), [#2141](https://github.com/autogluon/autogluon/pull/2141), [#2152](https://github.com/autogluon/autogluon/pull/2152), [#2164](https://github.com/autogluon/autogluon/pull/2164), [#2166](https://github.com/autogluon/autogluon/pull/2166), [#2192](https://github.com/autogluon/autogluon/pull/2192), [#2219](https://github.com/autogluon/autogluon/pull/2219), [#2250](https://github.com/autogluon/autogluon/pull/2250), [#2257](https://github.com/autogluon/autogluon/pull/2257), [#2280](https://github.com/autogluon/autogluon/pull/2280), [#2308](https://github.com/autogluon/autogluon/pull/2308), [#2315](https://github.com/autogluon/autogluon/pull/2315), [#2317](https://github.com/autogluon/autogluon/pull/2317), [#2321](https://github.com/autogluon/autogluon/pull/2321), [#2356](https://github.com/autogluon/autogluon/pull/2356), [#2388](https://github.com/autogluon/autogluon/pull/2388), [#2392](https://github.com/autogluon/autogluon/pull/2392), [#2413](https://github.com/autogluon/autogluon/pull/2413), [#2414](https://github.com/autogluon/autogluon/pull/2414), [#2417](https://github.com/autogluon/autogluon/pull/2417), [#2426](https://github.com/autogluon/autogluon/pull/2426), [#2028](https://github.com/autogluon/autogluon/pull/2028), [#2382](https://github.com/autogluon/autogluon/pull/2382), [#2415](https://github.com/autogluon/autogluon/pull/2415), [#2193](https://github.com/autogluon/autogluon/pull/2193), [#2213](https://github.com/autogluon/autogluon/pull/2213), [#2230](https://github.com/autogluon/autogluon/pull/2230))\n* CI improvements. [@yinweisu](https://github.com/yinweisu) ([#1965](https://github.com/autogluon/autogluon/pull/1965), [#1966](https://github.com/autogluon/autogluon/pull/1966), [#1972](https://github.com/autogluon/autogluon/pull/1972), [#1991](https://github.com/autogluon/autogluon/pull/1991), [#2002](https://github.com/autogluon/autogluon/pull/2002), [#2029](https://github.com/autogluon/autogluon/pull/2029), [#2137](https://github.com/autogluon/autogluon/pull/2137), [#2151](https://github.com/autogluon/autogluon/pull/2151), [#2156](https://github.com/autogluon/autogluon/pull/2156), [#2163](https://github.com/autogluon/autogluon/pull/2163), [#2191](https://github.com/autogluon/autogluon/pull/2191), [#2214](https://github.com/autogluon/autogluon/pull/2214), [#2369](https://github.com/autogluon/autogluon/pull/2369), [#2113](https://github.com/autogluon/autogluon/pull/2113), [#2118](https://github.com/autogluon/autogluon/pull/2118))\n\n\n#### Experimental Features\n\n- Support 11B-scale model finetuning with DeepSpeed. [@Raldir](https://github.com/Raldir) ([#2032](https://github.com/autogluon/autogluon/pull/2032))\n- Enable few-shot learning with 11B-scale model. [@Raldir](https://github.com/Raldir) ([#2197](https://github.com/autogluon/autogluon/pull/2197))\n- ONNX export example of hf_text model. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#2149](https://github.com/autogluon/autogluon/pull/2149))\n\n### Tabular\n\n#### New features\n\n- New experimental model `FT_TRANSFORMER`. [@bingzhaozhu](https://github.com/bingzhaozhu), [@innixma](https://github.com/innixma) ([#2085](https://github.com/autogluon/autogluon/pull/2085), [#2379](https://github.com/autogluon/autogluon/pull/2379), [#2389](https://github.com/autogluon/autogluon/pull/2389), [#2410](https://github.com/autogluon/autogluon/pull/2410)) \n - You can access it via specifying the `FT_TRANSFORMER` key\nin the `hyperparameters` dictionary or via `presets=\"experimental_best_quality\"`. \n - It is recommended to use GPU to train this model, but CPU training is also supported.\n - If given enough training time, this model generally improves the ensemble quality.\n\n- New experimental model compilation support via `predictor.compile_models()`. [@liangfu](https://github.com/liangfu), [@innixma](https://github.com/innixma) ([#2225](https://github.com/autogluon/autogluon/pull/2225), [#2260](https://github.com/autogluon/autogluon/pull/2260), [#2300](https://github.com/autogluon/autogluon/pull/2300)) \n - Currently only Random Forest and Extra Trees have compilation support.\n - You will need to install extra dependencies for this to work: `pip install autogluon.tabular[all,skl2onnx]`.\n - Compiling models dramatically speeds up inference time (~10x) when processing small batches of samples (<10000).\n - Note that a known bug exists in the current implementation: Refitting models after compilation will fail\nand cause a crash. To avoid this, ensure that `.compile_models` is called only at the very end.\n- Added `predictor.clone(...)` method to allow perfectly cloning a predictor object to a new directory. \nThis is useful to preserve the state of a predictor prior to altering it\n(such as prior to calling `.save_space`, `.distill`, `.compile_models`, or `.refit_full`. [@innixma](https://github.com/innixma) ([#2071](https://github.com/autogluon/autogluon/pull/2071))\n- Added simplified `num_gpus` and `num_cpus` arguments to `predictor.fit` to control total resources.\n[@yinweisu](https://github.com/yinweisu), [@innixma](https://github.com/innixma) ([#2263](https://github.com/autogluon/autogluon/pull/2263))\n- Improved stability and effectiveness of HPO functionality via various refactors regarding our usage of ray.\n[@yinweisu](https://github.com/yinweisu), [@innixma](https://github.com/innixma) ([#1974](https://github.com/autogluon/autogluon/pull/1974), [#1990](https://github.com/autogluon/autogluon/pull/1990), [#2094](https://github.com/autogluon/autogluon/pull/2094), [#2121](https://github.com/autogluon/autogluon/pull/2121), [#2133](https://github.com/autogluon/autogluon/pull/2133), [#2195](https://github.com/autogluon/autogluon/pull/2195), [#2253](https://github.com/autogluon/autogluon/pull/2253), [#2263](https://github.com/autogluon/autogluon/pull/2263), [#2330](https://github.com/autogluon/autogluon/pull/2330))\n- Upgraded dependency versions: XGBoost 1.7, CatBoost 1.1, Scikit-learn 1.1, Pandas 1.5, Scipy 1.9, Numpy 1.23.\n[@innixma](https://github.com/innixma) ([#2373](https://github.com/autogluon/autogluon/pull/2373))\n- Added python version compatibility check when loading a fitted TabularPredictor.\nWill now error if python versions are incompatible. [@innixma](https://github.com/innixma) ([#2054](https://github.com/autogluon/autogluon/pull/2054))\n- Added `fit_weighted_ensemble` argument to `predictor.fit`. This allows the user to disable the weighted ensemble.\n[@innixma](https://github.com/innixma) ([#2145](https://github.com/autogluon/autogluon/pull/2145))\n- Added cascade ensemble foundation logic. [@innixma](https://github.com/innixma) ([#1929](https://github.com/autogluon/autogluon/pull/1929)) \n\n#### Other Enhancements\n- Improved logging clarity when using `infer_limit`. [@innixma](https://github.com/innixma) ([#2014](https://github.com/autogluon/autogluon/pull/2014))\n- Significantly improved HPO search space of XGBoost. [@innixma](https://github.com/innixma) ([#2123](https://github.com/autogluon/autogluon/pull/2123))\n- Fixed HPO crashing when tuning Random Forest, Extra Trees, or KNN. [@innixma](https://github.com/innixma) ([#2070](https://github.com/autogluon/autogluon/pull/2070))\n- Optimized roc_auc metric scoring speed by 7x. [@innixma](https://github.com/innixma) ([#2318](https://github.com/autogluon/autogluon/pull/2318), [#2331](https://github.com/autogluon/autogluon/pull/2331))\n- Fixed bug with AutoMM Tabular model crashing if not trained last. [@innixma](https://github.com/innixma) ([#2309](https://github.com/autogluon/autogluon/pull/2309))\n- Refactored `Scorer` classes to be easier to use, plus added comprehensive unit tests for all metrics. [@innixma](https://github.com/innixma) ([#2242](https://github.com/autogluon/autogluon/pull/2242))\n- Sped up TextSpecial feature generation during preprocessing by 20% [@gidler](https://github.com/gidler) ([#2095](https://github.com/autogluon/autogluon/pull/2095))\n- imodels integration improvements [@Jalagarto](https://github.com/Jalagarto) ([#2062](https://github.com/autogluon/autogluon/pull/2062))\n- Fix crash when calling feature importance in quantile_regression. [@leloykun](https://github.com/leloykun) ([#1977](https://github.com/autogluon/autogluon/pull/1977))\n- Add FAQ section for missing value imputation. [@innixma](https://github.com/innixma) ([#2076](https://github.com/autogluon/autogluon/pull/2076))\n- Various minor fixes and cleanup [@innixma](https://github.com/innixma), [@yinweisu](https://github.com/yinweisu), [@gradientsky](https://github.com/gradientsky), [@gidler](https://github.com/gidler) ([#1997](https://github.com/autogluon/autogluon/pull/1997), [#2031](https://github.com/autogluon/autogluon/pull/2031), [#2124](https://github.com/autogluon/autogluon/pull/2124), [#2144](https://github.com/autogluon/autogluon/pull/2144), [#2178](https://github.com/autogluon/autogluon/pull/2178), [#2340](https://github.com/autogluon/autogluon/pull/2340), [#2342](https://github.com/autogluon/autogluon/pull/2342), [#2345](https://github.com/autogluon/autogluon/pull/2345), [#2374](https://github.com/autogluon/autogluon/pull/2374), [#2339](https://github.com/autogluon/autogluon/pull/2339), \n[#2348](https://github.com/autogluon/autogluon/pull/2348), [#2403](https://github.com/autogluon/autogluon/pull/2403), [#1981](https://github.com/autogluon/autogluon/pull/1981), [#1982](https://github.com/autogluon/autogluon/pull/1982), [#2234](https://github.com/autogluon/autogluon/pull/2234), [#2233](https://github.com/autogluon/autogluon/pull/2233), [#2243](https://github.com/autogluon/autogluon/pull/2243), [#2269](https://github.com/autogluon/autogluon/pull/2269), [#2288](https://github.com/autogluon/autogluon/pull/2288), [#2307](https://github.com/autogluon/autogluon/pull/2307), [#2367](https://github.com/autogluon/autogluon/pull/2367), [#2019](https://github.com/autogluon/autogluon/pull/2019))\n\n### Time Series\n\n#### New features\n\n- `TimeSeriesPredictor` now supports **static features** (a.k.a. time series metadata, static covariates) and **\n time-varying covariates** (a.k.a. dynamic features or related time series). [@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) ([#1986](https://github.com/autogluon/autogluon/pull/1986), [#2238](https://github.com/autogluon/autogluon/pull/2238),\n [#2276](https://github.com/autogluon/autogluon/pull/2276), [#2287](https://github.com/autogluon/autogluon/pull/2287))\n- AutoGluon-TimeSeries now uses **PyTorch** by default (for `DeepAR` and `SimpleFeedForward`), removing the dependency\n on MXNet. [@canerturkmen](https://github.com/canerturkmen) ([#2074](https://github.com/autogluon/autogluon/pull/2074), [#2205](https://github.com/autogluon/autogluon/pull/2205), [#2279](https://github.com/autogluon/autogluon/pull/2279))\n- New models! `AutoGluonTabular` relies on XGBoost, LightGBM and CatBoost under the hood via the `autogluon.tabular`\n module. `Naive` and `SeasonalNaive` forecasters are simple methods that provide strong baselines with no increase in\n training time. `TemporalFusionTransformerMXNet` brings the TFT transformer architecture to AutoGluon. [@shchur](https://github.com/shchur) ([#2106](https://github.com/autogluon/autogluon/pull/2106),\n [#2188](https://github.com/autogluon/autogluon/pull/2188), [#2258](https://github.com/autogluon/autogluon/pull/2258), [#2266](https://github.com/autogluon/autogluon/pull/2266))\n- Up to 20x faster parallel and memory-efficient training for statistical (local) forecasting models like `ETS`, `ARIMA`\n and `Theta`, as well as `WeightedEnsemble`. [@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) ([#2001](https://github.com/autogluon/autogluon/pull/2001), [#2033](https://github.com/autogluon/autogluon/pull/2033), [#2040](https://github.com/autogluon/autogluon/pull/2040), [#2067](https://github.com/autogluon/autogluon/pull/2067), [#2072](https://github.com/autogluon/autogluon/pull/2072), [#2073](https://github.com/autogluon/autogluon/pull/2073), [#2180](https://github.com/autogluon/autogluon/pull/2180),\n [#2293](https://github.com/autogluon/autogluon/pull/2293), [#2305](https://github.com/autogluon/autogluon/pull/2305))\n- Up to 3x faster training for GluonTS models with data caching. GPU training enabled by default on PyTorch models.\n [@shchur](https://github.com/shchur) ([#2323](https://github.com/autogluon/autogluon/pull/2323))\n- More accurate validation for time series models with multi-window backtesting. [@shchur](https://github.com/shchur) ([#2013](https://github.com/autogluon/autogluon/pull/2013), [#2038](https://github.com/autogluon/autogluon/pull/2038))\n- `TimeSeriesPredictor` now handles irregularly sampled time series with `ignore_index`. [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur) ([#1993](https://github.com/autogluon/autogluon/pull/1993),\n [#2322](https://github.com/autogluon/autogluon/pull/2322))\n- Improved and extended presets for more accurate forecasting. [@shchur](https://github.com/shchur) ([#2304](https://github.com/autogluon/autogluon/pull/2304))\n- 15x faster and more robust forecast evaluation with updates to `TimeSeriesEvaluator` [@shchur](https://github.com/shchur) ([#2147](https://github.com/autogluon/autogluon/pull/2147), [#2150](https://github.com/autogluon/autogluon/pull/2150))\n- Enabled Ray Tune backend for hyperparameter optimization of time series models. [@shchur](https://github.com/shchur) ([#2167](https://github.com/autogluon/autogluon/pull/2167), [#2203](https://github.com/autogluon/autogluon/pull/2203))\n\n#### More tutorials and examples\n\nImproved documentation and new tutorials:\n\n- Updated [Quickstart tutorial](https://auto.gluon.ai/0.6.0/tutorials/timeseries/forecasting-quickstart.html)\n- New! [In-depth tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-indepth.html)\n- New! [Overview of available models and hyperparameters](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-model-zoo.html)\n- Updated [API documentation](https://auto.gluon.ai/0.6.0/api/autogluon.predictor.html#module-5)\n\n[@shchur](https://github.com/shchur) ([#2120](https://github.com/autogluon/autogluon/pull/2120), [#2127](https://github.com/autogluon/autogluon/pull/2127), [#2146](https://github.com/autogluon/autogluon/pull/2146), [#2174](https://github.com/autogluon/autogluon/pull/2174), [#2187](https://github.com/autogluon/autogluon/pull/2187), [#2354](https://github.com/autogluon/autogluon/pull/2354))\n\n#### Miscellaneous\n\n[@shchur](https://github.com/shchur)\n- Deprecate passing quantile_levels to TimeSeriesPredictor.predict ([#2277](https://github.com/autogluon/autogluon/pull/2277))\n- Use static features in GluonTS forecasting models ([#2238](https://github.com/autogluon/autogluon/pull/2238))\n- Make sure that time series splitter doesn't trim training series shorter than prediction_length + 1 ([#2099](https://github.com/autogluon/autogluon/pull/2099))\n- Fix hyperparameter overloading in HPO for time series models ([#2189](https://github.com/autogluon/autogluon/pull/2189))\n- Clean up the TimeSeriesDataFrame public API ([#2105](https://github.com/autogluon/autogluon/pull/2105))\n- Fix item order in GluonTS models predictions ([#2092](https://github.com/autogluon/autogluon/pull/2092))\n- Implement hash_ts_dataframe_items ([#2060](https://github.com/autogluon/autogluon/pull/2060))\n- Speed up TimeSeriesDataFrame.slice_by_timestep ([#2020](https://github.com/autogluon/autogluon/pull/2020))\n- Speed up RandomForestQuantileRegressor and ExtraTreesQuantileRegressor ([#2204](https://github.com/autogluon/autogluon/pull/2204))\n- Various backend enhancements / refactoring / cleanup ([#2314](https://github.com/autogluon/autogluon/pull/2314), [#2294](https://github.com/autogluon/autogluon/pull/2294), [#2292](https://github.com/autogluon/autogluon/pull/2292), [#2278](https://github.com/autogluon/autogluon/pull/2278), [#1985](https://github.com/autogluon/autogluon/pull/1985), [#2398](https://github.com/autogluon/autogluon/pull/2398))\n\n[@canerturkmen](https://github.com/canerturkmen)\n- Increase the number of samples used by DeepAR at prediction time ([#2291](https://github.com/autogluon/autogluon/pull/2291)) \n- revise timeseries presets to minimum context length of 10 ([#2065](https://github.com/autogluon/autogluon/pull/2065)) \n- Fix timeseries daily frequency inferred period ([#2100](https://github.com/autogluon/autogluon/pull/2100)) \n- Various backend enhancements / refactoring / cleanup ([#2286](https://github.com/autogluon/autogluon/pull/2286), [#2302](https://github.com/autogluon/autogluon/pull/2302), [#2240](https://github.com/autogluon/autogluon/pull/2240), [#2093](https://github.com/autogluon/autogluon/pull/2093), [#2098](https://github.com/autogluon/autogluon/pull/2098), [#2044](https://github.com/autogluon/autogluon/pull/2044), [#2385](https://github.com/autogluon/autogluon/pull/2385), [#2355](https://github.com/autogluon/autogluon/pull/2355), [#2405](https://github.com/autogluon/autogluon/pull/2405))\n"
},
{
"path": "docs/whats_new/v0.6.1.md",
"content": "# Version 0.6.1\n\nv0.6.1 is a security fix / bug fix release.\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on. \nLoading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.6.0...v0.6.1\n\nSpecial thanks to [@lvwerra](https://github.com/lvwerra) who is first time contributors to AutoGluon this release!\n\nThis version supports Python versions 3.7 to 3.9. 0.6.x are the last releases that will support Python 3.7.\n\n## Changes\n\n### Documentation improvements\n\n- Fix object detection tutorial layout ([#2450](https://github.com/autogluon/autogluon/pull/2450)) - [@bryanyzhu](https://github.com/bryanyzhu)\n- Add multimodal cheatsheet ([#2467](https://github.com/autogluon/autogluon/pull/2467)) - [@sxjscience](https://github.com/sxjscience)\n- Refactoring detection inference quickstart and bug fix on fit->predict - [@yongxinw](https://github.com/yongxinw), [@zhiqiangdon](https://github.com/zhiqiangdon), [@Innixma](https://github.com/Innixma), [@BingzhaoZhu](https://github.com/BingzhaoZhu), [@tonyhoo](https://github.com/tonyhoo)\n- Use Pothole Dataset in Tutorial for AutoMM Detection ([#2468](https://github.com/autogluon/autogluon/pull/2468)) - [@FANGAreNotGnu](https://github.com/FANGAreNotGnu)\n- add time series cheat sheet, add time series to doc titles ([#2478](https://github.com/autogluon/autogluon/pull/2478)) - [@canerturkmen](https://github.com/canerturkmen)\n- Update all repo references to autogluon/autogluon ([#2463](https://github.com/autogluon/autogluon/pull/2463)) - [@gidler](https://github.com/gidler)\n- fix typo in object detection tutorial CI ([#2516](https://github.com/autogluon/autogluon/pull/2516)) - [@tonyhoo](https://github.com/tonyhoo)\n\n### Bug Fixes / Security\n\n- bump evaluate to 0.3.0 ([#2433](https://github.com/autogluon/autogluon/pull/2433)) - [@lvwerra](https://github.com/lvwerra)\n- Add finetune/eval tests for AutoMM detection ([#2441](https://github.com/autogluon/autogluon/pull/2441)) - [@FANGAreNotGnu](https://github.com/FANGAreNotGnu)\n- Adding Joint IA3_LoRA as efficient finetuning strategy ([#2451](https://github.com/autogluon/autogluon/pull/2451)) - [@Raldir](https://github.com/Raldir)\n- Fix AutoMM warnings about object detection ([#2458](https://github.com/autogluon/autogluon/pull/2458)) - [@zhiqiangdon](https://github.com/zhiqiangdon)\n- [Tabular] Speed up feature transform in tabular NN model ([#2442](https://github.com/autogluon/autogluon/pull/2442)) - [@liangfu](https://github.com/liangfu)\n- fix matcher cpu inference bug ([#2461](https://github.com/autogluon/autogluon/pull/2461)) - [@sxjscience](https://github.com/sxjscience)\n- [timeseries] Silence GluonTS JSON warning ([#2454](https://github.com/autogluon/autogluon/pull/2454)) - [@shchur](https://github.com/shchur)\n- [timeseries] Fix pandas groupby bug + GluonTS index bug ([#2420](https://github.com/autogluon/autogluon/pull/2420)) - [@shchur](https://github.com/shchur)\n- Simplified infer speed throughput calculation ([#2465](https://github.com/autogluon/autogluon/pull/2465)) - [@Innixma](https://github.com/Innixma)\n- [Tabular] make tabular nn dataset iterable ([#2395](https://github.com/autogluon/autogluon/pull/2395)) - [@liangfu](https://github.com/liangfu)\n- Remove old images and dataset download scripts ([#2471](https://github.com/autogluon/autogluon/pull/2471)) - [@Innixma](https://github.com/Innixma)\n- Support image bytearray in AutoMM ([#2490](https://github.com/autogluon/autogluon/pull/2490)) - [@suzhoum](https://github.com/suzhoum)\n- [NER] add an NER visualizer ([#2500](https://github.com/autogluon/autogluon/pull/2500)) - [@cheungdaven](https://github.com/cheungdaven)\n- [Cloud] Lazy load TextPredcitor and ImagePredictor which will be deprecated ([#2517](https://github.com/autogluon/autogluon/pull/2517)) - [@tonyhoo](https://github.com/tonyhoo)\n- Use detectron2 visualizer and update quickstart ([#2502](https://github.com/autogluon/autogluon/pull/2502)) - [@yongxinw](https://github.com/yongxinw), [@zhiqiangdon](https://github.com/zhiqiangdon), [@Innixma](https://github.com/Innixma), [@BingzhaoZhu](https://github.com/BingzhaoZhu), [@tonyhoo](https://github.com/tonyhoo)\n- fix df preprocessor properties ([#2512](https://github.com/autogluon/autogluon/pull/2512)) - [@zhiqiangdon](https://github.com/zhiqiangdon)\n- [timeseries] Fix info and fit_summary for TimeSeriesPredictor ([#2510](https://github.com/autogluon/autogluon/pull/2510)) - [@shchur](https://github.com/shchur)\n- [timeseries] Pass known_covariates to component models of the WeightedEnsemble - [@shchur](https://github.com/shchur)\n- [timeseries] Gracefully handle inconsistencies in static_features provided by user - [@shchur](https://github.com/shchur)\n- [security] update Pillow to >=9.3.0 ([#2519](https://github.com/autogluon/autogluon/pull/2519)) - [@gradientsky](https://github.com/gradientsky)\n- [CI] upgrade codeql v1 to v2 as v1 will be deprecated ([#2528](https://github.com/autogluon/autogluon/pull/2528)) - [@tonyhoo](https://github.com/tonyhoo)\n- Upgrade scikit-learn-intelex version ([#2466](https://github.com/autogluon/autogluon/pull/2466)) - [@Innixma](https://github.com/Innixma)\n- Save AutoGluonTabular model to the correct folder ([#2530](https://github.com/autogluon/autogluon/pull/2530)) - [@shchur](https://github.com/shchur)\n- support predicting with model fitted on v0.5.1 ([#2531](https://github.com/autogluon/autogluon/pull/2531)) - [@liangfu](https://github.com/liangfu)\n- [timeseries] Implement input validation for TimeSeriesPredictor and improve debug messages - [@shchur](https://github.com/shchur)\n- [timeseries] Ensure that timestamps are sorted when creating a TimeSeriesDataFrame - [@shchur](https://github.com/shchur)\n- Add tests for preprocessing mutation ([#2540](https://github.com/autogluon/autogluon/pull/2540)) - [@Innixma](https://github.com/Innixma)\n- Fix timezone datetime edgecase ([#2538](https://github.com/autogluon/autogluon/pull/2538)) - [@Innixma](https://github.com/Innixma), [@gradientsky](https://github.com/gradientsky)\n- Mmdet Fix Image Identifier ([#2492](https://github.com/autogluon/autogluon/pull/2492)) - [@FANGAreNotGnu](https://github.com/FANGAreNotGnu)\n- [timeseries] Warn if provided data has a frequency that is not supported - [@shchur](https://github.com/shchur)\n- Train and inference with different image data types ([#2535](https://github.com/autogluon/autogluon/pull/2535)) - [@suzhoum](https://github.com/suzhoum)\n- Remove pycocotools ([#2548](https://github.com/autogluon/autogluon/pull/2548)) - [@bryanyzhu](https://github.com/bryanyzhu)\n- avoid copying identical dataframes ([#2532](https://github.com/autogluon/autogluon/pull/2532)) - [@liangfu](https://github.com/liangfu)\n- Fix AutoMM Tokenizer ([#2550](https://github.com/autogluon/autogluon/pull/2550)) - [@FANGAreNotGnu](https://github.com/FANGAreNotGnu)\n- [Tabular] Resource Allocation Fix ([#2536](https://github.com/autogluon/autogluon/pull/2536)) - [@yinweisu](https://github.com/yinweisu)\n- imodels version cap ([#2557](https://github.com/autogluon/autogluon/pull/2557)) - [@yinweisu](https://github.com/yinweisu)\n- Fix int32/int64 difference between windows and other platforms; fix mutation issue - [@gradientsky](https://github.com/gradientsky)\n"
},
{
"path": "docs/whats_new/v0.6.2.md",
"content": "# Version 0.6.2\n\nv0.6.2 is a security and bug fix release.\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on.\nLoading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.6.1...v0.6.2\n\nSpecial thanks to [@daikikatsuragawa](https://github.com/daikikatsuragawa) and [@yzhliu](https://github.com/yzhliu) who were first time contributors to AutoGluon this release!\n\nThis version supports Python versions 3.7 to 3.9. 0.6.x are the last releases that will support Python 3.7.\n\n## Changes\n\n### Documentation improvements\n\n- Ray usage FAQ ([#2559](https://github.com/autogluon/autogluon/pull/2559)) - [@yinweisu](https://github.com/yinweisu)\n- Fix missing Predictor API doc ([#2573](https://github.com/autogluon/autogluon/pull/2573)) - [@gidler](https://github.com/gidler)\n- 2023 Roadmap Update ([#2590](https://github.com/autogluon/autogluon/pull/2590)) - [@Innixma](https://github.com/Innixma)\n- Image classifiction tutorial update for bytearray ([#2598](https://github.com/autogluon/autogluon/pull/2598)) - [@suzhoum](https://github.com/suzhoum)\n- Fix broken tutorial index links ([#2617](https://github.com/autogluon/autogluon/pull/2617)) - [@shchur](https://github.com/shchur)\n- Improve timeseries quickstart tutorial ([#2653](https://github.com/autogluon/autogluon/pull/2653)) - [@shchur](https://github.com/shchur)\n\n\n### Bug Fixes / Security\n\n- [multimodal] Refactoring and bug fixes([#2554](https://github.com/autogluon/autogluon/pull/2554), [#2541](https://github.com/autogluon/autogluon/pull/2541), [#2477](https://github.com/autogluon/autogluon/pull/2477), [#2569](https://github.com/autogluon/autogluon/pull/2569), [#2578](https://github.com/autogluon/autogluon/pull/2578), [#2613](https://github.com/autogluon/autogluon/pull/2613), [#2620](https://github.com/autogluon/autogluon/pull/2620), [#2630](https://github.com/autogluon/autogluon/pull/2630), [#2633](https://github.com/autogluon/autogluon/pull/2633), [#2635](https://github.com/autogluon/autogluon/pull/2635), [#2647](https://github.com/autogluon/autogluon/pull/2647), [#2645](https://github.com/autogluon/autogluon/pull/2645), [#2652](https://github.com/autogluon/autogluon/pull/2652), [#2659](https://github.com/autogluon/autogluon/pull/2659)) - [@zhiqiangdon](https://github.com/zhiqiangdon), [@yongxinw](https://github.com/yongxinw), [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@sxjscience](https://github.com/sxjscience), [@Innixma](https://github.com/Innixma)\n- [multimodal] Support of named entity recognition ([#2556](https://github.com/autogluon/autogluon/pull/2556)) - [@cheungdaven](https://github.com/cheungdaven)\n- [multimodal] bytearray support for image modality ([#2549](https://github.com/autogluon/autogluon/pull/2549)) - [@suzhoum](https://github.com/suzhoum)\n- [multimodal] Support HPO for matcher ([#2619](https://github.com/autogluon/autogluon/pull/2619)) - [@zhiqiangdon](https://github.com/zhiqiangdon)\n- [multimodal] Support Onnx export for timm image model ([#2564](https://github.com/autogluon/autogluon/pull/2564)) - [@liangfu](https://github.com/liangfu)\n- [tabular] Refactoring and bug fixes ([#2387](https://github.com/autogluon/autogluon/pull/2387), [#2595](https://github.com/autogluon/autogluon/pull/2595)īŧ[#2599](https://github.com/autogluon/autogluon/pull/2599), [#2589](https://github.com/autogluon/autogluon/pull/2589), [#2628](https://github.com/autogluon/autogluon/pull/2628), [#2376](https://github.com/autogluon/autogluon/pull/2376), [#2642](https://github.com/autogluon/autogluon/pull/2642), [#2646](https://github.com/autogluon/autogluon/pull/2646), [#2650](https://github.com/autogluon/autogluon/pull/2650), [#2657](https://github.com/autogluon/autogluon/pull/2657)) - [@Innixma](https://github.com/Innixma), [@liangfu](https://github.com/liangfu)īŧ [@yzhliu](https://github.com/yzhliu), [@daikikatsuragawa](https://github.com/daikikatsuragawa), [@yinweisu](https://github.com/yinweisu)\n- [tabular] Fix ensemble folding ([#2582](https://github.com/autogluon/autogluon/pull/2582)) - [@yinweisu](https://github.com/yinweisu)\n- [tabular] Convert ColumnTransformer in tabular NN from sklearn to onnx ([#2503](https://github.com/autogluon/autogluon/pull/2503)) - [@liangfu](https://github.com/liangfu) \n- [tabular] Throw error on non-finite values in label column ($2509) - [@gidler](https://github.com/gidler)\n- [timeseries] Refactoring and bug fixes ([#2584](https://github.com/autogluon/autogluon/pull/2584), [#2594](https://github.com/autogluon/autogluon/pull/2594), [#2605](https://github.com/autogluon/autogluon/pull/2605), [#2606](https://github.com/autogluon/autogluon/pull/2606)) - [@shchur](https://github.com/shchur)\n- [timeseries] Speed up data preparation for local models ([#2587](https://github.com/autogluon/autogluon/pull/2587)) - [@shchur](https://github.com/shchur)\n- [timeseries] Speed up prediction for GluonTS models ([#2593](https://github.com/autogluon/autogluon/pull/2593)) - [@shchur](https://github.com/shchur)\n- [timeseries] Speed up the train/val splitter ([#2586](https://github.com/autogluon/autogluon/pull/2586)) - [@shchur](https://github.com/shchur)\n [timeseries] Speed up TimeSeriesEnsembleSelection.fit ([#2602](https://github.com/autogluon/autogluon/pull/2602)) - [@shchur](https://github.com/shchur)\n- [security] Update torch ([#2588](https://github.com/autogluon/autogluon/pull/2588)) - [@gradientsky](https://github.com/gradientsky)\n"
},
{
"path": "docs/whats_new/v0.7.0.md",
"content": "# Version 0.7.0\n\nWe're happy to announce the AutoGluon 0.7 release. This release contains a new experimental module `autogluon.eda` for exploratory\ndata analysis. AutoGluon 0.7 offers **conda-forge support**, enhancements to Tabular, MultiModal, and Time Series\nmodules, and many quality of life improvements and fixes.\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on.\nLoading models trained in different versions of AutoGluon is not supported.\n\nThis release contains [**170** commits from **19** contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=2023-01-10&to=2023-02-16&type=c)!\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.6.2...v0.7.0\n\nSpecial thanks to [@MountPOTATO](https://github.com/MountPOTATO) who is a first time contributor to AutoGluon this release!\n\nFull Contributor List (ordered by # of commits):\n\n[@Innixma](https://github.com/Innixma), [@zhiqiangdon](https://github.com/zhiqiangdon), [@yinweisu](https://github.com/yinweisu), [@gradientsky](https://github.com/gradientsky), [@shchur](https://github.com/shchur), [@sxjscience](https://github.com/sxjscience), [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@yongxinw](https://github.com/yongxinw), [@cheungdaven](https://github.com/cheungdaven),\n[@liangfu](https://github.com/liangfu), [@tonyhoo](https://github.com/tonyhoo), [@bryanyzhu](https://github.com/bryanyzhu), [@suzhoum](https://github.com/suzhoum), [@canerturkmen](https://github.com/canerturkmen), [@giswqs](https://github.com/giswqs), [@gidler](https://github.com/gidler), [@yzhliu](https://github.com/yzhliu), [@Linuxdex](https://github.com/Linuxdex) and [@MountPOTATO](https://github.com/MountPOTATO)\n\nAutoGluon 0.7 supports Python versions 3.8, 3.9, and **3.10**. Python 3.7 is no longer supported as of this release. \n\n## Changes\n\n### NEW: AutoGluon available on conda-forge\n\nAs of AutoGluon 0.7 release, AutoGluon is now available on [conda-forge](https://anaconda.org/conda-forge/autogluon) ([#612](https://github.com/autogluon/autogluon/pull/612))!\n\nKudos to the following individuals for making this happen:\n * [@giswqs](https://github.com/giswqs) for leading the entire effort and being a 1-man army driving this forward.\n * [@h-vetinari](https://github.com/h-vetinari) for providing excellent advice for working with conda-forge and some truly exceptional feedback.\n * [@arturdaraujo](https://github.com/arturdaraujo), [@PertuyF](https://github.com/PertuyF), [@ngam](https://github.com/ngam) and [@priyanga24](https://github.com/priyanga24) for their encouragement, suggestions, and feedback.\n * The conda-forge team for their prompt and effective reviews of our (many) PRs.\n * [@gradientsky](https://github.com/gradientsky) for testing M1 support during the early stages.\n * [@sxjscience](https://github.com/sxjscience), [@zhiqiangdon](https://github.com/zhiqiangdon), [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur), and [@Innixma](https://github.com/Innixma) for helping upgrade our downstream dependency versions to be compatible with conda.\n * Everyone else who has supported this process either directly or indirectly.\n\n### NEW: `autogluon.eda` (Exploratory Data Analysis)\n\nWe are happy to announce AutoGluon Exploratory Data Analysis (EDA) toolkit. Starting with v0.7, AutoGluon now can analyze and visualize different aspects of data and models. We invite you to explore the following tutorials: [Quick Fit](https://auto.gluon.ai/0.7.0/tutorials/eda/eda-auto-quick-fit.html), [Dataset Overview](https://auto.gluon.ai/0.7.0/tutorials/eda/eda-auto-dataset-overview.html), [Target Variable Analysis](https://auto.gluon.ai/0.7.0/tutorials/eda/eda-auto-target-analysis.html), [Covariate Shift Analysis](https://auto.gluon.ai/0.7.0/tutorials/eda/eda-auto-covariate-shift.html). Other materials can be found in [EDA Section](https://auto.gluon.ai/0.7.0/tutorials/eda/index.html) of the website.\n\n### General\n\n- Added Python 3.10 support. [@Innixma](https://github.com/Innixma) ([#2721](https://github.com/autogluon/autogluon/pull/2721))\n- Dropped Python 3.7 support. [@Innixma](https://github.com/Innixma) ([#2722](https://github.com/autogluon/autogluon/pull/2722))\n- Removed `dask` and `distributed` dependencies. [@Innixma](https://github.com/Innixma) ([#2691](https://github.com/autogluon/autogluon/pull/2691))\n- Removed `autogluon.text` and `autogluon.vision` modules. We recommend using `autogluon.multimodal` for text and vision tasks going forward.\n\n### AutoMM\n\nAutoGluon MultiModal (a.k.a AutoMM) supports three new features: 1) document classification; 2) named entity recognition\nfor Chinese language; 3) few shot learning with SVM \n\nMeanwhile, we removed `autogluon.text` and `autogluon.vision` as these features are supported in `autogluon.multimodal`\n\n#### New features\n\n- Document Classification\n - Add scanned document classification (experimental).\n - Customers can train models for scanned document classification in a few lines of codes\n - See [tutorials](https://auto.gluon.ai/0.7.0/tutorials/multimodal/document/document_classification.html)\n - Contributors and commits: [@cheungdaven](https://github.com/cheungdaven) ([#2765](https://github.com/autogluon/autogluon/pull/2765), [#2826](https://github.com/autogluon/autogluon/pull/2826), [#2833](https://github.com/autogluon/autogluon/pull/2833), [#2928](https://github.com/autogluon/autogluon/pull/2928))\n- NER for Chinese Language\n - Support Chinese named entity recognition\n - See [tutorials](https://auto.gluon.ai/0.7.0/tutorials/multimodal/document/document_classification.html)\n - Contributors and commits: [@cheungdaven](https://github.com/cheungdaven) ([#2676](https://github.com/autogluon/autogluon/pull/2676), [#2709](https://github.com/autogluon/autogluon/pull/2709))\n- Few Shot Learning with SVM\n - Improved few shot learning by adding SVM support\n - See [tutorials](https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/few_shot_learning.html)\n - Contributors and commits: [@yongxinw](https://github.com/yongxinw) ([#2850](https://github.com/autogluon/autogluon/pull/2850))\n\n#### Other Enhancements\n\n- Add new loss function `FocalLoss`. [@yongxinw](https://github.com/yongxinw) ([#2860](https://github.com/autogluon/autogluon/pull/2860))\n- Add matcher realtime inference support. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#2613](https://github.com/autogluon/autogluon/pull/2613))\n- Add matcher HPO. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#2619](https://github.com/autogluon/autogluon/pull/2619))\n- Add YOLOX models (small, large, and x-large) and update presets for object detection. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#2644](https://github.com/autogluon/autogluon/pull/2644), [#2867](https://github.com/autogluon/autogluon/pull/2867), [#2927](https://github.com/autogluon/autogluon/pull/2927), [#2933](https://github.com/autogluon/autogluon/pull/2933))\n- Add AutoMM presets [@zhiqiangdon](https://github.com/zhiqiangdon). ([#2620](https://github.com/autogluon/autogluon/pull/2620), [#2749](https://github.com/autogluon/autogluon/pull/2749), [#2839](https://github.com/autogluon/autogluon/pull/2839))\n- Add model dump for models from HuggingFace, timm and mmdet. [@suzhoum](https://github.com/suzhoum) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@liangfu](https://github.com/liangfu) ([#2682](https://github.com/autogluon/autogluon/pull/2682), [#2700](https://github.com/autogluon/autogluon/pull/2700), [#2737](https://github.com/autogluon/autogluon/pull/2737), [#2840](https://github.com/autogluon/autogluon/pull/2840))\n- Bug fix / refactor for NER. [@cheungdaven](https://github.com/cheungdaven) ([#2659](https://github.com/autogluon/autogluon/pull/2659), [#2696](https://github.com/autogluon/autogluon/pull/2696), [#2759](https://github.com/autogluon/autogluon/pull/2759), [#2773](https://github.com/autogluon/autogluon/pull/2773))\n- MultiModalPredictor import time reduction. [@sxjscience](https://github.com/sxjscience) ([#2718](https://github.com/autogluon/autogluon/pull/2718))\n\n#### Bug Fixes / Code and Doc Improvements\n\n- NER example with visualization. [@sxjscience](https://github.com/sxjscience) ([#2698](https://github.com/autogluon/autogluon/pull/2698))\n- Bug fixes / Code and Doc Improvements. [@sxjscience](https://github.com/sxjscience) [@tonyhoo](https://github.com/tonyhoo) [@giswqs](https://github.com/giswqs) ([#2708](https://github.com/autogluon/autogluon/pull/2708), [#2714](https://github.com/autogluon/autogluon/pull/2714), [#2739](https://github.com/autogluon/autogluon/pull/2739), [#2782](https://github.com/autogluon/autogluon/pull/2782), [#2787](https://github.com/autogluon/autogluon/pull/2787), [#2857](https://github.com/autogluon/autogluon/pull/2857), [#2818](https://github.com/autogluon/autogluon/pull/2818), [#2858](https://github.com/autogluon/autogluon/pull/2858), [#2859](https://github.com/autogluon/autogluon/pull/2859), [#2891](https://github.com/autogluon/autogluon/pull/2891), [#2918](https://github.com/autogluon/autogluon/pull/2918), [#2940](https://github.com/autogluon/autogluon/pull/2940), [#2906](https://github.com/autogluon/autogluon/pull/2906), [#2907](https://github.com/autogluon/autogluon/pull/2907))\n- Support of [Label-Studio](https://labelstud.io/) file export in AutoMM and added [examples](https://github.com/autogluon/autogluon/tree/master/examples/automm/label_studio_export_reader). [@MountPOTATO](https://github.com/MountPOTATO) ([#2615](https://github.com/autogluon/autogluon/pull/2615))\n- Added example of few-shot memory bank model with feature extraction based on [Tip-adapter](https://arxiv.org/abs/2111.03930). [@Linuxdex](https://github.com/Linuxdex) ([#2822](https://github.com/autogluon/autogluon/pull/2822))\n\n#### Deprecations\n\n* `autogluon.vision` namespace is deprecated. [@bryanyzhu](https://github.com/bryanyzhu) ([#2790](https://github.com/autogluon/autogluon/pull/2790), [#2819](https://github.com/autogluon/autogluon/pull/2819), [#2832](https://github.com/autogluon/autogluon/pull/2832))\n* `autogluon.text` namespace is deprecated. [@sxjscience](https://github.com/sxjscience) [@Innixma](https://github.com/Innixma) ([#2695](https://github.com/autogluon/autogluon/pull/2695), [#2847](https://github.com/autogluon/autogluon/pull/2847))\n\n### Tabular\n\n1) TabularPredictorâs inference speed has been heavily optimized, with an average **250% speedup** for real-time inference. This means that TabularPredictor can satisfy <10 ms end-to-end latency on many datasets when using `infer_limit`, and the `high_quality` preset can satisfy <100 ms end-to-end latency on many datasets by default.\n2) TabularPredictorâs `\"multimodal\"` hyperparameter preset now leverages the full capabilities of MultiModalPredictor, resulting in stronger performance on datasets containing a mix of tabular, image, and text features.\n\n#### Performance Improvements\n\n- Upgraded versions of all dependency packages to use the latest releases. [@Innixma](https://github.com/Innixma) ([#2823](https://github.com/autogluon/autogluon/pull/2823), [#2829](https://github.com/autogluon/autogluon/pull/2829), [#2834](https://github.com/autogluon/autogluon/pull/2834), [#2887](https://github.com/autogluon/autogluon/pull/2887), [#2915](https://github.com/autogluon/autogluon/pull/2915))\n- Accelerated ensemble inference speed by 150% by removing TorchThreadManager context switching. [@liangfu](https://github.com/liangfu) ([#2472](https://github.com/autogluon/autogluon/pull/2472))\n- Accelerated FastAI neural network inference speed by 100x+ and training speed by 10x on datasets with many features. [@Innixma](https://github.com/Innixma) ([#2909](https://github.com/autogluon/autogluon/pull/2909))\n- (From 0.6.1) Avoid unnecessary DataFrame copies to accelerate feature preprocessing by 25%. [@liangfu](https://github.com/liangfu) ([#2532](https://github.com/autogluon/autogluon/pull/2532))\n- (From 0.6.1) Refactor `NN_TORCH` model to be dataset iterable, leading to a 100% inference speedup. [@liangfu](https://github.com/liangfu) ([#2395](https://github.com/autogluon/autogluon/pull/2395))\n- MultiModalPredictor is now used as a member of the ensemble when `TabularPredictor.fit` is passed `hyperparameters=\"multimodal\"`. [@Innixma](https://github.com/Innixma) ([#2890](https://github.com/autogluon/autogluon/pull/2890))\n\n#### API Enhancements\n\n- Added `predict_multi` and `predict_proba_multi` methods to `TabularPredictor` to efficiently get predictions from multiple models. [@Innixma](https://github.com/Innixma) ([#2727](https://github.com/autogluon/autogluon/pull/2727))\n- Allow label column to not be present in `leaderboard` calls when scoring is disabled. [@Innixma](https://github.com/Innixma) ([#2912](https://github.com/autogluon/autogluon/pull/2912))\n\n#### Deprecations\n\n- Added a deprecation warning when calling `predict_proba` with `problem_type=\"regression\"`. This will raise an exception in a future release. [@Innixma](https://github.com/Innixma) ([#2684](https://github.com/autogluon/autogluon/pull/2684))\n\n#### Bug Fixes / Doc Improvements\n\n- Fixed incorrect time_limit estimation in `NN_TORCH` model. [@Innixma](https://github.com/Innixma) ([#2909](https://github.com/autogluon/autogluon/pull/2909))\n- Fixed error when fitting with only text features. [@Innixma](https://github.com/Innixma) ([#2705](https://github.com/autogluon/autogluon/pull/2705))\n- Fixed error when `calibrate=True, use_bag_holdout=True` in `TabularPredictor.fit`. [@Innixma](https://github.com/Innixma) ([#2715](https://github.com/autogluon/autogluon/pull/2715))\n- Fixed error when tuning `n_estimators` with RandomForest / ExtraTrees models. [@Innixma](https://github.com/Innixma) ([#2735](https://github.com/autogluon/autogluon/pull/2735))\n- Fixed missing onnxruntime dependency on Linux/MacOS when installing optional dependency `skl2onnx`. [@liangfu](https://github.com/liangfu) ([#2923](https://github.com/autogluon/autogluon/pull/2923))\n- Fixed edge-case RandomForest error on Windows. [@yinweisu](https://github.com/yinweisu) ([#2851](https://github.com/autogluon/autogluon/pull/2851))\n- Added improved logging for `refit_full`. [@Innixma](https://github.com/Innixma) ([#2913](https://github.com/autogluon/autogluon/pull/2913))\n- Added `compile_models` to the deployment tutorial. [@liangfu](https://github.com/liangfu) ([#2717](https://github.com/autogluon/autogluon/pull/2717))\n- Various internal code refactoring. [@Innixma](https://github.com/Innixma) ([#2744](https://github.com/autogluon/autogluon/pull/2744), [#2887](https://github.com/autogluon/autogluon/pull/2887))\n- Various doc and logging improvements. [@Innixma](https://github.com/Innixma) ([#2668](https://github.com/autogluon/autogluon/pull/2668))\n\n## autogluon.timeseries\n\n### New features\n\n- `TimeSeriesPredictor` now supports **past covariates** (a.k.a.dynamic features or related time series which is not known for time steps to be predicted). [@shchur](https://github.com/shchur) ([#2665](https://github.com/autogluon/autogluon/pull/2665), [#2680](https://github.com/autogluon/autogluon/pull/2680))\n- New models from [StatsForecast](https://github.com/Nixtla/statsforecast) got introduced in `TimeSeriesPredictor` for various presets (`medium_quality`, `high_quality` and `best_quality`). [@shchur](https://github.com/shchur) ([#2758](https://github.com/autogluon/autogluon/pull/2758))\n- Support missing value imputation for TimeSeriesDataFrame which allows users to customize filling logics for missing values and fill gaps in an irregular sampled times series. [@shchur](https://github.com/shchur) ([#2781](https://github.com/autogluon/autogluon/pull/2781))\n- Improve quantile forecasting performance of the AutoGluon-Tabular forecaster using the empirical noise distribution. [@shchur](https://github.com/shchur) ([#2740](https://github.com/autogluon/autogluon/pull/2740))\n\n### Bug Fixes / Doc Improvements\n\n- Bug fixes and code improvements. [@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) ([#2703](https://github.com/autogluon/autogluon/pull/2703), [#2712](https://github.com/autogluon/autogluon/pull/2712), [#2713](https://github.com/autogluon/autogluon/pull/2713), [#2769](https://github.com/autogluon/autogluon/pull/2769), [#2771](https://github.com/autogluon/autogluon/pull/2771), [#2816](https://github.com/autogluon/autogluon/pull/2816), [#2817](https://github.com/autogluon/autogluon/pull/2817), [#2875](https://github.com/autogluon/autogluon/pull/2875), [#2877](https://github.com/autogluon/autogluon/pull/2877), [#2919](https://github.com/autogluon/autogluon/pull/2919))\n- Doc improvements. [@shchur](https://github.com/shchur) [@gidler](https://github.com/gidler) ([#2772](https://github.com/autogluon/autogluon/pull/2772), [#2783](https://github.com/autogluon/autogluon/pull/2783), [#2800](https://github.com/autogluon/autogluon/pull/2800))\n"
},
{
"path": "docs/whats_new/v0.8.0.md",
"content": "# Version 0.8.0\nWe're happy to announce the AutoGluon 0.8 release.\n\nNote: Loading models trained in different versions of AutoGluon is not supported.\n\nThis release contains 196 commits from 20 contributors!\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/0.7.0...0.8.0\n\nSpecial thanks to [@geoalgo](https://github.com/geoalgo) for the joint work in generating the experimental tabular Zeroshot-HPO portfolio this release!\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur), [@Innixma](https://github.com/Innixma), [@yinweisu](https://github.com/yinweisu), [@gradientsky](https://github.com/gradientsky), [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@zhiqiangdon](https://github.com/zhiqiangdon), [@gidler](https://github.com/gidler), [@liangfu](https://github.com/liangfu), [@tonyhoo](https://github.com/tonyhoo), [@cheungdaven](https://github.com/cheungdaven), [@cnpgs](https://github.com/cnpgs), [@giswqs](https://github.com/giswqs), [@suzhoum](https://github.com/suzhoum), [@yongxinw](https://github.com/yongxinw), [@isunli](https://github.com/isunli), [@jjaeyeon](https://github.com/jjaeyeon), [@xiaochenbin9527](https://github.com/xiaochenbin9527), [@yzhliu](https://github.com/yzhliu), [@jsharpna](https://github.com/jsharpna), [@sxjscience](https://github.com/sxjscience)\n\nAutoGluon 0.8 supports Python versions 3.8, 3.9, and 3.10.\n\n## Changes\n\n### Highlights\n* AutoGluon TimeSeries introduced several major improvements, including new models, upgraded presets that lead to better forecast accuracy, and optimizations that speed up training & inference.\n* AutoGluon Tabular now supports **[calibrating the decision threshold in binary classification](https://auto.gluon.ai/stable/tutorials/tabular/tabular-indepth.html#decision-threshold-calibration)** ([API](https://auto.gluon.ai/stable/api/autogluon.tabular.TabularPredictor.calibrate_decision_threshold.html)), leading to massive improvements in metrics such as `f1` and `balanced_accuracy`. It is not uncommon to see `f1` scores improve from `0.70` to `0.73` as an example. We **strongly** encourage all users who are using these metrics to try out the new decision threshold calibration logic.\n* AutoGluon MultiModal introduces two new features: 1) **PDF document classification**, and 2) **Open Vocabulary Object Detection**.\n* AutoGluon MultiModal upgraded the presets for object detection, now offering `medium_quality`, `high_quality`, and `best_quality` options. The empirical results demonstrate significant ~20% relative improvements in the mAP (mean Average Precision) metric, using the same preset.\n* AutoGluon Tabular has added an experimental **Zeroshot HPO config** which performs well on small datasets <10000 rows when at least an hour of training time is provided (~60% win-rate vs `best_quality`). To try it out, specify `presets=\"experimental_zeroshot_hpo_hybrid\"` when calling `fit()`.\n* AutoGluon EDA added support for **Anomaly Detection** and **Partial Dependence Plots**.\n* AutoGluon Tabular has added experimental support for **[TabPFN](https://github.com/automl/TabPFN)**, a pre-trained tabular transformer model. Try it out via `pip install autogluon.tabular[all,tabpfn]` (hyperparameter key is \"TABPFN\")!\n\n### General\n* General doc improvements [@tonyhoo](https://github.com/tonyhoo) [@Innixma](https://github.com/Innixma) [@yinweisu](https://github.com/yinweisu) [@gidler](https://github.com/gidler) [@cnpgs](https://github.com/cnpgs) [@isunli](https://github.com/isunli) [@giswqs](https://github.com/giswqs) ([#2940](https://github.com/autogluon/autogluon/pull/2940), [#2953](https://github.com/autogluon/autogluon/pull/2953), [#2963](https://github.com/autogluon/autogluon/pull/2963), [#3007](https://github.com/autogluon/autogluon/pull/3007), [#3027](https://github.com/autogluon/autogluon/pull/3027), [#3059](https://github.com/autogluon/autogluon/pull/3059), [#3068](https://github.com/autogluon/autogluon/pull/3068), [#3083](https://github.com/autogluon/autogluon/pull/3083), [#3128](https://github.com/autogluon/autogluon/pull/3128), [#3129](https://github.com/autogluon/autogluon/pull/3129), [#3130](https://github.com/autogluon/autogluon/pull/3130), [#3147](https://github.com/autogluon/autogluon/pull/3147), [#3174](https://github.com/autogluon/autogluon/pull/3174), [#3187](https://github.com/autogluon/autogluon/pull/3187), [#3256](https://github.com/autogluon/autogluon/pull/3256), [#3258](https://github.com/autogluon/autogluon/pull/3258), [#3280](https://github.com/autogluon/autogluon/pull/3280), [#3306](https://github.com/autogluon/autogluon/pull/3306), [#3307](https://github.com/autogluon/autogluon/pull/3307), [#3311](https://github.com/autogluon/autogluon/pull/3311), [#3313](https://github.com/autogluon/autogluon/pull/3313))\n* General code fixes and improvements [@yinweisu](https://github.com/yinweisu) [@Innixma](https://github.com/Innixma) ([#2921](https://github.com/autogluon/autogluon/pull/2921), [#3078](https://github.com/autogluon/autogluon/pull/3078), [#3113](https://github.com/autogluon/autogluon/pull/3113), [#3140](https://github.com/autogluon/autogluon/pull/3140), [#3206](https://github.com/autogluon/autogluon/pull/3206))\n* CI improvements [@yinweisu](https://github.com/yinweisu) [@gidler](https://github.com/gidler) [@yzhliu](https://github.com/yzhliu) [@liangfu](https://github.com/liangfu) [@gradientsky](https://github.com/gradientsky) ([#2965](https://github.com/autogluon/autogluon/pull/2965), [#3008](https://github.com/autogluon/autogluon/pull/3008), [#3013](https://github.com/autogluon/autogluon/pull/3013), [#3020](https://github.com/autogluon/autogluon/pull/3020), [#3046](https://github.com/autogluon/autogluon/pull/3046), [#3053](https://github.com/autogluon/autogluon/pull/3053), [#3108](https://github.com/autogluon/autogluon/pull/3108), [#3135](https://github.com/autogluon/autogluon/pull/3135), [#3159](https://github.com/autogluon/autogluon/pull/3159), [#3283](https://github.com/autogluon/autogluon/pull/3283), [#3185](https://github.com/autogluon/autogluon/pull/3185))\n* New AutoGluon Webpage [@gidler](https://github.com/gidler) [@shchur](https://github.com/shchur) ([#2924](https://github.com/autogluon/autogluon/pull/2924))\n* Support sample_weight in RMSE [@jjaeyeon](https://github.com/jjaeyeon) ([#3052](https://github.com/autogluon/autogluon/pull/3052))\n* Move AG search space to common [@yinweisu](https://github.com/yinweisu) ([#3192](https://github.com/autogluon/autogluon/pull/3192))\n* Deprecation utils [@yinweisu](https://github.com/yinweisu) ([#3206](https://github.com/autogluon/autogluon/pull/3206), [#3209](https://github.com/autogluon/autogluon/pull/3209))\n* Update namespace packages for PEP420 compatibility [@gradientsky](https://github.com/gradientsky) ([#3228](https://github.com/autogluon/autogluon/pull/3228))\n\n### Multimodal\n\nAutoGluon MultiModal (also known as AutoMM) introduces two new features: 1) PDF document classification, and 2) Open Vocabulary Object Detection. Additionally, we have upgraded the presets for object detection, now offering `medium_quality`, `high_quality`, and `best_quality` options. The empirical results demonstrate significant ~20% relative improvements in the mAP (mean Average Precision) metric, using the same preset.\n\n#### New Features\n* PDF Document Classification. See [tutorial](https://auto.gluon.ai/0.8.0/tutorials/multimodal/document/pdf_classification.html) [@cheungdaven](https://github.com/cheungdaven) ([#2864](https://github.com/autogluon/autogluon/pull/2864), [#3043](https://github.com/autogluon/autogluon/pull/3043))\n* Open Vocabulary Object Detection. See [tutorial](https://auto.gluon.ai/0.8.0/tutorials/multimodal/object_detection/quick_start/quick_start_ovd.html) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3164](https://github.com/autogluon/autogluon/pull/3164))\n\n#### Performance Improvements\n* Upgrade the detection engine from mmdet 2.x to mmdet 3.x, and upgrade our presets [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3262](https://github.com/autogluon/autogluon/pull/3262))\n * `medium_quality`: yolo-s -> yolox-l \n * `high_quality`: yolox-l -> DINO-Res50\n * `best_quality`: yolox-x -> DINO-Swin_l \n* Speedup fusion model training with deepspeed strategy. [@liangfu](https://github.com/liangfu) ([#2932](https://github.com/autogluon/autogluon/pull/2932))\n* Enable detection backbone freezing to boost finetuning speed and save GPU usage [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3220](https://github.com/autogluon/autogluon/pull/3220))\n\n#### Other Enhancements\n* Support passing data path to the fit() API [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3006](https://github.com/autogluon/autogluon/pull/3006))\n* Upgrade TIMM to the latest v0.9.* [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3282](https://github.com/autogluon/autogluon/pull/3282))\n* Support xywh output for object detection [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#2948](https://github.com/autogluon/autogluon/pull/2948))\n* Fusion model inference acceleration with TensorRT [@liangfu](https://github.com/liangfu) ([#2836](https://github.com/autogluon/autogluon/pull/2836), [#2987](https://github.com/autogluon/autogluon/pull/2987))\n* Support customizing advanced image data augmentation. Users can pass a list of [torchvision transform](https://pytorch.org/vision/stable/transforms.html#geometry) objects as image augmentation. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3022](https://github.com/autogluon/autogluon/pull/3022))\n* Add yoloxm and yoloxtiny [@FangAreNotGnu](https://github.com/FangAreNotGnu) ([#3038](https://github.com/autogluon/autogluon/pull/3038))\n* Add MultiImageMix Dataset for Object Detection [@FangAreNotGnu](https://github.com/FangAreNotGnu) ([#3094](https://github.com/autogluon/autogluon/pull/3094))\n* Support loading specific checkpoints. Users can load the intermediate checkpoints other than model.ckpt and last.ckpt. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3244](https://github.com/autogluon/autogluon/pull/3244))\n* Add some predictor properties for model statistics [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3289](https://github.com/autogluon/autogluon/pull/3289))\n * `trainable_parameters` returns the number of trainable parameters.\n * `total_parameters` returns the number of total parameters.\n * `model_size` returns the model size measured by megabytes.\n\n#### Bug Fixes / Code and Doc Improvements\n* General bug fixes and improvements [@zhiqiangdon](https://github.com/zhiqiangdon) [@liangfu](https://github.com/liangfu) [@cheungdaven](https://github.com/cheungdaven) [@xiaochenbin9527](https://github.com/xiaochenbin9527) [@Innixma](https://github.com/Innixma) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@gradientsky](https://github.com/gradientsky) [@yinweisu](https://github.com/yinweisu) [@yongxinw](https://github.com/yongxinw) ([#2939](https://github.com/autogluon/autogluon/pull/2939), [#2989](https://github.com/autogluon/autogluon/pull/2989), [#2983](https://github.com/autogluon/autogluon/pull/2983), [#2998](https://github.com/autogluon/autogluon/pull/2998), [#3001](https://github.com/autogluon/autogluon/pull/3001), [#3004](https://github.com/autogluon/autogluon/pull/3004), [#3006](https://github.com/autogluon/autogluon/pull/3006), [#3025](https://github.com/autogluon/autogluon/pull/3025), [#3026](https://github.com/autogluon/autogluon/pull/3026), [#3048](https://github.com/autogluon/autogluon/pull/3048), [#3055](https://github.com/autogluon/autogluon/pull/3055), [#3064](https://github.com/autogluon/autogluon/pull/3064), [#3070](https://github.com/autogluon/autogluon/pull/3070), [#3081](https://github.com/autogluon/autogluon/pull/3081), [#3090](https://github.com/autogluon/autogluon/pull/3090), [#3103](https://github.com/autogluon/autogluon/pull/3103), [#3106](https://github.com/autogluon/autogluon/pull/3106), [#3119](https://github.com/autogluon/autogluon/pull/3119), [#3155](https://github.com/autogluon/autogluon/pull/3155), [#3158](https://github.com/autogluon/autogluon/pull/3158), [#3167](https://github.com/autogluon/autogluon/pull/3167), [#3180](https://github.com/autogluon/autogluon/pull/3180), [#3188](https://github.com/autogluon/autogluon/pull/3188), [#3222](https://github.com/autogluon/autogluon/pull/3222), [#3261](https://github.com/autogluon/autogluon/pull/3261), [#3266](https://github.com/autogluon/autogluon/pull/3266), [#3277](https://github.com/autogluon/autogluon/pull/3277), [#3279](https://github.com/autogluon/autogluon/pull/3279), [#3261](https://github.com/autogluon/autogluon/pull/3261), [#3267](https://github.com/autogluon/autogluon/pull/3267))\n* General doc improvements [@suzhoum](https://github.com/suzhoum) ([#3295](https://github.com/autogluon/autogluon/pull/3295), [#3300](https://github.com/autogluon/autogluon/pull/3300))\n* Remove clip from fusion models [@liangfu](https://github.com/liangfu) ([#2946](https://github.com/autogluon/autogluon/pull/2946))\n* Refactor inferring problem type and output shape [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3227](https://github.com/autogluon/autogluon/pull/3227))\n* Log GPU info including GPU total memory, free memory, GPU card name, and CUDA version during training [@zhiqaingdon](https://github.com/zhiqaingdon) ([#3291](https://github.com/autogluon/autogluon/pull/3291))\n\n\n### Tabular\n\n#### New Features\n* Added `calibrate_decision_threshold` ([tutorial](https://auto.gluon.ai/stable/tutorials/tabular/tabular-indepth.html#decision-threshold-calibration)), which allows to optimize a given metric's decision threshold for predictions to strongly enhance the metric score. [@Innixma](https://github.com/Innixma) ([#3298](https://github.com/autogluon/autogluon/pull/3298))\n* We've added an experimental Zeroshot HPO config, which performs well on small datasets <10000 rows when at least an hour of training time is provided. To try it out, specify `presets=\"experimental_zeroshot_hpo_hybrid\"` when calling `fit()` [@Innixma](https://github.com/Innixma) [@geoalgo](https://github.com/geoalgo) ([#3312](https://github.com/autogluon/autogluon/pull/3312))\n* The [TabPFN model](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#tabpfnmodel) is now supported as an experimental model. TabPFN is a viable model option when inference speed is not a concern, and the number of rows of training data is less than 10,000. Try it out via `pip install autogluon.tabular[all,tabpfn]`! [@Innixma](https://github.com/Innixma) ([#3270](https://github.com/autogluon/autogluon/pull/3270))\n* Backend support for distributed training, which will be available with the next Cloud module release. [@yinweisu](https://github.com/yinweisu) ([#3054](https://github.com/autogluon/autogluon/pull/3054), [#3110](https://github.com/autogluon/autogluon/pull/3110), [#3115](https://github.com/autogluon/autogluon/pull/3115), [#3131](https://github.com/autogluon/autogluon/pull/3131), [#3142](https://github.com/autogluon/autogluon/pull/3142), [#3179](https://github.com/autogluon/autogluon/pull/3179), [#3216](https://github.com/autogluon/autogluon/pull/3216))\n#### Performance Improvements\n* Accelerate boolean preprocessing [@Innixma](https://github.com/Innixma) ([#2944](https://github.com/autogluon/autogluon/pull/2944))\n#### Other Enhancements\n* Add quantile regression support for CatBoost [@shchur](https://github.com/shchur) ([#3165](https://github.com/autogluon/autogluon/pull/3165))\n* Implement quantile regression for LGBModel [@shchur](https://github.com/shchur) ([#3168](https://github.com/autogluon/autogluon/pull/3168))\n* Log to file support [@yinweisu](https://github.com/yinweisu) ([#3232](https://github.com/autogluon/autogluon/pull/3232))\n* Add support for `included_model_types` [@yinweisu](https://github.com/yinweisu) ([#3239](https://github.com/autogluon/autogluon/pull/3239))\n* Add enable_categorical=True support to XGBoost [@Innixma](https://github.com/Innixma) ([#3286](https://github.com/autogluon/autogluon/pull/3286))\n#### Bug Fixes / Code and Doc Improvements\n* Cross-OS loading of a fit TabularPredictor should now work properly [@yinweisu](https://github.com/yinweisu) [@Innixma](https://github.com/Innixma)\n* General bug fixes and improvements [@Innixma](https://github.com/Innixma) [@cnpgs](https://github.com/cnpgs) [@shchur](https://github.com/shchur) [@yinweisu](https://github.com/yinweisu) [@gradientsky](https://github.com/gradientsky) ([#2865](https://github.com/autogluon/autogluon/pull/2865), [#2936](https://github.com/autogluon/autogluon/pull/2936), [#2990](https://github.com/autogluon/autogluon/pull/2990), [#3045](https://github.com/autogluon/autogluon/pull/3045), [#3060](https://github.com/autogluon/autogluon/pull/3060), [#3069](https://github.com/autogluon/autogluon/pull/3069), [#3148](https://github.com/autogluon/autogluon/pull/3148), [#3182](https://github.com/autogluon/autogluon/pull/3182), [#3199](https://github.com/autogluon/autogluon/pull/3199), [#3226](https://github.com/autogluon/autogluon/pull/3226), [#3257](https://github.com/autogluon/autogluon/pull/3257), [#3259](https://github.com/autogluon/autogluon/pull/3259), [#3268](https://github.com/autogluon/autogluon/pull/3268), [#3269](https://github.com/autogluon/autogluon/pull/3269), [#3287](https://github.com/autogluon/autogluon/pull/3287), [#3288](https://github.com/autogluon/autogluon/pull/3288), [#3285](https://github.com/autogluon/autogluon/pull/3285), [#3293](https://github.com/autogluon/autogluon/pull/3293), [#3294](https://github.com/autogluon/autogluon/pull/3294), [#3302](https://github.com/autogluon/autogluon/pull/3302))\n* Move interpretable logic to InterpretableTabularPredictor [@Innixma](https://github.com/Innixma) ([#2981](https://github.com/autogluon/autogluon/pull/2981))\n* Enhance drop_duplicates, enable by default [@Innixma](https://github.com/Innixma) ([#3010](https://github.com/autogluon/autogluon/pull/3010))\n* Refactor params_aux & memory checks [@Innixma](https://github.com/Innixma) ([#3033](https://github.com/autogluon/autogluon/pull/3033))\n* Raise regression `pred_proba` [@Innixma](https://github.com/Innixma) ([#3240](https://github.com/autogluon/autogluon/pull/3240))\n\n\n### TimeSeries\nIn v0.8 we introduce several major improvements to the Time Series module, including new models, upgraded presets that lead to better forecast accuracy, and optimizations that speed up training & inference.\n\n#### Highlights\n- New models: `PatchTST` and `DLinear` from GluonTS, and `RecursiveTabular` based on integration with the [`mlforecast`](https://github.com/Nixtla/mlforecast) library [@shchur](https://github.com/shchur) ([#3177](https://github.com/autogluon/autogluon/pull/3177), [#3184](https://github.com/autogluon/autogluon/pull/3184), [#3230](https://github.com/autogluon/autogluon/pull/3230))\n- Improved accuracy and reduced overall training time thanks to updated presets [@shchur](https://github.com/shchur) ([#3281](https://github.com/autogluon/autogluon/pull/3281), [#3120](https://github.com/autogluon/autogluon/pull/3120))\n- 3-6x faster training and inference for `AutoARIMA`, `AutoETS`, `Theta`, `DirectTabular`, `WeightedEnsemble` models [@shchur](https://github.com/shchur) ([#3062](https://github.com/autogluon/autogluon/pull/3062), [#3214](https://github.com/autogluon/autogluon/pull/3214), [#3252](https://github.com/autogluon/autogluon/pull/3252))\n\n#### New Features\n- Dramatically faster repeated calls to `predict()`, `leaderboard()` and `evaluate()` thanks to prediction caching [@shchur](https://github.com/shchur) ([#3237](https://github.com/autogluon/autogluon/pull/3237))\n- Reduce overfitting by using multiple validation windows with the `num_val_windows` argument to `fit()` [@shchur](https://github.com/shchur) ([#3080](https://github.com/autogluon/autogluon/pull/3080))\n- Exclude certain models from presets with the `excluded_model_types` argument to `fit()` [@shchur](https://github.com/shchur) ([#3231](https://github.com/autogluon/autogluon/pull/3231))\n- New method `refit_full()` that refits models on combined train and validation data [@shchur](https://github.com/shchur) ([#3157](https://github.com/autogluon/autogluon/pull/3157))\n- Train multiple configurations of the same model by providing lists in the `hyperparameters` argument [@shchur](https://github.com/shchur) ([#3183](https://github.com/autogluon/autogluon/pull/3183))\n- Time limit set by `time_limit` is now respected by all models [@shchur](https://github.com/shchur) ([#3214](https://github.com/autogluon/autogluon/pull/3214))\n\n#### Enhancements\n- Improvements to the `DirectTabular` model (previously called `AutoGluonTabular`): faster featurization, trained as a quantile regression model if `eval_metric` is set to `\"mean_wQuantileLoss\"` [@shchur](https://github.com/shchur) ([#2973](https://github.com/autogluon/autogluon/pull/2973), [#3211](https://github.com/autogluon/autogluon/pull/3211))\n- Use correct seasonal period when computing the MASE metric [@shchur](https://github.com/shchur) ([#2970](https://github.com/autogluon/autogluon/pull/2970))\n- Check the AutoGluon version when loading `TimeSeriesPredictor` from disk [@shchur](https://github.com/shchur) ([#3233](https://github.com/autogluon/autogluon/pull/3233))\n\n#### Minor Improvements / Documentation / Bug Fixes\n* Update documentation and tutorials [@shchur](https://github.com/shchur) ([#2960](https://github.com/autogluon/autogluon/pull/2960), [#2964](https://github.com/autogluon/autogluon/pull/2964), [#3296](https://github.com/autogluon/autogluon/pull/3296), [#3297](https://github.com/autogluon/autogluon/pull/3297))\n* General bug fixes and improvements [@shchur](https://github.com/shchur) ([#2977](https://github.com/autogluon/autogluon/pull/2977), [#3058](https://github.com/autogluon/autogluon/pull/3058), [#3066](https://github.com/autogluon/autogluon/pull/3066), [#3160](https://github.com/autogluon/autogluon/pull/3160), [#3193](https://github.com/autogluon/autogluon/pull/3193), [#3202](https://github.com/autogluon/autogluon/pull/3202), [#3236](https://github.com/autogluon/autogluon/pull/3236), [#3255](https://github.com/autogluon/autogluon/pull/3255), [#3275](https://github.com/autogluon/autogluon/pull/3275), [#3290](https://github.com/autogluon/autogluon/pull/3290))\n\n### Exploratory Data Analysis (EDA) tools\nIn 0.8 we introduce a few new tools to help with data exploration and feature engineering:\n* **Anomaly Detection** [@gradientsky](https://github.com/gradientsky) ([#3124](https://github.com/autogluon/autogluon/pull/3124), [#3137](https://github.com/autogluon/autogluon/pull/3137)) - helps to identify unusual patterns or behaviors in data that deviate significantly from the norm. It's best used when finding outliers, rare events, or suspicious activities that could indicate fraud, defects, or system failures. Check the [Anomaly Detection Tutorial](https://auto.gluon.ai/0.8.0/tutorials/eda/eda-auto-anomaly-detection.html) to explore the functionality.\n* **Partial Dependence Plots** [@gradientsky](https://github.com/gradientsky) ([#3071](https://github.com/autogluon/autogluon/pull/3071), [#3079](https://github.com/autogluon/autogluon/pull/3079)) - visualize the relationship between a feature and the model's output for each individual instance in the dataset. Two-way variant can visualize potential interactions between any two features. Please see this tutorial for more detail: [Using Interaction Charts To Learn Information About the Data](https://auto.gluon.ai/0.8.0/tutorials/eda/eda-auto-analyze-interaction.html#using-interaction-charts-to-learn-information-about-the-data)\n#### Bug Fixes / Code and Doc Improvements\n* Switch regression analysis in `quick_fit` to use residuals plot [@gradientsky](https://github.com/gradientsky) ([#3039](https://github.com/autogluon/autogluon/pull/3039))\n* Added `explain_rows` method to `autogluon.eda.auto` - Kernel SHAP visualization [@gradientsky](https://github.com/gradientsky) ([#3014](https://github.com/autogluon/autogluon/pull/3014))\n* General improvements and fixes [@gradientsky](https://github.com/gradientsky) ([#2991](https://github.com/autogluon/autogluon/pull/2991), [#3056](https://github.com/autogluon/autogluon/pull/3056), [#3102](https://github.com/autogluon/autogluon/pull/3102), [#3107](https://github.com/autogluon/autogluon/pull/3107), [#3138](https://github.com/autogluon/autogluon/pull/3138))\n"
},
{
"path": "docs/whats_new/v0.8.1.md",
"content": "# Version 0.8.1\n\nv0.8.1 is a bug fix release.\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on. \nLoading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.8.0...v0.8.1\n\nThis version supports Python versions 3.8, 3.9, and 3.10.\n\n## Changes\n\n### Documentation improvements\n\n* Update google analytics property [@gidler](https://github.com/gidler) ([#3330](https://github.com/autogluon/autogluon/pull/3330))\n* Add Discord Link [@Innixma](https://github.com/Innixma) ([#3332](https://github.com/autogluon/autogluon/pull/3332))\n* Add community section to website front page [@Innixma](https://github.com/Innixma) ([#3333](https://github.com/autogluon/autogluon/pull/3333))\n* Update Windows Conda install instructions [@gidler](https://github.com/gidler) ([#3346](https://github.com/autogluon/autogluon/pull/3346))\n* Add some missing Colab buttons in tutorials [@gidler](https://github.com/gidler) ([#3359](https://github.com/autogluon/autogluon/pull/3359))\n\n\n### Bug Fixes / General Improvements\n\n* Move PyMuPDF to optional [@Innixma](https://github.com/Innixma) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3331](https://github.com/autogluon/autogluon/pull/3331))\n* Remove TIMM in core setup [@Innixma](https://github.com/Innixma) ([#3334](https://github.com/autogluon/autogluon/pull/3334))\n* Update persist_models max_memory 0.1 -> 0.4 [@Innixma](https://github.com/Innixma) ([#3338](https://github.com/autogluon/autogluon/pull/3338))\n* Lint modules [@yinweisu](https://github.com/yinweisu) ([#3337](https://github.com/autogluon/autogluon/pull/3337), [#3339](https://github.com/autogluon/autogluon/pull/3339), [#3344](https://github.com/autogluon/autogluon/pull/3344), [#3347](https://github.com/autogluon/autogluon/pull/3347))\n* Remove fairscale [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3342](https://github.com/autogluon/autogluon/pull/3342))\n* Fix refit crash [@Innixma](https://github.com/Innixma) ([#3348](https://github.com/autogluon/autogluon/pull/3348))\n* Fix `DirectTabular` model failing for some metrics; hide warnings produced by `AutoARIMA` [@shchur](https://github.com/shchur) ([#3350](https://github.com/autogluon/autogluon/pull/3350))\n* Pin dependencies [@yinweisu](https://github.com/yinweisu) ([#3358](https://github.com/autogluon/autogluon/pull/3358))\n* Reduce per gpu batch size for AutoMM high_quality_hpo to avoid out of memory error for some corner cases [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3360](https://github.com/autogluon/autogluon/pull/3360))\n* Fix HPO crash by setting reuse_actor to False [@yinweisu](https://github.com/yinweisu) ([#3361](https://github.com/autogluon/autogluon/pull/3361))\n"
},
{
"path": "docs/whats_new/v0.8.2.md",
"content": "# Version 0.8.2\n\nv0.8.2 is a hot-fix release to pin `pydantic` version to avoid crashing during HPO\n\nAs always, only load previously trained models using the same version of AutoGluon that they were originally trained on. \nLoading models trained in different versions of AutoGluon is not supported.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.8.1...v0.8.2\n\nThis version supports Python versions 3.8, 3.9, and 3.10.\n\n## Changes\n\n* codespell: action, config + some typos fixed [@yarikoptic](https://github.com/yarikoptic) [@yinweisu](https://github.com/yinweisu) ([#3323](https://github.com/autogluon/autogluon/pull/3323))\n* Unpin sentencepiece [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3368](https://github.com/autogluon/autogluon/pull/3368))\n* Pin pydantic [@yinweisu](https://github.com/yinweisu) (3370)\n"
},
{
"path": "docs/whats_new/v0.8.3.md",
"content": "# Version 0.8.3\n\nv0.8.3 is a patch release to address security vulnerabilities.\n\nSee the full commit change-log here: https://github.com/autogluon/autogluon/compare/v0.8.2...v0.8.3\n\nThis version supports Python versions 3.8, 3.9, and 3.10.\n\n## Changes\n* `transformers` and other packages version upgrades + some fixes. [@suzhoum](https://github.com/suzhoum) ([#4155](https://github.com/autogluon/autogluon/pull/4155))\n"
},
{
"path": "docs/whats_new/v1.0.0.md",
"content": "# Version 1.0.0\n\nToday is finally the day... AutoGluon 1.0 has arrived!!\nAfter [over four years of development](https://automlpodcast.com/episode/autogluon-the-story)\nand [2061 commits from 111 contributors](https://github.com/autogluon/autogluon/graphs/contributors),\nwe are excited to share with you the culmination of our efforts to create and democratize the most powerful, easy to use, and feature rich automated machine learning system in the world.\nAutoGluon 1.0 comes with transformative enhancements to predictive quality resulting from the combination of multiple novel ensembling innovations, spotlighted below.\nBesides performance enhancements, many other improvements have been made that are detailed in the individual module sections.\n\nNote: Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.0.\n\nThis release supports Python versions 3.8, 3.9, 3.10, and 3.11.\n\nThis release contains 223 commits from 17 contributors!\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur), [@zhiqiangdon](https://github.com/zhiqiangdon), [@Innixma](https://github.com/Innixma), [@prateekdesai04](https://github.com/prateekdesai04), [@FANGAreNotGnu](https://github.com/FANGAreNotGnu), [@yinweisu](https://github.com/yinweisu), [@taoyang1122](https://github.com/taoyang1122), [@LennartPurucker](https://github.com/LennartPurucker), [@Harry-zzh](https://github.com/Harry-zzh), [@AnirudhDagar](https://github.com/AnirudhDagar), [@jaheba](https://github.com/jaheba), [@gradientsky](https://github.com/gradientsky), [@melopeo](https://github.com/melopeo), [@ddelange](https://github.com/ddelange), [@tonyhoo](https://github.com/tonyhoo), [@canerturkmen](https://github.com/canerturkmen), [@suzhoum](https://github.com/suzhoum)\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N) \nGet the latest updates: [](https://twitter.com/autogluon)\n\n## Spotlight\n\n### Tabular Performance Enhancements\n\nAutoGluon 1.0 features **major enhancements to predictive quality**, establishing a new state-of-the-art in Tabular modeling.\nTo the best of our knowledge, **AutoGluon 1.0 marks the largest leap forward in the state-of-the-art for tabular data since the [original AutoGluon paper](https://arxiv.org/abs/2003.06505) from March 2020**.\nThe enhancements come primarily from two features:\n[Dynamic stacking](https://github.com/autogluon/autogluon/pull/3616) to [mitigate stacked overfitting](https://github.com/autogluon/autogluon/issues/2779#issuecomment-1736468165),\nand a new [learned model hyperparameters portfolio](https://github.com/autogluon/autogluon/blob/1.0.0/tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2023.py) via Zeroshot-HPO, obtained from the newly released [TabRepo](https://github.com/autogluon/tabrepo) ensemble simulation library.\nTogether, they lead to a **75% win-rate compared to AutoGluon 0.8 with faster inference speed, lower disk usage, and higher stability.**\n\n### AutoML Benchmark Results\n\nOpenML released the [official 2023 AutoML Benchmark results](https://arxiv.org/abs/2207.12560) on November 16th, 2023.\nTheir results show AutoGluon 0.8 as the state-of-the-art in AutoML systems across a wide variety of tasks:\n\"Overall, in terms of model performance, AutoGluon consistently has the highest average rank in our benchmark.\"\nWe now showcase that AutoGluon 1.0 achieves far superior results even to AutoGluon 0.8!\n\nBelow is a comparison on the [OpenML AutoML Benchmark](https://openml.github.io/automlbenchmark/index.html) across 1040 tasks.\nLightGBM, XGBoost, and CatBoost results were obtained via AutoGluon, and other methods are from [the official AutoML Benchmark 2023 results](https://arxiv.org/abs/2207.12560).\nAutoGluon 1.0 has a 95%+ win-rate against traditional tabular models, including **a 99% win-rate vs LightGBM and a 100% win-rate vs XGBoost**.\nAutoGluon 1.0 has between an 82% and 94% win-rate against other AutoML systems.\nFor all methods, AutoGluon is able to achieve >10% average loss improvement (Ex: Going from 90% accuracy to 91% accuracy is a 10% loss improvement).\n**AutoGluon 1.0 achieves first place in 63% of tasks**, with lightautoml having the second most at 12% (AutoGluon 0.8 previously took first place 48% of the time).\nAutoGluon 1.0 even achieves a 7.4% average loss improvement over AutoGluon 0.8!\n\n| Method | AG Winrate | AG Loss Improvement | Rescaled Loss | Rank | Champion |\n|:-----------------------------|:-----------|:--------------------|--------------:|---------:|:---------|\n| AutoGluon 1.0 (Best, 4h8c) | **-** | **-** | **0.04** | **1.95** | **63%** |\n| lightautoml (2023, 4h8c) | 84% | 12.0% | 0.2 | 4.78 | 12% |\n| H2OAutoML (2023, 4h8c) | 94% | 10.8% | 0.17 | 4.98 | 1% |\n| FLAML (2023, 4h8c) | 86% | 16.7% | 0.23 | 5.29 | 5% |\n| MLJAR (2023, 4h8c) | 82% | 23.0% | 0.33 | 5.53 | 6% |\n| autosklearn (2023, 4h8c) | 91% | 12.5% | 0.22 | 6.07 | 4% |\n| GAMA (2023, 4h8c) | 86% | 15.4% | 0.28 | 6.13 | 5% |\n| CatBoost (2023, 4h8c) | 95% | 18.2% | 0.28 | 6.89 | 3% |\n| TPOT (2023, 4h8c) | 91% | 23.1% | 0.4 | 8.15 | 1% |\n| LightGBM (2023, 4h8c) | 99% | 23.6% | 0.4 | 8.95 | 0% |\n| XGBoost (2023, 4h8c) | 100% | 24.1% | 0.43 | 9.5 | 0% |\n| RandomForest (2023, 4h8c) | 97% | 25.1% | 0.53 | 9.78 | 1% |\n\nNot only is AutoGluon more accurate in 1.0,\nit is also more stable thanks to our [new usage of Ray subprocesses](https://github.com/autogluon/autogluon/pull/3614) during low-memory training,\nresulting in **0 task failures** on the AutoML Benchmark.\n\nAutoGluon 1.0 is capable of achieving the fastest inference throughput of any AutoML system while still obtaining state-of-the-art results.\nBy specifying the `infer_limit` fit argument, users can trade off between accuracy and inference speed to meet their needs.\n\nAs seen in the below plot, AutoGluon 1.0 sets the Pareto Frontier for quality and inference throughput, achieving Pareto Dominance compared to all other AutoML systems.\nAutoGluon 1.0 High achieves superior performance to AutoGluon 0.8 Best with 8x faster inference and 8x less disk usage!\n\n\n\nYou can get more details on the results [here](https://github.com/Innixma/autogluon-benchmark/tree/master/v1_results).\n\nWe are excited to see what our users can accomplish with AutoGluon 1.0's enhanced performance.\nAs always, we will continue to improve AutoGluon in future releases to push the boundaries of AutoML forward for all.\n\n### AutoGluon Multimodal (AutoMM) Highlights in One Figure\n\n\n### AutoMM Uniqueness\nAutoGluon Multimodal (AutoMM) distinguishes itself from other open-source AutoML toolboxes like [AutosSklearn](https://github.com/automl/auto-sklearn), [LightAutoML](https://github.com/sb-ai-lab/LightAutoML), [H2OAutoML](https://github.com/h2oai/h2o-3), [FLAML](https://github.com/microsoft/FLAML), [MLJAR](https://github.com/mljar/mljar-supervised), [TPOT](https://github.com/EpistasisLab/tpot) and [GAMA](https://github.com/openml-labs/gama), which mainly focus on tabular data for classification or regression. \nAutoMM is designed for fine-tuning [foundation models](https://en.wikipedia.org/wiki/Foundation_models) across multiple modalitiesâimage, text, tabular, and document, either individually or combined. \nIt offers extensive capabilities for tasks like classification, regression, object detection, named entity recognition, semantic matching, and image segmentation. \nIn contrast, other AutoML systems generally have limited support for image or text, typically using a few pretrained models like [EfficientNet](https://arxiv.org/pdf/1905.11946.pdf) or hand-crafted rules like [bag-of-words](https://en.wikipedia.org/wiki/Bag-of-words_model) as feature extractors.\nThey often rely on traditional models or simple neural networks.\nAutoMM provides a uniquely comprehensive and versatile approach to AutoML, being the only AutoML system to support flexible multimodality and support for a wide range of tasks.\nA comparative table detailing support for various data modalities, tasks, and model types is provided below.\n\n| | | | Data | | | | | | Task | | | | Model | |\n|-------------|:---------------:|:------------:|:-------------:|:---------:|:----------------:|:------------------:|:-------------:|:-----------------:|:------------------:|:--------------------------:|:-------------------:|:-------------------:|:--------------------:|:------------------:|\n| | image | text | tabular | document | any combination | classification | regression | object detection | semantic matching | named entity recognition | image segmentation | traditional models | deep learning models | foundation models |\n| LightAutoML | ✓ | ✓ | ✓ | | | ✓ | ✓ | | | | | ✓ | ✓ | |\n| H2OAutoML | | | ✓ | | | ✓ | ✓ | | | | | ✓ | | |\n| FLAML | | ✓ | ✓ | | | ✓ | ✓ | | | | | ✓ | ✓ | ✓ |\n| MLJAR | | | ✓ | | | ✓ | ✓ | | | | | ✓ | | |\n| AutoSklearn | | ✓ | ✓ | | | ✓ | ✓ | | | | | ✓ | | |\n| GAMA | | | ✓ | | | ✓ | ✓ | | | | | ✓ | | |\n| TPOT | ✓ | | ✓ | | | ✓ | ✓ | | | | | ✓ | ✓ | |\n| AutoMM | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ |\n\n### Special Thanks\n\nWe would like to conclude this spotlight by thanking Pieter Gijsbers, SÊbastien Poirier, Erin LeDell, Joaquin Vanschoren,\nand the rest of the AutoML Benchmark authors for their key role in \nproviding a shared and extensive benchmark to monitor the progress of the AutoML field.\nTheir support has been invaluable to the AutoGluon project's continued growth.\n\nWe would also like to thank Frank Hutter, who continues to be a leader within the AutoML field, for organizing\nthe AutoML Conference in 2022 and 2023 to bring the community together to share ideas and align on a compelling vision.\n\nFinally, we would like to thank Alex Smola and Mu Li for championing open source software at Amazon to make this project possible.\n\n#### Additional Special Thanks\n* Special thanks to [@LennartPurucker](https://github.com/LennartPurucker) for leading development of dynamic stacking\n* Special thanks to [@geoalgo](https://github.com/geoalgo) for co-authoring [TabRepo](https://github.com/autogluon/tabrepo) to enable Zeroshot-HPO\n* Special thanks to [@ddelange](https://github.com/ddelange) for helping to add Python 3.11 support\n* Special thanks to [@mglowacki100](https://github.com/mglowacki100) for providing numerous feedback and suggestions\n* Special thanks to [@Harry-zzh](https://github.com/Harry-zzh) for contributing the new semantic segmentation problem type\n\n## General\n\n### Highlights\n* Python 3.11 Support [@ddelange](https://github.com/ddelange) [@yinweisu](https://github.com/yinweisu) ([#3190](https://github.com/autogluon/autogluon/pull/3190))\n\n### Other Enhancements\n* Added system info logging utility [@Innixma](https://github.com/Innixma) ([#3718](https://github.com/autogluon/autogluon/pull/3718))\n\n### Dependency Updates\n* Upgraded torch to `>=2.0,<2.2` [@zhiqiangdon](https://github.com/zhiqiangdon) [@yinweisu](https://github.com/yinweisu) [@shchur](https://github.com/shchur) ([#3404](https://github.com/autogluon/autogluon/pull/3404), [#3587](https://github.com/autogluon/autogluon/pull/3587), [#3588](https://github.com/autogluon/autogluon/pull/3588))\n* Upgraded numpy to `>=1.21,<1.29` [@prateekdesai04](https://github.com/prateekdesai04) ([#3709](https://github.com/autogluon/autogluon/pull/3709))\n* Upgraded Pandas to `>=2.0,<2.2` [@yinweisu](https://github.com/yinweisu) [@tonyhoo](https://github.com/tonyhoo) [@shchur](https://github.com/shchur) ([#3498](https://github.com/autogluon/autogluon/pull/3498))\n* Upgraded scikit-learn to `>=1.3,<1.5` [@yinweisu](https://github.com/yinweisu) [@tonyhoo](https://github.com/tonyhoo) [@shchur](https://github.com/shchur) ([#3498](https://github.com/autogluon/autogluon/pull/3498))\n* Upgraded Pillow to `>=10.0.1,<11` [@jaheba](https://github.com/jaheba) ([#3688](https://github.com/autogluon/autogluon/pull/3688))\n* Upgraded scipy to `>=1.5.4,<1.13` [@prateekdesai04](https://github.com/prateekdesai04) ([#3709](https://github.com/autogluon/autogluon/pull/3709))\n* Upgraded LightGBM to `>=3.3,<4.2` [@mglowacki100](https://github.com/mglowacki100) [@prateekdesai04](https://github.com/prateekdesai04) [@Innixma](https://github.com/Innixma) ([#3427](https://github.com/autogluon/autogluon/pull/3427), [#3709](https://github.com/autogluon/autogluon/pull/3709), [#3733](https://github.com/autogluon/autogluon/pull/3733))\n* Upgraded XGBoost to `>=1.6,<2.1` [@Innixma](https://github.com/Innixma) ([#3768](https://github.com/autogluon/autogluon/pull/3768))\n* Various minor dependency updates [@jaheba](https://github.com/jaheba) ([#3689](https://github.com/autogluon/autogluon/pull/3689))\n\n## Tabular\n\n### Highlights\nAutoGluon 1.0 features major enhancements to predictive quality, establishing a new state-of-the-art in Tabular modeling. Refer to the spotlight section above for more details!\n\n### New Features\n* Added `dynamic_stacking` predictor fit argument to mitigate [stacked overfitting](https://github.com/autogluon/autogluon/issues/2779#issuecomment-1736468165) [@LennartPurucker](https://github.com/LennartPurucker) [@Innixma](https://github.com/Innixma) ([#3616](https://github.com/autogluon/autogluon/pull/3616))\n* Added [zeroshot-HPO learned portfolio](https://github.com/autogluon/autogluon/blob/master/tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2023.py) as new hyperparameters for `best_quality` and `high_quality` presets. [@Innixma](https://github.com/Innixma) [@geoalgo](https://github.com/geoalgo) ([#3750](https://github.com/autogluon/autogluon/pull/3750))\n* Added experimental scikit-learn API compatible wrappers to TabularPredictor. You can access them via `from autogluon.tabular.experimental import TabularClassifier, TabularRegressor`. [@Innixma](https://github.com/Innixma) ([#3769](https://github.com/autogluon/autogluon/pull/3769))\n* Added `predictor.model_failures()` [@Innixma](https://github.com/Innixma) ([#3421](https://github.com/autogluon/autogluon/pull/3421))\n* Added enhanced FT-Transformer [@taoyang1122](https://github.com/taoyang1122) [@Innixma](https://github.com/Innixma) ([#3621](https://github.com/autogluon/autogluon/pull/3621), [#3644](https://github.com/autogluon/autogluon/pull/3644), [#3692](https://github.com/autogluon/autogluon/pull/3692))\n* Added `predictor.simulation_artifact()` to support integration with [TabRepo](https://github.com/autogluon/tabrepo) [@Innixma](https://github.com/Innixma) ([#3555](https://github.com/autogluon/autogluon/pull/3555))\n\n### Performance Improvements\n* Enhanced FastAI model quality on regression via output clipping [@LennartPurucker](https://github.com/LennartPurucker) [@Innixma](https://github.com/Innixma) ([#3597](https://github.com/autogluon/autogluon/pull/3597))\n* Added Skip-connection Weighted Ensemble [@LennartPurucker](https://github.com/LennartPurucker) ([#3598](https://github.com/autogluon/autogluon/pull/3598))\n* Fix memory leaks by using ray processes for sequential fitting [@LennartPurucker](https://github.com/LennartPurucker) ([#3614](https://github.com/autogluon/autogluon/pull/3614))\n* Added dynamic parallel folds support to better utilize compute in low memory scenarios [@yinweisu](https://github.com/yinweisu) [@Innixma](https://github.com/Innixma) ([#3511](https://github.com/autogluon/autogluon/pull/3511))\n* Fixed linear model crashes during HPO and added search space for linear models [@Innixma](https://github.com/Innixma) ([#3571](https://github.com/autogluon/autogluon/pull/3571), [#3720](https://github.com/autogluon/autogluon/pull/3720))\n\n### Other Enhancements\n* Multi-layer stacking now produces deterministic results [@LennartPurucker](https://github.com/LennartPurucker) ([#3573](https://github.com/autogluon/autogluon/pull/3573))\n* Various model dependency updates [@mglowacki100](https://github.com/mglowacki100) ([#3373](https://github.com/autogluon/autogluon/pull/3373))\n* Various code cleanup and logging improvements [@Innixma](https://github.com/Innixma) ([#3408](https://github.com/autogluon/autogluon/pull/3408), [#3570](https://github.com/autogluon/autogluon/pull/3570), [#3652](https://github.com/autogluon/autogluon/pull/3652), [#3734](https://github.com/autogluon/autogluon/pull/3734))\n\n### Bug Fixes / Code and Doc Improvements\n* Fixed incorrect model memory usage calculation [@Innixma](https://github.com/Innixma) ([#3591](https://github.com/autogluon/autogluon/pull/3591))\n* Fixed `infer_limit` being used incorrectly when bagging [@Innixma](https://github.com/Innixma) ([#3467](https://github.com/autogluon/autogluon/pull/3467))\n* Fixed rare edge-case FastAI model crash [@Innixma](https://github.com/Innixma) ([#3416](https://github.com/autogluon/autogluon/pull/3416))\n* Various minor bug fixes [@Innixma](https://github.com/Innixma) ([#3418](https://github.com/autogluon/autogluon/pull/3418), [#3480](https://github.com/autogluon/autogluon/pull/3480))\n\n## AutoMM\n[AutoGluon Multimodal (AutoMM)](https://auto.gluon.ai/stable/tutorials/multimodal/index.html) is designed to simplify the fine-tuning of foundation models for downstream applications with just three lines of code. \nIt seamlessly integrates with popular model zoos such as [HuggingFace Transformers](https://github.com/huggingface/transformers), \n[TIMM](https://github.com/huggingface/pytorch-image-models), and [MMDetection](https://github.com/open-mmlab/mmdetection), \nproviding support for a diverse range of data modalities, \nincluding image, text, tabular, and document data, whether used individually or in combination.\n\n### New Features\n\n* Semantic Segmentation\n * Introducing the new problem type `semantic_segmentation`, \n for fine-tuning [Segment Anything Model (SAM)](https://segment-anything.com/) with three lines of code. [@Harry-zzh](https://github.com/Harry-zzh) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3645](https://github.com/autogluon/autogluon/pull/3645), [#3677](https://github.com/autogluon/autogluon/pull/3677), [#3697](https://github.com/autogluon/autogluon/pull/3697), [#3711](https://github.com/autogluon/autogluon/pull/3711), [#3722](https://github.com/autogluon/autogluon/pull/3722), [#3728](https://github.com/autogluon/autogluon/pull/3728)) \n * Added comprehensive benchmarks from diverse domains, \n including natural images, agriculture, remote sensing, and healthcare.\n * Utilizing parameter-efficient finetuning (PEFT) [LoRA](https://arxiv.org/abs/2106.09685), showcasing consistent superior performance over alternatives ([VPT](https://arxiv.org/abs/2203.12119), \n [adaptor](https://arxiv.org/abs/1902.00751), [BitFit](https://arxiv.org/abs/2106.10199),\n [SAM-adaptor](https://arxiv.org/abs/2304.09148), and [LST](https://arxiv.org/abs/2206.06522)) in the extensive benchmarks.\n * Added one [semantic segmentation tutorial](https://auto.gluon.ai/stable/tutorials/multimodal/image_segmentation/beginner_semantic_seg.html) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3716](https://github.com/autogluon/autogluon/pull/3716)).\n * Using [SAM-ViT Huge](https://huggingface.co/facebook/sam-vit-huge) by default (GPU memory > 25GB required).\n* Few Shot Classification\n * Added the new `few_shot_classification` problem type for training few shot classifiers on images or texts. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3662](https://github.com/autogluon/autogluon/pull/3662), [#3681](https://github.com/autogluon/autogluon/pull/3681), [#3695](https://github.com/autogluon/autogluon/pull/3695))\n * Leveraging image/text foundation models to extract features and train SVM classifiers.\n * Added one [few shot classification tutorial](https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/few_shot_learning.html). [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3662](https://github.com/autogluon/autogluon/pull/3662))\n* Supported [torch.compile](https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) for faster training (experimental and torch >=2.2 required) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3520](https://github.com/autogluon/autogluon/pull/3520)).\n\n### Performance Improvements\n\n* Improved default image backbones, achieving a 100% win-rate on the image benchmark. [@taoyang1122](https://github.com/taoyang1122) ([#3738](https://github.com/autogluon/autogluon/pull/3738))\n* Replaced MLPs with FT-Transformer as the default tabular backbones, resulting in a 67% win-rate on the text+tabular benchmark. [@taoyang1122](https://github.com/taoyang1122) ([#3732](https://github.com/autogluon/autogluon/pull/3732))\n* Using both the improved default image backbones and FT-Transformer achieves a 62% win-rate on the text+tabular+image benchmark. [@taoyang1122](https://github.com/taoyang1122) ([#3732](https://github.com/autogluon/autogluon/pull/3732), [#3738](https://github.com/autogluon/autogluon/pull/3738))\n\n### Stability Enhancements\n* Enabled rigorous multi-GPU CI testing. [@prateekdesai04](https://github.com/prateekdesai04) ([#3566](https://github.com/autogluon/autogluon/pull/3566))\n* Fixed multi-GPU issues. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3617](https://github.com/autogluon/autogluon/pull/3617) [#3665](https://github.com/autogluon/autogluon/pull/3665) [#3684](https://github.com/autogluon/autogluon/pull/3684) [#3691](https://github.com/autogluon/autogluon/pull/3691), [#3639](https://github.com/autogluon/autogluon/pull/3639), [#3618](https://github.com/autogluon/autogluon/pull/3618))\n\n### Enhanced Usability\n* Supported custom evaluation metrics, which allows defining custom\n [metric object](https://auto.gluon.ai/dev/tutorials/tabular/advanced/tabular-custom-metric.html) and passing it to the `eval_metric` argument. [@taoyang1122](https://github.com/taoyang1122) ([#3548](https://github.com/autogluon/autogluon/pull/3548))\n* Supported multi-GPU training in notebooks (experimental). [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3484](https://github.com/autogluon/autogluon/pull/3484))\n* Improved logging with system info. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3735](https://github.com/autogluon/autogluon/pull/3735))\n\n### Improved Scalability\n* The introduction of the new learner class design facilitates easier support for new tasks and data modalities within AutoMM, enhancing overall scalability. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3650](https://github.com/autogluon/autogluon/pull/3650), [#3685](https://github.com/autogluon/autogluon/pull/3685), [#3735](https://github.com/autogluon/autogluon/pull/3735))\n\n### Other Enhancements\n\n* Added the option `hf_text.use_fast` for customizing fast tokenizer usage in `hf_text` models. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3379](https://github.com/autogluon/autogluon/pull/3379)) \n* Added fallback evaluation/validation metric, supporting `f1_macro` `f1_micro`, and `f1_weighted`. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3696](https://github.com/autogluon/autogluon/pull/3696))\n* Supported multi-GPU inference with the DDP strategy. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3445](https://github.com/autogluon/autogluon/pull/3445), [#3451](https://github.com/autogluon/autogluon/pull/3451))\n* Upgraded torch to 2.0. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3404](https://github.com/autogluon/autogluon/pull/3404))\n* Upgraded lightning to 2.0 [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3419](https://github.com/autogluon/autogluon/pull/3419))\n* Upgraded torchmetrics to 1.0 [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3422](https://github.com/autogluon/autogluon/pull/3422))\n\n### Code Improvements\n\n* Refactored AutoMM with the learner class for improved design. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3650](https://github.com/autogluon/autogluon/pull/3650), [#3685](https://github.com/autogluon/autogluon/pull/3685), [#3735](https://github.com/autogluon/autogluon/pull/3735))\n* Refactored FT-Transformer. [@taoyang1122](https://github.com/taoyang1122) ([#3621](https://github.com/autogluon/autogluon/pull/3621), [#3700](https://github.com/autogluon/autogluon/pull/3700))\n* Refactored the visualizers of object detection, semantic segmentation, and NER. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3716](https://github.com/autogluon/autogluon/pull/3716))\n* Other code refactor/clean-up: [@zhiqiangdon](https://github.com/zhiqiangdon) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3383](https://github.com/autogluon/autogluon/pull/3383) [#3399](https://github.com/autogluon/autogluon/pull/3399) [#3434](https://github.com/autogluon/autogluon/pull/3434) [#3667](https://github.com/autogluon/autogluon/pull/3667) [#3684](https://github.com/autogluon/autogluon/pull/3684) [#3695](https://github.com/autogluon/autogluon/pull/3695))\n\n### Bug Fixes/Doc Improvements\n\n* Fixed HPO for focal loss. [@suzhoum](https://github.com/suzhoum) ([#3739](https://github.com/autogluon/autogluon/pull/3739))\n* Fixed one ONNX export issue. [@AnirudhDagar](https://github.com/AnirudhDagar) ([#3725](https://github.com/autogluon/autogluon/pull/3725))\n* Improved AutoMM introduction for clarity. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3388](https://github.com/autogluon/autogluon/pull/3388) [#3726](https://github.com/autogluon/autogluon/pull/3726))\n* Improved AutoMM API doc. [@zhiqiangdon](https://github.com/zhiqiangdon) [@AnirudhDagar](https://github.com/AnirudhDagar) ([#3772](https://github.com/autogluon/autogluon/pull/3772) [#3777](https://github.com/autogluon/autogluon/pull/3777))\n* Other bug fixes [@zhiqiangdon](https://github.com/zhiqiangdon) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@taoyang1122](https://github.com/taoyang1122) [@tonyhoo](https://github.com/tonyhoo) [@rsj123](https://github.com/rsj123) [@AnirudhDagar](https://github.com/AnirudhDagar) ([#3384](https://github.com/autogluon/autogluon/pull/3384), [#3424](https://github.com/autogluon/autogluon/pull/3424), [#3526](https://github.com/autogluon/autogluon/pull/3526), [#3593](https://github.com/autogluon/autogluon/pull/3593), [#3615](https://github.com/autogluon/autogluon/pull/3615), [#3638](https://github.com/autogluon/autogluon/pull/3638), [#3674](https://github.com/autogluon/autogluon/pull/3674), [#3693](https://github.com/autogluon/autogluon/pull/3693), [#3702](https://github.com/autogluon/autogluon/pull/3702), [#3690](https://github.com/autogluon/autogluon/pull/3690), [#3729](https://github.com/autogluon/autogluon/pull/3729), [#3736](https://github.com/autogluon/autogluon/pull/3736), [#3474](https://github.com/autogluon/autogluon/pull/3474), [#3456](https://github.com/autogluon/autogluon/pull/3456), [#3590](https://github.com/autogluon/autogluon/pull/3590), [#3660](https://github.com/autogluon/autogluon/pull/3660))\n* Other doc improvements [@zhiqiangdon](https://github.com/zhiqiangdon) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@taoyang1122](https://github.com/taoyang1122) ([#3397](https://github.com/autogluon/autogluon/pull/3397), [#3461](https://github.com/autogluon/autogluon/pull/3461), [#3579](https://github.com/autogluon/autogluon/pull/3579), [#3670](https://github.com/autogluon/autogluon/pull/3670), [#3699](https://github.com/autogluon/autogluon/pull/3699), [#3710](https://github.com/autogluon/autogluon/pull/3710), [#3716](https://github.com/autogluon/autogluon/pull/3716), [#3737](https://github.com/autogluon/autogluon/pull/3737), [#3744](https://github.com/autogluon/autogluon/pull/3744), [#3745](https://github.com/autogluon/autogluon/pull/3745), [#3680](https://github.com/autogluon/autogluon/pull/3680))\n\n## TimeSeries\n\n### Highlights\nAutoGluon 1.0 features numerous usability and performance improvements to the TimeSeries module. These include automatic handling of missing data and irregular time series, new forecasting metrics (including custom metric support), advanced time series cross-validation options, and new forecasting models. AutoGluon produces state-of-the-art results in forecast accuracy, achieving [70%+ win rate](https://openreview.net/forum?id=XHIY3cQ8Tew) compared to other popular forecasting frameworks.\n\n### New features\n- Support for custom forecasting metrics [@shchur](https://github.com/shchur) ([#3760](https://github.com/autogluon/autogluon/pull/3760), [#3602](https://github.com/autogluon/autogluon/pull/3602))\n- New forecasting metrics `WAPE`, `RMSSE`, `SQL` + improved [documentation for metrics](https://auto.gluon.ai/dev/tutorials/timeseries/forecasting-metrics.html) [@melopeo](https://github.com/melopeo) [@shchur](https://github.com/shchur) ([#3747](https://github.com/autogluon/autogluon/pull/3747), [#3632](https://github.com/autogluon/autogluon/pull/3632), [#3510](https://github.com/autogluon/autogluon/pull/3510), [#3490](https://github.com/autogluon/autogluon/pull/3490))\n- Improved robustness: `TimeSeriesPredictor` can now handle data with all [pandas frequencies](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases), irregular timestamps, or missing values represented by `NaN` [@shchur](https://github.com/shchur) ([#3563](https://github.com/autogluon/autogluon/pull/3563), [#3454](https://github.com/autogluon/autogluon/pull/3454))\n- New models: intermittent demand forecasting models based on conformal prediction (`ADIDA`, `CrostonClassic`, `CrostonOptimized`, `CrostonSBA`, `IMAPA`); `WaveNet` and `NPTS` from GluonTS; new baseline models (`Average`, `SeasonalAverage`, `Zero`) [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#3706](https://github.com/autogluon/autogluon/pull/3706), [#3742](https://github.com/autogluon/autogluon/pull/3742), [#3606](https://github.com/autogluon/autogluon/pull/3606), [#3459](https://github.com/autogluon/autogluon/pull/3459))\n- Advanced cross-validation options: avoid retraining the models for each validation window with `refit_every_n_windows` or adjust the step size between validation windows with `val_step_size` arguments to `TimeSeriesPredictor.fit` [@shchur](https://github.com/shchur) ([#3704](https://github.com/autogluon/autogluon/pull/3704), [#3537](https://github.com/autogluon/autogluon/pull/3537))\n\n### Enhancements\n- Enable Ray Tune for deep-learning forecasting models [@canerturkmen](https://github.com/canerturkmen) ([#3705](https://github.com/autogluon/autogluon/pull/3705))\n- Support passing multiple evaluation metrics to `TimeSeriesPredictor.evaluate` [@shchur](https://github.com/shchur) ([#3646](https://github.com/autogluon/autogluon/pull/3646))\n- Static features can now be passed directly to `TimeSeriesDataFrame.from_path` and `TimeSeriesDataFrame.from_data_frame` constructors [@shchur](https://github.com/shchur) ([#3635](https://github.com/autogluon/autogluon/pull/3635))\n\n### Performance improvements\n- Much more accurate forecasts at low time limits thanks to new presets and updated logic for splitting the training time across models [@shchur](https://github.com/shchur) ([#3749](https://github.com/autogluon/autogluon/pull/3749), [#3657](https://github.com/autogluon/autogluon/pull/3657), [#3741](https://github.com/autogluon/autogluon/pull/3741))\n- Faster training and prediction + lower memory usage for `DirectTabular` and `RecursiveTabular` models ([#3740](https://github.com/autogluon/autogluon/pull/3740), [#3620](https://github.com/autogluon/autogluon/pull/3620), [#3559](https://github.com/autogluon/autogluon/pull/3559))\n- Enable early stopping and improve inference speed for GluonTS models [@shchur](https://github.com/shchur) ([#3575](https://github.com/autogluon/autogluon/pull/3575))\n- Reduce import time for `autogluon.timeseries` by moving import statements inside model classes ([#3514](https://github.com/autogluon/autogluon/pull/3514))\n\n### Bug Fixes / Code and Doc Improvements\n- Improve log messages [@shchur](https://github.com/shchur) ([#3721](https://github.com/autogluon/autogluon/pull/3721))\n- Add reference to the publication on AutoGluon-TimeSeries to README [@shchur](https://github.com/shchur) ([#3482](https://github.com/autogluon/autogluon/pull/3482))\n- Align API of `TimeSeriesPredictor` with `TabularPredictor`, remove deprecated methods [@shchur](https://github.com/shchur) ([#3714](https://github.com/autogluon/autogluon/pull/3714), [#3655](https://github.com/autogluon/autogluon/pull/3655), [#3396](https://github.com/autogluon/autogluon/pull/3396))\n- General bug fixes and improvements [@shchur](https://github.com/shchur)([#3758](https://github.com/autogluon/autogluon/pull/3758), [#3756](https://github.com/autogluon/autogluon/pull/3756), [#3755](https://github.com/autogluon/autogluon/pull/3755), [#3754](https://github.com/autogluon/autogluon/pull/3754), [#3746](https://github.com/autogluon/autogluon/pull/3746), [#3743](https://github.com/autogluon/autogluon/pull/3743), [#3727](https://github.com/autogluon/autogluon/pull/3727), [#3698](https://github.com/autogluon/autogluon/pull/3698), [#3654](https://github.com/autogluon/autogluon/pull/3654), [#3653](https://github.com/autogluon/autogluon/pull/3653), [#3648](https://github.com/autogluon/autogluon/pull/3648), [#3628](https://github.com/autogluon/autogluon/pull/3628), [#3588](https://github.com/autogluon/autogluon/pull/3588), [#3560](https://github.com/autogluon/autogluon/pull/3560), [#3558](https://github.com/autogluon/autogluon/pull/3558), [#3536](https://github.com/autogluon/autogluon/pull/3536), [#3533](https://github.com/autogluon/autogluon/pull/3533), [#3523](https://github.com/autogluon/autogluon/pull/3523), [#3522](https://github.com/autogluon/autogluon/pull/3522), [#3476](https://github.com/autogluon/autogluon/pull/3476), [#3463](https://github.com/autogluon/autogluon/pull/3463))\n\n## EDA\n\nThe EDA module will be released at a later time, as it requires additional development effort before it is ready for 1.0.\nWe will make an announcement when EDA is ready for release. For now, please continue to use `\"autogluon.eda==0.8.2\"`.\n\n## Deprecations\n\n### General\n* `autogluon.core.spaces` has been deprecated. Please use `autogluon.common.spaces` instead [@Innixma](https://github.com/Innixma) ([#3701](https://github.com/autogluon/autogluon/pull/3701))\n\n### Tabular\nTabular will log warnings if using the deprecated methods. Deprecated methods are planned to be removed in AutoGluon 1.2 [@Innixma](https://github.com/Innixma) ([#3701](https://github.com/autogluon/autogluon/pull/3701))\n* `autogluon.tabular.TabularPredictor`\n * `predictor.get_model_names()` -> `predictor.model_names()`\n * `predictor.get_model_names_persisted()` -> `predictor.model_names(persisted=True)`\n * `predictor.compile_models()` -> `predictor.compile()`\n * `predictor.persist_models()` -> `predictor.persist()`\n * `predictor.unpersist_models()` -> `predictor.unpersist()`\n * `predictor.get_model_best()` -> `predictor.model_best`\n * `predictor.get_pred_from_proba()` -> `predictor.predict_from_proba()`\n * `predictor.get_oof_pred_proba()` -> `predictor.predict_proba_oof()`\n * `predictor.get_oof_pred()` -> `predictor.predict_oof()`\n * `predictor.get_model_full_dict()` -> `predictor.model_refit_map()`\n * `predictor.get_size_disk()` -> `predictor.disk_usage()`\n * `predictor.get_size_disk_per_file()` -> `predictor.disk_usage_per_file()`\n * `predictor.leaderboard()` `silent` argument deprecated, replaced by `display`, defaults to False\n * Same for `predictor.evaluate()` and `predictor.evaluate_predictions()`\n\n### AutoMM\n\n* Deprecated the `FewShotSVMPredictor` in favor of the new `few_shot_classification` problem type [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3699](https://github.com/autogluon/autogluon/pull/3699))\n* Deprecated the `AutoMMPredictor` in favor of `MultiModalPredictor` [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3650](https://github.com/autogluon/autogluon/pull/3650))\n* `autogluon.multimodal.MultiModalPredictor`\n * Deprecated the `config` argument in the fit API. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3679](https://github.com/autogluon/autogluon/pull/3679))\n * Deprecated the `init_scratch` and `pipeline` arguments in the init API [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3668](https://github.com/autogluon/autogluon/pull/3668))\n\n### TimeSeries\n* `autogluon.timeseries.TimeSeriesPredictor`\n * Deprecated argument `TimeSeriesPredictor(ignore_time_index: bool)`. Now, if the data contains irregular timestamps, either convert it to regular frequency with `data = data.convert_frequency(freq)` or provide frequency when creating the predictor as `TimeSeriesPredictor(freq=freq)`.\n * `predictor.evaluate()` now returns a dictionary (previously returned a float)\n * `predictor.score()` -> `predictor.evaluate()`\n * `predictor.get_model_names()` -> `predictor.model_names()`\n * `predictor.get_model_best()` -> `predictor.model_best`\n * Metric `\"mean_wQuantileLoss\"` has been renamed to `\"WQL\"`\n * `predictor.leaderboard()` `silent` argument deprecated, replaced by `display`, defaults to False\n * When setting `hyperparameters` to a string in `predictor.fit()`, supported values are now `\"default\"`, `\"light\"` and `\"very_light\"`\n* `autogluon.timeseries.TimeSeriesDataFrame`\n - `df.to_regular_index()` -> `df.convert_frequency()`\n - Deprecated method `df.get_reindexed_view()`. Please see deprecation notes for `ignore_time_index` under `TimeSeriesPredictor` above for information on how to deal with irregular timestamps\n- Models\n - All models based on MXNet (`DeepARMXNet`, `MQCNNMXNet`, `MQRNNMXNet`, `SimpleFeedForwardMXNet`, `TemporalFusionTransformerMXNet`, `TransformerMXNet`) have been removed \n - Statistical models from Statmodels (`ARIMA`, `Theta`, `ETS`) have been replaced by their counterparts from StatsForecast ([#3513](https://github.com/autogluon/autogluon/pull/3513)). Note that these models now have different hyperparameter names.\n - `DirectTabular` is now implemented using `mlforecast` backend (same as `RecursiveTabular`), most hyperparameter names for the model have changed.\n- `autogluon.timeseries.TimeSeriesEvaluator` has been deprecated. Please use metrics available in `autogluon.timeseries.metrics` instead.\n- `autogluon.timeseries.splitter.MultiWindowSplitter` and `autogluon.timeseries.splitter.LastWindowSplitter` have been deprecated. Please use `num_val_windows` and `val_step_size` arguments to `TimeSeriesPredictor.fit` instead (alternatively, use `autogluon.timeseries.splitter.ExpandingWindowSplitter`).\n\n## Papers\n\n### AutoGluon-TimeSeries: AutoML for Probabilistic Time Series Forecasting\n\nWe have published a paper on AutoGluon-TimeSeries at AutoML Conference 2023 ([Paper Link](https://openreview.net/forum?id=XHIY3cQ8Tew), [YouTube Video](https://www.youtube.com/watch?v=niLmfjXeHnE)).\nIn the paper, we benchmarked AutoGluon and popular open-source forecasting frameworks (including DeepAR, TFT, AutoARIMA, AutoETS, AutoPyTorch).\nAutoGluon produces SOTA results in point and probabilistic forecasting, and even **achieves 65% win rate against the best-in-hindsight combination of models**.\n\n### TabRepo: A Large Scale Repository of Tabular Model Evaluations and its AutoML Applications\n\nWe have published a paper on Tabular Zeroshot-HPO ensembling simulation to arXiv ([Paper Link](https://arxiv.org/pdf/2311.02971.pdf), [GitHub](https://github.com/autogluon/tabrepo)).\nThis paper is key to achieving the performance improvements seen in AutoGluon 1.0, and we plan to continue to develop the code-base to support future enhancements.\n\n### XTab: Cross-table Pretraining for Tabular Transformers\n\nWe have published a paper on tabular Transformer pre-training at ICML 2023 ([Paper Link](https://arxiv.org/abs/2305.06090), [GitHub](https://github.com/BingzhaoZhu/XTab)).\nIn the paper we demonstrate state-of-the-art performance for tabular deep learning models, including being able to match the performance of XGBoost and LightGBM models.\nWhile the pre-trained transformer is not yet incorporated into AutoGluon, we plan to integrate it in a future release.\n\n### Learning Multimodal Data Augmentation in Feature Space\n\nOur paper on learning multimodal data augmentation was accepted at ICLR 2023 ([Paper Link](https://arxiv.org/pdf/2212.14453.pdf), [GitHub](https://github.com/lzcemma/LeMDA/)).\nThis paper introduces a plug-and-play module to learn multimodal data augmentation in feature space, \n with no constraints on the identities of the modalities or the relationship between modalities.\nWe show that it can (1) improve the performance of multimodal deep learning architectures, \n(2) apply to combinations of modalities that have not been previously considered, \nand (3) achieve state-of-the-art results on a wide range of applications comprised of image, text,\nand tabular data. This work is not yet incorporated into AutoGluon, but we plan to integrate it in a future release.\n\n### Data Augmentation for Object Detection via Controllable Diffusion Models\n\nOur paper on generative object detection data augmentation has been accepted at WACV 2024 (Paper and GitHub link will be available soon). \nThis paper proposes a data augmentation pipeline based on controllable diffusion models and CLIP, \nwith visual prior generation to guide the generation and post-filtering by category-calibrated CLIP scores to control its quality.\nWe demonstrate that the performance improves across various tasks and settings when using our augmentation pipeline with different detectors. \nAlthough diffusion models are currently not integrated into AutoGluon, we plan to incorporate the data augmentation techniques in a future release.\n\n### Adapting Image Foundation Models for Video Understanding\n\nWe have published a paper on how to efficiently adapt image foundation models for video understanding at ICLR 2023 ([Paper Link](https://arxiv.org/pdf/2302.03024.pdf), [GitHub](https://github.com/taoyang1122/adapt-image-models)). This paper introduces spatial adaptation, temporal adaptation and joint adaptation to gradually equip a frozen image model with spatiotemporal reasoning capability. The proposed method achieves competitive or even better performance than traditional full finetuning while largely saving the training cost of large foundation models.\n"
},
{
"path": "docs/whats_new/v1.1.0.md",
"content": "# Version 1.1.0\n\nWe're happy to announce the AutoGluon 1.1 release.\n\nAutoGluon 1.1 contains major improvements to the TimeSeries module, achieving a 60% win-rate vs AutoGluon 1.0 through the addition of Chronos, a pretrained model for time series forecasting, along with numerous other enhancements.\nThe other modules have also been enhanced through new features such as Conv-LORA support and improved performance for large tabular datasets between 5 - 30 GB in size. \nFor a full breakdown of AutoGluon 1.1 features, please refer to the feature spotlights and the itemized enhancements below.\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N) \nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.8, 3.9, 3.10, and 3.11. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.1.\n\nThis release contains [121 commits from 20 contributors](https://github.com/autogluon/autogluon/compare/v1.0.0...v1.1.0)!\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur) [@prateekdesai04](https://github.com/prateekdesai04) [@Innixma](https://github.com/Innixma) [@canerturkmen](https://github.com/canerturkmen) [@zhiqiangdon](https://github.com/zhiqiangdon) [@tonyhoo](https://github.com/tonyhoo) [@AnirudhDagar](https://github.com/AnirudhDagar) [@Harry-zzh](https://github.com/Harry-zzh) [@suzhoum](https://github.com/suzhoum) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@nimasteryang](https://github.com/nimasteryang) [@lostella](https://github.com/lostella) [@dassaswat](https://github.com/dassaswat) [@afmkt](https://github.com/afmkt) [@npepin-hub](https://github.com/npepin-hub) [@mglowacki100](https://github.com/mglowacki100) [@ddelange](https://github.com/ddelange) [@LennartPurucker](https://github.com/LennartPurucker) [@taoyang1122](https://github.com/taoyang1122) [@gradientsky](https://github.com/gradientsky)\n\nSpecial thanks to [@ddelange](https://github.com/ddelange) for their continued assistance with Python 3.11 support and Ray version upgrades!\n\n## Spotlight\n\n### AutoGluon Achieves Top Placements in ML Competitions!\n\nAutoGluon has experienced [wide-spread adoption on Kaggle](https://www.kaggle.com/search?q=autogluon+sortBy%3Adate) since the AutoGluon 1.0 release. \nAutoGluon has been used in over 130 Kaggle notebooks and mentioned in over 100 discussion threads in the past 90 days!\nMost excitingly, AutoGluon has already been used to achieve top ranking placements in multiple competitions with thousands of competitors since the start of 2024:\n\n| Placement | Competition | Author | Date | AutoGluon Details | Notes |\n|:-----------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------|:-----------|:------------------|:-------------------------------|\n| :3rd_place_medal: Rank 3/2303 (Top 0.1%) | [Steel Plate Defect Prediction](https://www.kaggle.com/competitions/playground-series-s4e3/discussion/488127) | [Samvel Kocharyan](https://github.com/samvelkoch) | 2024/03/31 | v1.0, Tabular | Kaggle Playground Series S4E3 |\n| :2nd_place_medal: Rank 2/93 (Top 2%) | [Prediction Interval Competition I: Birth Weight](https://www.kaggle.com/competitions/prediction-interval-competition-i-birth-weight/leaderboard) | [Oleksandr Shchur](https://shchur.github.io/) | 2024/03/21 | v1.0, Tabular | |\n| :2nd_place_medal: Rank 2/1542 (Top 0.1%) | [WiDS Datathon 2024 Challenge #1](https://www.kaggle.com/competitions/widsdatathon2024-challenge1/discussion/482285) | [lazy_panda](https://www.kaggle.com/byteliberator) | 2024/03/01 | v1.0, Tabular | |\n| :2nd_place_medal: Rank 2/3746 (Top 0.1%) | [Multi-Class Prediction of Obesity Risk](https://www.kaggle.com/competitions/playground-series-s4e2/discussion/480939) | [Kirderf](https://twitter.com/kirderf9) | 2024/02/29 | v1.0, Tabular | Kaggle Playground Series S4E2 |\n| :2nd_place_medal: Rank 2/3777 (Top 0.1%) | [Binary Classification with a Bank Churn Dataset](https://www.kaggle.com/competitions/playground-series-s4e1/discussion/472496) | [lukaszl](https://www.kaggle.com/lukaszl) | 2024/01/31 | v1.0, Tabular | Kaggle Playground Series S4E1 |\n| Rank 4/1718 (Top 0.2%) | [Multi-Class Prediction of Cirrhosis Outcomes](https://www.kaggle.com/competitions/playground-series-s3e26/discussion/464863) | [Kirderf](https://twitter.com/kirderf9) | 2024/01/01 | v1.0, Tabular | Kaggle Playground Series S3E26 |\n\nWe are thrilled that the data science community is leveraging AutoGluon as their go-to method to quickly and effectively achieve top-ranking ML solutions! \nFor an up-to-date list of competition solutions using AutoGluon refer to our [AWESOME.md](https://github.com/autogluon/autogluon/blob/master/AWESOME.md#competition-solutions-using-autogluon), \nand don't hesitate to let us know if you used AutoGluon in a competition!\n\n### Chronos, a pretrained model for time series forecasting\n\nAutoGluon-TimeSeries now features [Chronos](https://arxiv.org/abs/2403.07815), a family of forecasting models pretrained on large collections of open-source time series datasets that can generate accurate zero-shot predictions for new unseen data. Check out the [new tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-chronos.html) to learn how to use Chronos through the familiar `TimeSeriesPredictor` API.\n\n\n## General\n\n- Refactor project README & project Tagline [@Innixma](https://github.com/Innixma) ([#3861](https://github.com/autogluon/autogluon/pull/3861), [#4066](https://github.com/autogluon/autogluon/pull/4066))\n- Add AWESOME.md competition results and other doc improvements. [@Innixma](https://github.com/Innixma) ([#4023](https://github.com/autogluon/autogluon/pull/4023))\n- Pandas version upgrade. [@shchur](https://github.com/shchur) [@Innixma](https://github.com/Innixma) ([#4079](https://github.com/autogluon/autogluon/pull/4079), [#4089](https://github.com/autogluon/autogluon/pull/4089))\n- PyTorch, CUDA, Lightning version upgrades. [@prateekdesai04](https://github.com/prateekdesai04) [@canerturkmen](https://github.com/canerturkmen) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3982](https://github.com/autogluon/autogluon/pull/3982), [#3984](https://github.com/autogluon/autogluon/pull/3984), [#3991](https://github.com/autogluon/autogluon/pull/3991), [#4006](https://github.com/autogluon/autogluon/pull/4006))\n- Ray version upgrade. [@ddelange](https://github.com/ddelange) [@tonyhoo](https://github.com/tonyhoo) ([#3774](https://github.com/autogluon/autogluon/pull/3774), [#3956](https://github.com/autogluon/autogluon/pull/3956))\n- Scikit-learn version upgrade. [@prateekdesai04](https://github.com/prateekdesai04) ([#3872](https://github.com/autogluon/autogluon/pull/3872), [#3881](https://github.com/autogluon/autogluon/pull/3881), [#3947](https://github.com/autogluon/autogluon/pull/3947))\n- Various dependency upgrades. [@Innixma](https://github.com/Innixma) [@tonyhoo](https://github.com/tonyhoo) ([#4024](https://github.com/autogluon/autogluon/pull/4024), [#4083](https://github.com/autogluon/autogluon/pull/4083))\n\n## TimeSeries\n\n### Highlights\nAutoGluon 1.1 comes with numerous new features and improvements to the time series module. These include highly requested functionality such as feature importance,\nsupport for categorical covariates, ability to visualize forecasts, and enhancements to logging.\nThe new release also comes with considerable improvements to forecast accuracy, achieving 60% win rate and 3% average error reduction compared to the previous AutoGluon version. These improvements are mostly attributed to the addition of Chronos, improved preprocessing logic, and native handling of missing values.\n\n\n### New Features\n- Add Chronos pretrained forecasting model ([tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-chronos.html)). [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) [@lostella](https://github.com/lostella) ([#3978](https://github.com/autogluon/autogluon/pull/3978), [#4013](https://github.com/autogluon/autogluon/pull/4013), [#4052](https://github.com/autogluon/autogluon/pull/4052), [#4055](https://github.com/autogluon/autogluon/pull/4055), [#4056](https://github.com/autogluon/autogluon/pull/4056), [#4061](https://github.com/autogluon/autogluon/pull/4061), [#4092](https://github.com/autogluon/autogluon/pull/4092), [#4098](https://github.com/autogluon/autogluon/pull/4098))\n- Measure the importance of features & covariates on the forecast accuracy with `TimeSeriesPredictor.feature_importance()`. [@canerturkmen](https://github.com/canerturkmen) ([#4033](https://github.com/autogluon/autogluon/pull/4033), [#4087](https://github.com/autogluon/autogluon/pull/4087))\n- Native missing values support (no imputation required). [@shchur](https://github.com/shchur) ([#3995](https://github.com/autogluon/autogluon/pull/3995), [#4068](https://github.com/autogluon/autogluon/pull/4068), [#4091](https://github.com/autogluon/autogluon/pull/4091))\n- Add support for categorical covariates. [@shchur](https://github.com/shchur) ([#3874](https://github.com/autogluon/autogluon/pull/3874), [#4037](https://github.com/autogluon/autogluon/pull/4037))\n- Improve inference speed by persisting models in memory with `TimeSeriesPredictor.persist()`. [@canerturkmen](https://github.com/canerturkmen) ([#4005](https://github.com/autogluon/autogluon/pull/4005))\n- Visualize forecasts with `TimeSeriesPredictor.plot()`. [@shchur](https://github.com/shchur) ([#3889](https://github.com/autogluon/autogluon/pull/3889))\n- Add `RMSLE` evaluation metric. [@canerturkmen](https://github.com/canerturkmen) ([#3938](https://github.com/autogluon/autogluon/pull/3938))\n- Enable logging to file. [@canerturkmen](https://github.com/canerturkmen) ([#3877](https://github.com/autogluon/autogluon/pull/3877))\n- Add option to keep lightning logs after training with `keep_lightning_logs` hyperparameter. [@shchur](https://github.com/shchur) ([#3937](https://github.com/autogluon/autogluon/pull/3937))\n\n### Fixes and Improvements\n- Automatically preprocess real-valued covariates [@shchur](https://github.com/shchur) ([#4042](https://github.com/autogluon/autogluon/pull/4042), [#4069](https://github.com/autogluon/autogluon/pull/4069))\n- Add option to skip model selection when only one model is trained. [@shchur](https://github.com/shchur) ([#4002](https://github.com/autogluon/autogluon/pull/4002))\n- Ensure all metrics handle missing values in target [@shchur](https://github.com/shchur) ([#3966](https://github.com/autogluon/autogluon/pull/3966))\n- Fix bug when loading a GPU trained model on a CPU machine [@shchur](https://github.com/shchur) ([#3979](https://github.com/autogluon/autogluon/pull/3979))\n- Fix inconsistent random seed. [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#3934](https://github.com/autogluon/autogluon/pull/3934), [#4099](https://github.com/autogluon/autogluon/pull/4099))\n- Fix crash when calling .info after load. [@afmkt](https://github.com/afmkt) ([#3900](https://github.com/autogluon/autogluon/pull/3900))\n- Fix leaderboard crash when no models trained. [@shchur](https://github.com/shchur) ([#3849](https://github.com/autogluon/autogluon/pull/3849))\n- Add prototype TabRepo simulation artifact generation. [@shchur](https://github.com/shchur) ([#3829](https://github.com/autogluon/autogluon/pull/3829))\n- Fix refit_full bug. [@shchur](https://github.com/shchur) ([#3820](https://github.com/autogluon/autogluon/pull/3820))\n- Documentation improvements, hide deprecated methods. [@shchur](https://github.com/shchur) ([#3764](https://github.com/autogluon/autogluon/pull/3764), [#4054](https://github.com/autogluon/autogluon/pull/4054), [#4098](https://github.com/autogluon/autogluon/pull/4098)) \n- Minor fixes. [@canerturkmen](https://github.com/canerturkmen), [@shchur](https://github.com/shchur), [@AnirudhDagar](https://github.com/AnirudhDagar) ([#4009](https://github.com/autogluon/autogluon/pull/4009), [#4040](https://github.com/autogluon/autogluon/pull/4040), [#4041](https://github.com/autogluon/autogluon/pull/4041), [#4051](https://github.com/autogluon/autogluon/pull/4051), [#4070](https://github.com/autogluon/autogluon/pull/4070), [#4094](https://github.com/autogluon/autogluon/pull/4094))\n\n## AutoMM\n\n### Highlights\n\nAutoMM 1.1 introduces the innovative Conv-LoRA, \na parameter-efficient fine-tuning (PEFT) method stemming from our latest paper presented at ICLR 2024, \ntitled \"[Convolution Meets LoRA: Parameter Efficient Finetuning for Segment Anything Model](https://arxiv.org/abs/2401.17868)\". \nConv-LoRA is designed for fine-tuning the Segment Anything Model, \nexhibiting superior performance compared to previous PEFT approaches, \nsuch as LoRA and visual prompt tuning, \nacross various semantic segmentation tasks in diverse domains \nincluding natural images, agriculture, remote sensing, and healthcare. Check out [our Conv-LoRA example](https://github.com/autogluon/autogluon/tree/master/examples/automm/Conv-LoRA).\n\n### New Features\n\n- Added Conv-LoRA, a new parameter efficient fine-tuning method. [@Harry-zzh](https://github.com/Harry-zzh) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3933](https://github.com/autogluon/autogluon/pull/3933), [#3999](https://github.com/autogluon/autogluon/pull/3999), [#4007](https://github.com/autogluon/autogluon/pull/4007), [#4022](https://github.com/autogluon/autogluon/pull/4022), [#4025](https://github.com/autogluon/autogluon/pull/4025))\n- Added support for new column type: 'image_base64_str'. [@Harry-zzh](https://github.com/Harry-zzh) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3867](https://github.com/autogluon/autogluon/pull/3867))\n- Added support for loading pre-trained weights in FT-Transformer. [@taoyang1122](https://github.com/taoyang1122) [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3859](https://github.com/autogluon/autogluon/pull/3859))\n\n### Fixes and Improvements\n\n- Fixed bugs in semantic segmentation. [@Harry-zzh](https://github.com/Harry-zzh) ([#3801](https://github.com/autogluon/autogluon/pull/3801), [#3812](https://github.com/autogluon/autogluon/pull/3812))\n- Fixed crashes when using F1 metric. [@suzhoum](https://github.com/suzhoum) ([#3822](https://github.com/autogluon/autogluon/pull/3822))\n- Fixed bugs in PEFT methods. [@Harry-zzh](https://github.com/Harry-zzh) ([#3840](https://github.com/autogluon/autogluon/pull/3840))\n- Accelerated object detection training by ~30\\% for the high_quality and best_quality presets. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3970](https://github.com/autogluon/autogluon/pull/3970))\n- Depreciated Grounding-DINO [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#3974](https://github.com/autogluon/autogluon/pull/3974))\n- Fixed lightning upgrade issues [@zhiqiangdon](https://github.com/zhiqiangdon) ([#3991](https://github.com/autogluon/autogluon/pull/3991))\n- Fixed using f1, f1_macro, f1_micro for binary classification in knowledge distillation. [@nimasteryang](https://github.com/nimasteryang) ([#3837](https://github.com/autogluon/autogluon/pull/3837))\n- Removed MyMuPDF from installation due to the license issue. Users need to install it by themselves to do document classification. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#4093](https://github.com/autogluon/autogluon/pull/4093))\n\n\n## Tabular\n\n### Highlights\nAutoGluon-Tabular 1.1 primarily focuses on bug fixes and stability improvements. In particular, we have greatly improved the runtime performance for large datasets between 5 - 30 GB in size through the usage of subsampling for decision threshold calibration and the weighted ensemble fitting to 1 million rows, maintaining the same quality while being far faster to execute. We also adjusted the default weighted ensemble iterations from 100 to 25, which will speedup all weighted ensemble fit times by 4x. We heavily refactored the `fit_pseudolabel` logic, and it should now achieve noticeably stronger results.\n\n### Fixes and Improvements\n- Fix return value in `predictor.fit_weighted_ensemble(refit_full=True)`. [@Innixma](https://github.com/Innixma) ([#1956](https://github.com/autogluon/autogluon/pull/1956))\n- Enhance performance on large datasets through subsampling. [@Innixma](https://github.com/Innixma) ([#3977](https://github.com/autogluon/autogluon/pull/3977))\n- Fix refit_full crash when out of memory. [@Innixma](https://github.com/Innixma) ([#3977](https://github.com/autogluon/autogluon/pull/3977))\n- Refactor and enhance `.fit_pseudolabel` logic. [@Innixma](https://github.com/Innixma) ([#3930](https://github.com/autogluon/autogluon/pull/3930))\n- Fix crash in memory check during HPO for LightGBM, CatBoost, and XGBoost. [@Innixma](https://github.com/Innixma) ([#3931](https://github.com/autogluon/autogluon/pull/3931))\n- Fix dynamic stacking on windows. [@Innixma](https://github.com/Innixma) ([#3893](https://github.com/autogluon/autogluon/pull/3893))\n- LightGBM version upgrade. [@mglowacki100](https://github.com/mglowacki100), [@Innixma](https://github.com/Innixma) ([#3427](https://github.com/autogluon/autogluon/pull/3427))\n- Fix memory-safe sub-fits being skipped if Ray is not initialized. [@LennartPurucker](https://github.com/LennartPurucker) ([#3868](https://github.com/autogluon/autogluon/pull/3868))\n- Logging improvements. [@AnirudhDagar](https://github.com/AnirudhDagar) ([#3873](https://github.com/autogluon/autogluon/pull/3873))\n- Hide deprecated methods. [@Innixma](https://github.com/Innixma) ([#3795](https://github.com/autogluon/autogluon/pull/3795))\n- Documentation improvements. [@Innixma](https://github.com/Innixma) [@AnirudhDagar](https://github.com/AnirudhDagar) ([#2024](https://github.com/autogluon/autogluon/pull/2024), [#3975](https://github.com/autogluon/autogluon/pull/3975), [#3976](https://github.com/autogluon/autogluon/pull/3976), [#3996](https://github.com/autogluon/autogluon/pull/3996))\n\n## Docs and CI\n- Add auto benchmarking report generation. [@prateekdesai04](https://github.com/prateekdesai04) ([#4038](https://github.com/autogluon/autogluon/pull/4038), [#4039](https://github.com/autogluon/autogluon/pull/4039))\n- Fix tabular tests for Windows. [@tonyhoo](https://github.com/tonyhoo) ([#4036](https://github.com/autogluon/autogluon/pull/4036))\n- Fix hanging tabular unit tests. [@prateekdesai04](https://github.com/prateekdesai04) ([#4031](https://github.com/autogluon/autogluon/pull/4031))\n- Fix CI evaluation. [@suzhoum](https://github.com/suzhoum) ([#4019](https://github.com/autogluon/autogluon/pull/4019))\n- Add package version comparison between CI runs [@prateekdesai04](https://github.com/prateekdesai04) ([#3962](https://github.com/autogluon/autogluon/pull/3962), [#3968](https://github.com/autogluon/autogluon/pull/3968), [#3972](https://github.com/autogluon/autogluon/pull/3972))\n- Update conf.py to reflect current year. [@dassaswat](https://github.com/dassaswat) ([#3932](https://github.com/autogluon/autogluon/pull/3932))\n- Avoid redundant unit test runs. [@prateekdesai04](https://github.com/prateekdesai04) ([#3942](https://github.com/autogluon/autogluon/pull/3942))\n- Fix colab notebook links [@prateekdesai04](https://github.com/prateekdesai04) ([#3926](https://github.com/autogluon/autogluon/pull/3926))\n"
},
{
"path": "docs/whats_new/v1.1.1.md",
"content": "# Version 1.1.1\n\nWe're happy to announce the AutoGluon 1.1.1 release.\n\nAutoGluon 1.1.1 contains bug fixes and logging improvements for Tabular, TimeSeries, and Multimodal modules, as well as support for PyTorch 2.2 and 2.3.\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N) \nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.8, 3.9, 3.10, and 3.11. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.1.1.\n\nThis release contains **[52 commits from 10 contributors](https://github.com/autogluon/autogluon/compare/v1.1.0...v1.1.1)**!\n\n## General\n- Add support for PyTorch 2.2. [@prateekdesai04](https://github.com/prateekdesai04) ([#4123](https://github.com/autogluon/autogluon/pull/4123))\n- Add support for PyTorch 2.3. [@suzhoum](https://github.com/suzhoum) ([#4239](https://github.com/autogluon/autogluon/pull/4239), [#4256](https://github.com/autogluon/autogluon/pull/4256))\n- Upgrade GluonTS to 0.15.1. [@shchur](https://github.com/shchur) ([#4231](https://github.com/autogluon/autogluon/pull/4231))\n\n## Tabular\nNote: Trying to load a TabularPredictor with a FastAI model trained on a previous AutoGluon release will raise an exception when calling `predict` due to a fix in the `model-interals.pkl` path. Please ensure matching versions.\n\n- Fix deadlock when `num_gpus>0` and dynamic_stacking is enabled. [@Innixma](https://github.com/Innixma) ([#4208](https://github.com/autogluon/autogluon/pull/4208))\n- Improve decision threshold calibration. [@Innixma](https://github.com/Innixma) ([#4136](https://github.com/autogluon/autogluon/pull/4136), [#4137](https://github.com/autogluon/autogluon/pull/4137))\n- Improve dynamic stacking logging. [@Innixma](https://github.com/Innixma) ([#4208](https://github.com/autogluon/autogluon/pull/4208), [#4262](https://github.com/autogluon/autogluon/pull/4262))\n- Fix regression metrics (other than RMSE and MSE) being calculated incorrectly for LightGBM early stopping. [@Innixma](https://github.com/Innixma) ([#4174](https://github.com/autogluon/autogluon/pull/4174))\n- Fix custom multiclass metrics being calculated incorrectly for LightGBM early stopping. [@Innixma](https://github.com/Innixma) ([#4250](https://github.com/autogluon/autogluon/pull/4250))\n- Fix HPO crashing with NN_TORCH and FASTAI models. [@Innixma](https://github.com/Innixma) ([#4232](https://github.com/autogluon/autogluon/pull/4232))\n- Improve NN_TORCH runtime estimate. [@Innixma](https://github.com/Innixma) ([#4247](https://github.com/autogluon/autogluon/pull/4247))\n- Add infer throughput logging. [@Innixma](https://github.com/Innixma) ([#4200](https://github.com/autogluon/autogluon/pull/4200))\n- Disable sklearnex for linear models due to observed performance degradation. [@Innixma](https://github.com/Innixma) ([#4223](https://github.com/autogluon/autogluon/pull/4223))\n- Improve sklearnex logging verbosity in Kaggle. [@Innixma](https://github.com/Innixma) ([#4216](https://github.com/autogluon/autogluon/pull/4216))\n- Rename cached version file to version.txt. [@Innixma](https://github.com/Innixma) ([#4203](https://github.com/autogluon/autogluon/pull/4203))\n- Add refit_full support for Linear models. [@Innixma](https://github.com/Innixma) ([#4222](https://github.com/autogluon/autogluon/pull/4222))\n- Add AsTypeFeatureGenerator detailed exception logging. [@Innixma](https://github.com/Innixma) ([#4251](https://github.com/autogluon/autogluon/pull/4251), [#4252](https://github.com/autogluon/autogluon/pull/4252))\n\n## TimeSeries\n- Ensure prediction_length is stored as an integer. [@shchur](https://github.com/shchur) ([#4160](https://github.com/autogluon/autogluon/pull/4160))\n- Fix tabular model preprocessing failure edge-case. [@shchur](https://github.com/shchur) ([#4175](https://github.com/autogluon/autogluon/pull/4175))\n- Fix loading of Tabular models failure if predictor moved to a different directory. [@shchur](https://github.com/shchur) ([#4171](https://github.com/autogluon/autogluon/pull/4171))\n- Fix cached predictions error when predictor saved on-top of an existing predictor. [@shchur](https://github.com/shchur) ([#4202](https://github.com/autogluon/autogluon/pull/4202))\n- Use AutoGluon forks of Chronos models. [@shchur](https://github.com/shchur) ([#4198](https://github.com/autogluon/autogluon/pull/4198))\n- Fix off-by-one bug in Chronos inference. [@canerturkmen](https://github.com/canerturkmen) ([#4205](https://github.com/autogluon/autogluon/pull/4205))\n- Rename cached version file to version.txt. [@Innixma](https://github.com/Innixma) ([#4203](https://github.com/autogluon/autogluon/pull/4203))\n- Use correct target and quantile_levels in fallback model for MLForecast. [@shchur](https://github.com/shchur) ([#4230](https://github.com/autogluon/autogluon/pull/4230))\n\n## Multimodal\n- Fix bug in CLIP's image feature normalization. [@Harry-zzh](https://github.com/Harry-zzh) ([#4114](https://github.com/autogluon/autogluon/pull/4114))\n- Fix bug in text augmentation. [@Harry-zzh](https://github.com/Harry-zzh) ([#4115](https://github.com/autogluon/autogluon/pull/4115))\n- Modify default fine-tuning tricks. [@Harry-zzh](https://github.com/Harry-zzh) ([#4166](https://github.com/autogluon/autogluon/pull/4166))\n- Add PyTorch version warning for object detection. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#4217](https://github.com/autogluon/autogluon/pull/4217))\n\n## Docs and CI\n- Add competition solutions to `AWESOME.md`. [@Innixma](https://github.com/Innixma) [@shchur](https://github.com/shchur) ([#4122](https://github.com/autogluon/autogluon/pull/4122), [#4163](https://github.com/autogluon/autogluon/pull/4163), [#4245](https://github.com/autogluon/autogluon/pull/4245))\n- Fix PDF classification tutorial. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#4127](https://github.com/autogluon/autogluon/pull/4127))\n- Add AutoMM paper citation. [@zhiqiangdon](https://github.com/zhiqiangdon) ([#4154](https://github.com/autogluon/autogluon/pull/4154))\n- Add pickle load warning in all modules and tutorials. [@shchur](https://github.com/shchur) ([#4243](https://github.com/autogluon/autogluon/pull/4243))\n- Various minor doc and test fixes and improvements. [@tonyhoo](https://github.com/tonyhoo) [@shchur](https://github.com/shchur) [@lovvge](https://github.com/lovvge) [@Innixma](https://github.com/Innixma) [@suzhoum](https://github.com/suzhoum) ([#4113](https://github.com/autogluon/autogluon/pull/4113), [#4176](https://github.com/autogluon/autogluon/pull/4176), [#4225](https://github.com/autogluon/autogluon/pull/4225), [#4233](https://github.com/autogluon/autogluon/pull/4233), [#4235](https://github.com/autogluon/autogluon/pull/4235), [#4249](https://github.com/autogluon/autogluon/pull/4249), [#4266](https://github.com/autogluon/autogluon/pull/4266))\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@Innixma](https://github.com/Innixma) [@shchur](https://github.com/shchur) [@Harry-zzh](https://github.com/Harry-zzh) [@suzhoum](https://github.com/suzhoum) [@zhiqiangdon](https://github.com/zhiqiangdon) [@lovvge](https://github.com/lovvge) [@rey-allan](https://github.com/rey-allan) [@prateekdesai04](https://github.com/prateekdesai04) [@canerturkmen](https://github.com/canerturkmen) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) \n\n### New Contributors\n* [@lovvge](https://github.com/lovvge) made their first contribution in https://github.com/autogluon/autogluon/commit/57a15fcfbbbc94514ff20ed2774cd447d9f4115f\n* [@rey-allan](https://github.com/rey-allan) made their first contribution in [#4145](https://github.com/autogluon/autogluon/pull/4145)"
},
{
"path": "docs/whats_new/v1.2.0.md",
"content": "# Version 1.2.0\n\nWe're happy to announce the AutoGluon 1.2.0 release.\n\nAutoGluon 1.2 contains massive improvements to both Tabular and TimeSeries modules, each achieving a 70% win-rate vs AutoGluon 1.1. This release additionally adds support for Python 3.12 and drops support for Python 3.8.\n\nThis release contains [186 commits from 19 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=2024-06-15&to=2024-11-29&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/v1.1.1...v1.2.0\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N) \nGet the latest updates: [](https://twitter.com/autogluon)\n\nLoading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.2.\n\nFor Tabular, we encompass the primary enhancements of the new [TabPFNMix tabular foundation model](https://huggingface.co/autogluon/tabpfn-mix-1.0-classifier) and parallel fit strategy into the new `\"experimental_quality\"` preset to ensure a smooth transition period for those who wish to try the new cutting edge features. We will be using this release to gather feedback prior to incorporating these features into the other presets. We also introduce a new stack layer model pruning technique that results in a 3x inference speedup on small datasets with zero performance loss and greatly improved post-hoc calibration across the board, particularly on small datasets.\n\nFor TimeSeries, we introduce [Chronos-Bolt](https://huggingface.co/autogluon/chronos-bolt-base), our latest foundation model integrated into AutoGluon, with massive improvements to both accuracy and inference speed compared to Chronos, along with fine-tuning capabilities. We also added covariate regressor support!\n\nWe are also excited to announce [AutoGluon-Assistant](https://github.com/autogluon/autogluon-assistant/) (AG-A), our first venture into the realm of Automated Data Science.\n\nSee more details in the Spotlights below!\n\n## Spotlight\n\n### AutoGluon Becomes the Golden Standard for Competition ML in 2024\n\nBefore diving into the new features of 1.2, we would like to start by highlighting the [wide-spread adoption](https://www.kaggle.com/search?q=autogluon+sortBy%3Adate) AutoGluon has received on competition ML sites like Kaggle in 2024. Across all of 2024, AutoGluon was used to achieve a top 3 finish in 15 out of 18 tabular Kaggle competitions, including 7 first place finishes, and was never outside the top 1% of private leaderboard placements, with an average of over 1000 competing human teams in each competition. In the $75,000 prize money [2024 Kaggle AutoML Grand Prix](https://www.kaggle.com/automl-grand-prix), AutoGluon was used by the 1st, 2nd, and 3rd place teams, with the 2nd place team led by two AutoGluon developers: [Lennart Purucker](https://github.com/LennartPurucker) and [Nick Erickson](https://github.com/Innixma)! For comparison, in 2023 AutoGluon achieved only 1 first place and 1 second place solution. We attribute the bulk of this increase to the improvements seen in AutoGluon 1.0 and beyond.\n\n\n![](\"https://autogluon.s3.amazonaws.com/images/autogluon_kaggle_results_2024.png\")
\n \n\nWe'd like to emphasize that these results are achieved via human expert interaction with AutoGluon and other tools, and often includes manual feature engineering and hyperparameter tuning to get the most out of AutoGluon. To see a live tracking of all AutoGluon solution placements on Kaggle, refer to our [AWESOME.md ML competition section](https://github.com/autogluon/autogluon/blob/master/AWESOME.md#kaggle) where we provide links to all solution write-ups.\n\n### AutoGluon-Assistant: Automating Data Science with AutoGluon and LLMs\n\nWe are excited to share the release of a new [AutoGluon-Assistant module](https://github.com/autogluon/autogluon-assistant/) (AG-A), powered by LLMs from AWS Bedrock or OpenAI. AutoGluon-Assistant empowers users to solve tabular machine learning problems using only natural language descriptions, in zero lines of code with our simple user interface. Fully autonomous AG-A outperforms 74% of human ML practitioners in Kaggle competitions and secured a live top 10 finish in the $75,000 prize money [2024 Kaggle AutoML Grand Prix](https://www.kaggle.com/automl-grand-prix) competition as Team AGA đ¤!\n\n### TabularPredictor presets=\"experimental_quality\"\n\nTabularPredictor has a new `\"experimental_quality\"` preset that offers even better predictive quality than `\"best_quality\"`. On [the AutoMLBenchmark](https://github.com/openml/automlbenchmark), we observe a 70% winrate vs `best_quality` when running for 4 hours on a 64 CPU machine. This preset is a testing ground for cutting edge features and models which we hope to incorporate into `best_quality` for future releases. We recommend to use a machine with at least 16 CPU cores, 64 GB of memory, and a 4 hour+ `time_limit` to get the most benefit out of `experimental_quality`. Please let us know via a GitHub issue if you run into any problems running the `experimental_quality` preset.\n\n#### TabPFNMix: A Foundation Model for Tabular Data\n\n[TabPFNMix]((https://huggingface.co/autogluon/tabpfn-mix-1.0-classifier)) is the first tabular foundation model created by the AutoGluon team, and was pre-trained exclusively on synthetic data.\nThe model builds upon the prior work of [TabPFN](https://arxiv.org/abs/2207.01848) and [TabForestPFN](https://arxiv.org/abs/2405.13396). TabPFNMix to the best of our knowledge achieves a new state-of-the-art for individual open source model performance on datasets between 1000 and 10000 samples, and also supports regression tasks! Across the 109 classification datasets with less than or equal to 10000 training samples in [TabRepo](https://github.com/autogluon/tabrepo), fine-tuned TabPFNMix outperforms all prior models, with a 64% win-rate vs the strongest tree model, CatBoost, and a 61% win-rate vs fine-tuned TabForestPFN.\n\nThe model is available via the `TABPFNMIX` hyperparameters key, and is used in the new `experimental_quality` preset. We recommend using this model for datasets smaller than 50,000 training samples, ideally with a large time limit and 64+ GB of memory. This work is still in the early stages, and we appreciate any feedback from the community to help us iterate and improve for future releases. You can learn more by going to our HuggingFace model page for the model ([tabpfn-mix-1.0-classifier](https://huggingface.co/autogluon/tabpfn-mix-1.0-classifier), [tabpfn-mix-1.0-regressor](https://huggingface.co/autogluon/tabpfn-mix-1.0-regressor)). Give us a like on HuggingFace if you want to see more! A paper is planned in future to provide more details about the model.\n\n#### fit_strategy=\"parallel\"\n\nAutoGluon's TabularPredictor now supports the new fit argument `fit_strategy` and the new `\"parallel\"` option, enabled by default in the new `experimental_quality` preset. For machines with 16 or more CPU cores, the parallel fit strategy offers a major speedup over the previous `\"sequential\"` strategy. We estimate with 64 CPU cores that most datasets will experience a 2-4x speedup, with the speedup getting larger as CPU cores increase.\n\n### Chronos-BoltâĄ: a 250x faster, more accurate Chronos model\n\nChronos-Bolt is our latest foundation model for forecasting that has been integrated into AutoGluon. It is based on the T5 encoder-decoder architecture and has been trained on nearly 100 billion time series observations. It chunks the historical time series context into patches of multiple observations, which are then input into the encoder. The decoder then uses these representations to directly generate quantile forecasts across multiple future stepsâ_a method known as direct multi-step forecasting_. Chronos-Bolt models are up to 250 times faster and 20 times more memory-efficient than the original Chronos models of the same size.\n\nThe following plot compares the inference time of Chronos-Bolt against the original Chronos models for forecasting 1024 time series with a context length of 512 observations and a prediction horizon of 64 steps.\n\n\n![](\"https://autogluon.s3.amazonaws.com/images/chronos_bolt_speed.svg\")
\n \n\nChronos-Bolt models are not only significantly faster but also more accurate than the original Chronos models. The following plot reports the probabilistic and point forecasting performance of Chronos-Bolt in terms of the [Weighted Quantile Loss (WQL)](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-metrics.html#autogluon.timeseries.metrics.WQL) and the [Mean Absolute Scaled Error (MASE)](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-metrics.html#autogluon.timeseries.metrics.MASE), respectively, aggregated over 27 datasets (see the [Chronos paper](https://arxiv.org/abs/2403.07815) for details on this benchmark). Remarkably, despite having no prior exposure to these datasets during training, the zero-shot Chronos-Bolt models outperform commonly used statistical models and deep learning models that have been trained on these datasets (highlighted by *). Furthermore, they also perform better than other FMs, denoted by a +, which indicates that these models were pretrained on certain datasets in our benchmark and are not entirely zero-shot. Notably, Chronos-Bolt (Base) also surpasses the original Chronos (Large) model in terms of the forecasting accuracy while being over 600 times faster.\n\n\n![](\"https://autogluon.s3.amazonaws.com/images/chronos_bolt_accuracy.svg\")
\n \n\nChronos-Bolt models are now available through AutoGluon in four sizesâTiny (9M), Mini (21M), Small (48M), and Base (205M)âand can also be used on the CPU. With the addition of Chronos-Bolt models and other enhancements, **AutoGluon v1.2 achieves a 70%+ win rate against the previous release**!\n\nIn addition to the new Chronos-Bolt models, we have also added support for effortless fine-tuning of Chronos and Chronos-Bolt models. Check out the updated [Chronos tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-chronos.html) to learn how to use and fine-tune Chronos-Bolt models.\n\n### Time Series Covariate Regressors\n\nWe have added support for covariate regressors for all forecasting models. Covariate regressors are tabular regression models that can be combined with univariate forecasting models to incorporate exogenous information. These are particularly useful for foundation models like Chronos-Bolt, which rely solely on the target time series' historical data and cannot directly use exogenous information (such as holidays or promotions). To improve the predictions of univariate models when covariates are available, a covariate regressor is first fit on the known covariates and static features to predict the target column at each time step. The predictions of the covariate regressor are then subtracted from the target column, and the univariate model then forecasts the residuals. The [Chronos tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-chronos.html) showcases how covariate regressors can be used with Chronos-Bolt.\n\n\n## General\n\n### Improvements\n* Update `full_install.sh` to install AutoGluon in parallel and to use `uv`, resulting in much faster source installation times. [@Innixma](https://github.com/Innixma) ([#4582](https://github.com/autogluon/autogluon/pull/4582), [#4587](https://github.com/autogluon/autogluon/pull/4587), [#4592](https://github.com/autogluon/autogluon/pull/4592))\n\n### Dependencies\n* Python 3.12 support added. [@suzhoum](https://github.com/suzhoum) ([#4536](https://github.com/autogluon/autogluon/pull/4536))\n* Python 3.8 support dropped. [@prateekdesai04](https://github.com/prateekdesai04) ([#4512](https://github.com/autogluon/autogluon/pull/4512))\n* Update numpy to `>=1.25.0,<2.1.4`. [@suzhoum](https://github.com/suzhoum) ([#4538](https://github.com/autogluon/autogluon/pull/4538))\n* Update scipy to `>=1.5.4,<1.16`. [@suzhoum](https://github.com/suzhoum) ([#4538](https://github.com/autogluon/autogluon/pull/4538))\n* Update torch to `>=2.2,<2.6`. [@tonyhoo](https://github.com/tonyhoo) ([#4360](https://github.com/autogluon/autogluon/pull/4360), [#4612](https://github.com/autogluon/autogluon/pull/4612))\n* Update ray to `>=2.10.0,<2.40`. [@suzhoum](https://github.com/suzhoum), [@Innixma](https://github.com/Innixma) ([#4302](https://github.com/autogluon/autogluon/pull/4302), [#4688](https://github.com/autogluon/autogluon/pull/4688))\n* Update scikit-learn to `>=1.4.0,<1.5.3`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4420](https://github.com/autogluon/autogluon/pull/4420), [#4570](https://github.com/autogluon/autogluon/pull/4570))\n* Update matplotlib to `>=3.7.0,<3.11`. [@suzhoum](https://github.com/suzhoum) ([#4511](https://github.com/autogluon/autogluon/pull/4511))\n* Update pyarrow to `>=15.0.0`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4520](https://github.com/autogluon/autogluon/pull/4520))\n* Update psutil to `>=5.7.3,<7.0.0`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4570](https://github.com/autogluon/autogluon/pull/4570))\n* Update Pillow to `>=10.0.1,<12`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4570](https://github.com/autogluon/autogluon/pull/4570))\n* Update xgboost to `>=1.6,<2.2`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4570](https://github.com/autogluon/autogluon/pull/4570))\n* Update torchvision to `>=0.16.0,<0.21.0`. [@Innixma](https://github.com/Innixma) ([#4579](https://github.com/autogluon/autogluon/pull/4579))\n* Update nltk to `>=3.4.5,<3.9`. [@tonyhoo](https://github.com/tonyhoo) ([#4604](https://github.com/autogluon/autogluon/pull/4604))\n* Update timm to `>=0.9.5,<1.0.7`. [@prateekdesai04](https://github.com/prateekdesai04) ([#4580](https://github.com/autogluon/autogluon/pull/4580))\n* Update lightning to `>=2.2,<2.6`. [@tonyhoo](https://github.com/tonyhoo) ([#4612](https://github.com/autogluon/autogluon/pull/4612))\n* Update async_timeout to `>=4.0,<6`. [@tonyhoo](https://github.com/tonyhoo) ([#4612](https://github.com/autogluon/autogluon/pull/4612))\n* Update transformers to `>4.38.0,<5`. [@tonyhoo](https://github.com/tonyhoo) ([#4612](https://github.com/autogluon/autogluon/pull/4612))\n* Update accelerate to `>=0.34.0,<1.0`. [@cheungdaven](https://github.com/cheungdaven) [@tonyhoo](https://github.com/tonyhoo) [@shchur](https://github.com/shchur) ([#4596](https://github.com/autogluon/autogluon/pull/4596), [#4612](https://github.com/autogluon/autogluon/pull/4612), [#4676](https://github.com/autogluon/autogluon/pull/4676))\n* Update lightgbm to `>=4.0,<4.6`. [@Innixma](https://github.com/Innixma) ([#4688](https://github.com/autogluon/autogluon/pull/4688))\n* Update scikit-learn-intelex to `>=2024.0,<2025.1`. [@Innixma](https://github.com/Innixma) ([#4688](https://github.com/autogluon/autogluon/pull/4688))\n\n### Documentation\n* Update install instructions to use proper torch and ray versions. [@Innixma](https://github.com/Innixma) ([#4581](https://github.com/autogluon/autogluon/pull/4581))\n* Add +cpu tag for cpu installation guide. [@tonyhoo](https://github.com/tonyhoo) ([#4554](https://github.com/autogluon/autogluon/pull/4554))\n* Add SECURITY.md for vulnerability reporting. [@tonyhoo](https://github.com/tonyhoo) ([#4298](https://github.com/autogluon/autogluon/pull/4298))\n\n### Fixes and Improvements\n* Speed up DropDuplicatesFeatureGenerator fit time by 2x+. [@shchur](https://github.com/shchur) ([#4543](https://github.com/autogluon/autogluon/pull/4543))\n* Add `compute_metric` as a replacement for `compute_weighted_metric` with improved compatibility across the project. [@Innixma](https://github.com/Innixma) ([#4631](https://github.com/autogluon/autogluon/pull/4631))\n* Enhanced `generate_train_test_split`. [@Innixma](https://github.com/Innixma) ([#4478](https://github.com/autogluon/autogluon/pull/4478))\n\n\n## Tabular\n\n### New Features\n* Add TabPFNMix model. Try it out with `presets=\"experimental\"`. [@xiyuanzh](https://github.com/xiyuanzh) [@Innixma](https://github.com/Innixma) ([#4671](https://github.com/autogluon/autogluon/pull/4671), [#4694](https://github.com/autogluon/autogluon/pull/4694))\n* Parallel model fit support. Try it out with `fit_strategy=\"parallel\"`. [@LennartPurucker](https://github.com/LennartPurucker) [@Innixma](https://github.com/Innixma) ([#4606](https://github.com/autogluon/autogluon/pull/4606))\n* Predictor callbacks support. [@Innixma](https://github.com/Innixma) ([#4327](https://github.com/autogluon/autogluon/pull/4327), [#4473](https://github.com/autogluon/autogluon/pull/4473))\n* Learning curve generation feature. [@adibiasio](https://github.com/adibiasio) [@Innixma](https://github.com/Innixma) ([#4411](https://github.com/autogluon/autogluon/pull/4411), [#4635](https://github.com/autogluon/autogluon/pull/4635))\n* Set `calibrate_decision_threshold=\"auto\"` by default, and improve decision threshold calibration. This dramatically improves results when the eval_metric is `f1` and `balanced_accuracy` for binary classification. [@Innixma](https://github.com/Innixma) ([#4632](https://github.com/autogluon/autogluon/pull/4632))\n* Add `roc_auc_ovo` and `roc_auc_ovr` metrics. [@Innixma](https://github.com/Innixma) ([#4248](https://github.com/autogluon/autogluon/pull/4248))\n* Add support for custom memory (soft) limits. [@LennartPurucker](https://github.com/LennartPurucker) ([#4333](https://github.com/autogluon/autogluon/pull/4333))\n* Add `ag.compile` hyperparameter to models to enable compiling at fit time rather than with `predictor.compile`. [@Innixma](https://github.com/Innixma) ([#4354](https://github.com/autogluon/autogluon/pull/4354))\n* Add AdaptiveES support to NN_TORCH and increase max_epochs from 500 to 1000, enabled by default. [@Innixma](https://github.com/Innixma) ([#4436](https://github.com/autogluon/autogluon/pull/4436))\n* Add support for controlling repeated cross-validation behavior via `delay_bag_sets` fit argument. Set default to False (previously True). [@LennartPurucker](https://github.com/LennartPurucker) ([#4552](https://github.com/autogluon/autogluon/pull/4552))\n* Make `positive_class` an init argument of TabularPredictor. [@Innixma](https://github.com/Innixma) ([#4445](https://github.com/autogluon/autogluon/pull/4445))\n* Add AdamW support to NN_TORCH model. [@Innixma](https://github.com/Innixma) ([#4610](https://github.com/autogluon/autogluon/pull/4610))\n\n### Documentation\n* Added a [tutorial](https://auto.gluon.ai/stable/tutorials/tabular/how-it-works.html) with a deep dive on how AutoGluon works. [@rey-allan](https://github.com/rey-allan) ([#4284](https://github.com/autogluon/autogluon/pull/4284))\n\n### Fixes and Improvements\n* (Major) Fix stacker max_models logic for a 3x inference speedup. [@Innixma](https://github.com/Innixma) ([#4290](https://github.com/autogluon/autogluon/pull/4290))\n* (Major) Speed up EnsembleSelection fitting speed by 2x+. [@nathanaelbosch](https://github.com/nathanaelbosch) ([#4367](https://github.com/autogluon/autogluon/pull/4367))\n* (Major) Dramatically improve temperature scaling performance by using the best iteration instead of the last iteration's temperature. [@LennartPurucker](https://github.com/LennartPurucker) ([#4396](https://github.com/autogluon/autogluon/pull/4396))\n* (Major) Automatically skip temperature scaling if negative temperature is found. [@Innixma](https://github.com/Innixma) ([#4397](https://github.com/autogluon/autogluon/pull/4397))\n* (Major) Fix `roc_auc` metric to use `macro` for multiclass instead of `weighted`. [@LennartPurucker](https://github.com/LennartPurucker) ([#4407](https://github.com/autogluon/autogluon/pull/4407))\n* (Major) Ensure `refit_full` respects user specified `num_cpus` and `num_gpus`. [@Innixma](https://github.com/Innixma) ([#4495](https://github.com/autogluon/autogluon/pull/4495))\n* (Major) Refactor TabularDataset. Now TabularDataset will always return a pandas DataFrame object when initialized, to simplify various documentation and improve IDE debugging visualization compatibility. [@Innixma](https://github.com/Innixma) ([#4613](https://github.com/autogluon/autogluon/pull/4613))\n* Fix bug where validation data is not used when in HPO mode when no search space is provided for the model. [@echowve](https://github.com/echowve) ([#4667](https://github.com/autogluon/autogluon/pull/4667))\n* Set `num_bag_sets=1` by default, to avoid `num_bag_sets>1` being used if the user doesn't use a preset and sets `num_bag_folds>=2`. [@Innixma](https://github.com/Innixma) ([#4446](https://github.com/autogluon/autogluon/pull/4446))\n* Fix FASTAI crash when a column contains only a single unique value + NaNs. [@Innixma](https://github.com/Innixma) ([#4584](https://github.com/autogluon/autogluon/pull/4584))\n* Fix torch seed accidentally being updated on model.score calls in NN_TORCH. [@adibiasio](https://github.com/adibiasio) ([#4391](https://github.com/autogluon/autogluon/pull/4391))\n* Fix LightGBM predict_proba quantile output dtype. [@Innixma](https://github.com/Innixma) ([#4272](https://github.com/autogluon/autogluon/pull/4272))\n* Fix incorrect return type for `predict_multi` for regression. [@Innixma](https://github.com/Innixma) ([#4450](https://github.com/autogluon/autogluon/pull/4450))\n* Improved error messages when given invalid hyperparameters. [@Innixma](https://github.com/Innixma) ([#4258](https://github.com/autogluon/autogluon/pull/4258))\n* Improved user specified `num_cpus` and `num_gpus` sanity checking. [@Innixma](https://github.com/Innixma) ([#4277](https://github.com/autogluon/autogluon/pull/4277))\n* Add readable error message for invalid models in `predictor.persist` calls. [@Innixma](https://github.com/Innixma) ([#4285](https://github.com/autogluon/autogluon/pull/4285))\n* Improve logging for invalid label columns. [@Innixma](https://github.com/Innixma) ([#4287](https://github.com/autogluon/autogluon/pull/4287))\n* Improve NN_TORCH timeout logging. [@Innixma](https://github.com/Innixma) ([#4382](https://github.com/autogluon/autogluon/pull/4382))\n* Add toggle `raise_on_no_models_fitted` to control if AutoGluon errors when no models are fit. [@LennartPurucker](https://github.com/LennartPurucker) ([#4389](https://github.com/autogluon/autogluon/pull/4389))\n* Make `raise_on_no_models_fitted=True` by default. Was False in previous release. [@Innixma](https://github.com/Innixma) ([#4400](https://github.com/autogluon/autogluon/pull/4400))\n* Add utility methods to FeatureMetadata. [@Innixma](https://github.com/Innixma) ([#4401](https://github.com/autogluon/autogluon/pull/4401))\n* Fix feature pruning crashing on Windows. [@Innixma](https://github.com/Innixma) ([#4405](https://github.com/autogluon/autogluon/pull/4405))\n* Add `valid_stacker` and `use_orig_features` model options. [@Innixma](https://github.com/Innixma) ([#4444](https://github.com/autogluon/autogluon/pull/4444))\n* Improve reliability of `predictor.predict_proba_multi` in edge-case scenarios. [@Innixma](https://github.com/Innixma) ([#4527](https://github.com/autogluon/autogluon/pull/4527))\n* Remove ensemble cascade support. [@Innixma](https://github.com/Innixma) ([#4548](https://github.com/autogluon/autogluon/pull/4548))\n* Fix edgecase crash during label column handling if it is a pandas category dtype with 0 instances of a category. [@Innixma](https://github.com/Innixma) ([#4583](https://github.com/autogluon/autogluon/pull/4583))\n* Enable aarch64 platform build. [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) ([#4663](https://github.com/autogluon/autogluon/pull/4663))\n* Minor fixes. [@Innixma](https://github.com/Innixma) [@LennartPurucker](https://github.com/LennartPurucker) [@shchur](https://github.com/shchur) [@rsj123](https://github.com/rsj123) ([#4224](https://github.com/autogluon/autogluon/pull/4224), [#4317](https://github.com/autogluon/autogluon/pull/4317), [#4335](https://github.com/autogluon/autogluon/pull/4335), [#4352](https://github.com/autogluon/autogluon/pull/4352), [#4353](https://github.com/autogluon/autogluon/pull/4353), [#4379](https://github.com/autogluon/autogluon/pull/4379), [#4384](https://github.com/autogluon/autogluon/pull/4384), [#4474](https://github.com/autogluon/autogluon/pull/4474), [#4485](https://github.com/autogluon/autogluon/pull/4485), [#4675](https://github.com/autogluon/autogluon/pull/4675), [#4682](https://github.com/autogluon/autogluon/pull/4682), [#4700](https://github.com/autogluon/autogluon/pull/4700))\n* Minor unit tests, documentation, and cleanup. [@Innixma](https://github.com/Innixma) [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) ([#4398](https://github.com/autogluon/autogluon/pull/4398), [#4399](https://github.com/autogluon/autogluon/pull/4399), [#4402](https://github.com/autogluon/autogluon/pull/4402), [#4498](https://github.com/autogluon/autogluon/pull/4498), [#4546](https://github.com/autogluon/autogluon/pull/4546), [#4547](https://github.com/autogluon/autogluon/pull/4547), [#4549](https://github.com/autogluon/autogluon/pull/4549), [#4687](https://github.com/autogluon/autogluon/pull/4687), [#4690](https://github.com/autogluon/autogluon/pull/4690), [#4692](https://github.com/autogluon/autogluon/pull/4692))\n\n\n## TimeSeries\n\n### New Features\n* Add fine-tuning support for Chronos and Chronos-Bolt models [@abdulfatir](https://github.com/abdulfatir) ([#4608](https://github.com/autogluon/autogluon/pull/4608), [#4645](https://github.com/autogluon/autogluon/pull/4645), [#4653](https://github.com/autogluon/autogluon/pull/4653), [#4655](https://github.com/autogluon/autogluon/pull/4655), [#4659](https://github.com/autogluon/autogluon/pull/4659), [#4661](https://github.com/autogluon/autogluon/pull/4661), [#4673](https://github.com/autogluon/autogluon/pull/4673), [#4677](https://github.com/autogluon/autogluon/pull/4677))\n* Add Chronos-Bolt [@canerturkmen](https://github.com/canerturkmen) ([#4625](https://github.com/autogluon/autogluon/pull/4625))\n* `TimeSeriesPredictor.leaderboard` now can compute extra metrics and return hyperparameters for each model [@shchur](https://github.com/shchur) ([#4481](https://github.com/autogluon/autogluon/pull/4481))\n* Add `target_scaler` support for all forecasting models [@shchur](https://github.com/shchur) ([#4460](https://github.com/autogluon/autogluon/pull/4460), [#4644](https://github.com/autogluon/autogluon/pull/4644))\n* Add `covariate_regressor` support for all forecasting models [@shchur](https://github.com/shchur) ([#4566](https://github.com/autogluon/autogluon/pull/4566), [#4641](https://github.com/autogluon/autogluon/pull/4641))\n* Add method to convert a TimeSeriesDataFrame to a regular pd.DataFrame [@shchur](https://github.com/shchur) ([#4415](https://github.com/autogluon/autogluon/pull/4415))\n* [experimental] Add the weighted cumulative error forecasting metric [@shchur](https://github.com/shchur) ([#4594](https://github.com/autogluon/autogluon/pull/4594))\n* [experimental] Allow custom ensemble model types for time series [@shchur](https://github.com/shchur) ([#4662](https://github.com/autogluon/autogluon/pull/4662))\n\n### Fixes and Improvements\n* Update presets [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#4656](https://github.com/autogluon/autogluon/pull/4656), [#4658](https://github.com/autogluon/autogluon/pull/4658), [#4666](https://github.com/autogluon/autogluon/pull/4666), [#4672](https://github.com/autogluon/autogluon/pull/4672))\n* Unify all Croston models into a single class [@shchur](https://github.com/shchur) ([#4564](https://github.com/autogluon/autogluon/pull/4564))\n* Bump `statsforecast` version to 1.7 [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#4194](https://github.com/autogluon/autogluon/pull/4194), [#4357](https://github.com/autogluon/autogluon/pull/4357))\n* Fix deep learning models failing if item_ids have StringDtype [@rsj123](https://github.com/rsj123) ([#4539](https://github.com/autogluon/autogluon/pull/4539))\n* Update logic for inferring the time series frequency [@shchur](https://github.com/shchur) ([#4540](https://github.com/autogluon/autogluon/pull/4540))\n* Speed up and reduce memory usage of the `TimeSeriesFeatureGenerator` preprocessing logic [@shchur](https://github.com/shchur) ([#4557](https://github.com/autogluon/autogluon/pull/4557))\n* Update to GluonTS v0.16.0 [@shchur](https://github.com/shchur) ([#4628](https://github.com/autogluon/autogluon/pull/4628))\n* Refactor GluonTS default parameter handling, update TiDE parameters [@canerturkmen](https://github.com/canerturkmen) ([#4640](https://github.com/autogluon/autogluon/pull/4640))\n* Move covariate scaling logic into a separate class [@shchur](https://github.com/shchur) ([#4634](https://github.com/autogluon/autogluon/pull/4634))\n* Prune timeseries unit and smoke tests [@canerturkmen](https://github.com/canerturkmen) ([#4650](https://github.com/autogluon/autogluon/pull/4650))\n* Minor fixes [@abdulfatir](https://github.com/abdulfatir) [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#4259](https://github.com/autogluon/autogluon/pull/4259), [#4299](https://github.com/autogluon/autogluon/pull/4299), [#4395](https://github.com/autogluon/autogluon/pull/4395), [#4386](https://github.com/autogluon/autogluon/pull/4386), [#4409](https://github.com/autogluon/autogluon/pull/4409), [#4533](https://github.com/autogluon/autogluon/pull/4533), [#4565](https://github.com/autogluon/autogluon/pull/4565), [#4633](https://github.com/autogluon/autogluon/pull/4633), [#4647](https://github.com/autogluon/autogluon/pull/4647))\n\n\n## Multimodal\n\n### Fixes and Improvements\n* Fix Missing Validation Metric While Resuming A Model Failed At Checkpoint Fusing Stage by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4449\n* Add coco_root for better support for custom dataset in COCO format. by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/3809\n* Add COCO Format Saving Support and Update Object Detection I/O Handling by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/3811\n* Skip MMDet Config Files While Checking with bandit by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4630\n* Fix Logloss Bug and Refine Compute Score Logics by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4629\n* Fix Index Typo in Tutorial by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4642\n* Fix Proba Metrics for Multiclass by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4643\n* Support torch 2.4 by [@tonyhoo](https://github.com/tonyhoo) in https://github.com/autogluon/autogluon/pull/4360\n* Add Installation Guide for Object Detection in Tutorial by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4430\n* Add Bandit Warning Mitigation for Internal `torch.save` and `torch.load` Usage by [@tonyhoo](https://github.com/tonyhoo) in https://github.com/autogluon/autogluon/pull/4502\n* update accelerate version range by [@cheungdaven](https://github.com/cheungdaven) in https://github.com/autogluon/autogluon/pull/4596\n* Bound nltk version to avoid verbose logging issue by [@tonyhoo](https://github.com/tonyhoo) in https://github.com/autogluon/autogluon/pull/4604\n* Upgrade TIMM by [@prateekdesai04](https://github.com/prateekdesai04) in https://github.com/autogluon/autogluon/pull/4580\n* Key dependency updates in _setup_utils.py for v1.2 release by [@tonyhoo](https://github.com/tonyhoo) in https://github.com/autogluon/autogluon/pull/4612\n* Configurable Number of Checkpoints to Keep per HPO Trial by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4615\n* Refactor Metrics for Each Problem Type by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4616\n* Fix Torch Version and Colab Installation for Object Detection by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in https://github.com/autogluon/autogluon/pull/4447\n\n## Special Thanks\n\n* [Xiyuan Zhang](https://xiyuanzh.github.io/) for leading the development of TabPFNMix!\n* The TabPFN author's [Noah Hollmann](https://twitter.com/noahholl), [Samuel Muller](https://twitter.com/SamuelMullr), [Katharina Eggensperger](https://twitter.com/KEggensperger), and [Frank Hutter](https://twitter.com/FrankRHutter) for unlocking the power of foundation models for tabular data, and the TabForestPFN author's [Felix den Breejen](https://github.com/FelixdenBreejen), [Sangmin Bae](https://scholar.google.com/citations?user=T5rHY14AAAAJ&hl=ko), [Stephen Cha](https://scholar.google.com/citations?user=jqLvFdIAAAAJ&hl=en), and [Se-Young Yun](https://fbsqkd.github.io/) for extending the idea to a more generic representation. Our TabPFNMix work builds upon the shoulders of giants.\n* [Lennart Purucker](https://x.com/LennartPurucker) for leading development of the [parallel model fit functionality](https://github.com/autogluon/autogluon/pull/4606) and pushing AutoGluon to its limits in the 2024 Kaggle AutoML Grand Prix.\n* [Robert Hatch](https://www.kaggle.com/roberthatch), [Tilii](https://www.kaggle.com/tilii7), [Optimistix](https://www.kaggle.com/optimistix), [Mart Preusse](https://www.kaggle.com/martinapreusse), [Ravi Ramakrishnan](https://www.kaggle.com/ravi20076), [Samvel Kocharyan](https://www.kaggle.com/samvelkoch), [Kirderf](https://www.kaggle.com/kirderf), [Carl McBride Ellis](https://www.kaggle.com/carlmcbrideellis), [Konstantin Dmitriev](https://www.kaggle.com/kdmitrie), and others for their insightful discussions and for championing AutoGluon on Kaggle!\n* [Eddie Bergman](https://x.com/edberg_wardman) for his insightful surprise code review of the [tabular callback support](https://github.com/autogluon/autogluon/pull/4327) feature.\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@Innixma](https://github.com/Innixma) [@shchur](https://github.com/shchur) [@prateekdesai04](https://github.com/prateekdesai04) [@tonyhoo](https://github.com/tonyhoo) [@FangAreNotGnu](https://github.com/FangAreNotGnu) [@suzhoum](https://github.com/suzhoum) [@abdulfatir](https://github.com/abdulfatir) [@canerturkmen](https://github.com/canerturkmen) [@LennartPurucker](https://github.com/LennartPurucker) [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) [@adibiasio](https://github.com/adibiasio) [@rsj123](https://github.com/rsj123) [@nathanaelbosch](https://github.com/nathanaelbosch) [@cheungdaven](https://github.com/cheungdaven) [@lostella](https://github.com/lostella) [@zkalson](https://github.com/zkalson) [@rey-allan](https://github.com/rey-allan) [@echowve](https://github.com/echowve) [@xiyuanzh](https://github.com/xiyuanzh)\n\n### New Contributors\n* [@nathanaelbosch](https://github.com/nathanaelbosch) made their first contribution in https://github.com/autogluon/autogluon/pull/4366\n* [@adibiasio](https://github.com/adibiasio) made their first contribution in https://github.com/autogluon/autogluon/pull/4391\n* [@abdulfatir](https://github.com/abdulfatir) made their first contribution in https://github.com/autogluon/autogluon/pull/4608\n* [@echowve](https://github.com/echowve) made their first contribution in https://github.com/autogluon/autogluon/pull/4667\n* [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) made their first contribution in https://github.com/autogluon/autogluon/pull/4685\n* [@xiyuanzh](https://github.com/xiyuanzh) made their first contribution in https://github.com/autogluon/autogluon/pull/4694\n"
},
{
"path": "docs/whats_new/v1.3.0.md",
"content": "# Version 1.3.0\n\nWe are happy to announce the AutoGluon 1.3.0 release!\n\nAutoGluon 1.3 focuses on stability & usability improvements, bug fixes, and dependency upgrades.\n\nThis release contains [144 commits from 20 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=11%2F29%2F2024&to=4%2F30%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/v1.2.0...v1.3.0\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nLoading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.3.\n\n--------\n\n## Highlights\n\n### AutoGluon-Tabular is the state of the art in the AutoML Benchmark 2025!\n\nThe [AutoML Benchmark 2025](https://arxiv.org/pdf/2504.01222), an independent large-scale evaluation of tabular AutoML frameworks, showcases AutoGluon 1.2 as the state of the art AutoML framework! Highlights include:\n- AutoGluon's rank statistically significantly outperforms all AutoML systems via the Nemenyi post-hoc test across all time constraints.\n- AutoGluon with a 5 minute training budget outperforms all other AutoML systems with a 1 hour training budget.\n- AutoGluon is pareto efficient in quality and speed across all evaluated presets and time constraints.\n- AutoGluon with `presets=\"high\", infer_limit=0.0001` (HQIL in the figures) achieves >10,000 samples/second inference throughput while outperforming all methods.\n- AutoGluon is the most stable AutoML system. For \"best\" and \"high\" presets, AutoGluon has 0 failures on all time budgets >5 minutes.\n\n\n\n### AutoGluon Multimodal's \"Bag of Tricks\" Update\n\nWe are pleased to announce the integration of a comprehensive \"Bag of Tricks\" update for AutoGluon's MultiModal (AutoMM). This significant enhancement substantially improves multimodal AutoML performance when working with combinations of image, text, and tabular data. The update implements various strategies including multimodal model fusion techniques, multimodal data augmentation, cross-modal alignment, tabular data serialization, better handling of missing modalities, and an ensemble learner that integrates these techniques for optimal performance.\n\nUsers can now access these capabilities through a simple parameter when initializing the MultiModalPredictor after following the instruction [here](https://github.com/autogluon/autogluon/blob/2b90eb0f4a848941d70cd387c2fdec67bc67706d/multimodal/src/autogluon/multimodal/learners/ensemble.py#L306-L322) to download the checkpoints:\n```python\nfrom autogluon.multimodal import MultiModalPredictor\npredictor = MultiModalPredictor(label=\"label\", use_ensemble=True)\npredictor.fit(train_data=train_data)\n```\n\nWe express our gratitude to [@zhiqiangdon](https://github.com/zhiqiangdon), for this substantial contribution that enhances AutoGluon's capabilities for handling complex multimodal datasets. Here is the corresponding research paper describing the technical details: [Bag of Tricks for Multimodal AutoML with Image, Text, and Tabular Data](https://arxiv.org/html/2412.16243v1).\n\n\n## Deprecations and Breaking Changes\n\nThe following deprecated TabularPredictor methods have been removed in the 1.3.0 release (deprecated in 1.0.0, raise in 1.2.0, removed in 1.3.0). Please use the new names:\n- `persist_models` -> `persist`, `unpersist_models` -> `unpersist`, `get_model_names` -> `model_names`, `get_model_best` -> `model_best`, `get_pred_from_proba` -> `predict_from_proba`, `get_model_full_dict` -> `model_refit_map`, `get_oof_pred_proba` -> `predict_proba_oof`, `get_oof_pred` -> `predict_oof`, `get_size_disk_per_file` -> `disk_usage_per_file`, `get_size_disk` -> `disk_usage`, `get_model_names_persisted` -> `model_names(persisted=True)`\n\nThe following logic has been deprecated starting in 1.3.0 and will log a `FutureWarning`. Functionality will be changed in a future release:\n\n- (**FutureWarning**) `TabularPredictor.delete_models()` will default to `dry_run=False` in a future release (currently `dry_run=True`). Please ensure you explicitly specify `dry_run=True` for the existing logic to remain in future releases. [@Innixma](https://github.com/Innixma) ([#4905](https://github.com/autogluon/autogluon/pull/4905))\n\n\n## General\n\n\n### Improvements\n- (**Major**) Internal refactor of `AbstractTrainer` class to improve extensibility and reduce code duplication. [@canerturkmen](https://github.com/canerturkmen) ([#4804](https://github.com/autogluon/autogluon/pull/4804), [#4820](https://github.com/autogluon/autogluon/pull/4820), [#4851](https://github.com/autogluon/autogluon/pull/4851))\n\n### Dependencies\n\n- Update numpy to `>=1.25.0,<2.3.0`. [@tonyhoo](https://github.com/tonyhoo), [@Innixma](https://github.com/Innixma), [@suzhoum](https://github.com/suzhoum) ([#5020](https://github.com/autogluon/autogluon/pull/5020), [#5056](https://github.com/autogluon/autogluon/pull/5056), [#5072](https://github.com/autogluon/autogluon/pull/5072))\n- Update spacy to `<3.9`. [@tonyhoo](https://github.com/tonyhoo) ([#5072](https://github.com/autogluon/autogluon/pull/5072))\n- Update scikit-learn to `>=1.4.0,<1.7.0`. [@tonyhoo](https://github.com/tonyhoo), [@Innixma](https://github.com/Innixma) ([#5029](https://github.com/autogluon/autogluon/pull/5029), [#5045](https://github.com/autogluon/autogluon/pull/5045))\n- Update psutil to `>=5.7.3,<7.1.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update s3fs to `>=2024.2,<2026`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update ray to `>=2.10.0,<2.45`. [@suzhoum](https://github.com/suzhoum), [@celestinoxp](https://github.com/celestinoxp), [@tonyhoo](https://github.com/tonyhoo) ([#4714](https://github.com/autogluon/autogluon/pull/4714), [#4887](https://github.com/autogluon/autogluon/pull/4887), [#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update tabpfn to `>=0.1.11,<0.2`. [@Innixma](https://github.com/Innixma) ([#4787](https://github.com/autogluon/autogluon/pull/4787))\n- Update torch to `>=2.2,<2.7`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update lightning to `>=2.2,<2.7`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update torchmetrics to `>=1.2.0,<1.8`. [@zkalson](https://github.com/zkalson), [@tonyhoo](https://github.com/tonyhoo) ([#4720](https://github.com/autogluon/autogluon/pull/4720), [#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update torchvision to `>=0.16.0,<0.22.0`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update accelerate to `>=0.34.0,<2.0`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update lightgbm to `>=4.0,<4.7`. [@tonyhoo](https://github.com/tonyhoo) ([#4960](https://github.com/autogluon/autogluon/pull/4960))\n- Update fastai to `>=2.3.1,<2.9`. [@Innixma](https://github.com/Innixma) ([#4988](https://github.com/autogluon/autogluon/pull/4988))\n- Update jsonschema to `>=4.18,<4.24`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update scikit-image to `>=0.19.1,<0.26.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update omegaconf to `>=2.1.1,<2.4.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update pytorch-metric-learning to `>=1.3.0,<2.9`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update nltk to `>=3.4.5,<4.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update pytesseract to `>=0.3.9,<0.4`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update nvidia-ml-py3 to `>=7.352.0,<8.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update datasets to `>=2.16.0,<3.6.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update onnxruntime to `>=1.17.0,<1.22.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update tensorrt to `>=8.6.0,<10.9.1`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update xgboost to `>=2.0,<3.1`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update imodels to `>=1.3.10,<2.1.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update statsforecast to `>=1.7.0,<2.0.2`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n\n\n### Documentation\n- Updating documented python version's in CONTRIBUTING.md. [@celestinoxp](https://github.com/celestinoxp) ([#4796](https://github.com/autogluon/autogluon/pull/4796))\n- Refactored CONTRIBUTING.md to have up-to-date information. [@Innixma](https://github.com/Innixma) ([#4798](https://github.com/autogluon/autogluon/pull/4798))\n- Fix various typos. [@celestinoxp](https://github.com/celestinoxp) ([#4819](https://github.com/autogluon/autogluon/pull/4819))\n- Minor doc improvements. [@tonyhoo](https://github.com/tonyhoo) ([#4894](https://github.com/autogluon/autogluon/pull/4894), [#4929](https://github.com/autogluon/autogluon/pull/4929))\n\n### Fixes and Improvements\n- Fix colab AutoGluon source install with `uv`. [@tonyhoo](https://github.com/tonyhoo) ([#4943](https://github.com/autogluon/autogluon/pull/4943), [#4964](https://github.com/autogluon/autogluon/pull/4964))\n- Make `full_install.sh` use the script directory instead of the working directory. [@Innixma](https://github.com/Innixma) ([#4933](https://github.com/autogluon/autogluon/pull/4933))\n- Add `test_version.py` to ensure proper version format for releases. [@Innixma](https://github.com/Innixma) ([#4799](https://github.com/autogluon/autogluon/pull/4799))\n- Fix `setup_outputdir` to work with s3 paths. [@suzhoum](https://github.com/suzhoum) ([#4734](https://github.com/autogluon/autogluon/pull/4734))\n- Ensure `setup_outputdir` always makes a new directory if `path_suffix != None` and `path=None`. [@Innixma](https://github.com/Innixma) ([#4903](https://github.com/autogluon/autogluon/pull/4903))\n- Check `cuda.is_available()` before calling `cuda.device_count()` to avoid warnings. [@Innixma](https://github.com/Innixma) ([#4902](https://github.com/autogluon/autogluon/pull/4902))\n- Log a warning if mlflow autologging is enabled. [@shchur](https://github.com/shchur) ([#4925](https://github.com/autogluon/autogluon/pull/4925))\n- Fix rare ZeroDivisionError edge-case in `get_approximate_df_mem_usage`. [@shchur](https://github.com/shchur) ([#5083](https://github.com/autogluon/autogluon/pull/5083))\n- Minor fixes & improvements. [@suzhoum](https://github.com/suzhoum) [@Innixma](https://github.com/Innixma) [@canerturkmen](https://github.com/canerturkmen) [@PGijsbers](https://github.com/PGijsbers) [@tonyhoo](https://github.com/tonyhoo) ([#4744](https://github.com/autogluon/autogluon/pull/4744), [#4785](https://github.com/autogluon/autogluon/pull/4785), [#4822](https://github.com/autogluon/autogluon/pull/4822), [#4860](https://github.com/autogluon/autogluon/pull/4860), [#4891](https://github.com/autogluon/autogluon/pull/4891), [#5012](https://github.com/autogluon/autogluon/pull/5012), [#5047](https://github.com/autogluon/autogluon/pull/5047))\n\n--------\n\n## Tabular\n\n### Removed Models\n- Removed vowpalwabbit model (key: `VW`) and optional dependency (`autogluon.tabular[vowpalwabbit]`), as the model implemented in AutoGluon was not widely used and was largely unmaintained. [@Innixma](https://github.com/Innixma) ([#4975](https://github.com/autogluon/autogluon/pull/4975))\n- Removed TabTransformer model (key: `TRANSF`), as the model implemented in AutoGluon was heavily outdated, unmaintained since 2020, and generally outperformed by FT-Transformer (key: `FT_TRANSFORMER`). [@Innixma](https://github.com/Innixma) ([#4976](https://github.com/autogluon/autogluon/pull/4976))\n- Removed tabpfn from `autogluon.tabular[tests]` install in preparation for future `tabpfn>=2.x` support. [@Innixma](https://github.com/Innixma) ([#4974](https://github.com/autogluon/autogluon/pull/4974))\n\n### New Features\n- Add support for regression stratified splits via binning. [@Innixma](https://github.com/Innixma) ([#4586](https://github.com/autogluon/autogluon/pull/4586))\n- Add `TabularPredictor.model_hyperparameters(model)` that returns the hyperparameters of a model. [@Innixma](https://github.com/Innixma) ([#4901](https://github.com/autogluon/autogluon/pull/4901))\n- Add `TabularPredictor.model_info(model)` that returns the metadata of a model. [@Innixma](https://github.com/Innixma) ([#4901](https://github.com/autogluon/autogluon/pull/4901))\n- (Experimental) Add `plot_leaderboard.py` to visualize performance over training time of the predictor. [@Innixma](https://github.com/Innixma) ([#4907](https://github.com/autogluon/autogluon/pull/4907))\n- (**Major**) Add internal `ag_model_registry` to improve the tracking of supported model families and their capabilities. [@Innixma](https://github.com/Innixma) ([#4913](https://github.com/autogluon/autogluon/pull/4913), [#5057](https://github.com/autogluon/autogluon/pull/5057), [#5107](https://github.com/autogluon/autogluon/pull/5107))\n- Add `raise_on_model_failure` `TabularPredictor.fit` argument, default to False. If True, will immediately raise the original exception if a model raises an exception during fit instead of continuing to the next model. Setting to True is very helpful when using a debugger to try to figure out why a model is failing, as otherwise exceptions are handled by AutoGluon which isn't desired while debugging. [@Innixma](https://github.com/Innixma) ([#4937](https://github.com/autogluon/autogluon/pull/4937), [#5055](https://github.com/autogluon/autogluon/pull/5055))\n\n### Documentation\n- Minor tutorial doc improvements/fixes. [@kbulygin](https://github.com/kbulygin) [@Innixma](https://github.com/Innixma) ([#4779](https://github.com/autogluon/autogluon/pull/4779), [#4777](https://github.com/autogluon/autogluon/pull/4777))\n- Add Kaggle competition results. [@Innixma](https://github.com/Innixma) ([#4717](https://github.com/autogluon/autogluon/pull/4717), [#4770](https://github.com/autogluon/autogluon/pull/4770))\n\n### Fixes and Improvements\n- (**Major**) Ensure bagged refits in refit_full works properly (crashed in v1.2.0 due to a bug). [@Innixma](https://github.com/Innixma) ([#4870](https://github.com/autogluon/autogluon/pull/4870))\n- Improve XGBoost and CatBoost memory estimates. [@Innixma](https://github.com/Innixma) ([#5090](https://github.com/autogluon/autogluon/pull/5090))\n- Improve LightGBM memory estimates. [@Innixma](https://github.com/Innixma) ([#5101](https://github.com/autogluon/autogluon/pull/5101))\n- Fixed plot_tabular_models save path. [@everdark](https://github.com/everdark) ([#4711](https://github.com/autogluon/autogluon/pull/4711))\n- Fixed balanced_accuracy metric edge-case exception + added unit tests to ensure future bugs don't occur. [@Innixma](https://github.com/Innixma) ([#4775](https://github.com/autogluon/autogluon/pull/4775))\n- Fix HPO logging verbosity. [@Innixma](https://github.com/Innixma) ([#4781](https://github.com/autogluon/autogluon/pull/4781))\n- Improve logging for use_child_oof=True. [@Innixma](https://github.com/Innixma) ([#4780](https://github.com/autogluon/autogluon/pull/4780))\n- Fix crash when NN_TORCH trains with fewer than 8 samples. [@Innixma](https://github.com/Innixma) ([#4790](https://github.com/autogluon/autogluon/pull/4790))\n- Improve logging and documentation in CatBoost memory_check callback. [@celestinoxp](https://github.com/celestinoxp) ([#4802](https://github.com/autogluon/autogluon/pull/4802))\n- Improve code formatting to satisfy PEP585. [@celestinoxp](https://github.com/celestinoxp) ([#4823](https://github.com/autogluon/autogluon/pull/4823))\n- Remove deprecated TabularPredictor methods: [@Innixma](https://github.com/Innixma) ([#4906](https://github.com/autogluon/autogluon/pull/4906))\n- (**FutureWarning**) `TabularPredictor.delete_models()` will default to `dry_run=False` in a future release (currently `dry_run=True`). Please ensure you explicitly specify `dry_run=True` for the existing logic to remain in future releases. [@Innixma](https://github.com/Innixma) ([#4905](https://github.com/autogluon/autogluon/pull/4905))\n- Sped up tabular unit tests by 4x through various optimizations (3060s -> 743s). [@Innixma](https://github.com/Innixma) ([#4944](https://github.com/autogluon/autogluon/pull/4944))\n- Major tabular unit test refactor to avoid using fixtures. [@Innixma](https://github.com/Innixma) ([#4949](https://github.com/autogluon/autogluon/pull/4949))\n- Fix XGBoost GPU warnings. [@Innixma](https://github.com/Innixma) ([#4866](https://github.com/autogluon/autogluon/pull/4866))\n- Fix `TabularPredictor.refit_full(train_data_extra)` failing when categorical features exist. [@Innixma](https://github.com/Innixma) ([#4948](https://github.com/autogluon/autogluon/pull/4948))\n- Reduced memory usage of artifact created by `convert_simulation_artifacts_to_tabular_predictions_dict` by 4x. [@Innixma](https://github.com/Innixma) ([#5024](https://github.com/autogluon/autogluon/pull/5024))\n- Minor fixes. [@shchur](https://github.com/shchur) ([#5030](https://github.com/autogluon/autogluon/pull/5030))\n- Ensure that max model resources is respected during holdout model fit. [@Innixma](https://github.com/Innixma) ([#5067](https://github.com/autogluon/autogluon/pull/5067))\n- Remove unintended setting of global random seed during LightGBM model fit. [@Innixma](https://github.com/Innixma) ([#5095](https://github.com/autogluon/autogluon/pull/5095))\n\n\n------\n\n## TimeSeries\n\nThe new v1.3 release brings numerous usability improvements and bug fixes to the TimeSeries module.\nInternally, we completed a major refactor of the core classes and introduced static type checking to simplify future contributions, accelerate development, and catch potential bugs earlier.\n\n\n### API Changes and Deprecations\n* As part of the refactor, we made several changes to the internal `AbstractTimeSeriesModel` class. If you maintain a **custom model** implementation, you will likely need to update it. Please refer to the [custom forecasting model tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/advanced/forecasting-custom-model.html) for details.\n\n No action is needed from the users that rely solely on the public API of the `timeseries` module (`TimeSeriesPredictor` and `TimeSeriesDataFrame`).\n\n\n### New Features\n* New tutorial on adding custom forecasting models by [@shchur](https://github.com/shchur) in [#4749](https://github.com/autogluon/autogluon/pull/4749)\n* Add `cutoff` support in `evaluate` and `leaderboard` by [@abdulfatir](https://github.com/abdulfatir) in [#5078](https://github.com/autogluon/autogluon/pull/5078)\n* Add `horizon_weight` support for `TimeSeriesPredictor` by [@shchur](https://github.com/shchur) in [#5084](https://github.com/autogluon/autogluon/pull/5084)\n* Add `make_future_data_frame` method to TimeSeriesPredictor by [@shchur](https://github.com/shchur) in [#5051](https://github.com/autogluon/autogluon/pull/5051)\n* Refactor ensemble base class and add new ensembles by [@canerturkmen](https://github.com/canerturkmen) in [#5062](https://github.com/autogluon/autogluon/pull/5062)\n\n### Code Quality\n* Add static type checking for the `timeseries` module by [@canerturkmen](https://github.com/canerturkmen) in [#4712](https://github.com/autogluon/autogluon/pull/4712) [#4788](https://github.com/autogluon/autogluon/pull/4788) [#4801](https://github.com/autogluon/autogluon/pull/4801) [#4821](https://github.com/autogluon/autogluon/pull/4821) [#4969](https://github.com/autogluon/autogluon/pull/4969) [#5086](https://github.com/autogluon/autogluon/pull/5086) [#5085](https://github.com/autogluon/autogluon/pull/5085)\n* Refactor the `AbstractTimeSeriesModel` class by [@canerturkmen](https://github.com/canerturkmen) in [#4868](https://github.com/autogluon/autogluon/pull/4868) [#4909](https://github.com/autogluon/autogluon/pull/4909) [#4946](https://github.com/autogluon/autogluon/pull/4946) [#4958](https://github.com/autogluon/autogluon/pull/4958) [#5008](https://github.com/autogluon/autogluon/pull/5008) [#5038](https://github.com/autogluon/autogluon/pull/5038)\n* Improvements to the unit tests by [@canerturkmen](https://github.com/canerturkmen) in [#4773](https://github.com/autogluon/autogluon/pull/4773) [#4828](https://github.com/autogluon/autogluon/pull/4828) [#4877](https://github.com/autogluon/autogluon/pull/4877) [#4872](https://github.com/autogluon/autogluon/pull/4872) [#4884](https://github.com/autogluon/autogluon/pull/4884) [#4888](https://github.com/autogluon/autogluon/pull/4888)\n\n### Fixes and Improvements\n* Allow using custom `distr_output` with the TFT model by [@shchur](https://github.com/shchur) in [#4899](https://github.com/autogluon/autogluon/pull/4899)\n* Update version ranges for `statsforecast` & `coreforecast` by [@shchur](https://github.com/shchur) in [#4745](https://github.com/autogluon/autogluon/pull/4745)\n* Fix feature importance calculation for models that use a `covariate_regressor` by [@canerturkmen](https://github.com/canerturkmen) in [#4845](https://github.com/autogluon/autogluon/pull/4845)\n* Fix hyperparameter tuning for Chronos and other models by [@abdulfatir](https://github.com/abdulfatir) [@shchur](https://github.com/shchur) in [#4838](https://github.com/autogluon/autogluon/pull/4838) [#5075](https://github.com/autogluon/autogluon/pull/5075) [#5079](https://github.com/autogluon/autogluon/pull/5079)\n* Fix frequency inference for `TimeSeriesDataFrame` by [@abdulfatir](https://github.com/abdulfatir) [@shchur](https://github.com/shchur) in [#4834](https://github.com/autogluon/autogluon/pull/4834) [#5066](https://github.com/autogluon/autogluon/pull/5066)\n* Fix minor CovariateRegressor bugs by [@shchur](https://github.com/shchur) in [#4849](https://github.com/autogluon/autogluon/pull/4849)\n* Update docs for custom `distr_output` by [@Killer3048](https://github.com/Killer3048) in [#5068](https://github.com/autogluon/autogluon/pull/5068)\n* Minor documentation updates by [@shchur](https://github.com/shchur) in [#4928](https://github.com/autogluon/autogluon/pull/4928) [#5092](https://github.com/autogluon/autogluon/pull/5092)\n* Raise informative error message if invalid model name is provided by [@shchur](https://github.com/shchur) in [#5004](https://github.com/autogluon/autogluon/pull/5004)\n* Gracefully handle corrupted cached predictions by [@shchur](https://github.com/shchur) in [#5005](https://github.com/autogluon/autogluon/pull/5005)\n* Chronos-Bolt: Fix scaling that affects constant series by [@abdulfatir](https://github.com/abdulfatir) in [#5013](https://github.com/autogluon/autogluon/pull/5013)\n* Fix deprecated `evaluation_strategy` kwarg in `transformers` by [@abdulfatir](https://github.com/abdulfatir) in [#5019](https://github.com/autogluon/autogluon/pull/5019)\n* Fix time_limit when val_data is provided [#5046](https://github.com/autogluon/autogluon/pull/5046) by [@shchur](https://github.com/shchur) in [#5059](https://github.com/autogluon/autogluon/pull/5059)\n* Rename covariate metadata by [@canerturkmen](https://github.com/canerturkmen) in [#5064](https://github.com/autogluon/autogluon/pull/5064)\n* Fix NaT timestamp values during resampling by [@shchur](https://github.com/shchur) in [#5080](https://github.com/autogluon/autogluon/pull/5080)\n* Fix typing compatibility for py39 by [@suzhoum](https://github.com/suzhoum) [@shchur](https://github.com/shchur) in [#5094](https://github.com/autogluon/autogluon/pull/5094) [#5097](https://github.com/autogluon/autogluon/pull/5097)\n* Warn if an S3 path is provided to the `TimeSeriesPredictor` by [@shchur](https://github.com/shchur) in [#5091](https://github.com/autogluon/autogluon/pull/5091)\n\n\n\n------\n\n## Multimodal\n\n### New Features\nAutoGluon's MultiModal module has been enhanced with a comprehensive \"Bag of Tricks\" update that significantly improves performance when working with combined image, text, and tabular data through advanced fusion techniques, data augmentation, and an integrated ensemble learner now accessible via a simple `use_ensemble=True` parameter after following the instruction [here](https://github.com/autogluon/autogluon/blob/2b90eb0f4a848941d70cd387c2fdec67bc67706d/multimodal/src/autogluon/multimodal/learners/ensemble.py#L306-L322) to download the checkpoints.\n\n* [AutoMM] Bag of Tricks by [@zhiqiangdon](https://github.com/zhiqiangdon) in [#4737](https://github.com/autogluon/autogluon/pull/4737)\n\n### Documentation\n* [Tutorial] categorical convert_to_text default value by [@cheungdaven](https://github.com/cheungdaven) in [#4699](https://github.com/autogluon/autogluon/pull/4699)\n* [AutoMM] Fix and Update Object Detection Tutorials by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in [#4889](https://github.com/autogluon/autogluon/pull/4889)\n\n### Fixes and Improvements\n* Update s3 path to public URL for AutoMM unit tests by [@suzhoum](https://github.com/suzhoum) in [#4809](https://github.com/autogluon/autogluon/pull/4809)\n* Fix object detection tutorial and default behavior of predict by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in [#4865](https://github.com/autogluon/autogluon/pull/4865)\n* Fix NLTK tagger path in download function by [@k-ken-t4g](https://github.com/k-ken-t4g) in [#4982](https://github.com/autogluon/autogluon/pull/4982)\n* Fix AutoMM model saving logic by capping transformer range by [@tonyhoo](https://github.com/tonyhoo) in [#5007](https://github.com/autogluon/autogluon/pull/5007)\n* fix: account for distributed training in learning rate schedule by [@tonyhoo](https://github.com/tonyhoo) in [#5003](https://github.com/autogluon/autogluon/pull/5003)\n\n--------\n\n## Special Thanks\n* [Zhiqiang Tang](https://github.com/zhiqiangdon) for implementing \"Bag of Tricks\" for AutoGluon's MultiModal, which significantly enhances the multimodal performance.\n* [Caner Turkmen](https://github.com/canerturkmen) for leading the efforts on refactoring and improving the internal logic in the `timeseries` module.\n* [Celestino](https://github.com/celestinoxp) for providing numerous bug reports, suggestions, and code cleanup as a new contributor.\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@Innixma](https://github.com/Innixma) [@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) [@tonyhoo](https://github.com/tonyhoo) [@abdulfatir](https://github.com/abdulfatir) [@celestinoxp](https://github.com/celestinoxp) [@suzhoum](https://github.com/suzhoum) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@prateekdesai04](https://github.com/prateekdesai04) [@zhiqiangdon](https://github.com/zhiqiangdon)\n[@cheungdaven](https://github.com/cheungdaven) [@LennartPurucker](https://github.com/LennartPurucker) [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) [@zkalson](https://github.com/zkalson) [@nathanaelbosch](https://github.com/nathanaelbosch) [@Killer3048](https://github.com/Killer3048) [@FireballDWF](https://github.com/FireballDWF) [@timostrunk](https://github.com/timostrunk) [@everdark](https://github.com/everdark) [@kbulygin](https://github.com/kbulygin) [@PGijsbers](https://github.com/PGijsbers) [@k-ken-t4g](https://github.com/k-ken-t4g)\n\n\n### New Contributors\n* [@everdark](https://github.com/everdark) made their first contribution in [#4711](https://github.com/autogluon/autogluon/pull/4711)\n* [@kbulygin](https://github.com/kbulygin) made their first contribution in [#4777](https://github.com/autogluon/autogluon/pull/4777)\n* [@celestinoxp](https://github.com/celestinoxp) made their first contribution in [#4796](https://github.com/autogluon/autogluon/pull/4796)\n* [@PGijsbers](https://github.com/PGijsbers) made their first contribution in [#4891](https://github.com/autogluon/autogluon/pull/4891)\n* [@k-ken-t4g](https://github.com/k-ken-t4g) made their first contribution in [#4982](https://github.com/autogluon/autogluon/pull/4982)\n* [@FireballDWF](https://github.com/FireballDWF) made their first contribution in [#5000](https://github.com/autogluon/autogluon/pull/5000)\n* [@Killer3048](https://github.com/Killer3048) made their first contribution in [#5068](https://github.com/autogluon/autogluon/pull/5068)\n"
},
{
"path": "docs/whats_new/v1.3.1.md",
"content": "# Version 1.3.1\n\nWe are happy to announce the AutoGluon 1.3.1 release!\n\nAutoGluon 1.3.1 contains several bug fixes and logging improvements for Tabular, TimeSeries, and Multimodal modules.\n\nThis release contains [9 commits from 5 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=5%2F1%2F2025&to=5%2F20%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/1.3.0...1.3.1\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.8, 3.9, 3.10, and 3.11. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.3.1.\n\n--------\n\n## General\n- Version update. [@tonyhoo](https://github.com/tonyhoo) [#5112](https://github.com/autogluon/autogluon/pull/5112)\n\n--------\n\n## Tabular\n\n### Fixes and Improvements\n- Fix TabPFN dependency. [@fplein](https://github.com/fplein) [#5119](https://github.com/autogluon/autogluon/pull/5119)\n- Fix incorrect reference to positive_class in TabularPredictor constructor. [@celestinoxp](https://github.com/celestinoxp) [#5129](https://github.com/autogluon/autogluon/pull/5129)\n\n--------\n\n## TimeSeries\n\n### Fixes and Improvements\n- Fix ensemble weights format for printing. [@shchur](https://github.com/shchur) [#5132](https://github.com/autogluon/autogluon/pull/5132)\n- Avoid masking the `scaler` param with the default `target_scaler` value for `DirectTabular` and `RecursiveTabular` models. [@shchur](https://github.com/shchur) [#5131](https://github.com/autogluon/autogluon/pull/5131)\n- Fix `FutureWarning` in leaderboard and evaluate methods. [@shchur](https://github.com/shchur) [#5126](https://github.com/autogluon/autogluon/pull/5126)\n\n--------\n\n## Multimodal\n\n### Fixes and Improvements\n- Fix multimodal tutorial issue after 1.3 release [@tonyhoo](https://github.com/tonyhoo) [#5121](https://github.com/autogluon/autogluon/pull/5121)\n\n--------\n\n## Documentation and CI\n- Add release instructions for pasting whats_new release notes. [@Innixma](https://github.com/Innixma) [#5111](https://github.com/autogluon/autogluon/pull/5111)\n- Update docker image to use 1.3 release base. [@tonyhoo](https://github.com/tonyhoo) [#5130](https://github.com/autogluon/autogluon/pull/5130)\n\n--------\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur) [@tonyhoo](https://github.com/tonyhoo) [@celestinoxp](https://github.com/celestinoxp)\n\n\n### New Contributors\n- [@fplein](https://github.com/fplein) made their first contribution in [#5119](https://github.com/autogluon/autogluon/pull/5119)\n"
},
{
"path": "docs/whats_new/v1.4.0.md",
"content": "# Version 1.4.0\n\nWe are happy to announce the AutoGluon 1.4.0 release!\n\nAutoGluon 1.4.0 introduces massive new features and improvements to both tabular and time series modules. In particular, we introduce the `extreme` preset to TabularPredictor, which sets a new state of the art for predictive performance **by a massive margin** on datasets with fewer than 30000 samples. We have also added 5 new tabular model families in this release: RealMLP, TabM, TabPFNv2, TabICL, and Mitra. We also release **MLZero 1.0**, aka AutoGluon-Assistant, an end-to-end automated data science agent that brings AutoGluon from 3 lines of code to 0. For more details, refer to the highlights section below.\n\nThis release contains [69 commits from 18 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=5%2F21%2F2025&to=7%2F26%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/1.3.1...1.4.0\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.9, 3.10, 3.11, and 3.12. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.4.0.\n\n--------\n\n## Spotlight\n\n### AutoGluon Tabular Extreme Preset\n\n![](\"https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/refs/heads/main/docs/whats_new/v1.4.0/AG14_TabArena.png\")
\n\nAutoGluon 1.4.0 introduces a new tabular preset, `extreme_quality` aka `extreme`.\nAutoGluon's [extreme preset](https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets) is **the largest singular improvement to AutoGluon's predictive performance in the history of the package**, even larger than the improvement seen in AutoGluon 1.0 compared to 0.8.\nThis preset achieves an **88% win-rate** vs Autogluon 1.3 `best_quality` for datasets with fewer than 10000 samples, and a 290 Elo improvement overall on [TabArena](https://tabarena.ai) (shown in the figure above).\n\nTry it out in 3 lines of code:\n\n```python\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(label=\"class\").fit(\"train.csv\", presets=\"extreme\")\npredictions = predictor.predict(\"test.csv\")\n```\n\nThe `extreme` preset leverages a [new model portfolio](https://github.com/autogluon/autogluon/blob/master/tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2025.py), which is an improved version of the `TabArena ensemble` shown in Figure 6a of the [TabArena paper](https://arxiv.org/abs/2506.16791).\nIt consists of many new model families added in this release: TabPFNv2, TabICL, Mitra, TabM, as well as tree methods: CatBoost, LightGBM, XGBoost.\nThis preset is not only more accurate, it also requires much less training time. AutoGluon's `extreme` preset in 5 minutes is able to outperform `best` ran for 4 hours.\n\nIn order to get the most out of the `extreme` preset, a CUDA compatible GPU is required, ideally with 32+ GB vRAM.\nNote that inference time can be longer than `best`, but with a GPU it is very reasonable.\nThe `extreme` portfolio is only leveraged for datasets with at most 30000 samples. For larger datasets, we continue to use the `best_quality` portfolio.\nThe preset requires downloading foundation model weights for TabPFNv2, TabICL, and Mitra during fit. If you don't have an internet connection,\nensure that you pre-download the weights of the models to be able to use them during fit.\nThis preset is considered experimental for this release, and may change without warning in a future release.\n\n### TabArena and new models: TabPFNv2, TabICL, TabM, RealMLP\n\nđ¨What is SOTA on tabular data, really? We are excited to introduce [TabArena](https://tabarena.ai), a living benchmark for machine learning on IID tabular data with:\n\nđ an online leaderboard accepting submissions \nđ carefully curated datasets (real, predictive, tabular, IID, permissive license) \nđ strong tree-based, deep learning, and foundation models \nâī¸ best practices for evaluation (inner CV, outer CV, early stopping) \n\nâšī¸ đđ¯đđĢđ¯đĸđđ° \nLeaderboard: https://tabarena.ai \nPaper: https://arxiv.org/abs/2506.16791 \nCode: https://tabarena.ai/code \n\nđĄ đđđĸđ§ đĸđ§đŦđĸđ đĄđđŦ: \nâĄī¸ Recent deep learning models, RealMLP and TabM, have marginally overtaken boosted trees with weighted ensembling, although they have slower train+inference times. With defaults or regular tuning, CatBoost takes the #1 spot. \nâĄī¸ Foundation models TabPFNv2 and TabICL are only applicable to a subset of datasets, but perform very strongly on these. They have a large inference time and still need tuning/ensembling to get the top spot (for TabPFNv2). \nâĄī¸ The winner does NOT take it all. By using a weighted ensemble of different model types from TabArena, we can significantly outperform the current state of the art on tabular data, AutoGluon 1.3. \nâĄī¸ These insights have been directly incorporated into the AutoGluon 1.4 release with the [extreme preset](https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets), dramatically advancing the state of the art! \nâĄī¸ The models TabPFNv2, TabICL, TabM, and RealMLP have been added to AutoGluon! To use them, run `pip install autogluon[tabarena]` and use the `extreme` preset.\n\nđ¯TabArena is a living benchmark. With the community, we will continually update it! \n\nTabArena Authors: [Nick Erickson](https://github.com/Innixma), [Lennart Purucker](https://github.com/LennartPurucker), [Andrej Tschalzev](https://github.com/atschalz), [David HolzmÃŧller](https://github.com/dholzmueller), [Prateek Mutalik Desai](https://github.com/prateekdesai04), [David Salinas](https://github.com/geoalgo), [Frank Hutter](https://github.com/frank-hutter)\n\n\n### AutoGluon Assistant (MLZero)\n> *Multi-Agent System Powered by LLMs for End-to-end Multimodal ML Automation*\n\nWe are excited to present the [AutoGluon Assistant](https://github.com/autogluon/autogluon-assistant) 1.0 release. Level up from v0.1: v1.0 expands beyond tabular data to robustly support any and many modalities, including **image, text, tabular, audio and mixed-data pipelines**. This aligns precisely with the MLZero vision of comprehensive, modality-agnostic ML automation.\n\nAutoGluon Assistant v1.0 is now synonymous with **\"MLZero: A Multi-Agent System for End-to-end Machine Learning Automation\"** ([arXiv:2505.13941](https://arxiv.org/abs/2505.13941)), the end-to-end, zero-human-intervention AutoML agent framework for multimodal data. Built on a novel **multi-agent architecture** using LLMs, MLZero handles perception, memory (semantic & episodic), code generation, execution, and iterative debugging â seamlessly transforming raw multimodal inputs into high-quality ML/DL pipelines.\n\n- **No-code**: Users define tasks purely through natural language (\"classify images of cats vs dogs with custom labels\"), and MLZero delivers complete solutions with zero manual configuration or technical expertise required.\n- **Built on proven foundations**: MLZero generates code using established, high-performance ML libraries rather than reinventing the wheel, ensuring robust solutions while maintaining the flexibility to easily integrate new libraries as they emerge.\n- **Research-grade performance**: MLZero is extensively validated across 25 challenging tasks spanning diverse data modalities, MLZero outperforms the competing methods by a large margin with a success rate of 0.92 (+263.6\\%) and an average rank of 2.42. \n \n\n```\n\nTo use webUI:\n```bash\nmlzero-backend # command to start backend\nmlzero-frontend # command to start frontend on 8509 (default)\n```\n\nTo use MCP:\n```bash\n# server\nmlzero-backend # command to start backend\nbash ./src/autogluon/mcp/server/start_services.sh # This will start the serviceârun it in a new terminal.\n# client\npython ./src/autogluon/mcp/client/server.py\n```\n\nMLZero Authors: [Haoyang Fang](https://github.com/FANGAreNotGnu), [Boran Han](https://github.com/boranhan), [Steven Shen](https://github.com/HuawenShen), [Nick Erickson](https://github.com/Innixma), [Xiyuan Zhang](https://xiyuanzh.github.io/), [Su Zhou](https://github.com/suzhoum), [Anirudh Dagar](https://github.com/AnirudhDagar), [Jiani Zhang](https://jennyzhang0215.github.io/), [Ali Caner Turkmen](https://github.com/canerturkmen), [Cuixiong Hu](https://github.com/tonyhoo), [Huzefa Rangwala](https://cs.gmu.edu/~hrangwal/), [Ying Nian Wu](https://scholar.google.com/citations?user=7k_1QFIAAAAJ&hl=en), [Bernie Wang](https://www.mit.edu/~ywang02/), [George Karypis](https://karypis.github.io/)\n\n### Mitra\n\nđ [Mitra](https://huggingface.co/autogluon/mitra-classifier) is a new state-of-the-art tabular foundation model developed by the AutoGluon team, natively supported in AutoGluon with just **three lines of code** via `predictor.fit(train_data, hyperparameters={\"MITRA\": {}})`. Built on the in-context learning paradigm and **pretrained exclusively on synthetic data**, Mitra introduces a principled pretraining approach by carefully selecting and mixing diverse synthetic priors to promote robust generalization across a wide range of real-world tabular datasets. Mitra is incorporated into the new `extreme` preset. \n\nđ Mitra achieves **state of the art performance** on major benchmarks including TabRepo, TabZilla, AMLB, and TabArena, especially excelling on small tabular datasets with fewer than 5,000 samples and 100 features, for both **classification** and **regression** tasks.\n\nđ§ Mitra supports both **zero-shot** and **fine-tuning** modes and runs seamlessly on both **GPU** and **CPU**. Its weights are fully open-sourced under the Apache-2.0 license, making it a privacy-conscious and production-ready solution for enterprises concerned about data sharing and hosting.\n\nđ Learn more by reading the [Mitra release blog post](https://www.amazon.science/blog/mitra-mixed-synthetic-priors-for-enhancing-tabular-foundation-models) and on HuggingFace:\n\n* Classification model: [autogluon/mitra-classifier](https://huggingface.co/autogluon/mitra-classifier)\n* Regression model: [autogluon/mitra-regressor](https://huggingface.co/autogluon/mitra-regressor)\n\nWe welcome community feedback for future iterations. Give us a like on HuggingFace if you want to see more cutting-edge foundation models for structured data!\n\nMitra Authors: [Xiyuan Zhang](https://xiyuanzh.github.io/), [Danielle Robinson](https://dcmaddix.github.io/), [Junming Yin](https://github.com/junmingy), [Nick Erickson](https://github.com/Innixma), [Abdul Fatir Ansari](https://github.com/abdulfatir), [Boran Han](https://github.com/boranhan), [Shuai Zhang](https://github.com/cheungdaven), [Leman Akoglu](https://scholar.google.com/citations?user=4ITkr_kAAAAJ&hl=en), [Christos Faloutsos](https://www.cs.cmu.edu/~christos/), [Michael W. Mahoney](https://www.stat.berkeley.edu/~mmahoney/), [Cuixiong Hu](https://github.com/tonyhoo), [Huzefa Rangwala](https://cs.gmu.edu/~hrangwal/), [George Karypis](https://karypis.github.io/), [Bernie Wang](https://www.mit.edu/~ywang02/)\n\n--------\n\n## General\n- Add CPU utility functions for better CPU detection in restrained env such as docker and slurm cluster. [@tonyhoo](https://github.com/tonyhoo) ([#5197](https://github.com/autogluon/autogluon/pull/5197))\n- Use joblib instead of loky for cpu detection. [@shchur](https://github.com/shchur) ([#5215](https://github.com/autogluon/autogluon/pull/5215))\n- Support Apple Silicon and log it in the system info. [@tonyhoo](https://github.com/tonyhoo) ([#5141](https://github.com/autogluon/autogluon/pull/5141))\n- Add load pickle from url support, fix save_str if root path. [@Innixma](https://github.com/Innixma) ([#5142](https://github.com/autogluon/autogluon/pull/5142))\n- Use pyarrow by default, remove fastparquet. [@Innixma](https://github.com/Innixma) ([#5150](https://github.com/autogluon/autogluon/pull/5150))\n- Resolve AttributeError in LinearModel when using RAPIDS cuML models. [@tonyhoo](https://github.com/tonyhoo) ([#5157](https://github.com/autogluon/autogluon/pull/5157))\n- add kwargs option to upload_file. [@Innixma](https://github.com/Innixma) ([#5161](https://github.com/autogluon/autogluon/pull/5161))\n- prioritize the CUDA libraries from PyTorch wheel instead of the system/DLC. [@FireballDWF](https://github.com/FireballDWF) ([#5163](https://github.com/autogluon/autogluon/pull/5163))\n- update numpy cap, thus 2.3.0 is allowed. [@FireballDWF](https://github.com/FireballDWF) ([#5170](https://github.com/autogluon/autogluon/pull/5170))\n- Replace pkg_resources.parse_version with packaging.version.parse. [@shchur](https://github.com/shchur) ([#5182](https://github.com/autogluon/autogluon/pull/5182))\n- Update pandas, scikit-learn, and scipy version caps in setup utils. [@tonyhoo](https://github.com/tonyhoo) ([#5194](https://github.com/autogluon/autogluon/pull/5194))\n- Enhance spunge_augment and munge_augment functions for model distillation. [@tonyhoo](https://github.com/tonyhoo) ([#5208](https://github.com/autogluon/autogluon/pull/5208))\n- Spunge Augmentation Speed-Up. [@mwhol](https://github.com/mwhol) ([#5217](https://github.com/autogluon/autogluon/pull/5217))\n- Increase pytorch cap to 2.8 to enable 2.7. [@FireballDWF](https://github.com/FireballDWF) ([#5089](https://github.com/autogluon/autogluon/pull/5089))\n- Resolve datetime deprecation warnings. [@emmanuel-ferdman](https://github.com/emmanuel-ferdman) ([#5069](https://github.com/autogluon/autogluon/pull/5069))\n\n\n--------\n\n## Tabular\n\n### New Presets\n\n- Add [extreme preset](https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets) with meta-learned [TabArena](https://tabarena.ai) portfolio. [@Innixma](https://github.com/Innixma) ([#5211](https://github.com/autogluon/autogluon/pull/5211))\n- The `extreme` preset is **the largest singular improvement to AutoGluon's predictive performance in the history of the package**, even larger than the improvement seen in AutoGluon 1.0 compared to 0.8.\n- For more information, refer to the highlights section above.\n\n### New Models\n\n#### Mitra\n\n- Add [Mitra](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#autogluon.tabular.models.MitraModel) Model (key: `\"MITRA\"`). [@xiyuanzh](https://github.com/xiyuanzh), [@dcmaddix](https://github.com/dcmaddix), [@junmingy](https://github.com/junmingy), [@Innixma](https://github.com/Innixma), [@tonyhoo](https://github.com/tonyhoo) ([#5195](https://github.com/autogluon/autogluon/pull/5195), [#5218](https://github.com/autogluon/autogluon/pull/5218), [#5232](https://github.com/autogluon/autogluon/pull/5232), [#5221](https://github.com/autogluon/autogluon/pull/5221))\n- Install via `pip install autogluon.tabular[all,mitra]` (and natively in `pip install autogluon`)\n- Blog Post: [Mitra: Mixed synthetic priors for enhancing tabular foundation models](https://www.amazon.science/blog/mitra-mixed-synthetic-priors-for-enhancing-tabular-foundation-models)\n- [Usage Tutorial](https://auto.gluon.ai/dev/tutorials/tabular/tabular-foundational-models.html)\n\n#### TabPFNv2\n\n- Add [TabPFNv2](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#autogluon.tabular.models.TabPFNV2Model) Model (key: `\"TABPFNV2\"`). [@LennartPurucker](https://github.com/LennartPurucker), [@Innixma](https://github.com/Innixma) ([#5191](https://github.com/autogluon/autogluon/pull/5191))\n- Install via `pip install autogluon.tabular[all,tabpfn]` (or `pip install autogluon[tabarena]`)\n- Paper: [Accurate predictions on small data with a tabular foundation model](https://www.nature.com/articles/s41586-024-08328-6)\n- [Usage Tutorial](https://auto.gluon.ai/dev/tutorials/tabular/tabular-foundational-models.html)\n\n#### TabICL\n\n- Add [TabICL](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#autogluon.tabular.models.TabICLModel) Model (key: `\"TABICL\"`). [@LennartPurucker](https://github.com/LennartPurucker), [@Innixma](https://github.com/Innixma) ([#5193](https://github.com/autogluon/autogluon/pull/5193))\n- Install via `pip install autogluon.tabular[all,tabicl]` (or `pip install autogluon[tabarena]`)\n- Paper: [TabICL: A Tabular Foundation Model for In-Context Learning on Large Data](https://arxiv.org/abs/2502.05564)\n- [Usage Tutorial](https://auto.gluon.ai/dev/tutorials/tabular/tabular-foundational-models.html)\n\n#### RealMLP\n\n- Add [RealMLP](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#autogluon.tabular.models.RealMLPModel) Model (key: `\"REALMLP\"`). [@dholzmueller](https://github.com/dholzmueller), [@Innixma](https://github.com/Innixma), [@LennartPurucker](https://github.com/LennartPurucker) ([#5190](https://github.com/autogluon/autogluon/pull/5190))- \n- Install via `pip install autogluon.tabular[all,realmlp]` (or `pip install autogluon[tabarena]`)\n- Paper: [Better by Default: Strong Pre-Tuned MLPs and Boosted Trees on Tabular Data](https://arxiv.org/abs/2407.04491)\n\n#### TabM\n\n- Add [TabM](https://auto.gluon.ai/stable/api/autogluon.tabular.models.html#autogluon.tabular.models.TabMModel) Model (key: `\"TABM\"`). [@LennartPurucker](https://github.com/LennartPurucker), [@dholzmueller](https://github.com/dholzmueller), [@Innixma](https://github.com/Innixma) ([#5196](https://github.com/autogluon/autogluon/pull/5196))\n- Install via `pip install autogluon.tabular[all,tabm]` (and natively in `pip install autogluon`)\n- Paper: [TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling](https://arxiv.org/abs/2410.24210)\n\n#### Removals\n\n- Removed TabPFNv1 model, as it is incompatible with TabPFNv2. [@Innixma](https://github.com/Innixma) ([#5191](https://github.com/autogluon/autogluon/pull/5191))\n- Removed KNN model from `best_quality` and `high_quality` preset portfolios, as it did not generally improve results. [@Innixma](https://github.com/Innixma) ([#5211](https://github.com/autogluon/autogluon/pull/5211))\n\n### Fixes and Improvements\n\n- Respect num_cpus/num_gpus in sequential_local fit. [@Innixma](https://github.com/Innixma) ([#5203](https://github.com/autogluon/autogluon/pull/5203))\n- Switch to loky for get_cpu_count in all places. [@Innixma](https://github.com/Innixma) ([#5204](https://github.com/autogluon/autogluon/pull/5204))\n- Add support for max_rows, max_features, max_classes, problem_types. [@Innixma](https://github.com/Innixma) ([#5181](https://github.com/autogluon/autogluon/pull/5181))\n- Add ag.ens. shortcut for ag_args_ensemble. [@Innixma](https://github.com/Innixma) ([#5143](https://github.com/autogluon/autogluon/pull/5143))\n- Fix CatBoost crashing for problem_type=\"quantile\" if len(quantile_levels) == 1. [@shchur](https://github.com/shchur) ([#5201](https://github.com/autogluon/autogluon/pull/5201))\n- Add tabular foundational model cache from s3 to benchmark to avoid rate limit issue from HF. [@tonyhoo](https://github.com/tonyhoo) ([#5214](https://github.com/autogluon/autogluon/pull/5214))\n- Fix default loss_function for CatBoostModel with problem_type='regression'. [@shchur](https://github.com/shchur) ([#5216](https://github.com/autogluon/autogluon/pull/5216))\n- Remove fobj in Softclass. [@rsj123](https://github.com/rsj123) ([#5219](https://github.com/autogluon/autogluon/pull/5219))\n- Minor enhancements and fixes. [@adibiasio](https://github.com/adibiasio), [@Innixma](https://github.com/Innixma) ([#5158](https://github.com/autogluon/autogluon/pull/5158))\n--------\n\n## TimeSeries\n\n### Highlights\n\n- Major [efficiency improvements](https://github.com/autogluon/autogluon/pull/5159) to the core `TimeSeriesDataFrame` methods, resulting in up to 7x lower end-to-end `predictor.fit()` and `predict()` time when working with large datasets (>10M rows).\n\n- New tabular forecasting model [`PerStepTabular`](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-model-zoo.html#autogluon.timeseries.models.PerStepTabularModel) that fits a separate tabular regression model for each time step in the forecast horizon. Both fitting and inference for the model are parallelized across cores, resulting in one of the most efficient and accurate implementations of this model among open-source Python packages.\n\n\n### API Changes and Deprecations\n- `DirectTabular` and `RecursiveTabular` models: hyperparameters `tabular_hyperparameters` and `tabular_fit_kwargs` are now deprecated in favor of `model_name` and `model_hyperparameters`.\n\n These models now fit a single regression model from `autogluon.tabular` under the hood instead of creating an entire `TabularPredictor`. This results in lower disk usage and API better aligned with the rest of the `timeseries` module.\n\n \n \n
\n\n\n
\n\n#### New Model: RealTabPFN-2.5\n\nTech Report: [TabPFN-2.5: Advancing the State of the Art in Tabular Foundation Models](https://arxiv.org/pdf/2511.08667v1)\n\nAutoGluon 1.5 adds support for fitting the RealTabPFN-2.5 model, the current strongest individual model on TabArena. Unlike TabPFN-2 which has a permissive license, RealTabPFN-2.5 comes with a non-commercial license and requires the user to authenticate with HuggingFace and accept a terms of use agreement before being able to download the weights. The user will be automatically prompted to perform these steps during AutoGluon's fit call if RealTabPFN-2.5 is specified, and the model will be skipped until the weights have been downloaded by the user. RealTabPFN-2.5 is not currently used in any AutoGluon preset, and must be manually specified.\n\nAll TabPFN user telemetry is disabled when used with AutoGluon.\n\nTo use RealTabPFN-2.5 (non-commercial use only):\n\n```python\n# pip install autogluon.tabular[all,tabpfn]\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(...).fit(\n train_data,\n hyperparameters={\"REALTABPFN-V2.5\": [{}]},\n) # GPU required, non-commercial\n```\n\nTo use RealTabPFN-2 (permissive license):\n\n```python\n# pip install autogluon.tabular[all,tabpfn]\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(...).fit(train_data, hyperparameters={\"REALTABPFN-V2\": [{}]}) # GPU required\n```\n\nFor users who previously were using `\"TABPFNV2\"`, we strongly recommend switching to `\"REALTABPFN-V2\"` to avoid breaking changes in the latest TabPFN releases.\n\n#### New Model: TabDPT\n\nPaper: [TabDPT: Scaling Tabular Foundation Models on Real Data](https://arxiv.org/pdf/2410.18164)\n\nTo use TabDPT (permissive license):\n\n```python\n# pip install autogluon.tabular[all,tabdpt]\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(...).fit(train_data, hyperparameters={\"TABDPT\": [{}]}) # GPU recommended\n```\n\n#### New Model: TabPrep-LightGBM\n\nTabPrep-LightGBM is an experimental model that uses a custom data preprocessing pipeline to enhance the performance of LightGBM. It represents a working snapshot of an in-progress research effort. Further details will be shared as part of an upcoming paper.\n\nTabPrep-LightGBM achieves a new state-of-the-art for model performance on TabArena's 15 largest datasets (10k - 100k training samples), exceeding RealTabPFN-2.5 by 100 Elo while fitting 3x faster using just 8 CPU cores. TabPrep-LightGBM is also incorporated into the AutoGluon 1.5 extreme preset.\n\n##### TabArena Medium (10k - 100k samples, 15 datasets)\n\n| Model | Elo [âŦī¸] | Imp (%) [âŦī¸] | Train Time (s/1K) [âŦī¸] | Predict Time (s/1K) [âŦī¸] |\n|:----------------------------------------|-----------:|-------------------------:|--------------------------------:|----------------------------------:|\n| AutoGluon 1.5 (extreme, 4h) | 1965 | 1.876 | 191.18 | 2.207 |\n| AutoGluon 1.4 (extreme, 4h) | 1813 | 3.016 | 289.53 | 3.187 |\n| AutoGluon 1.4 (best, 4h) | 1794 | 3.122 | 432.35 | 4.085 |\n| TabPrep-LightGBM (tuned + ensembled) | 1787 | 3.573 | 256.12 | 2.281 |\n| RealTabPFN-v2.5 (tuned + ensembled) | 1680 | 5.818 | 735.58 | 11.736 |\n| RealMLP (tuned + ensembled) | 1649 | 6.102 | 1719.82 | 1.675 |\n| ModernNCA (tuned + ensembled) | 1636 | 6.189 | 2526.28 | 6.013 |\n| CatBoost (tuned + ensembled) | 1616 | 6.011 | 777.59 | 0.25 |\n| LightGBM (tuned + ensembled) | 1598 | 7.77 | 131.56 | 2.639 |\n\nTo use TabPrep-LightGBM, we recommend trying the presets it is used in: `\"extreme\", \"best_v150\", \"high_v150\"`. Fitting TabPrep-LightGBM outside of the use of presets is currently complicated.\n\n--------\n\n## General\n\n### Dependencies\n- Update torch to `>=2.6,<2.10` [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@shchur](https://github.com/shchur) ([#5270](https://github.com/autogluon/autogluon/pull/5270)) ([#5425](https://github.com/autogluon/autogluon/pull/5425))\n- Update seaborn to `>=0.12.0,<0.14`. [@Innixma](https://github.com/Innixma) ([#5378](https://github.com/autogluon/autogluon/pull/5378))\n- Update onnx to `>=1.13.0,<1.21.0` [@shchur](https://github.com/shchur) ([#5439](https://github.com/autogluon/autogluon/pull/5439))\n- Update ray to `>=2.43.0,<2.53` [@shchur](https://github.com/shchur) [@prateekdesai04](https://github.com/prateekdesai04) ([#5442](https://github.com/autogluon/autogluon/pull/5442)) ([#5312](https://github.com/autogluon/autogluon/pull/5312))\n- Update transformers to `\">=4.51.0,<4.58\"` [@shchur](https://github.com/shchur) ([#5439](https://github.com/autogluon/autogluon/pull/5439))\n- Update lightning to `>=2.5.1,<2.6` [@canerturkmen](https://github.com/canerturkmen) ([#5432](https://github.com/autogluon/autogluon/pull/5432))\n- Update psutil to `>=5.7.3,<7.2.0` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Update xgboost to `>=2.0,<3.2` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Update pytabkit to `1.7.2,<1.8` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Update tabpfn to `>=6.1.0,<6.1.1` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Update tabicl to `0.1.4,<0.2` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Update scikit-learn-intelex to `2025.0,<2025.10` [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Add experimental support for Python 3.13 [@shchur](https://github.com/shchur) [@shou10152208](https://github.com/shou10152208) ([#5073](https://github.com/autogluon/autogluon/pull/5073)) ([#5423](https://github.com/autogluon/autogluon/pull/5423))\n\n### Fixes and Improvements\n- Minor typing fixes. [@canerturkmen](https://github.com/canerturkmen) ([#5292](https://github.com/autogluon/autogluon/pull/5292))\n- Fix conda install instructions for ray version. [@Innixma](https://github.com/Innixma) ([#5323](https://github.com/autogluon/autogluon/pull/5323))\n- Use standalone uv in full_install.sh. [@Innixma](https://github.com/Innixma) ([#5328](https://github.com/autogluon/autogluon/pull/5328))\n- Cleanup load_pd and save_pd. [@Innixma](https://github.com/Innixma) ([#5359](https://github.com/autogluon/autogluon/pull/5359))\n- Remove LICENSE and NOTICE files from common. [@prateekdesai04](https://github.com/prateekdesai04) ([#5396](https://github.com/autogluon/autogluon/pull/5396))\n- Fix upload python package. [@prateekdesai04](https://github.com/prateekdesai04) ([#5397](https://github.com/autogluon/autogluon/pull/5397))\n- Change build order. [@prateekdesai04](https://github.com/prateekdesai04) ([#5398](https://github.com/autogluon/autogluon/pull/5398))\n- Decouple and enable module-wise installation. [@prateekdesai04](https://github.com/prateekdesai04) ([#5399](https://github.com/autogluon/autogluon/pull/5399))\n- Fix get_smallest_valid_dtype_int for negative values. [@Innixma](https://github.com/Innixma) ([#5421](https://github.com/autogluon/autogluon/pull/5421))\n\n\n--------\n\n## Tabular\n\nAutoGluon-Tabular v1.5 introduces several improvements focused on accuracy, robustness, and usability. The release adds new foundation models, updates the feature preprocessing pipeline, and improves GPU stability and memory estimation. New model portfolios are provided for both CPU and GPU workloads.\n\n### Highlights\n- **New models**: RealTabPFN-2, RealTabPFN-2.5, TabDPT, TabPrep-LightGBM, and EBM are now available in AutoGluon-Tabular.\n- **Updated preprocessing pipeline** with more consistent feature handling across models.\n- **Improved GPU stability** and more reliable memory estimation during training.\n- **New CPU and GPU portfolios** tuned for better performance across a wide range of datasets: `\"extreme\", \"best_v150\", \"high_v150\"`.\n- **Stronger benchmark results**: with the new presets, AutoGluon-Tabular v1.5 Extreme achieves a **70% win rate** over AutoGluon v1.4 Extreme on the 51 TabArena datasets, with a **2.8% reduction in mean relative error**.\n\n### New Features\n- New preprocessors for tabular data. [@atschalz](https://github.com/atschalz) [@Innixma](https://github.com/Innixma) ([#5441](https://github.com/autogluon/autogluon/pull/5441))\n- New model: LightGBMPrep. [@atschalz](https://github.com/atschalz) [@Innixma](https://github.com/Innixma) ([#5490](https://github.com/autogluon/autogluon/pull/5490))\n- New models: TabPFN-2.5, TabDPT. [@Innixma](https://github.com/Innixma) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- New model: Explainable Boosting Machine. [@paulbkoch](https://github.com/paulbkoch) ([#4480](https://github.com/autogluon/autogluon/pull/4480))\n- Add v1.5.0 presets [@Innixma](https://github.com/Innixma) ([#5505](https://github.com/autogluon/autogluon/pull/5505))\n\n### Fixes and Improvements\n- Fix bug if pred is inf and weight is 0 in weighted ensemble. [@Innixma](https://github.com/Innixma) ([#5317](https://github.com/autogluon/autogluon/pull/5317))\n- Default TabularPredictor.delete_models dry_run=False. [@Innixma](https://github.com/Innixma) ([#5260](https://github.com/autogluon/autogluon/pull/5260))\n- Remove redundant TabPFNv2 CPU log. [@Innixma](https://github.com/Innixma) ([#5259](https://github.com/autogluon/autogluon/pull/5259))\n- Add einops in mitra install. [@xiyuanzh](https://github.com/xiyuanzh) ([#5266](https://github.com/autogluon/autogluon/pull/5266))\n- Support different random seeds per fold. [@LennartPurucker](https://github.com/LennartPurucker) ([#5267](https://github.com/autogluon/autogluon/pull/5267))\n- Changing the default output dir's base path. [@LennartPurucker](https://github.com/LennartPurucker) ([#5285](https://github.com/autogluon/autogluon/pull/5285))\n- Add Mitra download_default_weights. [@Innixma](https://github.com/Innixma) ([#5271](https://github.com/autogluon/autogluon/pull/5271))\n- Ensure compatibility of flash attention unpad_input. [@xiyuanzh](https://github.com/xiyuanzh) ([#5298](https://github.com/autogluon/autogluon/pull/5298))\n- Refactor of validation technique selection. [@LennartPurucker](https://github.com/LennartPurucker) ([#4585](https://github.com/autogluon/autogluon/pull/4585))\n- Mitra HF Args. [@xiyuanzh](https://github.com/xiyuanzh) ([#5272](https://github.com/autogluon/autogluon/pull/5272))\n- Gracefully handle ray exceptions. [@Innixma](https://github.com/Innixma) ([#5327](https://github.com/autogluon/autogluon/pull/5327))\n- Add logs for LightGBM CUDA device. [@Innixma](https://github.com/Innixma) ([#5325](https://github.com/autogluon/autogluon/pull/5325))\n- Add Load/Save to TabularDataset. [@Innixma](https://github.com/Innixma) ([#5357](https://github.com/autogluon/autogluon/pull/5357))\n- Fix model random state. [@Innixma](https://github.com/Innixma) ([#5369](https://github.com/autogluon/autogluon/pull/5369))\n- Add AbstractModel type hints. [@Innixma](https://github.com/Innixma) ([#5358](https://github.com/autogluon/autogluon/pull/5358))\n- MakeOneFeatureGenerator pass check_is_fitted test. [@betatim](https://github.com/betatim) ([#5386](https://github.com/autogluon/autogluon/pull/5386))\n- Enable CPU loading of models trained on GPU [@Innixma](https://github.com/Innixma) ([#5403](https://github.com/autogluon/autogluon/pull/5403)) ([#5434](https://github.com/autogluon/autogluon/pull/5434))\n- Remove unused variable val_improve_epoch in TabularNeuralNetTorchModel. [@celestinoxp](https://github.com/celestinoxp) ([#5466](https://github.com/autogluon/autogluon/pull/5466))\n- Fix memory estimation for RF/XT in parallel mode. [@celestinoxp](https://github.com/celestinoxp) ([#5467](https://github.com/autogluon/autogluon/pull/5467))\n- Pass label cleaner to model for semantic encodings. [@LennartPurucker](https://github.com/LennartPurucker) ([#5482](https://github.com/autogluon/autogluon/pull/5482))\n- Fix time_epoch_average calculation in TabularNeuralNetTorch. [@celestinoxp](https://github.com/celestinoxp) ([#5484](https://github.com/autogluon/autogluon/pull/5484))\n- GPU optimization, scheduling for parallel_local fitting strategy. [@prateekdesai04](https://github.com/prateekdesai04) ([#5388](https://github.com/autogluon/autogluon/pull/5388))\n- Fix XGBoost crashing on eval metric name in HPs. [@LennartPurucker](https://github.com/LennartPurucker) ([#5493](https://github.com/autogluon/autogluon/pull/5493))\n\n\n--------\n\n## TimeSeries\n\nAutoGluon v1.5 introduces substantial improvements to the time series module, with clear gains in both accuracy and usability. Across our benchmarks, v1.5 achieves up to an 80% win rate compared to v1.4. The release adds new models, more flexible ensembling options, and numerous bug fixes and quality-of-life improvements.\n\n### Highlights\n- **Chronos-2** is now available in AutoGluon, with support for zero-shot inference as well as full and LoRA fine-tuning ([tutorial](https://auto.gluon.ai/dev/tutorials/timeseries/forecasting-chronos.html)).\n- **Customizable ensembling logic**: Adds item-level ensembling, multi-layer stack ensembles, and other advanced forecast combination methods ([documentation](https://auto.gluon.ai/dev/tutorials/timeseries/forecasting-ensembles.html)).\n- **New presets** leading to major gains in accuracy & efficiency. AG-TS v1.5 achieves up to **80% win rate** over v1.4 on point and probabilistic forecasting tasks. With just a 10 minute time limit, v1.5 outperforms v1.4 running for 2 hours.\n- **Usability improvements**: Automatically determine an appropriate backtesting configuration by setting `num_val_windows=\"auto\"` and `refit_every_n_windows=\"auto\"`. Easily access the validation predictions and perform rolling evaluation on custom data with new predictor methods [`backtest_predictions`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_predictions.html) and [`backtest_targets`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_targets.html).\n\n\n### New Features\n- Add multi-layer stack ensembling support [@canerturkmen](https://github.com/canerturkmen) ([#5459](https://github.com/autogluon/autogluon/pull/5459)) ([#5472](https://github.com/autogluon/autogluon/pull/5472)) ([#5463](https://github.com/autogluon/autogluon/pull/5463)) ([#5456](https://github.com/autogluon/autogluon/pull/5456)) ([#5436](https://github.com/autogluon/autogluon/pull/5436)) ([#5422](https://github.com/autogluon/autogluon/pull/5422)) ([#5391](https://github.com/autogluon/autogluon/pull/5391))\n- Add new advanced ensembling methods [@canerturkmen](https://github.com/canerturkmen) [@shchur](https://github.com/shchur) ([#5465](https://github.com/autogluon/autogluon/pull/5465)) ([#5420](https://github.com/autogluon/autogluon/pull/5420)) ([#5401](https://github.com/autogluon/autogluon/pull/5401)) ([#5389](https://github.com/autogluon/autogluon/pull/5389)) ([#5376](https://github.com/autogluon/autogluon/pull/5376))\n- Add Chronos-2 model. [@abdulfatir](https://github.com/abdulfatir) [@canerturkmen](https://github.com/canerturkmen) ([#5427](https://github.com/autogluon/autogluon/pull/5427)) ([#5447](https://github.com/autogluon/autogluon/pull/5447)) ([#5448](https://github.com/autogluon/autogluon/pull/5448)) ([#5449](https://github.com/autogluon/autogluon/pull/5449)) ([#5454](https://github.com/autogluon/autogluon/pull/5454)) ([#5455](https://github.com/autogluon/autogluon/pull/5455)) ([#5450](https://github.com/autogluon/autogluon/pull/5450)) ([#5458](https://github.com/autogluon/autogluon/pull/5458)) ([#5492](https://github.com/autogluon/autogluon/pull/5492)) ([#5495](https://github.com/autogluon/autogluon/pull/5495)) ([#5487](https://github.com/autogluon/autogluon/pull/5487)) ([#5486](https://github.com/autogluon/autogluon/pull/5486))\n- Update Chronos-2 tutorial. [@abdulfatir](https://github.com/abdulfatir) ([#5481](https://github.com/autogluon/autogluon/pull/5481))\n- Add Toto model. [@canerturkmen](https://github.com/canerturkmen) ([#5321](https://github.com/autogluon/autogluon/pull/5321)) ([#5390](https://github.com/autogluon/autogluon/pull/5390)) ([#5475](https://github.com/autogluon/autogluon/pull/5475))\n- Fine-tune Chronos-Bolt on user-provided `quantile_levels`. [@shchur](https://github.com/shchur) ([#5315](https://github.com/autogluon/autogluon/pull/5315))\n- Add backtesting methods for the TimeSeriesPredictor. [@shchur](https://github.com/shchur) ([#5356](https://github.com/autogluon/autogluon/pull/5356))\n\n- Update predictor presets. [@shchur](https://github.com/shchur) ([#5480](https://github.com/autogluon/autogluon/pull/5480)) ([#5480](https://github.com/autogluon/autogluon/pull/5494))\n\n### API Changes and Deprecations\n- Remove outdated presets related to the original Chronos model: `chronos`, `chronos_large`, `chronos_base`, `chronos_small`, `chronos_mini`, `chronos_tiny`, `chronos_ensemble`. We recommend to use the new presets `chronos2`, `chronos2_small` and `chronos2_ensemble` instead.\n\n### Fixes and Improvements\n- Replace `inf` values with `NaN` inside `_check_and_prepare_data_frame`. [@shchur](https://github.com/shchur) ([#5240](https://github.com/autogluon/autogluon/pull/5240))\n- Add model registry and fix presets typing. [@canerturkmen](https://github.com/canerturkmen) ([#5100](https://github.com/autogluon/autogluon/pull/5100))\n- Fix broken unittests for time series. [@shchur](https://github.com/shchur) ([#5361](https://github.com/autogluon/autogluon/pull/5361))\n- Move ITEMID and TIMESTAMP to dataset namespace. [@canerturkmen](https://github.com/canerturkmen) ([#5363](https://github.com/autogluon/autogluon/pull/5363))\n- Remove deprecated arguments and classes. [@shchur](https://github.com/shchur) ([#5354](https://github.com/autogluon/autogluon/pull/5354))\n- Replace Chronos code with a dependency on `chronos-forecasting` [@canerturkmen](https://github.com/canerturkmen) ([#5380](https://github.com/autogluon/autogluon/pull/5380)) ([#5383](https://github.com/autogluon/autogluon/pull/5383))\n- Avoid errors if date_feature clashes with known_covariates. [@shchur](https://github.com/shchur) ([#5414](https://github.com/autogluon/autogluon/pull/5414))\n- Make `ray` an optional dependency for `autogluon.timeseries`. [@shchur](https://github.com/shchur) ([#5430](https://github.com/autogluon/autogluon/pull/5430))\n- Sort feature importance df. [@shchur](https://github.com/shchur) ([#5468](https://github.com/autogluon/autogluon/pull/5468))\n- Make NPTS model deterministic. [@shchur](https://github.com/shchur) ([#5471](https://github.com/autogluon/autogluon/pull/5471))\n- Store cardinality inside CovariateMetadata. [@shchur](https://github.com/shchur) ([#5476](https://github.com/autogluon/autogluon/pull/5476))\n- Minor fixes and improvements [@shchur](https://github.com/shchur) [@abdulfatir](https://github.com/abdulfatir) [@canerturkmen](https://github.com/canerturkmen) ([#5489](https://github.com/autogluon/autogluon/pull/5489)) ([#5452](https://github.com/autogluon/autogluon/pull/5452)) ([#5444](https://github.com/autogluon/autogluon/pull/5444)) ([#5416](https://github.com/autogluon/autogluon/pull/5416)) ([#5413](https://github.com/autogluon/autogluon/pull/5413)) ([#5410](https://github.com/autogluon/autogluon/pull/5410)) ([#5406](https://github.com/autogluon/autogluon/pull/5406))\n\n### Code Quality\n- Refactor trainable model set build logic. [@canerturkmen](https://github.com/canerturkmen) ([#5297](https://github.com/autogluon/autogluon/pull/5297))\n- Typing improvements to multiwindow model. [@canerturkmen](https://github.com/canerturkmen) ([#5308](https://github.com/autogluon/autogluon/pull/5308))\n- Move prediction cache out of trainer. [@canerturkmen](https://github.com/canerturkmen) ([#5313](https://github.com/autogluon/autogluon/pull/5313))\n- Refactor trainer methods with ensemble logic. [@canerturkmen](https://github.com/canerturkmen) ([#5375](https://github.com/autogluon/autogluon/pull/5375))\n- Use builtin generics for typing, remove types in internal docstrings. [@canerturkmen](https://github.com/canerturkmen) ([#5300](https://github.com/autogluon/autogluon/pull/5300))\n- Reorganize ensembles, add base class for array-based ensemble learning. [@canerturkmen](https://github.com/canerturkmen) ([#5332](https://github.com/autogluon/autogluon/pull/5332))\n- Separate ensemble training logic from trainer. [@canerturkmen](https://github.com/canerturkmen) ([#5384](https://github.com/autogluon/autogluon/pull/5384))\n- Clean up typing and documentation for Chronos. [@canerturkmen](https://github.com/canerturkmen) ([#5392](https://github.com/autogluon/autogluon/pull/5392))\n- Add timer utility, fix time limit in ensemble regressors, clean up tests. [@canerturkmen](https://github.com/canerturkmen) ([#5393](https://github.com/autogluon/autogluon/pull/5393))\n- upgrade type annotations to Python3.10. [@canerturkmen](https://github.com/canerturkmen) ([#5431](https://github.com/autogluon/autogluon/pull/5431))\n\n\n\n--------\n\n## Multimodal\n\n### Fixes and Improvements\n- Bug Fix and Update AutoMM Tutorials. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#5167](https://github.com/autogluon/autogluon/pull/5167))\n- Fix Focal Loss. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#5496](https://github.com/autogluon/autogluon/pull/5496))\n- Fix false positive document detection for images with incidental text. [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) ([#5469](https://github.com/autogluon/autogluon/pull/5469))\n\n--------\n\n## Documentation and CI\n\n- [doc] Clarify tuning_data documentation. [@Innixma](https://github.com/Innixma) ([#5296](https://github.com/autogluon/autogluon/pull/5296))\n- [Test] Fix CI + Upgrade Ray. [@prateekdesai04](https://github.com/prateekdesai04) ([#5306](https://github.com/autogluon/autogluon/pull/5306))\n- Fix notebook build failures. [@prateekdesai04](https://github.com/prateekdesai04) ([#5348](https://github.com/autogluon/autogluon/pull/5348))\n- ci: scope down GitHub Token permissions. [@AdnaneKhan](https://github.com/AdnaneKhan) ([#5351](https://github.com/autogluon/autogluon/pull/5351))\n- Fix CodeQL GitHub action. [@shchur](https://github.com/shchur) ([#5367](https://github.com/autogluon/autogluon/pull/5367))\n- [CI] Fix docker build. [@prateekdesai04](https://github.com/prateekdesai04) ([#5402](https://github.com/autogluon/autogluon/pull/5402))\n- [docs] Reorder modules in docs. [@shchur](https://github.com/shchur) ([#5404](https://github.com/autogluon/autogluon/pull/5404))\n- remove ROADMAP.md. [@canerturkmen](https://github.com/canerturkmen) ([#5405](https://github.com/autogluon/autogluon/pull/5405))\n- [docs] Add citations for Chronos-2 and multi-layer stacking for TS. [@shchur](https://github.com/shchur) ([#5412](https://github.com/autogluon/autogluon/pull/5412))\n- Fix permissions for platform_tests action. [@shchur](https://github.com/shchur) ([#5418](https://github.com/autogluon/autogluon/pull/5418))\n- Revert \"Fix permissions for platform_tests action\". [@shchur](https://github.com/shchur) ([#5419](https://github.com/autogluon/autogluon/pull/5419))\n- Fix torch<2.10 issues in the CI. [@shchur](https://github.com/shchur) ([#5435](https://github.com/autogluon/autogluon/pull/5435))\n\n\n--------\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) [@Innixma](https://github.com/Innixma) [@prateekdesai04](https://github.com/prateekdesai04) [@abdulfatir](https://github.com/abdulfatir) [@LennartPurucker](https://github.com/LennartPurucker) [@celestinoxp](https://github.com/celestinoxp) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@xiyuanzh](https://github.com/xiyuanzh) [@nathanaelbosch](https://github.com/nathanaelbosch) [@betatim](https://github.com/betatim) [@AdnaneKhan](https://github.com/AdnaneKhan) [@paulbkoch](https://github.com/paulbkoch) [@shou10152208](https://github.com/shou10152208) [@ryuichi-ichinose](https://github.com/ryuichi-ichinose) [@atschalz](https://github.com/atschalz) [@colesussmeier](https://github.com/colesussmeier)\n\n\n### New Contributors\n\n- [@betatim](https://github.com/betatim) made their first contribution in ([#5386](https://github.com/autogluon/autogluon/pull/5386))\n- [@AdnaneKhan](https://github.com/AdnaneKhan) made their first contribution in ([#5351](https://github.com/autogluon/autogluon/pull/5351))\n- [@paulbkoch](https://github.com/paulbkoch) made their first contribution in ([#4480](https://github.com/autogluon/autogluon/pull/4480))\n- [@shou10152208](https://github.com/shou10152208) made their first contribution in ([#5073](https://github.com/autogluon/autogluon/pull/5073))\n- [@ryuichi-ichinose](https://github.com/ryuichi-ichinose) made their first contribution in ([#5458](https://github.com/autogluon/autogluon/pull/5073))\n- [@atschalz](https://github.com/atschalz) made their first contribution in ([#5441](https://github.com/autogluon/autogluon/pull/5441))\n- [@colesussmeier](https://github.com/colesussmeier) made their first contribution in ([#5452](https://github.com/autogluon/autogluon/pull/5452))\n"
},
{
"path": "eda/setup.cfg",
"content": "[pycodestyle]\nmax_line_length = 160\n\n[flake8]\nmax-line-length = 160\nper-file-ignores =\n */__init__.py: F401\n\n[mypy]\nwarn_unused_configs = True\nshow_error_context = True\npretty = True\ncheck_untyped_defs = True\n\n\n[tool:pytest]\ntestpaths = tests\naddopts = --cov --strict-markers\nxfail_strict = True\n\n[coverage:run]\nsource = autogluon.eda\nbranch = True\n\n[coverage:report]\nshow_missing = True\nskip_covered = True\nfail_under = 90\n\n[coverage:paths]\nsource =\n src/autogluon/eda\n */site-packages/autogluon/eda\n\n[tox:tox]\nenvlist =\n py38\n; py39 - Not supported in the container\n; py310 - Not supported in the container\nisolated_build = True\n\n[pkg-imports]\ndeps =\n ../common\n ../features\n ../core\n ../tabular\n .[tests]\n\n[testenv]\nusedevelop=True\ndeps =\n pytest\n flake8\n pytest-cov\n {[pkg-imports]deps}\ncommands =\n pytest {posargs}\n\n[testenv:typecheck]\ndeps =\n pytest\n pytest-mypy\n types-requests\n types-setuptools\n {[pkg-imports]deps}\ncommands =\n mypy --ignore-missing-imports {posargs:src}\n\n[testenv:format]\nskip_install = True\ndeps =\n ruff\ncommands =\n ruff check {posargs:--diff --color src tests}\n\n[testenv:lint]\nallowlist_externals = ruff\nskip_install = True\ndeps =\n flake8\n flake8-bugbear\ncommands =\n flake8 {posargs:src tests}\n ruff check --select I --diff {posargs:src tests}\n\n"
},
{
"path": "eda/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport importlib.util\nimport os\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n \nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec) # type: ignore\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag) # type: ignore\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = \"eda\"\ninstall_requires = [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"scipy\", # version range defined in `core/_setup_utils.py`\n \"scikit-learn\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"matplotlib\", # version range defined in `core/_setup_utils.py`\n \"missingno>=0.5.1,<0.6\",\n \"phik>=0.12.2,<0.13\",\n \"seaborn>=0.12.0,<0.14\",\n \"ipywidgets>=7.7.1,<9.0\", # min versions guidance: 7.7.1 collab/kaggle\n \"shap>=0.44,<0.47\",\n \"yellowbrick>=1.5,<1.6\",\n \"pyod>=1.1,<1.2\",\n \"suod>=0.0.8,<0.1\",\n \"ipython>7.16,<8.13\", # IPython 8.13+ supports Python 3.9 and above; Python 3.7 is supported with IPython >7.16\n f\"autogluon.core=={version}\",\n f\"autogluon.common=={version}\",\n f\"autogluon.features=={version}\",\n f\"autogluon.tabular=={version}\",\n]\n\nextras_require = dict()\n\ntest_requirements = [\n \"tox\",\n \"pytest\",\n \"pytest-cov\",\n \"types-requests\",\n \"types-setuptools\",\n \"pytest-mypy\",\n \"PyHamcrest\",\n]\n\ntest_requirements = list(set(test_requirements))\nextras_require[\"tests\"] = test_requirements\n\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\nfor key in extras_require:\n extras_require[key] = ag.get_dependency_version_ranges(extras_require[key])\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "eda/src/autogluon/eda/__init__.py",
"content": "try:\n from .version import __version__\nexcept ImportError: # pragma: no cover\n pass\n\nfrom .state import AnalysisState\n"
},
{
"path": "eda/src/autogluon/eda/analysis/__init__.py",
"content": "from .anomaly import AnomalyDetector, AnomalyDetectorAnalysis\nfrom .base import Namespace\nfrom .dataset import (\n LabelInsightsAnalysis,\n ProblemTypeControl,\n RawTypesAnalysis,\n Sampler,\n SpecialTypesAnalysis,\n TrainValidationSplit,\n VariableTypeAnalysis,\n)\nfrom .explain import ShapAnalysis\nfrom .interaction import Correlation, CorrelationSignificance, DistributionFit, FeatureInteraction\nfrom .missing import MissingValuesAnalysis\nfrom .model import AutoGluonModelEvaluator, AutoGluonModelQuickFit\nfrom .shift import XShiftDetector\nfrom .transform import ApplyFeatureGenerator\n"
},
{
"path": "eda/src/autogluon/eda/analysis/anomaly.py",
"content": "from __future__ import print_function\n\nimport builtins as __builtin__\nimport contextlib\nimport logging\nfrom functools import partial\nfrom typing import Any, Dict, List, Optional\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nfrom pyod.models.base import BaseDetector\nfrom pyod.models.copod import COPOD\nfrom pyod.models.iforest import IForest\nfrom pyod.models.lof import LOF\nfrom pyod.models.suod import SUOD\n\nfrom autogluon.common.utils.cv_splitter import CVSplitter\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom .. import AnalysisState\nfrom .base import AbstractAnalysis\n\n__all__ = [\"AnomalyDetector\", \"AnomalyDetectorAnalysis\"]\n\nlogger = logging.getLogger(__name__)\n\n\n@contextlib.contextmanager\ndef _suod_silent_print(silent=True): # pragma: no cover\n \"\"\"\n Workaround to suppress log clutter from SUOD\n\n See Also\n --------\n https://github.com/yzhao062/SUOD/pull/7\n https://github.com/yzhao062/SUOD/pull/12\n \"\"\"\n\n orig_fn = joblib.Parallel._print\n orig_print = __builtin__.print\n\n def silent_print(*args, **kwargs):\n return\n\n def _silent_print(*args, **kwargs):\n if args != () and kwargs != {}:\n orig_print(*args, **kwargs)\n\n if silent:\n joblib.Parallel._print = silent_print\n __builtin__.print = _silent_print # type: ignore[assignment]\n try:\n yield\n finally:\n if silent:\n joblib.Parallel._print = orig_fn\n __builtin__.print = orig_print\n\n\nclass AnomalyDetector:\n \"\"\"\n Wrapper for anomaly detector algorithms.\n\n :py:meth:`~autogluon.eda.analysis.anomaly.AnomalyDetector.fit_transform` automatically creates\n cross-validation splits and fits detectors on each of them. The scores produced for the training\n data are produced using out-of-folds predictions\n\n :py:meth:`~autogluon.eda.analysis.anomaly.AnomalyDetector.transform` uses average of scores from\n the detectors trained on the folds.\n\n Please note: the data passed into the fit/transform must be already pre-processed;\n numeric columns must have no NaNs.\n\n Parameters\n ----------\n label: str\n dataset's label column name\n n_folds: int, default = 5,\n number of folds to use when training detectors\n detector_list: Optional[List[BaseDetector]], default = None\n list of detectors to ensemble. If `None`, then use the standard list:\n - LOF(n_neighbors=15)\n - LOF(n_neighbors=20)\n - LOF(n_neighbors=25)\n - LOF(n_neighbors=35)\n - COPOD\n - IForest(n_estimators=100)\n - IForest(n_estimators=200)\n See `pyod `_ documentation for the full model list.\n silent: bool, default = True\n Suppress SUOD logs if `True`\n detector_kwargs\n kwargs to pass into detector\n \"\"\"\n\n def __init__(\n self,\n label: str,\n n_folds: int = 5,\n detector_list: Optional[List[BaseDetector]] = None,\n silent: bool = True,\n **detector_kwargs,\n ) -> None:\n self.label = label\n self.n_folds = n_folds\n self.silent = silent\n if detector_list is None:\n detector_list = AnomalyDetector._get_default_detector_list()\n self.detector_list = detector_list\n\n # Can't go beyond 4 - SUOD is throwing errors\n num_cpus = min(ResourceManager.get_cpu_count(), 4)\n\n # Don't use `bps_flag=True` - it's using pre-trained models which aren't loading in newer versions of sklearn\n suod_defaults = dict(\n base_estimators=self.detector_list, n_jobs=num_cpus, combination=\"average\", bps_flag=False, verbose=False\n )\n self._suod_kwargs = {**suod_defaults, **detector_kwargs}\n self._detectors: Optional[List[BaseDetector]] = None\n self._train_index_to_detector: Optional[Dict[int, Any]] = None\n self.original_features: Optional[List[str]] = None\n\n @staticmethod\n def _get_default_detector_list():\n return [\n LOF(n_neighbors=15),\n LOF(n_neighbors=20),\n LOF(n_neighbors=25),\n LOF(n_neighbors=35),\n COPOD(),\n IForest(n_estimators=100),\n IForest(n_estimators=200),\n ]\n\n @property\n def problem_type(self):\n return \"regression\"\n\n def fit_transform(self, train_data: pd.DataFrame) -> pd.Series:\n \"\"\"\n Automatically creates cross-validation splits and fits detectors on each of them.\n The scores produced for the training data are produced using out-of-folds predictions\n\n Parameters\n ----------\n train_data: pd.DataFrame\n training data; must be already pre-processed; numeric columns must have NaNs filled\n\n Returns\n -------\n out-of-folds anomaly scores for the training data\n\n \"\"\"\n self._detectors = []\n self._train_index_to_detector = {}\n splitter = CVSplitter(n_splits=self.n_folds)\n x, y = train_data.drop(columns=self.label), train_data[self.label]\n self.original_features = x.columns\n\n folds_scores = []\n for i, (train_idx, val_idx) in enumerate(splitter.split(x, y)):\n x_train = x.iloc[train_idx]\n x_val = x.iloc[val_idx]\n\n with _suod_silent_print(self.silent):\n detector = SUOD(**self._suod_kwargs)\n np.int = int # type: ignore[attr-defined] # workaround to address shap's use of old numpy APIs\n self._detectors.append(detector.fit(x_train))\n self._train_index_to_detector = {**self._train_index_to_detector, **{idx: i for idx in x_train.index}}\n val_scores = detector.decision_function(x_val) # outlier scores\n folds_scores.append(pd.Series(name=\"score\", data=val_scores, index=x_val.index))\n return pd.concat(folds_scores, axis=0)[x.index]\n\n def transform(self, x: pd.DataFrame):\n \"\"\"\n Predict anomaly scores for the provided inputs.\n This method uses average of scores produced by all the detectors trained on folds.\n\n Parameters\n ----------\n x: pd.DataFrame\n data to score; must be already pre-processed; numeric columns must have NaNs filled\n\n Returns\n -------\n anomaly scores for the passed data\n \"\"\"\n assert self._detectors is not None, \"Detector is not fit - call `fit_transform` before calling `transform`\"\n\n folds_scores = []\n for detector in self._detectors:\n with _suod_silent_print(self.silent):\n y_test_scores = detector.decision_function(x[self.original_features])\n folds_scores.append(pd.DataFrame({\"score\": y_test_scores}, index=x.index))\n score = pd.concat([df.score for df in folds_scores], axis=1).mean(axis=1)\n score.name = \"score\"\n\n return score[x.index]\n\n def predict(self, x):\n \"\"\"\n API-compatibility wrapper for :py:meth:`~autogluon.eda.analysis.anomaly.AnomalyDetector.transform`\n \"\"\"\n return self.transform(x)\n\n\nclass AnomalyDetectorAnalysis(AbstractAnalysis):\n \"\"\"\n Anomaly detection analysis.\n\n The analysis automatically creates cross-validation splits and fits detectors on each of them using\n `train_data` input. The scores produced for the training data are produced using out-of-folds predictions.\n All other datasets scores are produced using average of scores from detectors trained on individual folds (bag).\n\n Please note, the analysis expects the data is ready to for fitting; all numeric columns must not have NaNs.\n Pre-processing can be performed using :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n and :py:class:`~autogluon.eda.analysis.dataset.ProblemTypeControl` (see example for more details).\n\n State attributes\n\n - `anomaly_detection.scores.`\n scores for each of the datasets passed into analysis (i.e. `train_data`, `test_data`)\n - `anomaly_detection.explain_rows_fns.`\n if `store_explainability_data=True`, then analysis will store helper functions into this\n variable. The function can be used later via :py:meth:`~autogluon.eda.auto.simple.explain_rows`\n and automatically pre-populates `train_data`, `model` and `rows` parameters when called\n (see example for more details)\n\n\n Parameters\n ----------\n n_folds: int, default = 5\n number of folds to use when training detectors; default is 5 folds.\n store_explainability_data: bool, default = False\n if `True` analysis will store helper functions into this variable.\n The function can be used later via :py:meth:`~autogluon.eda.auto.simple.explain_rows`\n and automatically pre-populates `train_data`, `model` and `rows` parameters when called\n (see example for more details)\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: List[AbstractAnalysis], default = []\n wrapped analyses; these will receive sampled `args` during `fit` call\n state: Optional[AnalysisState], default = None\n state to be updated by this fit function\n anomaly_detector_kwargs\n kwargs for :py:class:`~autogluon.eda.analysis.anomaly.AnomalyDetector`\n\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.anomaly.AnomalyDetector`\n :py:class:`~autogluon.eda.visualization.anomaly.AnomalyScoresVisualization`\n :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n :py:class:`~autogluon.eda.analysis.dataset.ProblemTypeControl`\n\n \"\"\"\n\n def __init__(\n self,\n n_folds: int = 5,\n store_explainability_data: bool = False,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n **anomaly_detector_kwargs,\n ) -> None:\n super().__init__(parent, children, state, **anomaly_detector_kwargs)\n self.n_folds = n_folds\n self.store_explainability_data = store_explainability_data\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n args_present = self.all_keys_must_be_present(args, \"train_data\", \"label\")\n no_nans = True\n if args_present:\n for ds, df in self.available_datasets(args):\n cols_with_nas = [c for c in df.columns if df[c].dtype != \"object\" and df[c].hasnans]\n if len(cols_with_nas) > 0:\n self.logger.warning(\n f\"{ds}: NaNs are present in the following columns: {cols_with_nas};\"\n f\" please fill them before calling this method.\"\n )\n no_nans = False\n\n return args_present and no_nans\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n det = self._create_detector(args)\n scores = det.fit_transform(args.train_data)\n s = {\"scores\": {\"train_data\": scores}}\n if self.store_explainability_data:\n s[\"explain_rows_fns\"] = {\n \"train_data\": partial(AnomalyDetectorAnalysis.explain_rows_fn, args, det, \"train_data\")\n }\n\n for ds, df in self.available_datasets(args):\n if ds == \"train_data\":\n continue\n s[\"scores\"][ds] = det.transform(df)\n if self.store_explainability_data:\n s[\"explain_rows_fns\"][ds] = partial(AnomalyDetectorAnalysis.explain_rows_fn, args, det, ds)\n\n state[\"anomaly_detection\"] = s\n\n def _create_detector(self, args) -> AnomalyDetector:\n return AnomalyDetector(label=args.label, n_folds=self.n_folds, **self.args)\n\n @staticmethod\n def explain_rows_fn(args: AnalysisState, detector: AnomalyDetector, dataset: str, dataset_row_ids: List[Any]):\n \"\"\"\n Prepares arguments for :py:meth:`~autogluon.eda.auto.simple.explain_rows` call to explain anomaly scores contributions\n\n Parameters\n ----------\n args: AnalysisState,\n args from the analysis call (will be pre-populated)\n detector: AnomalyDetector\n detector to use for the prediction (will be pre-populated)\n dataset: str\n dataset to use (will be pre-populated)\n dataset_row_ids: List[any]\n list of row ids to explain from the specified `dataset`\n\n Returns\n -------\n Dict of arguments to pass into\n\n See Also\n --------\n :py:meth:`~autogluon.eda.auto.simple.explain_rows`\n\n\n \"\"\"\n missing_ids = [item for item in dataset_row_ids if item not in args[dataset].index]\n assert len(missing_ids) == 0, f\"The following ids are missing in `{dataset_row_ids}`: {missing_ids}\"\n logger.info(\n \"Please note that the feature values shown on the charts are transformed into an internal representation; \"\n \"they may be encoded or modified based on internal preprocessing. Refer to the original datasets for the actual feature values.\"\n )\n if dataset == \"train_data\":\n logger.warning(\n \"Warning: The `train_data` dataset is used for explanation. The detector has seen the data, and estimates may be overly optimistic. \"\n \"Although the anomaly score in the explanation might not match, the magnitude of the features can still be utilized to \"\n \"evaluate the impact of the feature on the anomaly score.\"\n )\n return dict(\n train_data=args.train_data,\n model=detector,\n rows=args[dataset].loc[dataset_row_ids],\n )\n"
},
{
"path": "eda/src/autogluon/eda/analysis/base.py",
"content": "from __future__ import annotations\n\nimport logging\nfrom abc import ABC, abstractmethod\nfrom typing import Dict, Generator, List, Optional, Tuple\n\nfrom pandas import DataFrame\n\nfrom ..state import AnalysisState, StateCheckMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass AbstractAnalysis(ABC, StateCheckMixin):\n \"\"\"\n Base class for analysis functionality.\n\n Provides basic functionality for state/args management in analysis hierarchy\n and helper method to access frequently-used methods.\n\n Analyses can be nested; the hierarchical relationships can be navigated via `parent` and `children` properties.\n\n The main entry method of analysis is `fit` function. This `_fit` method is designed to be overridden\n by the component developer and should encapsulate all the outputs into `state` object provided.\n When called, the execution flow is the following:\n - gather `args` from the parent levels of analysis hierarchy; this is done to avoid referencing same args on each\n nested component (i.e. `train_data` can be specified at the top and all the children will be able to access it\n via `args` on all levels (unless overridden by one of the components in the hierarchy)\n - call `_fit` function for each component that returned `True` from `can_handle` call\n\n Please note: `state` is shared across the whole analysis hierarchy. If two components change the same space, then\n it will be overridden by each consecutive update. If same component have to be reused and requires writing different\n outputs, please use :py:class:`~autogluon.eda.analysis.base.Namespace` wrapper to isolate the components.\n\n Parameters\n ----------\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n state: Optional[AnalysisState], default = None\n state object to perform check on; if not provided a new state will be created during the `fit` call\n kwargs\n arguments to pass into the component\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.base.Namespace`\n\n \"\"\"\n\n def __init__(\n self,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n **kwargs,\n ) -> None:\n self.parent = parent\n self.children: List[AbstractAnalysis] = [] if children is None else children\n self.state: Optional[AnalysisState] = state\n for c in self.children:\n c.parent = self\n c.state = self.state\n self.args = kwargs\n\n def _gather_args(self) -> AnalysisState:\n chain = [self]\n while chain[0].parent is not None:\n chain.insert(0, chain[0].parent)\n args = AnalysisState()\n for node in chain:\n args = AnalysisState({**args, **node.args})\n return args\n\n @staticmethod\n def available_datasets(args: AnalysisState) -> Generator[Tuple[str, DataFrame], None, None]:\n \"\"\"\n Generator which iterates only through the datasets provided in arguments\n\n Parameters\n ----------\n args: AnalysisState\n arguments passed into the call. These are different from `self.args` in a way that it's arguments assembled from the\n parents and shadowed via children (allows to isolate reused parameters in upper arguments declarations.\n\n Returns\n -------\n tuple of dataset name (train_data, test_data or tuning_data) and dataset itself\n\n \"\"\"\n for ds in [\"train_data\", \"test_data\", \"tuning_data\", \"val_data\"]:\n if ds in args and args[ds] is not None:\n df: DataFrame = args[ds]\n yield ds, df\n\n def _get_state_from_parent(self) -> AnalysisState:\n state = self.state\n if state is None:\n if self.parent is None:\n state = AnalysisState()\n else:\n state = self.parent.state\n return state # type: ignore\n\n @abstractmethod\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n \"\"\"\n Checks if state and args has all the required parameters for fitting.\n See also :func:`at_least_one_key_must_be_present` and :func:`all_keys_must_be_present` helpers\n to construct more complex logic.\n\n Parameters\n ----------\n state: AnalysisState\n state to be updated by this fit function\n args: AnalysisState\n analysis properties assembled from root of analysis hierarchy to this component (with lower levels shadowing upper level args).\n\n Returns\n -------\n `True` if all the pre-requisites for fitting are present\n\n \"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n \"\"\"\n @override\n Component-specific internal processing.\n This method is designed to be overridden by the component developer\n\n Parameters\n ----------\n state: AnalysisState\n state to be updated by this fit function\n args: AnalysisState\n analysis properties assembled from root of analysis hierarchy to this component (with lower levels shadowing upper level args).\n fit_kwargs\n arguments passed into fit call\n \"\"\"\n raise NotImplementedError\n\n def fit(self, **kwargs) -> AnalysisState:\n \"\"\"\n Fit the analysis tree.\n\n Parameters\n ----------\n kwargs\n fit arguments\n\n Returns\n -------\n state produced by fit\n\n \"\"\"\n self.state = self._get_state_from_parent()\n if self.parent is not None:\n assert (\n self.state is not None\n ), \"Inner analysis fit() is called while parent has no state. Please call top-level analysis fit instead\"\n _args = self._gather_args()\n if self.can_handle(self.state, _args):\n self._fit(self.state, _args, **kwargs)\n for c in self.children:\n c.fit(**kwargs)\n return self.state\n\n\nclass BaseAnalysis(AbstractAnalysis):\n \"\"\"\n Simple implementation of :py:class:`~autogluon.eda.analysis.base.AbstractAnalysis`\n\n Parameters\n ----------\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.base.AbstractAnalysis`\n\n \"\"\"\n\n def __init__(\n self, parent: Optional[AbstractAnalysis] = None, children: Optional[List[AbstractAnalysis]] = None, **kwargs\n ) -> None:\n super().__init__(parent, children, **kwargs)\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n pass\n\n\nclass Namespace(AbstractAnalysis):\n \"\"\"\n Creates a nested namespace in state. All the components within `children` will have relative root of the state moved into this subspace.\n To instruct visualization facets to use a specific subspace, please use `namespace` argument (see the example).\n\n Parameters\n ----------\n namespace: Optional[str], default = None\n namespace to use; use root if not specified\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> auto.analyze(\n >>> train_data=..., label=...,\n >>> anlz_facets=[\n >>> # Puts output into the root namespace\n >>> eda.interaction.Correlation(),\n >>> # Puts output into the focus namespace\n >>> eda.Namespace(namespace='focus', children=[\n >>> eda.interaction.Correlation(focus_field='Fare', focus_field_threshold=0.3),\n >>> ])\n >>> ],\n >>> viz_facets=[\n >>> # Renders correlations from the root namespace\n >>> viz.interaction.CorrelationVisualization(),\n >>> # Renders correlations from the focus namespace\n >>> viz.interaction.CorrelationVisualization(namespace='focus'),\n >>> ]\n >>> )\n\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def __init__(\n self,\n namespace: Optional[str] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n self.namespace = namespace\n\n def fit(self, **kwargs) -> AnalysisState:\n assert (\n self.parent is not None\n ), \"Namespace must be wrapped into other analysis. You can use BaseAnalysis of one is needed\"\n return super().fit(**kwargs)\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n pass\n\n def _get_state_from_parent(self) -> AnalysisState:\n state = super()._get_state_from_parent()\n if self.namespace not in state:\n state[self.namespace] = {}\n return state[self.namespace]\n\n\nclass SaveArgsToState(AbstractAnalysis):\n \"\"\"\n Helper to copy parameters from args to state.\n\n Can be helpful if some other transformation were performed on the args (i.e. feature generator)\n and the modified args has to be stored for other purposes\n\n Parameters\n ----------\n params_mapping: Dict[str, str]\n mapping between args and state keys to be copied. I.e. `{'a': 'b'}` means `state.b = args.a`\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n state: AnalysisState\n state to be updated by this fit function\n kwargs\n \"\"\"\n\n def __init__(\n self,\n params_mapping: Dict[str, str],\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, state, **kwargs)\n self.params_mapping = params_mapping\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, *self.params_mapping.keys())\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n for key_args, key_state in self.params_mapping.items():\n state[key_state] = args[key_args]\n"
},
{
"path": "eda/src/autogluon/eda/analysis/dataset.py",
"content": "from __future__ import annotations\n\nfrom typing import Any, Dict, List, Optional, Set, Union\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.infer_types import get_type_group_map_special, get_type_map_raw\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT, R_OBJECT\n\nfrom ..state import AnalysisState\nfrom .base import AbstractAnalysis\n\n__all__ = [\n \"DatasetSummary\",\n \"RawTypesAnalysis\",\n \"Sampler\",\n \"SpecialTypesAnalysis\",\n \"VariableTypeAnalysis\",\n \"TrainValidationSplit\",\n \"ProblemTypeControl\",\n \"LabelInsightsAnalysis\",\n]\n\nfrom autogluon.core.constants import (\n BINARY,\n MULTICLASS,\n PROBLEM_TYPES_CLASSIFICATION,\n PROBLEM_TYPES_REGRESSION,\n REGRESSION,\n)\nfrom autogluon.core.utils import generate_train_test_split_combined, infer_problem_type\n\n\nclass Sampler(AbstractAnalysis):\n \"\"\"\n Sampler is a wrapper that provides sampling capabilities for the wrapped analyses.\n The sampling is performed for all datasets in `args` and passed to all `children` during `fit` call shadowing outer parameters.\n\n Parameters\n ----------\n sample: Union[None, int, float], default = None\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n\n Examples\n --------\n >>> from autogluon.eda.analysis.base import BaseAnalysis\n >>> from autogluon.eda.analysis import Sampler\n >>> import pandas as pd\n >>> import numpy as np\n >>>\n >>> df_train = pd.DataFrame(np.random.randint(0, 100, size=(10, 4)), columns=list('ABCD'))\n >>> df_test = pd.DataFrame(np.random.randint(0, 100, size=(20, 4)), columns=list('EFGH'))\n >>> analysis = BaseAnalysis(train_data=df_train, test_data=df_test, children=[\n >>> Sampler(sample=5, children=[\n >>> # Analysis here will be performed on a sample of 5 for both train_data and test_data\n >>> ])\n >>> ])\n \"\"\"\n\n def __init__(\n self,\n sample: Union[None, int, float] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n if sample is not None and isinstance(sample, float):\n assert 0.0 < sample < 1.0, \"sample must be within the range (0.0, 1.0)\"\n self.sample = sample\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n if self.sample is not None:\n for ds, df in self.available_datasets(args):\n arg = \"n\"\n if self.sample is not None and isinstance(self.sample, float):\n arg = \"frac\"\n if len(df) > self.sample:\n df = df.sample(**{arg: self.sample}, random_state=0)\n if state.sample_size is None:\n state.sample_size = {}\n state.sample_size[ds] = self.sample\n self.args[ds] = df\n\n\nclass ProblemTypeControl(AbstractAnalysis):\n \"\"\"\n Helper component to control problem type. Autodetect if `problem_type = 'auto'`.\n\n Parameters\n ----------\n problem_type: str, default = 'auto'\n problem type to use. Valid problem_type values include ['auto', 'binary', 'multiclass', 'regression', 'quantile', 'softclass']\n auto means it will be Auto-detected using AutoGluon methods.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n \"\"\"\n\n def __init__(\n self,\n problem_type: str = \"auto\",\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n valid_problem_types = [\"auto\"] + PROBLEM_TYPES_REGRESSION + PROBLEM_TYPES_CLASSIFICATION\n assert problem_type in valid_problem_types, f\"Valid problem_type values include {valid_problem_types}\"\n self.problem_type = problem_type\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"label\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n if self.problem_type == \"auto\":\n state.problem_type = infer_problem_type(args.train_data[args.label], silent=True)\n else:\n state.problem_type = self.problem_type\n\n\nclass TrainValidationSplit(AbstractAnalysis):\n \"\"\"\n This wrapper splits `train_data` into training and validation sets stored in `train_data` and `val_data` for the wrapped analyses.\n The split is performed for datasets in `args` and passed to all `children` during `fit` call shadowing outer parameters.\n\n This component requires :py:class:`~autogluon.eda.visualization.dataset.ProblemTypeControl` present in the analysis call to set `problem_type`.\n\n Parameters\n ----------\n val_size: float, default = 0.3\n fraction of training set to be assigned as validation set during the split.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n Examples\n --------\n >>> from autogluon.eda.analysis.base import BaseAnalysis\n >>> from autogluon.eda.analysis import Sampler\n >>> import pandas as pd\n >>> import numpy as np\n >>>\n >>> df_train = pd.DataFrame(np.random.randint(0, 100, size=(100, 4)), columns=list(\"ABCD\"))\n >>> analysis = BaseAnalysis(train_data=df_train, label=\"D\", children=[\n >>> Namespace(namespace=\"ns_val_split_specified\", children=[\n >>> ProblemTypeControl(),\n >>> TrainValidationSplit(val_pct=0.4, children=[\n >>> # This analysis sees 60/40 split of df_train between train_data and val_data\n >>> SomeAnalysis()\n >>> ])\n >>> ]),\n >>> Namespace(namespace=\"ns_val_split_default\", children=[\n >>> ProblemTypeControl(),\n >>> TrainValidationSplit(children=[\n >>> # This analysis sees 70/30 split (default) of df_train between train_data and val_data\n >>> SomeAnalysis()\n >>> ])\n >>> ]),\n >>> Namespace(namespace=\"ns_no_split\", children=[\n >>> # This analysis sees only original train_data\n >>> SomeAnalysis()\n >>> ]),\n >>> ],\n >>> )\n >>>\n >>> state = analysis.fit()\n >>>\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.dataset.ProblemTypeControl`\n\n \"\"\"\n\n def __init__(\n self,\n val_size: float = 0.3,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n\n assert 0 < val_size < 1.0, \"val_size must be between 0 and 1\"\n self.val_size = val_size\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"problem_type\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n train_data, val_data = generate_train_test_split_combined(\n args.train_data, args.label, state.problem_type, test_size=self.val_size, **self.args\n )\n self.args[\"train_data\"] = train_data\n self.args[\"val_data\"] = val_data\n\n\nclass DatasetSummary(AbstractAnalysis):\n \"\"\"\n Generates dataset summary including counts, number of unique elements, most frequent, dtypes and 7-figure summary (std/mean/min/max/quartiles)\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.DatasetSummary(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n s = {}\n for ds, df in self.available_datasets(args):\n summary = df.describe(include=\"all\").T\n summary = summary.join(pd.DataFrame({\"dtypes\": df.dtypes}))\n summary[\"unique\"] = args[ds].nunique()\n summary[\"count\"] = summary[\"count\"].astype(int)\n summary = summary.sort_index()\n s[ds] = summary.to_dict()\n state.dataset_stats = s\n\n\nclass RawTypesAnalysis(AbstractAnalysis):\n \"\"\"\n Infers autogluon raw types for the column.\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.RawTypesAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.raw_type = {}\n for ds, df in self.available_datasets(args):\n state.raw_type[ds] = get_type_map_raw(df)\n\n\nclass VariableTypeAnalysis(AbstractAnalysis):\n \"\"\"\n Infers variable types for the column: numeric vs category.\n\n This analysis depends on :func:`RawTypesAnalysis`.\n\n Parameters\n ----------\n numeric_as_categorical_threshold: int, default = 20\n if numeric column has less than this value, then the variable should be considered as categorical\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.RawTypesAnalysis(),\n >>> eda.dataset.VariableTypeAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis`\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n \"\"\"\n\n def __init__(\n self,\n parent: Union[None, AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n numeric_as_categorical_threshold: int = 20,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n self.numeric_as_categorical_threshold = numeric_as_categorical_threshold\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"raw_type\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.variable_type = {}\n for ds, df in self.available_datasets(args):\n state.variable_type[ds] = {\n c: self.map_raw_type_to_feature_type(c, t, df, self.numeric_as_categorical_threshold)\n for c, t in state.raw_type[ds].items()\n }\n\n @staticmethod\n def map_raw_type_to_feature_type(\n col: Optional[str], raw_type: str, df: pd.DataFrame, numeric_as_categorical_threshold: int = 20\n ) -> Union[None, str]:\n if col is None:\n return None\n elif df[col].nunique() <= numeric_as_categorical_threshold:\n return \"category\"\n elif raw_type in [R_INT, R_FLOAT]:\n return \"numeric\"\n elif raw_type in [R_OBJECT, R_CATEGORY, R_BOOL]:\n return \"category\"\n else:\n return None\n\n\nclass SpecialTypesAnalysis(AbstractAnalysis):\n \"\"\"\n Infers autogluon special types for the column (i.e. text).\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.SpecialTypesAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.special_types = {}\n for ds, df in self.available_datasets(args):\n state.special_types[ds] = self.infer_special_types(df)\n\n @staticmethod\n def infer_special_types(ds):\n special_types: Dict[str, Set[str]] = {}\n for t, cols in get_type_group_map_special(ds).items():\n for col in cols:\n if col not in special_types:\n special_types[col] = set()\n special_types[col].add(t)\n result: Dict[str, str] = {}\n for col, types in special_types.items():\n result[col] = \", \".join(sorted(types))\n return result\n\n\nclass LabelInsightsAnalysis(AbstractAnalysis):\n \"\"\"\n Analyze label for insights:\n\n - classification: low cardinality classes detection\n - classification: classes present in test data, but not in the train data\n - regression: out-of-domain labels detection\n\n Note: this Analysis requires `problem_type` present in state.\n It can be detected/set via :py:class:`~autogluon.eda.analysis.dataset.ProblemTypeControl` component\n\n Parameters\n ----------\n low_cardinality_classes_threshold: int, default = 50\n Minimum class instances present in the dataset to consider marking a class as low-cardinality\n regression_ood_threshold: float, default = 0.01\n mark results as out-of-domain when test label range in regression task is beyond train data range + regression_ood_threshold margin,\n This is performed because some algorithms can't extrapolate beyond training data.\n class_imbalance_ratio_threshold: float, default = 0.4\n minority class proportion to detect as imbalance.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n state: AnalysisState\n state object to perform check on\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> auto.analyze(\n >>> auto.analyze(train_data=..., test_data=..., label=..., anlz_facets=[\n >>> eda.dataset.ProblemTypeControl(),\n >>> eda.dataset.LabelInsightsAnalysis(low_cardinality_classes_threshold=50, regression_ood_threshold=0.01),\n >>> ], viz_facets=[\n >>> viz.dataset.LabelInsightsVisualization()\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.ProblemTypeControl`\n :py:class:`~autogluon.eda.visualization.dataset.LabelInsightsVisualization`\n\n \"\"\"\n\n def __init__(\n self,\n low_cardinality_classes_threshold: int = 50,\n regression_ood_threshold: float = 0.01,\n class_imbalance_ratio_threshold: float = 0.4,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, state, **kwargs)\n assert low_cardinality_classes_threshold > 0, \"low_cardinality_classes_threshold must be greater than 0\"\n self.low_cardinality_classes_threshold = low_cardinality_classes_threshold\n\n assert 0 < class_imbalance_ratio_threshold < 1, \"class_imbalance_ratio_threshold must be between 0 and 1\"\n self.class_imbalance_ratio_threshold = class_imbalance_ratio_threshold\n\n assert 0 < regression_ood_threshold < 1, \"regression_ood_threshold must be between 0 and 1\"\n self.regression_ood_threshold = regression_ood_threshold\n\n assert regression_ood_threshold >= 0, \"regression_ood_threshold must be non-negative\"\n self.regression_ood_threshold = regression_ood_threshold\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"label\") and self.all_keys_must_be_present(\n state, \"problem_type\"\n )\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n label = args.label\n train_data = args.train_data\n\n s: Dict[str, Any] = {}\n\n if state.problem_type in [BINARY, MULTICLASS]:\n label_counts = train_data[label].value_counts()\n minority_class = label_counts[label_counts == label_counts.min()].index.values[0]\n majority_class = label_counts[label_counts == label_counts.max()].index.values[0]\n minority_class_imbalance_ratio = min(label_counts) / max(label_counts)\n\n # Low-cardinality class detection\n label_counts = label_counts[label_counts < self.low_cardinality_classes_threshold].to_dict()\n if len(label_counts) > 0:\n s[\"low_cardinality_classes\"] = {\n \"instances\": label_counts,\n \"threshold\": self.low_cardinality_classes_threshold,\n }\n\n # Class imbalance detection\n if minority_class_imbalance_ratio < self.class_imbalance_ratio_threshold:\n s[\"minority_class_imbalance\"] = {\n \"majority_class\": majority_class,\n \"minority_class\": minority_class,\n \"ratio\": minority_class_imbalance_ratio,\n }\n\n # Classes not found in test_data\n if self._test_data_with_label_present(args, label):\n train_labels = set(train_data[label].unique())\n test_labels = set(args.test_data[label].unique())\n if sorted(train_labels) != sorted(test_labels):\n missing_classes = test_labels.difference(train_labels)\n s[\"not_present_in_train\"] = missing_classes\n elif (state.problem_type in [REGRESSION]) and self._test_data_with_label_present(args, label):\n # Out-of-domain range detection\n test_data = args.test_data\n label_min, label_max = np.min(train_data[label]), np.max(train_data[label])\n padding = np.abs(label_max - label_min) * self.regression_ood_threshold\n df_ood = args.test_data[\n (test_data[label] < label_min - padding) | (test_data[label] > label_max + padding)\n ]\n\n if len(df_ood) > 0:\n s[\"ood\"] = {\n \"count\": len(df_ood),\n \"train_range\": [label_min, label_max],\n \"test_range\": [np.min(test_data[label]), np.max(test_data[label])],\n \"threshold\": self.regression_ood_threshold,\n }\n if len(s) > 0:\n state.label_insights = s\n\n def _test_data_with_label_present(self, args, label):\n return (args.test_data is not None) and (label in args.test_data.columns)\n"
},
{
"path": "eda/src/autogluon/eda/analysis/explain.py",
"content": "import logging\nimport warnings\nfrom typing import List, Optional\n\nimport numpy as np\nimport pandas as pd\nimport shap\n\nfrom autogluon.core.constants import BINARY, REGRESSION\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis.base import AbstractAnalysis\n\n__all__ = [\"ShapAnalysis\"]\n\nlogger = logging.getLogger(__name__)\n\n\nclass _ShapAutoGluonWrapper:\n def __init__(self, predictor, feature_names, target_class=None):\n self.ag_model = predictor\n self.feature_names = feature_names\n self.target_class = target_class\n if target_class is None and predictor.problem_type != REGRESSION:\n logging.warning(\"Since target_class not specified, SHAP will explain predictions for each class\")\n\n def predict_proba(self, X):\n if isinstance(X, pd.Series):\n X = X.values.reshape(1, -1)\n if not isinstance(X, pd.DataFrame):\n X = pd.DataFrame(X, columns=self.feature_names)\n if self.ag_model.problem_type == REGRESSION:\n preds = self.ag_model.predict(X)\n else:\n preds = self.ag_model.predict_proba(X)\n if self.ag_model.problem_type == REGRESSION or self.target_class is None:\n return preds\n else:\n return preds[self.target_class]\n\n\nclass ShapAnalysis(AbstractAnalysis):\n \"\"\"\n Perform Shapley values calculation using `shap` package for the given rows.\n\n Parameters\n ----------\n rows: pd.DataFrame,\n rows to explain\n baseline_sample: int, default = 100\n The background dataset size to use for integrating out features. To determine the impact\n of a feature, that feature is set to \"missing\" and the change in the model output\n is observed.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: List[AbstractAnalysis], default = []\n wrapped analyses; these will receive sampled `args` during `fit` call\n state: AnalysisState\n state to be updated by this fit function\n random_state: int, default = 0\n random state for sampling\n kwargs\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> auto.analyze(\n >>> train_data=..., model=...,\n >>> anlz_facets=[\n >>> eda.explain.ShapAnalysis(rows, baseline_sample=200),\n >>> ],\n >>> viz_facets=[\n >>> # Visualize the given SHAP values with an additive force layout\n >>> viz.explain.ExplainForcePlot(),\n >>> # Visualize the given SHAP values with a waterfall layout\n >>> viz.explain.ExplainWaterfallPlot(),\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~shap.KernelExplainer`\n :py:class:`~autogluon.eda.visualization.explain.ExplainForcePlot`\n :py:class:`~autogluon.eda.visualization.explain.ExplainWaterfallPlot`\n \"\"\"\n\n def __init__(\n self,\n rows: pd.DataFrame,\n baseline_sample: int = 100,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n random_state: int = 0,\n positive_class: Optional = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, state, **kwargs)\n self.rows = rows\n self.baseline_sample = baseline_sample\n self.random_state = random_state\n self.positive_class = positive_class\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"model\", \"train_data\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n if self.baseline_sample <= len(args.train_data):\n _baseline_sample = self.baseline_sample\n else:\n _baseline_sample = len(args.train_data)\n\n baseline = args.train_data.sample(_baseline_sample, random_state=self.random_state)\n shap_data = []\n for _, row in self.rows.iterrows():\n _row = pd.DataFrame([row])\n predicted_class = self.positive_class\n if self.positive_class is None:\n if args.model.problem_type == REGRESSION:\n predicted_class = None\n elif args.model.problem_type == BINARY:\n predicted_class = args.model.positive_class\n else:\n # use the last column in predict proba\n predicted_class = args.model.predict_proba(_row).columns[-1]\n state.positive_class = predicted_class\n ag_wrapper = _ShapAutoGluonWrapper(args.model, args.train_data.columns, predicted_class)\n explainer = shap.KernelExplainer(ag_wrapper.predict_proba, baseline)\n with warnings.catch_warnings():\n warnings.filterwarnings(\"ignore\")\n # Suppress sklearn pipeline warnings\n np.int = int # type: ignore[attr-defined] # workaround to address shap's use of old numpy APIs\n ke_shap_values = explainer.shap_values(_row[args.train_data.columns], silent=True)\n shap_data.append(\n AnalysisState(\n row=_row,\n expected_value=explainer.expected_value,\n shap_values=ke_shap_values[0],\n features=row[args.model.original_features],\n feature_names=None,\n )\n )\n state.explain = {\"shapley\": shap_data}\n"
},
{
"path": "eda/src/autogluon/eda/analysis/interaction.py",
"content": "import warnings\nfrom typing import Any, Dict, List, Optional, Union\n\nimport numpy as np\nimport pandas as pd\nimport phik # noqa - required for significance_matrix instrumentation on pandas dataframes\nfrom pandas.core.dtypes.common import is_numeric_dtype\nfrom scipy import stats\nfrom scipy.cluster import hierarchy as hc\nfrom scipy.stats import spearmanr\n\nfrom .. import AnalysisState\nfrom .base import AbstractAnalysis\n\n__all__ = [\n \"Correlation\",\n \"CorrelationSignificance\",\n \"FeatureInteraction\",\n \"DistributionFit\",\n \"FeatureDistanceAnalysis\",\n]\n\nfrom autogluon.common.features.types import R_FLOAT, R_INT\n\n\nclass Correlation(AbstractAnalysis):\n \"\"\"\n Correlation analysis.\n\n Note: it is recommended to apply AutoGluon standard pre-processing - this will allow to include categorical variables into the analysis.\n This can be done via wrapping analysis into :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n\n\n Parameters\n ----------\n method: str {'pearson', 'kendall', 'spearman', 'phik'}, default='spearman'\n Method of correlation:\n * pearson : standard correlation coefficient\n * kendall : Kendall Tau correlation coefficient\n * spearman : Spearman rank correlation\n * phik : phi_k correlation\n Correlation matrix of bivariate gaussian derived from chi2-value Chi2-value\n gets converted into correlation coefficient of bivariate gauss with correlation\n value rho, assuming given binning and number of records. Correlation coefficient\n value is between 0 and 1. Bivariate gaussian's range is set to [-5,5] by construction.\n See Also `phik `_ documentation.\n\n focus_field: Optional[str], default = None\n field name to focus. Specifying a field would filter all correlations only when they are >= `focus_field_threshold`\n This is helpful when dealing with a large number of variables.\n focus_field_threshold: float, default = 0.5\n a cut-off threshold when `focus_field` is specified\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: List[AbstractAnalysis], default = []\n wrapped analyses; these will receive sampled `args` during `fit` call\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> import pandas as pd\n >>> import numpy as np\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> auto.analyze(return_sttrain_data=df_train, label=target_col, anlz_facets=[\n >>> # Apply standard AutoGluon pre-processing to transform categorical variables to numbers to ensure correlation includes them.\n >>> eda.transform.ApplyFeatureGenerator(category_to_numbers=True, children=[\n >>> # We use `spearman` correlation to capture non-linear interactions because it is based on the order rank.\n >>> eda.interaction.Correlation(method='spearman', focus_field=target_col, focus_field_threshold=0.3),\n >>> ])\n >>> ], viz_facets=[\n >>> viz.interaction.CorrelationVisualization(fig_args=dict(figsize=(12,8)), **common_args),\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n\n \"\"\"\n\n def __init__(\n self,\n method: str = \"spearman\",\n focus_field: Optional[str] = None,\n focus_field_threshold: float = 0.5,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n assert method in [\"pearson\", \"kendall\", \"spearman\", \"phik\"]\n self.method = method\n self.focus_field = focus_field\n self.focus_field_threshold = focus_field_threshold\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.correlations = {}\n state.correlations_method = self.method\n for ds, df in self.available_datasets(args):\n if args.label in df.columns and df[args.label].dtype not in [R_INT, R_FLOAT]:\n df[args.label] = df[args.label].astype(\"category\").cat.codes\n\n if self.method == \"phik\":\n state.correlations[ds] = df.phik_matrix(**self.args, verbose=False)\n else:\n state.correlations[ds] = df.corr(method=self.method, numeric_only=True, **self.args)\n\n if self.focus_field is not None and self.focus_field in state.correlations[ds].columns:\n state.correlations_focus_field = self.focus_field\n state.correlations_focus_field_threshold = self.focus_field_threshold\n state.correlations_focus_high_corr = {}\n df_corr = state.correlations[ds]\n df_corr = df_corr[df_corr[self.focus_field].abs() >= self.focus_field_threshold]\n keep_cols = df_corr.index.values\n state.correlations[ds] = df_corr[keep_cols]\n\n high_corr = (\n state.correlations[ds][[self.focus_field]]\n .sort_values(self.focus_field, ascending=False)\n .drop(self.focus_field)\n )\n state.correlations_focus_high_corr[ds] = high_corr\n\n\nclass CorrelationSignificance(AbstractAnalysis):\n \"\"\"\n Significance of correlation of all variable combinations in the DataFrame.\n\n See :py:meth:`~phik.significance.significance_matrix` for more details.\n This analysis requires :py:class:`~autogluon.eda.analysis.interaction.Correlation` results to be\n available in the state.\n\n Note: it is recommended to apply AutoGluon standard pre-processing - this will allow to include categorical variables into the analysis.\n This can be done via wrapping analysis into :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> import pandas as pd\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> auto.analyze(return_sttrain_data=df_train, label=target_col, anlz_facets=[\n >>> # Apply standard AutoGluon pre-processing to transform categorical variables to numbers to ensure correlation includes them.\n >>> eda.transform.ApplyFeatureGenerator(category_to_numbers=True, children=[\n >>> # We use `spearman` correlation to capture non-linear interactions because it is based on the order rank.\n >>> eda.interaction.Correlation(method='spearman', focus_field=target_col, focus_field_threshold=0.3),\n >>> eda.interaction.CorrelationSignificance()\n >>> ])\n >>> ], viz_facets=[\n >>> viz.interaction.CorrelationSignificanceVisualization(fig_args=dict(figsize=(12,8))),\n >>> ])\n\n See Also\n --------\n :py:meth:`~phik.significance.significance_matrix`\n :py:class:`~autogluon.eda.analysis.interaction.Correlation`\n :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"correlations\", \"correlations_method\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.significance_matrix = {}\n for ds, df in self.available_datasets(args):\n state.significance_matrix[ds] = df[state.correlations[ds].columns].significance_matrix(\n **self.args, verbose=False\n )\n\n\nclass FeatureInteraction(AbstractAnalysis):\n \"\"\"\n Feature interaction analysis\n\n Parameters\n ----------\n x: Optional[str], default = None\n variable to analyse which would be placed on x-axis\n y: Optional[str], default = None\n variable to analyse which would be placed on y-axis\n hue: Optional[str], default = None\n variable to use as hue in x/y-analysis.\n key: Optional[str], default = None\n key to use to store the analysis in the state; the value is later to be used by FeatureInteractionVisualization.\n If the key is not provided, then use one of theform: 'x:A|y:B|hue:C' (omit corresponding x/y/hue if the value not provided)\n See also :py:class:`~autogluon.eda.visualization.interaction.FeatureInteractionVisualization`\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n Examples\n --------\n >>> import pandas as pd\n >>> import numpy as np\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> state = auto.analyze(\n >>> train_data=df_train, label='Survived',\n >>> anlz_facets=[\n >>> eda.dataset.RawTypesAnalysis(),\n >>> eda.interaction.FeatureInteraction(key='target_col', x='Survived'),\n >>> eda.interaction.FeatureInteraction(key='target_col_vs_age', x='Survived', y='Age')\n >>> ],\n >>> viz_facets=[\n >>> # Bar Plot with counts per each of the values in Survived\n >>> viz.interaction.FeatureInteractionVisualization(key='target_col', headers=True),\n >>> # Box Plot Survived vs Age\n >>> viz.interaction.FeatureInteractionVisualization(key='target_col_vs_age', headers=True),\n >>> ]\n >>> )\n >>>\n >>> # Simplified shortcut for interactions: scatter plot of Fare vs Age colored based on Survived values.\n >>> auto.analyze_interaction(x='Fare', y='Age', hue='Survived', train_data=df_train)\n \"\"\"\n\n def __init__(\n self,\n x: Optional[str] = None,\n y: Optional[str] = None,\n hue: Optional[str] = None,\n key: Optional[str] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n self.x = x\n self.y = y\n self.hue = hue\n self.key = key\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"raw_type\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs):\n cols = {\n \"x\": self.x,\n \"y\": self.y,\n \"hue\": self.hue,\n }\n\n self.key = self._generate_key_if_not_provided(self.key, cols)\n\n cols = {k: v for k, v in cols.items() if v is not None}\n\n interactions: Dict[str, Dict[str, Any]] = state.get(\"interactions\", {})\n for ds, df in self.available_datasets(args):\n missing_cols = [c for c in cols.values() if c not in df.columns]\n if len(missing_cols) == 0:\n df = df[cols.values()]\n interaction = {\n \"features\": cols,\n \"data\": df,\n }\n if ds not in interactions:\n interactions[ds] = {}\n interactions[ds][self.key] = interaction\n state.interactions = interactions\n\n def _generate_key_if_not_provided(self, key: Optional[str], cols: Dict[str, Optional[str]]) -> str:\n # if key is not provided, then convert to form: 'x:A|y:B|hue:C'; if values is not provided, then skip the value\n if key is None:\n key_parts = []\n for k, v in cols.items():\n if v is not None:\n key_parts.append(f\"{k}:{v}\")\n key = \"|\".join(key_parts)\n return key\n\n\nclass DistributionFit(AbstractAnalysis):\n \"\"\"\n This component attempts to fit various distributions for further plotting via\n :py:class:`~autogluon.eda.visualization.interaction.FeatureInteractionVisualization`.\n\n The data specified in `columns` must be numeric to be considered for fitting (categorical variables are not supported).\n\n Only the distributions with statistical significance above `pvalue_min` threshold will be included in the results.\n\n Note: this analysis is an augmentation for :py:class:`~autogluon.eda.analysis.interaction.FeatureInteraction` and should be used in pair\n to be visualized via :py:class:`~autogluon.eda.visualization.interaction.FeatureInteractionVisualization`.\n\n Parameters\n ----------\n columns: Union[str, List[str]]\n column to be included into analysis. Can be passed as a string or a list of strings.\n pvalue_min: float = 0.01,\n min pvalue to consider including distribution fit in the results.\n keep_top_n: Optional[int] = None,\n how many distributions exceeding `pvalue_min` to include in the results. I.e. if `keep_top_n=3`,\n but 10 distributions satisfied `pvalue_min`, only top 3 will be included.\n If not specified and `distributions_to_fit` is not provided, then only top 3 will be included in the results.\n distributions_to_fit: Optional[Union[str, List[str]]] = None,\n list of distributions to fit. See `DistributionFit.AVAILABLE_DISTRIBUTIONS` for the list of supported values.\n See `scipy `_ documentation for each distribution details.\n If not specified, then all supported distributions will be attempted to fit.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> import pandas as pd\n >>> import numpy as np\n >>>\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> auto.analyze(\n >>> train_data=df_train, label=target_col,\n >>> anlz_facets=[\n >>> eda.dataset.RawTypesAnalysis(),\n >>> eda.interaction.DistributionFit(columns=['Fare', 'Age'], distributions_to_fit=['lognorm', 'beta', 'gamma', 'fisk']),\n >>> eda.interaction.FeatureInteraction(key='age-chart', x='Age'),\n >>>\n >>> ],\n >>> viz_facets=[\n >>> viz.interaction.FeatureInteractionVisualization(key='age-chart', headers=True),\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.interaction.FeatureInteraction`\n :py:class:`~autogluon.eda.visualization.interaction.FeatureInteractionVisualization`\n\n \"\"\"\n\n # Getting the list of distributions: https://docs.scipy.org/doc/scipy/tutorial/stats.html#getting-help\n AVAILABLE_DISTRIBUTIONS = sorted(\n [\n dist\n for dist in dir(stats)\n if isinstance(getattr(stats, dist), stats.rv_continuous)\n # kstwo can't be fit on a single variable\n # levy_stable, studentized_range are too slow\n and dist not in [\"kstwo\", \"levy_stable\", \"studentized_range\"]\n ]\n )\n\n def __init__(\n self,\n columns: Union[str, List[str]],\n pvalue_min: float = 0.01,\n keep_top_n: Optional[int] = None,\n distributions_to_fit: Optional[Union[str, List[str]]] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n\n if keep_top_n is None and distributions_to_fit is None:\n keep_top_n = 3\n\n if isinstance(columns, str):\n columns = [columns]\n self.columns = columns\n\n self.pvalue_min = pvalue_min\n self.keep_top_n = keep_top_n\n\n if distributions_to_fit is None:\n distributions_to_fit = self.AVAILABLE_DISTRIBUTIONS\n if isinstance(distributions_to_fit, str):\n distributions_to_fit = [distributions_to_fit]\n not_supported = [d for d in distributions_to_fit if d not in self.AVAILABLE_DISTRIBUTIONS]\n if len(not_supported) > 0:\n raise ValueError(\n f\"The following distributions are not supported: {sorted(not_supported)}. \"\n f\"Supported distributions are {sorted(self.AVAILABLE_DISTRIBUTIONS)}\"\n )\n self.distributions_to_fit = distributions_to_fit\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.distributions_fit = {}\n state.distributions_fit_pvalue_min = self.pvalue_min\n for ds, df in self.available_datasets(args):\n state.distributions_fit[ds] = {}\n for c in self.columns:\n if c in df.columns:\n col = df[c]\n col = col[col.notna()] # skip NaNs\n dist = self._fit_dist(col, self.pvalue_min)\n if dist is not None:\n state.distributions_fit[ds][c] = dist\n\n def _fit_dist(self, series, pvalue_min=0.01):\n results = {}\n if not is_numeric_dtype(series):\n self.logger.warning(f\"{series.name}: distribution cannot be fit; only numeric columns are supported\")\n return None\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n for i in self.distributions_to_fit:\n dist = getattr(stats, i)\n param = dist.fit(series)\n statistic, pvalue = stats.kstest(series, i, args=param)\n if pvalue >= pvalue_min:\n results[i] = {\n \"param\": param,\n \"shapes\": self._list_parameters(dist),\n \"statistic\": statistic,\n \"pvalue\": pvalue,\n }\n if len(results) == 0:\n return None\n df = pd.DataFrame(results).T.sort_values(\"pvalue\", ascending=False)\n if self.keep_top_n is not None:\n df = df[: self.keep_top_n]\n results = df.T.to_dict()\n return results\n\n def _list_parameters(self, distribution):\n \"\"\"List parameters for scipy.stats.distribution.\n # Arguments\n distribution: a string or scipy.stats distribution object.\n # Returns\n A list of distribution parameter strings.\n \"\"\"\n if isinstance(distribution, str):\n distribution = getattr(stats, distribution)\n if distribution.shapes:\n parameters = [name.strip() for name in distribution.shapes.split(\",\")]\n else:\n parameters = []\n if distribution.name in stats._discrete_distns._distn_names:\n parameters += [\"loc\"]\n elif distribution.name in stats._continuous_distns._distn_names:\n parameters += [\"loc\", \"scale\"]\n return parameters\n\n\nclass FeatureDistanceAnalysis(AbstractAnalysis):\n \"\"\"\n The component performs feature correlation distance analysis using Spearman rank correlation and hierarchical clustering\n for the data passed in `train_data` excluding `label`.\n The near duplicates grouping is automatically suggested given `near_duplicates_threshold`.\n The results can be visualized using :py:class:`~autogluon.eda.visualization.interaction.FeatureDistanceAnalysisVisualization`.\n\n Note: it is recommended to apply :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator` before the analysis\n to ensure correlations are calculated for categorical variables.\n\n Parameters\n ----------\n near_duplicates_threshold: float, default = 0.01\n defines feature distance to be considered as near duplicates\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> import pandas as pd\n >>> import numpy as np\n >>>\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> auto.analyze(\n >>> train_data=df_train, label=target_col,\n >>> anlz_facets=[\n >>> eda.transform.ApplyFeatureGenerator(category_to_numbers=True, children=[\n >>> eda.interaction.FeatureDistanceAnalysis(near_duplicates_threshold=0.7),\n >>> ])\n >>> ],\n >>> viz_facets=[\n >>> viz.interaction.FeatureDistanceAnalysisVisualization(fig_args=dict(figsize=(12,6))),\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.transform.ApplyFeatureGenerator`\n :py:class:`~autogluon.eda.visualization.interaction.FeatureDistanceAnalysisVisualization`\n `Removing redundant features `_ section of\n Jeremy Howard's \"Deep Learning for Coders with Fastai and PyTorch\" book.\n\n \"\"\"\n\n def __init__(\n self,\n near_duplicates_threshold: float = 0.01,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n self.near_duplicates_threshold = near_duplicates_threshold\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"label\", \"feature_generator\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n x = args.train_data\n if args.label is not None:\n x = x.drop(labels=[args.label], axis=1)\n corr = np.round(spearmanr(x).correlation, 4)\n np.fill_diagonal(corr, 1)\n corr_condensed = hc.distance.squareform(1 - np.nan_to_num(corr))\n z = hc.linkage(corr_condensed, method=\"average\")\n columns = list(x.columns)\n s = {\n \"columns\": columns,\n \"linkage\": z,\n \"near_duplicates_threshold\": self.near_duplicates_threshold,\n \"near_duplicates\": self.__get_linkage_clusters(z, columns, self.near_duplicates_threshold),\n }\n state[\"feature_distance\"] = s\n\n @staticmethod\n def __get_linkage_clusters(linkage, columns, threshold: float):\n idx_to_col = {i: v for i, v in enumerate(columns)}\n idx_to_dist: Dict[int, float] = {}\n clusters: Dict[int, List[int]] = {}\n for (f1, f2, d, _l), i in zip(linkage, np.arange(len(idx_to_col), len(idx_to_col) + len(linkage))):\n idx_to_dist[i] = d\n f1 = int(f1)\n f2 = int(f2)\n if d <= threshold:\n clusters[i] = [*clusters.pop(f1, [f1]), *clusters.pop(f2, [f2])]\n\n results = []\n for i, nodes in clusters.items():\n d = idx_to_dist[i]\n nodes = [idx_to_col[n] for n in nodes]\n results.append(\n {\n \"nodes\": sorted(nodes),\n \"distance\": d,\n }\n )\n\n return results\n"
},
{
"path": "eda/src/autogluon/eda/analysis/missing.py",
"content": "from typing import Any, Dict\n\nfrom .base import AbstractAnalysis, AnalysisState\n\n__all__ = [\"MissingValuesAnalysis\"]\n\n\nclass MissingValuesAnalysis(AbstractAnalysis):\n \"\"\"\n Analyze dataset's missing value counts and frequencies\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=...,\n >>> anlz_facets=[\n >>> eda.missing.MissingValuesAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> viz.missing.MissingValues()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n :py:class:`~autogluon.eda.visualization.missing.MissingValues`\n \"\"\"\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n s: Dict[str, Any] = {}\n for ds, df in self.available_datasets(args):\n s[ds] = {\n \"count\": {},\n \"ratio\": {},\n }\n na = df.isna().sum()\n na = na[na > 0]\n s[ds][\"count\"] = na.to_dict()\n s[ds][\"ratio\"] = (na / len(df)).to_dict()\n s[ds][\"data\"] = df\n\n state.missing_statistics = s\n"
},
{
"path": "eda/src/autogluon/eda/analysis/model.py",
"content": "from typing import Any, Dict, List, Optional, Union\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import confusion_matrix\n\nfrom autogluon.core.constants import (\n BINARY,\n MULTICLASS,\n PROBLEM_TYPES_CLASSIFICATION,\n PROBLEM_TYPES_REGRESSION,\n REGRESSION,\n)\nfrom autogluon.tabular import TabularPredictor\n\nfrom .base import AbstractAnalysis, AnalysisState\n\n__all__ = [\"AutoGluonModelEvaluator\", \"AutoGluonModelQuickFit\"]\n\n\nclass AutoGluonModelQuickFit(AbstractAnalysis):\n \"\"\"\n Fit a quick model using AutoGluon.\n\n `train_data`, `val_data` and `label` must be present in args.\n\n Note: this component can be wrapped into :py:class:`~autogluon.eda.analysis.dataset.TrainValidationSplit` and `~autogluon.eda.analysis.dataset.Sampler`\n to perform automated sampling and train-test split. This whole logic is implemented in :py:meth:`~autogluon.eda.auto.simple.quick_fit` shortcut.\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>>\n >>> # Quick fit\n >>> state = auto.quick_fit(\n >>> train_data=..., label=...,\n >>> return_state=True, # return state object from call\n >>> hyperparameters={'GBM': {}} # train specific model\n >>> )\n >>>\n >>> # Using quick fit model\n >>> model = state.model\n >>> y_pred = model.predict(test_data)\n\n Parameters\n ----------\n problem_type: str, default = 'auto'\n problem type to use. Valid problem_type values include ['auto', 'binary', 'multiclass', 'regression', 'quantile', 'softclass']\n auto means it will be Auto-detected using AutoGluon methods.\n estimator_args: Optional[Dict[str, Any]], default = None,\n kwargs to pass into estimator constructor (`TabularPredictor`)\n save_model_to_state: bool, default = True,\n save fitted model into `state` under `model` key.\n This functionality might be helpful in cases when the fitted model could be usable for other purposes (i.e. imputers)\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n kwargs\n\n See Also\n --------\n :py:meth:`~autogluon.eda.auto.simple.quick_fit`\n :py:class:`~autogluon.eda.analysis.dataset.TrainValidationSplit`\n :py:class:`~autogluon.eda.analysis.dataset.Sampler`\n\n \"\"\"\n\n def __init__(\n self,\n problem_type: str = \"auto\",\n estimator_args: Optional[Dict[str, Any]] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n save_model_to_state: bool = True,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n\n valid_problem_types = [\"auto\"] + PROBLEM_TYPES_REGRESSION + PROBLEM_TYPES_CLASSIFICATION\n assert problem_type in valid_problem_types, f\"Valid problem_type values include {valid_problem_types}\"\n self.problem_type: Optional[str] = None if problem_type == \"auto\" else problem_type\n\n self.save_model_to_state = save_model_to_state\n\n if estimator_args is not None:\n self.estimator_args = estimator_args\n else:\n self.estimator_args = {}\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"val_data\", \"label\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n estimator: TabularPredictor = TabularPredictor(\n label=args.label, problem_type=self.problem_type, **self.estimator_args\n )\n estimator.fit(train_data=args.train_data, **self.args)\n self.args[\"model\"] = estimator\n\n if self.save_model_to_state:\n state[\"model\"] = estimator\n\n\nclass AutoGluonModelEvaluator(AbstractAnalysis):\n \"\"\"\n Evaluate AutoGluon model performance.\n\n This analysis requires a trained classifier passed in `model` arg and uses 'val_data' dataset to assess model performance.\n\n It is assumed that the validation dataset should follow the same column names seen by the model and has not been used during the training process.\n\n Parameters\n ----------\n model: TabularPredictor, required\n fitted AutoGluon model to analyze\n val_data: pd.DataFrame, required\n validation dataset to use.\n Warning: do not use data used for training as a validation data.\n Predictions on data used by the model during training tend to be optimistic and might not generalize on unseen data.\n normalize : {'true', 'pred', 'all'}, default=None\n Normalizes confusion matrix over the true (rows), predicted (columns)\n conditions or all the population. If None, confusion matrix will not be\n normalized.\n Note: applicable only for binary and multiclass classification; ignored for regression models.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = ...\n >>> df_test = ...\n >>> predictor = ...\n >>>\n >>> auto.analyze(model=predictor, val_data=df_test, anlz_facets=[\n >>> eda.model.AutoGluonModelEvaluator(),\n >>> ], viz_facets=[\n >>> viz.layouts.MarkdownSectionComponent(markdown=f'### Model Prediction for {predictor.label}'),\n >>> viz.model.ConfusionMatrix(fig_args=dict(figsize=(3,3)), annot_kws={\"size\": 12}),\n >>> viz.model.RegressionEvaluation(fig_args=dict(figsize=(6,6)), chart_args=dict(marker='o', scatter_kws={'s':5})),\n >>> viz.layouts.MarkdownSectionComponent(markdown=f'### Feature Importance for Trained Model'),\n >>> viz.model.FeatureImportance(show_barplots=True)\n >>> ])\n \"\"\"\n\n def __init__(\n self,\n normalize: Union[None, str] = None,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n self.normalize = normalize\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n keys_present = self.all_keys_must_be_present(args, \"model\", \"val_data\")\n data_cols = sorted(args.val_data.columns.values)\n model_cols = sorted(args.model.original_features + [args.model.label])\n columns_the_same = data_cols == model_cols if keys_present else False\n if not columns_the_same:\n self.logger.warning(\n f\"val_data columns {data_cols} are not matching original features model was trained on: {model_cols}\"\n )\n\n return keys_present and columns_the_same\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs):\n predictor: TabularPredictor = args.model\n val_data = args.val_data\n problem_type = predictor.problem_type\n label = predictor.label\n y_true_val, y_pred_val, highest_error, undecided = self._predict(problem_type, predictor, val_data)\n test_data = args.test_data\n test_data_present = args.test_data is not None and label in args.test_data.columns\n\n y_true_test = None\n y_pred_test = None\n if test_data_present:\n test_data = args.test_data\n y_true_test, y_pred_test, highest_error, undecided = self._predict(problem_type, predictor, test_data)\n\n _data = test_data if test_data_present else val_data\n importance = predictor.feature_importance(_data.reset_index(drop=True), silent=True)\n leaderboard = predictor.leaderboard(_data)\n\n labels = predictor.class_labels\n s = {\n \"problem_type\": predictor.problem_type,\n \"importance\": importance,\n \"leaderboard\": leaderboard,\n \"labels\": labels,\n \"y_true_val\": y_true_val,\n \"y_pred_val\": y_pred_val,\n }\n\n try:\n y_pred_train = args.model.predict_proba_oof()\n s[\"y_true_train\"] = args.train_data[args.label]\n s[\"y_pred_train\"] = y_pred_train\n except AssertionError:\n # OOF is not available - don't use it\n pass\n\n if test_data_present:\n s[\"y_true_test\"] = y_true_test\n s[\"y_pred_test\"] = y_pred_test\n\n if undecided is not None:\n s[\"undecided\"] = undecided\n if highest_error is not None:\n s[\"highest_error\"] = highest_error\n\n if problem_type in [BINARY, MULTICLASS]:\n cm = confusion_matrix(y_true_val, y_pred_val, normalize=self.normalize, labels=labels)\n s[\"confusion_matrix_normalized\"] = self.normalize is not None\n s[\"confusion_matrix\"] = cm\n\n state.model_evaluation = s\n\n def _predict(self, problem_type, predictor, val_data):\n label = predictor.label\n y_true_val = val_data[label]\n y_pred_val = predictor.predict(val_data)\n highest_error = None\n undecided = None\n if predictor.problem_type in [BINARY, MULTICLASS]:\n y_proba = predictor.predict_proba(val_data)\n\n misclassified = y_proba[y_true_val != y_pred_val]\n expected_value = misclassified.join(y_true_val).apply(lambda row: row.loc[row[label]], axis=1)\n predicted_value = misclassified.max(axis=1)\n highest_error = predicted_value - expected_value\n highest_error.name = \"error\"\n\n scores = np.sort(misclassified.values, axis=1)\n diff = scores[:, -1] - expected_value\n undecided = pd.Series(index=y_pred_val.index, data=diff, name=\"error\").sort_values(ascending=True)\n undecided = val_data.join(y_proba).join(undecided).sort_values(by=\"error\")\n highest_error = (\n val_data.join(y_proba, rsuffix=\"_pred\")\n .join(highest_error, how=\"inner\")\n .sort_values(by=\"error\", ascending=False)\n )\n elif problem_type == REGRESSION:\n highest_error = np.abs(y_pred_val - y_true_val).sort_values(ascending=False)\n highest_error.name = \"error\"\n highest_error = (\n val_data.join(y_pred_val, rsuffix=\"_pred\")\n .join(highest_error, how=\"inner\")\n .sort_values(by=\"error\", ascending=False)\n )\n return y_true_val, y_pred_val, highest_error, undecided\n"
},
{
"path": "eda/src/autogluon/eda/analysis/shift.py",
"content": "import copy\nfrom typing import Any, Dict, List, Optional, Union\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.core.constants import BINARY\nfrom autogluon.core.metrics import BINARY_METRICS, roc_auc\nfrom autogluon.core.utils import generate_train_test_split\nfrom autogluon.tabular import TabularPredictor\n\nfrom .. import AnalysisState\nfrom ..state import StateCheckMixin\nfrom .base import AbstractAnalysis\n\n__all__ = [\"XShiftDetector\"]\n\n\nclass XShiftDetector(AbstractAnalysis, StateCheckMixin):\n \"\"\"Detect a change in covariate (X) distribution between training and test, which we call XShift. It can tell you\n if your training set is not representative of your test set distribution. This is done with a Classifier 2\n Sample Test.\n\n State attributes\n\n - `xshift_results.detection_status`:\n bool, True if detected\n - `xshift_results.test_statistic`: float\n Classifier Two-Sample Test (C2ST) statistic. It is a measure how well a classifier distinguishes between the samples from the training and test sets.\n If the classifier can accurately separate the samples, it suggests that the input distributions differ significantly, indicating the presence of\n covariate shift. A C2ST value close to 0.5 implies that the classifier struggles to differentiate between the sets, indicating minimal covariate shift.\n In contrast, a value significantly different from 0.5 suggests the presence of covariate shift, warranting further investigation and potential\n adjustments to the model or data preprocessing.\n - `xshift_results.pvalue`: float\n p-value using permutation test\n - `xshift_results.pvalue_threshold`: float,\n decision boundary of p-value threshold\n - `xshift_results.feature_importance`: DataFrame,\n the feature importance dataframe, if computed\n - `xshift_results.shift_features`\n list of features whose contribution is statistically significant; only present if `xshift_results.detection_status = True`\n\n Parameters\n ----------\n classifier_class : an AutoGluon predictor, such as autogluon.tabular.TabularPredictor (default)\n The predictor that will be fit on training set and predict the test set\n compute_fi : bool, default = True\n To compute the feature importances set to True, this can be computationally intensive\n pvalue_thresh : float, default = 0.01\n The threshold for the pvalue\n eval_metric : str, default = 'balanced_accuracy'\n The metric used for the C2ST, it must be one of the binary metrics from autogluon.core.metrics\n sample_label : str, default = '__label__'\n The label internally used for the classifier 2 sample test, the only reason to change it is in the off chance\n that the default value is a column in the data.\n classifier_kwargs : dict, default = {}\n The kwargs passed to the classifier, a member of classifier_class\n classifier_fit_kwargs : dict, default = {}\n The kwargs passed to the classifier's `fit` call, a member of classifier_class\n num_permutations: int, default = 1000\n The number of permutations used for any permutation based method\n test_size_2st: float, default = 0.3\n The size of the test set in the training test split in 2ST\n\n \"\"\"\n\n def __init__(\n self,\n classifier_class: Any = TabularPredictor,\n compute_fi: bool = True,\n pvalue_thresh: float = 0.01,\n eval_metric: str = \"roc_auc\",\n sample_label: str = \"__label__\",\n classifier_kwargs: Optional[dict] = None,\n classifier_fit_kwargs: Optional[dict] = None,\n num_permutations: int = 1000,\n test_size_2st: float = 0.3,\n parent: Union[None, AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, **kwargs)\n if classifier_kwargs is None:\n classifier_kwargs = {}\n if classifier_fit_kwargs is None:\n classifier_fit_kwargs = {}\n self.classifier_kwargs = classifier_kwargs\n self.classifier_fit_kwargs = classifier_fit_kwargs\n self.classifier_class = classifier_class\n self.compute_fi = compute_fi\n named_metrics = BINARY_METRICS\n assert eval_metric in named_metrics.keys(), (\n \"eval_metric must be one of [\" + \", \".join(named_metrics.keys()) + \"]\"\n )\n self.eval_metric = named_metrics[eval_metric]\n self.C2ST = Classifier2ST(\n classifier_class,\n sample_label=sample_label,\n eval_metric=self.eval_metric,\n compute_fi=compute_fi,\n classifier_kwargs=classifier_kwargs,\n test_size_2st=test_size_2st,\n )\n self.fi_scores = None\n self.compute_fi = compute_fi\n self.pvalue_thresh = pvalue_thresh\n self.num_permutations = num_permutations\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"test_data\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n \"\"\"Fit method. `args` can contain\n - 'train_data': pd.DataFrame, required\n - 'test_data': pd.DataFrame, required\n - 'label': str, optional\n The Y variable that is to be predicted (if it appears in the train/test data then it will be removed)\n \"\"\"\n X = args[\"train_data\"].copy()\n X_test = args[\"test_data\"].copy()\n assert (\n self.C2ST.sample_label not in X.columns\n ), f\"your data columns contain {self.C2ST.sample_label} which is used internally\"\n if \"label\" in args:\n label = args[\"label\"]\n if label in X.columns:\n X = X.drop(columns=[label])\n if label in X_test.columns:\n X_test = X_test.drop(columns=[label])\n self.C2ST.fit((X, X_test), **self.classifier_fit_kwargs, **fit_kwargs)\n # Feature importance\n if self.C2ST.has_fi and self.compute_fi:\n fi_scores = self.C2ST.feature_importance()\n else:\n fi_scores = None\n pvalue = self.C2ST.pvalue(num_permutations=self.num_permutations)\n\n detection_status = bool(pvalue <= self.pvalue_thresh) # numpy.bool_ -> bool\n\n state.xshift_results = {\n \"detection_status\": detection_status,\n \"test_statistic\": self.C2ST.test_stat,\n \"pvalue\": pvalue,\n \"pvalue_threshold\": self.pvalue_thresh,\n \"eval_metric\": self.eval_metric.name,\n \"feature_importance\": fi_scores,\n }\n\n if detection_status:\n fi_scores = fi_scores[fi_scores.p_value <= self.pvalue_thresh]\n shift_features = fi_scores.index.tolist()\n state.xshift_results[\"shift_features\"] = shift_features\n\n\ndef post_fit(func):\n \"\"\"decorator for post-fit methods\"\"\"\n\n def pff_wrapper(self, *args, **kwargs):\n assert self._is_fit, f\".fit needs to be called prior to .{func.__name__}\"\n return func(self, *args, **kwargs)\n\n return pff_wrapper\n\n\nclass Classifier2ST:\n \"\"\"A classifier 2 sample test, which tests for a difference between a source and target dataset. It fits a\n classifier to predict if a sample is in the source and target dataset, then computes an evaluation metric on a\n holdout which becomes the test statistic.\n\n Parameters\n ----------\n classifier_class : an AutoGluon predictor, such as autogluon.tabular.TabularPredictor\n The predictor (classifier) class to classify the source from target dataset, predictor class needs to support\n binary classification\n sample_label : str, default = 'xshift_label'\n The label that will be used to indicate if the sample is from training or test\n eval_metric : callable, default = autogluon.core.metrics.balanced_accuracy\n Binary classification metric to use for the classifier 2 sample test, currently only metrics that accept binary\n predictions are supported, such as balanced_accuracy\n compute_fi : bool, default = True\n To compute the feature importances set to True, this can be computationally intensive\n split : float, default = 0.5\n Training/test split proportion for classifier 2 sample test\n classifier_kwargs : dict, default = {}\n The kwargs passed to the classifier, a member of classifier_class\n test_size_2st: float, default = 0.3\n The size of the test set in the training test split in 2ST\n \"\"\"\n\n def __init__(\n self,\n classifier_class,\n sample_label=\"xshift_label\",\n eval_metric=roc_auc,\n split=0.5,\n compute_fi=True,\n classifier_kwargs: Optional[Dict] = None,\n test_size_2st=0.3,\n ):\n if classifier_kwargs is None:\n classifier_kwargs = {}\n else:\n classifier_kwargs = copy.deepcopy(classifier_kwargs)\n classifier_kwargs.update({\"label\": sample_label, \"eval_metric\": eval_metric})\n self.classifier = classifier_class(**classifier_kwargs)\n self.classifier_class = classifier_class\n self.split = split\n self.sample_label = sample_label\n self.eval_metric = eval_metric\n self._is_fit = False\n self._test = None\n self.test_stat = None\n self.has_fi: Optional[bool] = None\n self.compute_fi = compute_fi\n self.test_size_2st = test_size_2st\n\n @staticmethod\n def _make_source_target_label(data, sample_label):\n \"\"\"Turn a source, target pair into a single dataframe with label column\"\"\"\n source, target = data[0].copy(), data[1].copy()\n source.loc[:, sample_label] = 0\n target.loc[:, sample_label] = 1\n data = pd.concat((source, target), ignore_index=True)\n return data\n\n def fit(self, data, **kwargs):\n \"\"\"Fit the classifier for predicting if source or target and compute the 2-sample test statistic.\n\n Parameters\n ----------\n data : pd.DataFrame, or tuple\n either\n - a dataframe with a label column where 1 = target and 0 = source\n - a tuple of source dataframe and target dataframe\n \"\"\"\n if isinstance(data, pd.DataFrame):\n sample_label = self.sample_label\n assert sample_label in data.columns, \"sample_label needs to be a column of data\"\n assert self.split, \"sample_label requires the split parameter\"\n data = data.copy() # makes a copy\n else:\n assert len(data) == 2, \"Data needs to be tuple/list of (source, target) if sample_label is None\"\n data = self._make_source_target_label(data, self.sample_label) # makes a copy\n if data.index.has_duplicates:\n data = data.reset_index(drop=True)\n train, test, y_train, y_test = generate_train_test_split(\n data.drop(columns=[self.sample_label]), data[self.sample_label], BINARY, test_size=self.test_size_2st\n )\n train[self.sample_label] = y_train\n test[self.sample_label] = y_test\n self.classifier.fit(train, **kwargs)\n yhat = self.classifier.predict_proba(test)[1]\n self.test_stat = self.eval_metric(test[self.sample_label], yhat)\n self.has_fi = getattr(self.classifier, \"feature_importance\", None) is not None\n if self.has_fi and self.compute_fi:\n self._test = test # for feature importance\n self._is_fit = True\n\n @post_fit\n def _pvalue_half_permutation(self, num_permutations=1000):\n \"\"\"The half permutation method for computing p-values.\n See Section 9.2 of https://arxiv.org/pdf/1602.02210.pdf\n \"\"\"\n perm_stats = [self.test_stat]\n yhat = self.classifier.predict_proba(self._test)[1]\n for _ in range(num_permutations):\n perm_yhat = np.random.permutation(yhat)\n perm_test_stat = self.eval_metric(self._test[self.sample_label], perm_yhat) # type: ignore\n perm_stats.append(perm_test_stat)\n pval = (self.test_stat <= np.array(perm_stats)).mean()\n return pval\n\n @post_fit\n def pvalue(self, num_permutations: int = 1000):\n \"\"\"Compute the p-value which measures the significance level for the test statistic\n\n Parameters\n ----------\n num_permutations: int, default = 1000\n The number of permutations used for any permutation based method\n\n Returns\n -------\n float of the p-value for the 2-sample test\n \"\"\"\n pval = self._pvalue_half_permutation(num_permutations=num_permutations)\n return pval\n\n @post_fit\n def feature_importance(self):\n \"\"\"Returns the feature importances for the trained classifier for source v. target\n\n Returns\n -------\n pd.DataFrame of feature importances\n \"\"\"\n assert self.has_fi, \"Classifier class does not have feature_importance method\"\n assert self.compute_fi, \"Set compute_fi to True to compute feature importances\"\n fi_scores = self.classifier.feature_importance(self._test)\n return fi_scores\n"
},
{
"path": "eda/src/autogluon/eda/analysis/transform.py",
"content": "import logging\nfrom typing import List, Optional\n\nimport pandas as pd\n\nfrom autogluon.features import AbstractFeatureGenerator, AutoMLPipelineFeatureGenerator\n\nfrom ..state import AnalysisState, StateCheckMixin\nfrom .base import AbstractAnalysis\n\nlogger = logging.getLogger(__name__)\n\n__all__ = [\"ApplyFeatureGenerator\"]\n\n\nclass ApplyFeatureGenerator(AbstractAnalysis, StateCheckMixin):\n \"\"\"\n This wrapper provides transformed features to all `children` shadowing outer datasets with the updated one after application of FeatureGenerator.\n\n Parameters\n ----------\n category_to_numbers: bool, default = False\n if `True', then transform `category` variables into their codes. This is useful when wrapped analyses expect numeric values\n feature_generator: Optional[AbstractFeatureGenerator], default = None\n feature generator to use for the transformation. If `None` is provided then `AutoMLPipelineFeatureGenerator` is applied.\n parent: Optional[AbstractAnalysis], default = None\n parent Analysis\n children: Optional[List[AbstractAnalysis]], default None\n wrapped analyses; these will receive sampled `args` during `fit` call\n verbosity: int, default = 0,\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n kwargs\n\n See also :func:`autogluon.features.AbstractFeatureGenerator`\n\n Examples\n --------\n >>> from autogluon.eda.analysis.base import BaseAnalysis, Namespace\n >>> import pandas as pd\n >>> import numpy as np\n >>> df_train = pd.DataFrame(...)\n >>> df_test = pd.DataFrame(...)\n >>>\n >>> analysis = BaseAnalysis(train_data=df_train, test_data=df_test, label='D', children=[\n >>> Namespace(namespace='feature_generator_numbers', children=[\n >>> ApplyFeatureGenerator(category_to_numbers=True, children=[\n >>> # SomeAnalysis() # This analysis will be called with transformed `train_data` and `test_data`\n >>> ])\n >>> ]),\n >>> # SomeAnalysis() # This analysis will be called with the original features\n >>> ])\n\n \"\"\"\n\n def __init__(\n self,\n parent: Optional[AbstractAnalysis] = None,\n children: Optional[List[AbstractAnalysis]] = None,\n state: Optional[AnalysisState] = None,\n category_to_numbers: bool = False,\n feature_generator: Optional[AbstractFeatureGenerator] = None,\n verbosity: int = 0,\n **kwargs,\n ) -> None:\n super().__init__(parent, children, state, **kwargs)\n self.category_to_numbers = category_to_numbers\n if feature_generator is None:\n feature_generator = AutoMLPipelineFeatureGenerator(\n enable_numeric_features=True,\n enable_categorical_features=True,\n enable_datetime_features=False,\n enable_text_special_features=False,\n enable_text_ngram_features=False,\n enable_raw_text_features=False,\n enable_vision_features=False,\n verbosity=verbosity,\n **kwargs,\n )\n self.feature_generator = feature_generator\n\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return self.all_keys_must_be_present(args, \"train_data\", \"label\")\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n x = args.train_data\n if (args.label is not None) and (args.label in x.columns):\n x = x.drop(columns=args.label)\n self.feature_generator.fit(x)\n self.args[\"feature_generator\"] = True\n for ds, df in self.available_datasets(args):\n x = df\n y = None\n if args.label in df.columns:\n x = df.drop(columns=args.label)\n y = df[args.label]\n x_tx = self.feature_generator.transform(x)\n if self.category_to_numbers:\n for col, dtype in x_tx.dtypes.items():\n if dtype == \"category\":\n x_tx[col] = x_tx[col].cat.codes\n if y is not None:\n x_tx = pd.concat([x_tx, y], axis=1)\n self.args[ds] = x_tx\n"
},
{
"path": "eda/src/autogluon/eda/auto/__init__.py",
"content": "from .simple import (\n analyze,\n analyze_interaction,\n covariate_shift_detection,\n dataset_overview,\n detect_anomalies,\n explain_rows,\n missing_values_analysis,\n partial_dependence_plots,\n quick_fit,\n target_analysis,\n)\n"
},
{
"path": "eda/src/autogluon/eda/auto/simple.py",
"content": "import logging\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nimport pandas as pd\n\nfrom autogluon.common.utils.log_utils import verbosity2loglevel\nfrom autogluon.features import CategoryFeatureGenerator\nfrom autogluon.tabular import TabularPredictor\n\nfrom .. import AnalysisState\nfrom ..analysis import (\n AnomalyDetectorAnalysis,\n ApplyFeatureGenerator,\n AutoGluonModelEvaluator,\n AutoGluonModelQuickFit,\n Correlation,\n DistributionFit,\n FeatureInteraction,\n MissingValuesAnalysis,\n ProblemTypeControl,\n ShapAnalysis,\n TrainValidationSplit,\n XShiftDetector,\n)\nfrom ..analysis.base import AbstractAnalysis, BaseAnalysis, SaveArgsToState\nfrom ..analysis.dataset import (\n DatasetSummary,\n LabelInsightsAnalysis,\n RawTypesAnalysis,\n Sampler,\n SpecialTypesAnalysis,\n VariableTypeAnalysis,\n)\nfrom ..analysis.interaction import FeatureDistanceAnalysis\nfrom ..state import is_key_present_in_state\nfrom ..utils.common import expand_nested_args_into_nested_maps, get_empty_dict_if_none\nfrom ..utils.defaults import QuickFitDefaults\nfrom ..visualization import (\n AnomalyScoresVisualization,\n ConfusionMatrix,\n CorrelationVisualization,\n DatasetStatistics,\n DatasetTypeMismatch,\n ExplainForcePlot,\n ExplainWaterfallPlot,\n FeatureImportance,\n FeatureInteractionVisualization,\n LabelInsightsVisualization,\n MarkdownSectionComponent,\n MissingValues,\n ModelLeaderboard,\n PropertyRendererComponent,\n RegressionEvaluation,\n XShiftSummary,\n)\nfrom ..visualization.base import AbstractVisualization\nfrom ..visualization.interaction import FeatureDistanceAnalysisVisualization, PDPInteractions\nfrom ..visualization.layouts import SimpleVerticalLinearLayout\n\nlogger = logging.getLogger(__name__)\n\n__all__ = [\n \"analyze\",\n \"analyze_interaction\",\n \"covariate_shift_detection\",\n \"dataset_overview\",\n \"detect_anomalies\",\n \"explain_rows\",\n \"missing_values_analysis\",\n \"partial_dependence_plots\",\n \"quick_fit\",\n \"target_analysis\",\n]\n\nDEFAULT_SAMPLE_SIZE = 10000\n\n\ndef analyze(\n train_data: Optional[pd.DataFrame] = None,\n test_data: Optional[pd.DataFrame] = None,\n val_data: Optional[pd.DataFrame] = None,\n model=None,\n label: Optional[str] = None,\n state: Union[None, dict, AnalysisState] = None,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n anlz_facets: Optional[List[AbstractAnalysis]] = None,\n viz_facets: Optional[List[AbstractVisualization]] = None,\n return_state: bool = False,\n verbosity: int = 2,\n **kwargs,\n) -> Optional[AnalysisState]:\n \"\"\"\n This helper creates `BaseAnalysis` wrapping passed analyses into\n `Sampler` if needed, then fits and renders produced state with\n specified visualizations.\n\n Parameters\n ----------\n train_data\n training dataset\n test_data\n test dataset\n val_data\n validation dataset\n model\n trained `Predictor`\n label: str\n target variable\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n anlz_facets: List[AbstractAnalysis]\n analyses to add to this composite analysis\n viz_facets: List[AbstractVisualization]\n visualizations to add to this composite analysis\n return_state: bool, default = False\n return state if `True`\n verbosity: int, default = 2,\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> # Add analysis chain here\n >>> ],\n >>> viz_facets=[\n >>> # Add visualization facets here\n >>> ]\n >>> )\n\n \"\"\"\n\n if viz_facets is None:\n viz_facets = []\n\n if anlz_facets is None:\n anlz_facets = []\n\n if state is not None:\n assert isinstance(state, (dict, AnalysisState))\n\n if not isinstance(state, AnalysisState):\n state = AnalysisState(state)\n\n root_logger = logging.getLogger(\"autogluon\")\n root_log_level = root_logger.level\n log_level = verbosity2loglevel(verbosity)\n root_logger.setLevel(log_level)\n\n analysis = BaseAnalysis(\n state=state,\n train_data=train_data,\n test_data=test_data,\n val_data=val_data,\n model=model,\n label=label,\n children=[\n Sampler(sample=sample, children=anlz_facets),\n ],\n )\n\n state = analysis.fit()\n\n SimpleVerticalLinearLayout(\n facets=viz_facets,\n ).render(state)\n\n root_logger.setLevel(root_log_level) # Reset log level\n\n return state if return_state else None\n\n\ndef analyze_interaction(\n train_data: pd.DataFrame,\n x: Optional[str] = None,\n y: Optional[str] = None,\n hue: Optional[str] = None,\n fit_distributions: Union[bool, str, List[str]] = False,\n fig_args: Optional[Dict[str, Any]] = None,\n chart_args: Optional[Dict[str, Any]] = None,\n **analysis_args,\n):\n \"\"\"\n This helper performs simple feature interaction analysis.\n\n Parameters\n ----------\n train_data: pd.DataFrame\n training dataset\n x: Optional[str], default = None\n y: Optional[str], default = None\n hue: Optional[str], default = None\n fit_distributions: Union[bool, str, List[str]], default = False,\n If `True`, or list of distributions is provided, then fit distributions. Performed only if `y` and `hue` are not present.\n chart_args: Optional[dict], default = None\n kwargs to pass into visualization component\n fig_args: Optional[Dict[str, Any]], default = None,\n kwargs to pass into visualization component\n\n Examples\n --------\n >>> import pandas as pd\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = pd.DataFrame(...)\n >>>\n >>> auto.analyze_interaction(x='Age', hue='Survived', train_data=df_train, chart_args=dict(headers=True, alpha=0.2))\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.interaction.FeatureInteraction`\n :py:class:`~autogluon.eda.visualization.interaction.FeatureInteractionVisualization`\n \"\"\"\n assert (\n (x is not None) or (y is not None) or (hue is not None)\n ), \"At least one of the parameters must be specified: x, y or hue\"\n fig_args = get_empty_dict_if_none(fig_args).copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (12, 6)\n\n chart_args = get_empty_dict_if_none(chart_args)\n\n key = \"__analysis__\"\n\n _analysis_args = analysis_args.copy()\n _analysis_args.pop(\"return_state\", None)\n\n pvalue_min = _analysis_args.pop(\"pvalue_min\", 0.01)\n keep_top_n = _analysis_args.pop(\"keep_top_n\", 5)\n numeric_as_categorical_threshold = _analysis_args.pop(\"numeric_as_categorical_threshold\", 20)\n\n state: AnalysisState = analyze(\n train_data=train_data,\n return_state=True,\n **_analysis_args,\n anlz_facets=[\n RawTypesAnalysis(),\n VariableTypeAnalysis(numeric_as_categorical_threshold=numeric_as_categorical_threshold),\n ],\n ) # type: ignore\n\n analysis_facets: List[AbstractAnalysis] = [\n FeatureInteraction(key=key, x=x, y=y, hue=hue),\n ]\n\n if x is not None:\n x_type = state.variable_type.train_data[x]\n if _is_single_numeric_variable(x, y, hue, x_type) and (fit_distributions is not False):\n dists: Optional[List[str]] # fit all\n if fit_distributions is True:\n dists = None\n elif isinstance(fit_distributions, str):\n dists = [fit_distributions]\n else:\n dists = fit_distributions\n\n analysis_facets.append(\n DistributionFit(columns=x, keep_top_n=keep_top_n, pvalue_min=pvalue_min, distributions_to_fit=dists)\n ) # type: ignore # x is always present\n\n _analysis_args = analysis_args.copy()\n _analysis_args.pop(\"state\", None)\n\n return analyze(\n train_data=train_data,\n **_analysis_args,\n state=state,\n anlz_facets=analysis_facets,\n viz_facets=[\n FeatureInteractionVisualization(\n key=key,\n fig_args=fig_args,\n numeric_as_categorical_threshold=numeric_as_categorical_threshold,\n **chart_args,\n ),\n ],\n )\n\n\ndef _is_single_numeric_variable(x, y, hue, x_type):\n return (x is not None) and (y is None) and (hue is None) and (x_type == \"numeric\")\n\n\ndef quick_fit(\n train_data: pd.DataFrame,\n label: str,\n test_data: Optional[pd.DataFrame] = None,\n path: Optional[str] = None,\n val_size: float = 0.3,\n problem_type: str = \"auto\",\n fit_bagging_folds: int = 0,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n state: Union[None, dict, AnalysisState] = None,\n return_state: bool = False,\n save_model_to_state: bool = True,\n verbosity: int = 0,\n show_feature_importance_barplots: bool = False,\n estimator_args: Optional[Dict[str, Dict[str, Any]]] = None,\n fig_args: Optional[Dict[str, Dict[str, Any]]] = None,\n chart_args: Optional[Dict[str, Dict[str, Any]]] = None,\n render_analysis: bool = True,\n **fit_args,\n):\n \"\"\"\n This helper performs quick model fit analysis and then produces a composite report of the results.\n\n The analysis is structured in a sequence of operations:\n - Sample if `sample` is specified.\n - Perform train-test split using `val_size` ratio\n - Fit AutoGluon estimator given `fit_args`; if `hyperparameters` not present in args, then use default ones\n (Random Forest by default - because it is interpretable)\n - Display report\n\n The reports include:\n - confusion matrix for classification problems; predictions vs actual for regression problems\n - model leaderboard\n - feature importance\n - samples with the highest prediction error - candidates for inspection\n - samples with the least distance from the other class - candidates for labeling\n\n Supported `fig_args`/`chart_args` keys:\n - `confusion_matrix.` - confusion matrix chart for classification predictor\n - `regression_eval.` - regression predictor results chart\n - `feature_importance.` - feature importance barplot chart\n\n State attributes\n\n - `model`\n trained model\n - `model_evaluation.importance`\n feature importance calculated using the trained model\n - `model_evaluation.leaderboard`\n trained models leaderboard\n - `model_evaluation.highest_error`\n misclassified rows with the highest error between prediction and ground truth\n - `model_evaluation.undecided` (classification only)\n misclassified rows with the prediction closest to the decision boundary\n - `model_evaluation.confusion_matrix` (classification only)\n confusion matrix values\n\n Parameters\n ----------\n train_data: DataFrame\n training dataset\n test_data: DataFrame\n test dataset\n label: str\n target variable\n path: Optional[str], default = None,\n path for models saving\n problem_type: str, default = 'auto'\n problem type to use. Valid problem_type values include ['auto', 'binary', 'multiclass', 'regression', 'quantile', 'softclass']\n auto means it will be Auto-detected using AutoGluon methods.\n fit_bagging_folds: int, default = 0,\n shortcut to enable training with bagged folds; disabled if 0 (default)\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n val_size: float, default = 0.3\n fraction of training set to be assigned as validation set during the split.\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n return_state: bool, default = False\n return state if `True`\n save_model_to_state: bool, default = True,\n save fitted model into `state` under `model` key.\n This functionality might be helpful in cases when the fitted model could be usable for other purposes (i.e. imputers)\n verbosity: int, default = 0\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n show_feature_importance_barplots: bool, default = False\n if `True`, then barplot char will ba added with feature importance visualization\n estimator_args: Optional[Dict[str, Dict[str, Any]]], default = None,\n args to pass into the estimator constructor\n fit_args: Optional[Dict[str, Dict[str, Any]]], default = None,\n kwargs to pass into `TabularPredictor` fit.\n fig_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component figure. The args are supporting nested\n dot syntax: 'a.b.c'.\n chart_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component chart. The args are supporting nested\n dot syntax: 'a.b.c'.\n render_analysis: bool, default = True\n if `False`, then don't render any visualizations; this can be used if user just needs to train a model. It is recommended to use this option\n with `save_model_to_state=True` and `return_state=True` options.\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.auto as auto\n >>>\n >>> # Quick fit\n >>> state = auto.quick_fit(\n >>> train_data=..., label=...,\n >>> return_state=True, # return state object from call\n >>> fig_args={\"regression_eval.figsize\": (8,6)}, # customize regression evaluation `figsize`\n >>> chart_args={\"regression_eval.residuals_plot_mode\": \"hist\"} # customize regression evaluation `residuals_plot_mode`\n >>> hyperparameters={'GBM': {}} # train specific model\n >>> )\n >>>\n >>> # Using quick fit model\n >>> model = state.model\n >>> y_pred = model.predict(test_data)\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.model.ConfusionMatrix`\n :py:class:`~autogluon.eda.visualization.model.RegressionEvaluation`\n :py:class:`~autogluon.eda.visualization.model.ModelLeaderboard`\n :py:class:`~autogluon.eda.visualization.model.FeatureImportance`\n\n \"\"\"\n if state is not None:\n assert isinstance(state, (dict, AnalysisState))\n\n if not isinstance(state, AnalysisState):\n state = AnalysisState(state)\n\n fig_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(fig_args))\n chart_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(chart_args))\n\n estimator_args = get_empty_dict_if_none(estimator_args)\n assert fit_bagging_folds >= 0, \"fit_bagging_folds must be non-negative\"\n fit_args = get_default_estimator_if_not_specified(fit_args, fit_bagging_folds)\n\n if \"path\" not in estimator_args:\n estimator_args[\"path\"] = path # type: ignore\n\n if (test_data is not None) and (label not in test_data.columns):\n test_data = None\n\n if render_analysis:\n viz = [\n MarkdownSectionComponent(markdown=f\"### Model Prediction for {label}\"),\n MarkdownSectionComponent(\n condition_fn=(lambda state: is_key_present_in_state(state, \"model_evaluation.y_pred_test\")),\n markdown=\"Using `test_data` for `Test` points\",\n ),\n MarkdownSectionComponent(\n condition_fn=(lambda state: not is_key_present_in_state(state, \"model_evaluation.y_pred_test\")),\n markdown=\"Using validation data for `Test` points\",\n ),\n ConfusionMatrix(\n fig_args=fig_args.get(\"confusion_matrix\", {}),\n **chart_args.get(\"confusion_matrix\", dict(annot_kws={\"size\": 12})),\n ),\n RegressionEvaluation(\n fig_args=fig_args.get(\"regression_eval\", {}),\n **chart_args.get(\"regression_eval\", {}),\n ),\n MarkdownSectionComponent(markdown=\"### Model Leaderboard\"),\n ModelLeaderboard(),\n MarkdownSectionComponent(markdown=\"### Feature Importance for Trained Model\"),\n FeatureImportance(\n show_barplots=show_feature_importance_barplots,\n fig_args=fig_args.get(\"feature_importance\", {}),\n **chart_args.get(\"feature_importance\", {}),\n ),\n MarkdownSectionComponent(markdown=\"### Rows with the highest prediction error\"),\n MarkdownSectionComponent(markdown=\"Rows in this category worth inspecting for the causes of the error\"),\n PropertyRendererComponent(\"model_evaluation.highest_error\", transform_fn=(lambda df: df.head(10))),\n MarkdownSectionComponent(\n condition_fn=(lambda state: is_key_present_in_state(state, \"model_evaluation.undecided\")),\n markdown=\"### Rows with the least distance vs other class\",\n ),\n MarkdownSectionComponent(\n condition_fn=(lambda state: is_key_present_in_state(state, \"model_evaluation.undecided\")),\n markdown=\"Rows in this category are the closest to the decision boundary vs the other class \"\n \"and are good candidates for additional labeling\",\n ),\n PropertyRendererComponent(\"model_evaluation.undecided\", transform_fn=(lambda df: df.head(10))),\n ]\n else:\n viz = []\n\n return analyze(\n train_data=train_data,\n test_data=test_data,\n label=label,\n sample=sample,\n state=state,\n return_state=return_state,\n anlz_facets=[\n ProblemTypeControl(problem_type=problem_type),\n TrainValidationSplit(\n val_size=val_size,\n children=[\n AutoGluonModelQuickFit(\n estimator_args=estimator_args,\n verbosity=verbosity,\n problem_type=problem_type,\n save_model_to_state=save_model_to_state,\n children=[\n AutoGluonModelEvaluator(),\n ],\n **fit_args,\n ),\n ],\n ),\n ],\n viz_facets=viz,\n )\n\n\ndef dataset_overview(\n train_data: Optional[pd.DataFrame] = None,\n test_data: Optional[pd.DataFrame] = None,\n val_data: Optional[pd.DataFrame] = None,\n label: Optional[str] = None,\n state: Union[None, dict, AnalysisState] = None,\n return_state: bool = False,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n fig_args: Optional[Dict[str, Dict[str, Any]]] = None,\n chart_args: Optional[Dict[str, Dict[str, Any]]] = None,\n):\n \"\"\"\n Shortcut to perform high-level datasets summary overview (counts, frequencies, missing statistics, types info).\n\n Supported `fig_args`/`chart_args` keys:\n - `feature_distance.` - feature distance dendrogram chart\n - `chart..` - near-duplicate groups visualizations chart. If chart is labeled as a relationship /, then is \n\n Parameters\n ----------\n train_data: Optional[DataFrame], default = None\n training dataset\n test_data: Optional[DataFrame], default = None\n test dataset\n val_data: Optional[DataFrame], default = None\n validation dataset\n label: : Optional[str], default = None\n target variable\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n return_state: bool, default = False\n return state if `True`\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n fig_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component figure\n chart_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component chart\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>>\n >>> auto.dataset_overview(\n >>> train_data=df_train, test_data=df_test, label=target_col,\n >>> chart_args={'feature_distance.orientation': 'left'},\n >>> fig_args={'feature_distance.figsize': (6,6)},\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.DatasetSummary`\n :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis`\n :py:class:`~autogluon.eda.analysis.dataset.SpecialTypesAnalysis`\n :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis`\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n :py:class:`~autogluon.eda.visualization.dataset.DatasetTypeMismatch`\n\n \"\"\"\n\n fig_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(fig_args))\n chart_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(chart_args))\n\n state = analyze(\n train_data=train_data,\n test_data=test_data,\n val_data=val_data,\n label=label,\n sample=sample,\n state=state,\n return_state=True,\n anlz_facets=[\n DatasetSummary(),\n MissingValuesAnalysis(),\n RawTypesAnalysis(),\n VariableTypeAnalysis(),\n SpecialTypesAnalysis(),\n ApplyFeatureGenerator(category_to_numbers=True, children=[FeatureDistanceAnalysis()]),\n ],\n viz_facets=[\n DatasetStatistics(headers=True),\n DatasetTypeMismatch(headers=True),\n MarkdownSectionComponent(\"### Feature Distance\"),\n FeatureDistanceAnalysisVisualization(\n fig_args=fig_args.get(\"feature_distance\", {}), **chart_args.get(\"feature_distance\", {})\n ),\n ],\n )\n\n # Groups analysis\n distance = state.feature_distance # type: ignore # state is always present\n if len(distance.near_duplicates) > 0: # type: ignore # state is always present\n for group in distance.near_duplicates:\n nodes = group[\"nodes\"]\n\n interactions: List[AbstractVisualization] = []\n for n in nodes[1:]:\n if state.variable_type.train_data[n] != \"category\": # type: ignore\n interactions.append(MarkdownSectionComponent(f\"Feature interaction between `{nodes[0]}`/`{n}`\"))\n interactions.append(\n FeatureInteractionVisualization(\n key=f\"{nodes[0]}:{n}\",\n fig_args=fig_args.get(\"chart\", {}).get(n, {}),\n **chart_args.get(\"chart\", {}).get(n, {}),\n )\n )\n\n analyze(\n train_data=train_data,\n state=state,\n anlz_facets=[FeatureInteraction(key=f\"{nodes[0]}:{n}\", x=nodes[0], y=n) for n in nodes[1:]],\n viz_facets=[\n *interactions,\n ],\n )\n\n return state if return_state else None\n\n\ndef covariate_shift_detection(\n train_data: pd.DataFrame,\n test_data: pd.DataFrame,\n label: str,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n path: Optional[str] = None,\n state: Union[None, dict, AnalysisState] = None,\n return_state: bool = False,\n verbosity: int = 0,\n fig_args: Optional[Dict[str, Any]] = None,\n chart_args: Optional[Dict[str, Any]] = None,\n **fit_args,\n):\n \"\"\"\n Shortcut for covariate shift detection analysis.\n\n Detects a change in covariate (X) distribution between training and test, which we call XShift. It can tell you\n if your training set is not representative of your test set distribution. This is done with a Classifier 2\n Sample Test.\n\n Supported `fig_args`/`chart_args` keys:\n - `chart..` - properties for charts rendered during the analysis\n\n Parameters\n ----------\n train_data: Optional[DataFrame]\n training dataset\n test_data: Optional[DataFrame]\n test dataset\n label: : Optional[str]\n target variable\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n path: Optional[str], default = None,\n path for models saving\n return_state: bool, default = False\n return state if `True`\n verbosity: int, default = 0\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n fit_args\n kwargs to pass into `TabularPredictor` fit\n fig_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component figure. The args are supporting nested\n dot syntax: 'a.b.c'. Charts args are following the convention of `.`\n (i.e. `chart.PassengerId.figsize` will result in setting `figsize` on `PassengerId` figure.\n chart_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component chart. The args are supporting nested\n dot syntax: 'a.b.c'. Charts args are following the convention of `.`\n (i.e. `chart.PassengerId.fill` will result in setting `fill` on `PassengerId` chart.\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.auto as auto\n >>>\n >>> # use default settings\n >>> auto.covariate_shift_detection(train_data=..., test_data=..., label=...)\n >>>\n >>> # customize classifier and verbosity level\n >>> auto.covariate_shift_detection(train_data=..., test_data=..., label=..., verbosity=2, hyperparameters = {'GBM': {}})\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.shift.XShiftDetector`\n :py:class:`~autogluon.eda.visualization.shift.XShiftSummary`\n\n \"\"\"\n fit_args = get_default_estimator_if_not_specified(fit_args)\n fig_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(fig_args))\n chart_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(chart_args))\n\n state = analyze(\n train_data=train_data,\n test_data=test_data,\n label=label,\n sample=sample,\n state=state,\n return_state=True,\n anlz_facets=[\n RawTypesAnalysis(),\n VariableTypeAnalysis(),\n XShiftDetector(classifier_kwargs=dict(path=path, verbosity=verbosity), classifier_fit_kwargs=fit_args),\n ],\n viz_facets=[XShiftSummary()],\n )\n\n # Plot distribution differences between datasets\n # TODO: move `vars_to_plot` calculation to analysis\n # type: ignore # state is always present\n xshift: AnalysisState = state.xshift_results # type: ignore[union-attr] # state is not none\n if xshift.detection_status:\n vars_to_plot = xshift.shift_features[: XShiftSummary.MAX_FEATURES_TO_DISPLAY]\n if len(vars_to_plot) > 0:\n _train_data = train_data[vars_to_plot].copy()\n _train_data[\"__dataset__\"] = \"train_data\"\n _test_data = test_data[vars_to_plot].copy()\n _test_data[\"__dataset__\"] = \"test_data\"\n df_all = pd.concat([_train_data, _test_data], ignore_index=True)\n\n for var in vars_to_plot:\n if state.variable_type.train_data[var] != \"category\": # type: ignore\n pvalue = xshift.feature_importance.loc[var][\"p_value\"]\n analyze(\n viz_facets=[\n MarkdownSectionComponent(\n f\"**`{var}` values distribution between datasets; p-value: `{pvalue:.4f}`**\"\n )\n ]\n )\n\n analyze_interaction(\n train_data=df_all,\n state=state,\n x=var,\n hue=\"__dataset__\",\n fig_args=fig_args.get(\"chart\", {}).get(var, {}),\n chart_args=chart_args.get(\"chart\", {}).get(var, {}),\n )\n\n return state if return_state else None\n\n\ndef _is_lightgbm_available() -> bool:\n try:\n import lightgbm # noqa\n\n return True\n except (ImportError, OSError):\n return False\n\n\ndef get_default_estimator_if_not_specified(fit_args, fit_bagging_folds: int = 0):\n if (\"hyperparameters\" not in fit_args) and (\"presets\" not in fit_args):\n fit_args = fit_args.copy()\n\n fit_args[\"fit_weighted_ensemble\"] = False\n if fit_bagging_folds > 0:\n fit_args = {**dict(num_bag_folds=fit_bagging_folds, num_bag_sets=1, num_stack_levels=0), **fit_args}\n if (\"ag_args_ensemble\" not in fit_args) or (\"fold_fitting_strategy\" not in fit_args[\"ag_args_ensemble\"]):\n fit_args[\"ag_args_ensemble\"] = {\"fold_fitting_strategy\": \"sequential_local\"}\n if _is_lightgbm_available():\n fit_args[\"hyperparameters\"] = QuickFitDefaults.DEFAULT_LGBM_CONFIG\n else:\n fit_args[\"hyperparameters\"] = QuickFitDefaults.DEFAULT_RF_CONFIG\n return fit_args\n\n\ndef target_analysis(\n train_data: pd.DataFrame,\n label: str,\n test_data: Optional[pd.DataFrame] = None,\n problem_type: str = \"auto\",\n fit_distributions: Union[bool, str, List[str]] = True,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n state: Union[None, dict, AnalysisState] = None,\n return_state: bool = False,\n fig_args: Optional[Dict[str, Any]] = None,\n chart_args: Optional[Dict[str, Any]] = None,\n) -> Optional[AnalysisState]:\n \"\"\"\n Target variable composite analysis.\n\n Performs the following analysis components of the label field:\n - basic summary stats\n - feature values distribution charts; adds fitted distributions for numeric targets\n - target correlations analysis; with interaction charts of target vs high-correlated features\n\n Supported `fig_args`/`chart_args` keys:\n - `correlation.` - properties for correlation heatmap\n - `chart..` - properties for charts rendered during the analysis.\n If is matching `label` value, then this will modify the top chart; all other values will be affecting label/\n interaction charts\n\n Parameters\n ----------\n train_data: Optional[DataFrame]\n training dataset\n test_data: Optional[DataFrame], default = None\n test dataset\n label: : Optional[str]\n target variable\n problem_type: str, default = 'auto'\n problem type to use. Valid problem_type values include ['auto', 'binary', 'multiclass', 'regression', 'quantile', 'softclass']\n auto means it will be Auto-detected using AutoGluon methods.\n fit_distributions: Union[bool, str, List[str]], default = False,\n If `True`, or list of distributions is provided, then fit distributions. Performed only if `y` and `hue` are not present.\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n return_state: bool, default = False\n return state if `True`\n fig_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component figure. The args are supporting nested\n dot syntax: 'a.b.c'. Charts args are following the convention of `.`\n (i.e. `chart.PassengerId.figsize` will result in setting `figsize` on ``/`PassengerId` figure.\n chart_args: Optional[Dict[str, Any]], default = None,\n figures args for visualizations; key == component; value = dict of kwargs for component chart. The args are supporting nested\n dot syntax: 'a.b.c'. Charts args are following the convention of `.`\n (i.e. `chart.PassengerId.fill` will result in setting `fill` on ``/`PassengerId` chart.\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>>\n >>> auto.target_analysis(train_data=..., label=...)\n\n \"\"\"\n\n assert label in train_data.columns, f\"label `{label}` is not in `train_data` columns: `{train_data.columns}`\"\n\n fig_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(fig_args))\n chart_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(chart_args))\n\n if (test_data is not None) and (label in test_data.columns):\n _test_data = test_data[[label]]\n else:\n _test_data = None\n\n # Basic variable information table\n state: AnalysisState = analyze( # type: ignore # state is always present\n train_data=train_data[[label]],\n test_data=_test_data,\n label=label,\n state=state,\n sample=sample,\n return_state=True,\n anlz_facets=[\n DatasetSummary(),\n MissingValuesAnalysis(),\n RawTypesAnalysis(),\n SpecialTypesAnalysis(),\n ProblemTypeControl(problem_type=problem_type),\n LabelInsightsAnalysis(),\n ],\n viz_facets=[\n MarkdownSectionComponent(\"## Target variable analysis\"),\n MarkdownSectionComponent(\n \"### Label Insights\",\n condition_fn=(lambda s: is_key_present_in_state(s, \"label_insights\")),\n ),\n LabelInsightsVisualization(),\n DatasetStatistics(),\n ],\n )\n\n # Distribution chart\n state = analyze_interaction(\n train_data=train_data,\n sample=sample,\n x=label,\n state=state,\n return_state=True,\n fit_distributions=fit_distributions,\n fig_args=fig_args.get(\"chart\", {}).get(label, {}),\n chart_args=chart_args.get(\"chart\", {}).get(label, {}),\n )\n\n state = _render_distribution_fit_information_if_available(state, label)\n state = _render_correlation_analysis(state, train_data, label, sample, fig_args, chart_args)\n state = _render_features_highly_correlated_with_target(state, train_data, label, sample, fig_args, chart_args)\n\n return state if return_state else None\n\n\ndef _render_features_highly_correlated_with_target(\n state, train_data, label, sample, fig_args, chart_args\n) -> AnalysisState:\n fields = state.correlations_focus_high_corr.train_data.index.tolist() # type: ignore\n analyze(\n train_data=train_data,\n state=state,\n sample=sample,\n return_state=True,\n anlz_facets=[FeatureInteraction(key=f\"{f}:{label}\", x=f, y=label) for f in fields],\n viz_facets=[\n FeatureInteractionVisualization(\n headers=True,\n key=f\"{f}:{label}\",\n fig_args=fig_args.get(\"chart\", {}).get(f, {}),\n **chart_args.get(\"chart\", {}).get(f, {}),\n )\n for f in fields\n ],\n )\n return state\n\n\ndef _render_correlation_analysis(state, train_data, label, sample, fig_args, chart_args) -> AnalysisState:\n state = analyze(\n train_data=train_data,\n sample=sample,\n state=state,\n return_state=True,\n label=label,\n anlz_facets=[ApplyFeatureGenerator(category_to_numbers=True, children=[Correlation(focus_field=label)])],\n )\n corr_info = [\"### Target variable correlations\"]\n if len(state.correlations_focus_high_corr.train_data) < 1: # type: ignore\n corr_info.append(\n f\" - â ī¸ no fields with absolute correlation greater than \" # type: ignore\n f\"`{state.correlations_focus_field_threshold}` found for target variable `{label}`.\"\n )\n analyze(\n state=state,\n viz_facets=[\n MarkdownSectionComponent(\"\\n\".join(corr_info)),\n CorrelationVisualization(\n headers=True, fig_args=fig_args.get(\"correlation\", {}), **chart_args.get(\"correlation\", {})\n ),\n ],\n )\n return state\n\n\ndef _render_distribution_fit_information_if_available(state, label) -> Optional[AnalysisState]:\n if state.distributions_fit is not None: # type: ignore # state is always present\n dist_fit_state = state.distributions_fit.train_data # type: ignore\n dist_info = [\"### Distribution fits for target variable\"]\n if (label in dist_fit_state) and (len(dist_fit_state[label]) > 0):\n for d, p in state.distributions_fit.train_data[label].items(): # type: ignore\n dist_info.append(\n f\" - [{d}](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.{d}.html)\"\n )\n if p.param is not None and len(p.param) > 0:\n params = \", \".join([f\"{shape}: {param}\" for shape, param in zip(p.shapes, p.param)])\n dist_info.append(f' - p-value: {p[\"pvalue\"]:.3f}')\n dist_info.append(f\" - Parameters: ({params})\")\n else:\n dist_info.append(\n f\" - â ī¸ none of the [attempted](https://docs.scipy.org/doc/scipy/reference/stats.html#continuous-distributions) \" # type: ignore\n f\"distribution fits satisfy specified minimum p-value threshold: `{state.distributions_fit_pvalue_min}`\"\n )\n analyze(viz_facets=[MarkdownSectionComponent(\"\\n\".join(dist_info))])\n return state\n\n\ndef missing_values_analysis(\n train_data: Optional[pd.DataFrame] = None,\n test_data: Optional[pd.DataFrame] = None,\n val_data: Optional[pd.DataFrame] = None,\n graph_type: str = \"matrix\",\n state: Union[None, dict, AnalysisState] = None,\n return_state: bool = False,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n **chart_args,\n):\n \"\"\"\n Perform quick analysis of missing values across datasets.\n\n Parameters\n ----------\n train_data: Optional[DataFrame]\n training dataset\n test_data: Optional[DataFrame], default = None\n test dataset\n val_data\n validation dataset\n graph_type: str, default = 'matrix'\n One of the following visualization types:\n - matrix - nullity matrix is a data-dense display which lets you quickly visually pick out patterns in data completion\n This visualization will comfortably accommodate up to 50 labelled variables.\n Past that range labels begin to overlap or become unreadable, and by default large displays omit them.\n - bar - visualizes how many rows are non-null vs null in the column. Logarithmic scale can by specifying `log=True` in `kwargs`\n - heatmap - correlation heatmap measures nullity correlation: how strongly the presence or absence of one\n variable affects the presence of another. Nullity correlation ranges from -1\n (if one variable appears the other definitely does not) to 0 (variables appearing or not appearing have no effect on one another)\n to 1 (if one variable appears the other definitely also does).\n Entries marked <1 or >-1 have a correlation that is close to being exactingly negative or positive but is still not quite perfectly so.\n - dendrogram - the dendrogram allows to more fully correlate variable completion, revealing trends deeper than the pairwise ones\n visible in the correlation heatmap. The dendrogram uses a hierarchical clustering algorithm (courtesy of scipy) to bin variables\n against one another by their nullity correlation (measured in terms of binary distance).\n At each step of the tree the variables are split up based on which combination minimizes the distance of the remaining clusters.\n The more monotone the set of variables, the closer their total distance is to zero, and the closer their average distance (the y-axis) is to zero.\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n return_state: bool, default = False\n return state if `True`\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.auto as auto\n >>>\n >>> auto.missing_values_analysis(train_data=...)\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis`\n :py:class:`~autogluon.eda.visualization.dataset.DatasetStatistics`\n :py:class:`~autogluon.eda.visualization.missing.MissingValues`\n\n \"\"\"\n # TODO add null equivalents: i.e. >50% of values are the same (i.e. 0 is frequently used as null equivalent)\n return analyze(\n train_data=train_data,\n test_data=test_data,\n val_data=val_data,\n state=state,\n return_state=return_state,\n sample=sample,\n anlz_facets=[\n MissingValuesAnalysis(),\n ],\n viz_facets=[\n MarkdownSectionComponent(\"### Missing Values Analysis\"),\n DatasetStatistics(),\n MissingValues(graph_type=graph_type, **chart_args),\n ],\n )\n\n\ndef explain_rows(\n train_data: pd.DataFrame,\n model: TabularPredictor,\n rows: pd.DataFrame,\n positive_class: Optional = None,\n display_rows: bool = False,\n plot: Optional[str] = \"force\",\n baseline_sample: int = 100,\n return_state: bool = False,\n fit_args: Optional[Dict[str, Any]] = None,\n **kwargs,\n) -> Optional[AnalysisState]:\n \"\"\"\n Kernel SHAP is a method that uses a special weighted linear regression\n to compute the importance of each feature. The computed importance values\n are Shapley values from game theory and also coefficients from a local linear\n regression values analysis for the given rows.\n\n The results are rendered either as force plot or waterfall plot.\n\n Parameters\n ----------\n train_data: DataFrame\n training dataset\n model: TabularPredictor\n trained AutoGluon predictor\n rows: pd.DataFrame,\n rows to explain\n positive_class: Optional\n Optionally specify positive class to explain; if not provided, the value will be autodetected.\n For binary it's derived from `model.positive_class`.\n For multiclass it's the last column in prediction probabilities.\n display_rows: bool, default = False\n if `True` then display the row before the explanation chart\n plot: Optional[str], default = 'force'\n type of plot to visualize the Shapley values. Supported keys:\n - `force` - Visualize the given SHAP values with an additive force layout\n - `waterfall` - Visualize the given SHAP values with a waterfall layout\n - `None` - do not use any visualization\n baseline_sample: int, default = 100\n The background dataset size to use for integrating out features. To determine the impact\n of a feature, that feature is set to \"missing\" and the change in the model output\n is observed.\n return_state: bool, default = False\n return state if `True`\n fit_args: Optional[Dict[str, Any]], default = None,\n kwargs for `ShapAnalysis`.\n kwargs\n\n Examples\n --------\n >>> import autogluon.eda.auto as auto\n >>>\n >>> state = auto.quick_fit(\n >>> train_data=...,\n >>> label=...,\n >>> return_state=True,\n >>> )\n >>>\n >>> # quick_fit stored model in `state.model`, and can be passed here.\n >>> # This will visualize 1st row of rows with the highest errors;\n >>> # these rows are stored under `state.model_evaluation.highest_error`\n >>> auto.explain_rows(\n >>> train_data=...,\n >>> model=state.model,\n >>> display_rows=True,\n >>> rows=state.model_evaluation.highest_error[:1],\n >>> plot='waterfall', # visualize as waterfall plot\n >>> )\n\n See Also\n --------\n :py:class:`~shap.KernelExplainer`\n :py:class:`~autogluon.eda.analysis.explain.ShapAnalysis`\n :py:class:`~autogluon.eda.visualization.explain.ExplainForcePlot`\n :py:class:`~autogluon.eda.visualization.explain.ExplainWaterfallPlot`\n \"\"\"\n\n if fit_args is None:\n fit_args = {}\n\n if plot is None:\n viz_facets = None\n else:\n supported_plots = {\n \"force\": ExplainForcePlot,\n \"waterfall\": ExplainWaterfallPlot,\n }\n viz_cls = supported_plots.get(plot, None)\n assert viz_cls is not None, (\n f\"plot must be one of the following values: {','.join(supported_plots.keys())}. \"\n f\"If no visualization required, then `None` can be passed.\"\n )\n viz_facets = [viz_cls(display_rows=display_rows, **kwargs)]\n\n return analyze(\n train_data=train_data[model.original_features],\n model=model,\n return_state=return_state,\n anlz_facets=[ShapAnalysis(rows, positive_class=positive_class, baseline_sample=baseline_sample, **fit_args)], # type: ignore\n viz_facets=viz_facets, # type: ignore\n )\n\n\ndef partial_dependence_plots(\n train_data: pd.DataFrame,\n label: str,\n target: Optional[Any] = None,\n features: Optional[Union[str, List[str]]] = None,\n two_way: bool = False,\n path: Optional[str] = None,\n max_ice_lines: int = 300,\n sample: Optional[Union[int, float]] = DEFAULT_SAMPLE_SIZE,\n fig_args: Optional[Dict[str, Dict[str, Any]]] = None,\n chart_args: Optional[Dict[str, Dict[str, Any]]] = None,\n show_help_text: bool = True,\n return_state: bool = False,\n col_number_warning: int = 20,\n **fit_args,\n):\n \"\"\"\n Partial Dependence Plot (PDP)\n\n Analyze and interpret the relationship between a target variable and a specific feature in a machine learning model.\n PDP helps in understanding the marginal effect of a feature on the predicted outcome while holding other features constant\n\n The visualizations have two modes:\n - Display Partial Dependence Plots (PDP) with Individual Conditional Expectation (ICE) - this is the default mode of operation\n - Two-Way PDP plots - this mode can be selected via passing two `features` and setting `two_way = True`\n\n ICE plots complement PDP by showing the relationship between a feature and the model's output for each individual instance in the dataset.\n ICE lines (blue) can be overlaid on PDPs (red) to provide a more detailed view of how the model behaves for specific instances.\n Here are some points on how to interpret PDPs with ICE lines:\n\n - `Central tendency`\n The PDP line represents the average prediction for different values of the feature of interest.\n Look for the overall trend of the PDP line to understand the average effect of the feature on the model's output.\n - `Variability`\n The ICE lines represent the predicted outcomes for individual instances as the feature of interest changes.\n Examine the spread of ICE lines around the PDP line to understand the variability in predictions for different instances.\n - `Non-linear relationships`\n Look for any non-linear patterns in the PDP and ICE lines.\n This may indicate that the model captures a non-linear relationship between the feature and the model's output.\n - `Heterogeneity`\n Check for instances where ICE lines have widely varying slopes, indicating different relationships between the feature and\n the model's output for individual instances. This may suggest interactions between the feature of interest and other features.\n - `Outliers`\n Look for any ICE lines that are very different from the majority of the lines.\n This may indicate potential outliers or instances that have unique relationships with the feature of interest.\n - `Confidence intervals`\n If available, examine the confidence intervals around the PDP line. Wider intervals may indicate a less certain relationship\n between the feature and the model's output, while narrower intervals suggest a more robust relationship.\n - `Interactions`\n By comparing PDPs and ICE plots for different features, you may detect potential interactions between features.\n If the ICE lines change significantly when comparing two features, this might suggest an interaction effect.\n\n Two-way PDP can visualize potential interactions between any two features. Here are a few cases when two-way PDP can give good results:\n\n - `Suspected interactions`: Even if two features are not highly correlated, they may still interact in the context of the model.\n If you suspect that there might be interactions between any two features, two-way PDP can help to verify the hypotheses.\n - `Moderate to high correlation`: If two features have a moderate to high correlation,\n a two-way PDP can show how the combined effect of these features influences the model's predictions.\n In this case, the plot can help reveal whether the relationship between the features is additive, multiplicative, or more complex.\n - `Complementary features`: If two features provide complementary information, a two-way PDP can help illustrate how the joint effect\n of these features impacts the model's predictions.\n For example, if one feature measures the length of an object and another measures its width, a two-way PDP could show how the\n combination of these features affects the predicted outcome.\n - `Domain knowledge`: If domain knowledge suggests that the relationship between two features might be important for the model's output,\n a two-way PDP can help to explore and validate these hypotheses.\n - `Feature importance`: If feature importance analysis ranks both features high in the leaderboard, it might be beneficial\n to examine their joint effect on the model's predictions.\n\n State attributes\n\n - `pdp_id_to_category_mappings`\n Categorical are represented in charts as numbers; id to value mappings are available in this property.\n\n Parameters\n ----------\n train_data: DataFrame\n training dataset\n label: str\n target variable\n target: Optional[Any], default = None\n In a multiclass setting, specifies the class for which the PDPs should be computed.\n Ignored in binary classification or classical regression settings\n features: Optional[Union[str, List[str]]], default = None\n feature subset to display; `None` means all features will be rendered.\n two_way: bool, default = False\n render two-way PDP; this mode works only when two `features` are specified\n path: Optional[str], default = None\n location to store the model trained for this task\n max_ice_lines: int, default = 300\n max number of ice lines to display for each sub-plot\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n fig_args: Optional[Dict[str, Any]], default = None\n kwargs to pass into chart figure\n chart_args: Optional[dict], default = None\n kwargs to pass into visualization component\n show_help_text:bool, default = True\n if `True` shows additional information how to interpret the data\n return_state: bool, default = False\n return state if `True`\n col_number_warning: int, default = 20\n number of features to visualize after which the warning will be displayed to warn about rendering time\n fit_args: Optional[Dict[str, Dict[str, Any]]], default = None,\n kwargs to pass into `TabularPredictor` fit.\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n Examples\n --------\n >>> import autogluon.eda.auto as auto\n >>>\n >>> # Plot all features in a grid\n >>> auto.partial_dependence_plots(train_data=..., label=...)\n >>>\n >>> # Plot two-way feature interaction for features `feature_a` and `feature_b`\n >>> auto.partial_dependence_plots(train_data=..., label=..., features=['feature_a', 'feature_b'], two_way=True)\n\n See Also\n --------\n :py:class:`~autogluon.eda.visualization.interaction.PDPInteractions`\n \"\"\"\n\n chart_args, fig_args, features = _validate_and_normalize_pdp_args(\n train_data, features, fig_args, chart_args, col_number_warning\n )\n pdp_data, state, id_to_category_mappings = _prepare_pdp_data(train_data, label, sample, features)\n\n state = quick_fit(\n path=path,\n train_data=state.pdp_train_data,\n label=label,\n return_state=True,\n render_analysis=False,\n sample=sample,\n **fit_args,\n )\n state.pdp_data = pdp_data\n state.pdp_id_to_category_mappings = id_to_category_mappings\n\n if features is None:\n features = state.model_evaluation.importance.index.tolist()\n\n if len(id_to_category_mappings) > 0:\n cats = \", \".join([f\"`{c}`\" for c in id_to_category_mappings.keys()])\n else:\n cats = \"\"\n\n analyze(\n model=state.model,\n state=state,\n viz_facets=[\n MarkdownSectionComponent(\"### Two-way Partial Dependence Plots\", condition_fn=lambda _: two_way),\n MarkdownSectionComponent(\n \"Two-Way partial dependence plots (PDP) are useful for visualizing the relationship between a pair of features and the predicted outcome \"\n \"in a machine learning model. There are several things to look for when exploring the two-way plot:\\n\\n\"\n \"* **Shape of the interaction**: Look at the shape of the plot to understand the nature of the interaction. \"\n \"It could be linear, non-linear, or more complex.\\n\"\n \"* **Feature value range**: Observe the range of the feature values on both axes to understand the domain of the interaction. \"\n \"This can help you identify whether the model is making predictions within reasonable bounds or if there are extrapolations \"\n \"beyond the training data.\\n\"\n \"* **Areas of high uncertainty**: Look for areas in the plot where the predictions are less certain, which may be indicated by \"\n \"larger confidence intervals, higher variance, or fewer data points. These areas may require further investigation or additional data.\\n\"\n \"* **Outliers and anomalies**: Check for any outliers or anomalies in the plot that may indicate issues with the model or the data. \"\n \"These could be regions of the plot with unexpected patterns or values that do not align with the overall trend.\\n\"\n \"* **Sensitivity to feature values**: Assess how sensitive the predicted outcome is to changes in the feature values.\\n\\n\"\n \"Use `show_help_text=False` to hide this information when calling this function.\",\n condition_fn=lambda _: show_help_text and two_way,\n ),\n MarkdownSectionComponent(\"### Partial Dependence Plots\", condition_fn=lambda _: not two_way),\n MarkdownSectionComponent(\n \"Individual Conditional Expectation (ICE) plots complement Partial Dependence Plots (PDP) by showing the \"\n \"relationship between a feature and the model's output for each individual instance in the dataset. ICE lines (blue) \"\n \"can be overlaid on PDPs (red) to provide a more detailed view of how the model behaves for specific instances. \"\n \"Here are some points on how to interpret PDPs with ICE lines:\\n\\n\"\n \"* **Central tendency**: The PDP line represents the average prediction for different values of the feature of interest. \"\n \"Look for the overall trend of the PDP line to understand the average effect of the feature on the model's output.\\n\"\n \"* **Variability**: The ICE lines represent the predicted outcomes for individual instances as the feature of interest changes. \"\n \"Examine the spread of ICE lines around the PDP line to understand the variability in predictions for different instances.\\n\"\n \"* **Non-linear relationships**: Look for any non-linear patterns in the PDP and ICE lines. This may indicate that the model \"\n \"captures a non-linear relationship between the feature and the model's output.\\n\"\n \"* **Heterogeneity**: Check for instances where ICE lines have widely varying slopes, indicating different relationships \"\n \"between the feature and the model's output for individual instances. This may suggest interactions between the feature \"\n \"of interest and other features.\\n\"\n \"* **Outliers**: Look for any ICE lines that are very different from the majority of the lines. This may indicate potential \"\n \"outliers or instances that have unique relationships with the feature of interest.\\n\"\n \"* **Confidence** intervals: If available, examine the confidence intervals around the PDP line. Wider intervals may indicate \"\n \"a less certain relationship between the feature and the model's output, while narrower intervals suggest a more robust relationship.\\n\"\n \"* **Interactions**: By comparing PDPs and ICE plots for different features, you may detect potential interactions between features. \"\n \"If the ICE lines change significantly when comparing two features, this might suggest an interaction effect.\\n\\n\"\n \"Use `show_help_text=False` to hide this information when calling this function.\",\n condition_fn=lambda _: show_help_text and not two_way,\n ),\n PDPInteractions(\n features=features,\n two_way=two_way,\n fig_args=fig_args,\n sample=max_ice_lines,\n target=target,\n **chart_args,\n ), # type: ignore\n MarkdownSectionComponent(\n f\"The following variable(s) are categorical: {cats}. They are represented as the numbers in the figures above. \"\n f\"Mappings are available in `state.pdp_id_to_category_mappings`. The`state` can be returned from this call via adding `return_state=True`.\",\n condition_fn=lambda _: len(id_to_category_mappings) > 0,\n ),\n ],\n )\n\n s = AnalysisState({\"pdp_id_to_category_mappings\": id_to_category_mappings})\n\n return s if return_state else None\n\n\ndef _validate_and_normalize_pdp_args(\n train_data: pd.DataFrame,\n features: Optional[Union[str, List[str]]] = None,\n fig_args: Optional[Dict[str, Dict[str, Any]]] = None,\n chart_args: Optional[Dict[str, Dict[str, Any]]] = None,\n col_number_warning: int = 20,\n) -> Tuple[Dict[str, Any], Dict[str, Any], Optional[List[str]]]:\n fig_args = get_empty_dict_if_none(fig_args)\n chart_args = get_empty_dict_if_none(chart_args)\n\n if features is not None:\n if type(features) is not list:\n features = [features] # type: ignore\n features_not_present = [f for f in features if f not in train_data.columns]\n assert (\n len(features_not_present) == 0\n ), f\"Features {', '.join(features_not_present)} are not present in train_data: {', '.join(train_data.columns)}\"\n if features is None and len(train_data.columns) > col_number_warning:\n logger.warning(\n f\"This visualization will render {len(train_data.columns)} charts. \"\n f\"This can take a while. This warning can be disabled by setting `col_number_warning` to a higher value.\"\n )\n return chart_args, fig_args, features\n\n\ndef _prepare_pdp_data(\n train_data: pd.DataFrame,\n label: str,\n sample: Optional[Union[int, float]] = None,\n features: Optional[List[str]] = None,\n) -> Tuple[pd.DataFrame, AnalysisState, Dict[str, Dict[int, str]]]:\n apply_gen = ApplyFeatureGenerator(\n category_to_numbers=True,\n children=[\n SaveArgsToState(\n params_mapping={\n \"train_data\": \"pdp_train_data\",\n }\n )\n ],\n )\n state = analyze(\n train_data=train_data,\n label=label,\n return_state=True,\n anlz_facets=[apply_gen],\n sample=sample,\n )\n pdp_data = state.pdp_train_data # type: ignore\n id_to_category_mappings: Dict[str, Dict[int, str]] = {}\n for gen in [item for sublist in apply_gen.feature_generator.generators for item in sublist]:\n if type(gen) is CategoryFeatureGenerator:\n id_to_category_mappings = {\n k: {i: v for i, v in enumerate(v.tolist())} for k, v in gen.category_map.items()\n }\n\n if features is not None:\n id_to_category_mappings = {k: v for k, v in id_to_category_mappings.items() if k in features}\n\n return pdp_data, state, id_to_category_mappings # type: ignore\n\n\ndef detect_anomalies(\n train_data: pd.DataFrame,\n label: str,\n test_data: Optional[pd.DataFrame] = None,\n val_data: Optional[pd.DataFrame] = None,\n explain_top_n_anomalies: Optional[int] = None,\n show_top_n_anomalies: Optional[int] = 10,\n threshold_stds: float = 3,\n show_help_text: bool = True,\n state: Union[None, dict, AnalysisState] = None,\n sample: Union[None, int, float] = DEFAULT_SAMPLE_SIZE,\n return_state: bool = False,\n fig_args: Optional[Dict[str, Any]] = None,\n chart_args: Optional[Dict[str, Any]] = None,\n **anomaly_detector_kwargs,\n) -> Optional[AnalysisState]:\n \"\"\"\n Anomaly Detection\n\n This method is used to identify unusual patterns or behaviors in data that deviate significantly from the norm.\n It's best used when finding outliers, rare events, or suspicious activities that could indicate fraud, defects, or system failures.\n\n When interpreting anomaly scores, consider:\n\n - `Threshold`:\n Determine a suitable threshold to separate normal from anomalous data points, based on domain knowledge or statistical methods.\n - `Context`:\n Examine the context of anomalies, including time, location, and surrounding data points, to identify possible causes.\n - `False positives/negatives`:\n Be aware of the trade-offs between false positives (normal points classified as anomalies) and false negatives (anomalies missed).\n - `Feature relevance`:\n Ensure the features used for anomaly detection are relevant and contribute to the model's performance.\n - `Model performance`:\n Regularly evaluate and update the model to maintain its accuracy and effectiveness.\n\n It's important to understand the context and domain knowledge before deciding on an appropriate approach to deal with anomalies.\n The choice of method depends on the data's nature, the cause of anomalies, and the problem being addressed.\n The common ways to deal with anomalies:\n\n - `Removal`:\n If an anomaly is a result of an error, noise, or irrelevance to the analysis, it can be removed from the dataset\n to prevent it from affecting the model's performance.\n - `Imputation`:\n Replace anomalous values with appropriate substitutes, such as the mean, median, or mode of the feature,\n or by using more advanced techniques like regression or k-nearest neighbors.\n - `Transformation`:\n Apply transformations like log, square root, or z-score to normalize the data and reduce the impact of extreme values.\n Absolute dates might be transformed into relative features like age of the item.\n - `Capping`:\n Set upper and lower bounds for a feature, and replace values outside these limits with the bounds themselves.\n This method is also known as winsorizing.\n - `Separate modeling`:\n Treat anomalies as a distinct group and build a separate model for them, or use specialized algorithms designed\n for handling outliers, such as robust regression or one-class SVM.\n - `Incorporate as a feature`:\n Create a new binary feature indicating the presence of an anomaly, which can be useful if anomalies have predictive value.\n\n State attributes\n\n - `anomaly_detection.scores.`\n scores for each of the datasets passed into analysis (i.e. `train_data`, `test_data`)\n - `state.anomaly_detection.anomalies.`\n data points considered as anomalies - original rows with added `score` column sorted in descending score order.\n defined by `threshold_stds` parameter\n - `anomaly_detection.anomaly_score_threshold`\n anomaly score threshold above which data points are considered as anomalies;\n defined by `threshold_stds` parameter\n\n Parameters\n ----------\n train_data: DataFrame\n training dataset\n label: str\n target variable\n test_data: Optional[pd.DataFrame], default = None\n test dataset\n val_data: Optional[pd.DataFrame], default = None\n validation dataset\n explain_top_n_anomalies: Optional[int], default = None\n explain the anomaly scores for n rows with the highest scores; don't perform analysis if value is `None` or `0`\n show_top_n_anomalies: Optional[int], default = 10\n display n rows with highest anomaly scores\n threshold_stds: float, default = 3\n specifies how many standard deviations above mean anomaly score considered as anomalies\n (only needed for visualization, does not affect scores calculation)\n show_help_text:bool, default = True\n if `True` shows additional information how to interpret the data\n state: Union[None, dict, AnalysisState], default = None\n pass prior state if necessary; the object will be updated during `anlz_facets` `fit` call.\n sample: Union[None, int, float], default = 10000\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n return_state: bool, default = False\n return state if `True`\n fig_args: Optional[Dict[str, Any]], default = None,\n kwargs to pass into visualization component\n chart_args: Optional[dict], default = None\n kwargs to pass into visualization component\n anomaly_detector_kwargs\n kwargs to pass into :py:class:`~autogluon.eda.analysis.anomaly.AnomalyDetectorAnalysis`\n\n >>> import autogluon.eda.auto as auto\n >>>\n >>> state = auto.detect_anomalies(\n >>> train_data=...,\n >>> test_data=..., # optional\n >>> label=...,\n >>> threshold_stds=3,\n >>> show_top_n_anomalies=5,\n >>> explain_top_n_anomalies=3,\n >>> return_state=True,\n >>> chart_args={\n >>> 'normal.color': 'lightgrey',\n >>> 'anomaly.color': 'orange',\n >>> }\n >>> )\n >>>\n >>> # Getting anomaly scores from the analysis\n >>> train_anomaly_scores = state.anomaly_detection.scores.train_data\n >>> test_anomaly_scores = state.anomaly_detection.scores.test_data\n >>>\n >>> # Anomaly score threshold for specified level - see `threshold_stds` parameter\n >>> anomaly_score_threshold = state.anomaly_detection.anomaly_score_threshold\n\n Returns\n -------\n state after `fit` call if `return_state` is `True`; `None` otherwise\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.anomaly.AnomalyDetectorAnalysis`\n :py:class:`~autogluon.eda.visualization.anomaly.AnomalyScoresVisualization`\n \"\"\"\n fig_args_defaults = {\"figsize\": (12, 6)}\n fig_args = {**fig_args_defaults, **get_empty_dict_if_none(fig_args).copy()}\n\n chart_args = get_empty_dict_if_none(chart_args).copy()\n\n store_explainability_data = (explain_top_n_anomalies is not None) and explain_top_n_anomalies > 0\n _state: AnalysisState = analyze( # type: ignore[assignment] # always has value: return_state=True\n train_data=train_data,\n test_data=test_data,\n val_data=val_data,\n label=label,\n state=state,\n sample=sample,\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(),\n ApplyFeatureGenerator(\n category_to_numbers=True,\n children=[\n AnomalyDetectorAnalysis(\n store_explainability_data=store_explainability_data, **anomaly_detector_kwargs\n ),\n ],\n ),\n ],\n )\n\n analyze(\n state=_state,\n viz_facets=[\n MarkdownSectionComponent(\"### Anomaly Detection Report\"),\n MarkdownSectionComponent(\n \"When interpreting anomaly scores, consider:\\n\"\n \"* **Threshold**: Determine a suitable threshold to separate normal from anomalous data points, \"\n \" based on domain knowledge or statistical methods.\\n\"\n \"* **Context**: Examine the context of anomalies, including time, location, and surrounding data points, to identify possible causes.\\n\"\n \"* **False positives/negatives**: Be aware of the trade-offs between false positives (normal points classified as anomalies) \"\n \" and false negatives (anomalies missed).\\n\"\n \"* **Feature relevance**: Ensure the features used for anomaly detection are relevant and contribute to the model's performance.\\n\"\n \"* **Model performance**: Regularly evaluate and update the model to maintain its accuracy and effectiveness.\\n\\n\"\n \"It's important to understand the context and domain knowledge before deciding on an appropriate approach to deal with anomalies.\"\n \"The choice of method depends on the data's nature, the cause of anomalies, and the problem being addressed.\"\n \"The common ways to deal with anomalies:\\n\\n\"\n \"* **Removal**: If an anomaly is a result of an error, noise, or irrelevance to the analysis, it can be removed from the dataset \"\n \" to prevent it from affecting the model's performance.\\n\"\n \"* **Imputation**: Replace anomalous values with appropriate substitutes, such as the mean, median, or mode of the feature,\"\n \" or by using more advanced techniques like regression or k-nearest neighbors.\\n\"\n \"* **Transformation**: Apply transformations like log, square root, or z-score to normalize the data and reduce the impact of extreme values.\\n\"\n \" Absolute dates might be transformed into relative features like age of the item.\\n\"\n \"* **Capping**: Set upper and lower bounds for a feature, and replace values outside these limits with the bounds themselves.\"\n \" This method is also known as winsorizing.\\n\"\n \"* **Separate modeling**: Treat anomalies as a distinct group and build a separate model for them, or use specialized algorithms designed\"\n \" for handling outliers, such as robust regression or one-class SVM.\\n\"\n \"* **Incorporate as a feature**: Create a new binary feature indicating the presence of an anomaly, \"\n \" which can be useful if anomalies have predictive value.\\n\\n\"\n \"Use `show_help_text=False` to hide this information when calling this function.\",\n condition_fn=lambda _: show_help_text,\n ),\n AnomalyScoresVisualization(threshold_stds=threshold_stds, headers=True, fig_args=fig_args, **chart_args),\n ],\n )\n\n # Store anomalies with the scores into the state\n _state.anomaly_detection.anomalies = {}\n anomaly_score_threshold = _state.anomaly_detection.scores.train_data.std() * threshold_stds\n _state.anomaly_detection.anomaly_score_threshold = anomaly_score_threshold\n for ds, df in AbstractAnalysis.available_datasets(\n AnalysisState({\"train_data\": train_data, \"test_data\": test_data, \"val_data\": val_data})\n ):\n anomaly_scores = _state.anomaly_detection.scores[ds]\n anomaly_idx = anomaly_scores[anomaly_scores >= anomaly_score_threshold].sort_values(ascending=False).index\n _state.anomaly_detection.anomalies[ds] = df.loc[anomaly_idx].join(anomaly_scores)\n\n if (show_top_n_anomalies is not None) and (show_top_n_anomalies > 0) and (len(anomaly_idx) > 0):\n analyze(\n state=_state,\n viz_facets=[\n MarkdownSectionComponent(\n markdown=f\"**Top-{show_top_n_anomalies} `{ds}` anomalies (total: {len(anomaly_idx)})**\"\n ),\n PropertyRendererComponent(\n f\"anomaly_detection.anomalies.{ds}\", transform_fn=(lambda d: d.head(show_top_n_anomalies))\n ),\n ],\n )\n\n if store_explainability_data:\n analyze(\n state=_state,\n viz_facets=[\n MarkdownSectionComponent(\n markdown=\"â ī¸ Please note that the feature values shown on the charts below are transformed \"\n \"into an internal representation; they may be encoded or modified based on internal preprocessing. \"\n \"Refer to the original datasets for the actual feature values.\"\n ),\n MarkdownSectionComponent(\n markdown=\"â ī¸ The detector has seen this dataset; the may result in overly optimistic estimates. \"\n \"Although the anomaly score in the explanation might not match, the magnitude of the feature scores \"\n \"can still be utilized to evaluate the impact of the feature on the anomaly score.\",\n condition_fn=(lambda _: ds == \"train_data\"), # noqa: B023\n ),\n ],\n )\n\n explain_rows(\n **_state.anomaly_detection.explain_rows_fns[ds](anomaly_idx[:explain_top_n_anomalies]),\n plot=\"waterfall\",\n )\n\n return _state if return_state else None\n"
},
{
"path": "eda/src/autogluon/eda/state.py",
"content": "import logging\n\n__all__ = [\"AnalysisState\", \"StateCheckMixin\", \"is_key_present_in_state\"]\n\nfrom typing import Any\n\n\nclass AnalysisState(dict):\n \"\"\"Enabling dot.notation access to dictionary attributes and dynamic code assist in jupyter\"\"\"\n\n _getattr__ = dict.get\n __delattr__ = dict.__delitem__ # type: ignore\n\n def __getattr__(self, item) -> Any: # needed for mypy checks\n return self._getattr__(item)\n\n def __init__(self, *args, **kwargs) -> None:\n for arg in args:\n if isinstance(arg, dict):\n for k, v in arg.items():\n self[k] = v\n\n for k, v in kwargs.items():\n self[k] = v\n\n def __setattr__(self, name: str, value) -> None:\n if isinstance(value, dict):\n value = AnalysisState(value)\n self[name] = value\n\n def __setitem__(self, key, value) -> None:\n if isinstance(value, dict):\n value = AnalysisState(value)\n super().__setitem__(key, value)\n\n @property\n def __dict__(self):\n return self\n\n\nclass StateCheckMixin:\n logger = logging.getLogger(__name__)\n\n def at_least_one_key_must_be_present(self, state: AnalysisState, *keys) -> bool:\n \"\"\"\n Checks if at least one key is present in the state\n\n Parameters\n ----------\n state: AnalysisState\n state object to perform check on\n keys:\n list of the keys to check\n\n Returns\n -------\n True if at least one key from the `keys` list is present in the state\n \"\"\"\n for k in keys:\n if state.get(k, None) is not None:\n return True\n self.logger.warning(f\"{self.__class__.__name__}: at least one of the following keys must be present: {keys}\")\n return False\n\n def all_keys_must_be_present(self, state: AnalysisState, *keys) -> bool:\n \"\"\"\n Checks if all the keys are present in the state\n\n Parameters\n ----------\n state: AnalysisState\n state object to perform check on\n keys:\n list of the keys to check\n\n Returns\n -------\n True if all the key from the `keys` list are present in the state\n \"\"\"\n keys_not_present = [k for k in keys if state.get(k, None) is None]\n can_handle = len(keys_not_present) == 0\n if not can_handle:\n self.logger.warning(\n f'{self.__class__.__name__}: all of the following keys must be present: [{\", \".join(keys)}]. '\n f'The following keys are missing: [{\", \".join(keys_not_present)}]'\n )\n return can_handle\n\n\ndef is_key_present_in_state(state: AnalysisState, key: str):\n \"\"\"\n Check if the nested key represented with dot notation (`a.b.c`) is present in the state\n Parameters\n ----------\n state: AnalysisState\n state to check the key in\n key: str\n the key to check for presence\n\n\n Returns\n -------\n `True` if the key is present\n\n \"\"\"\n path = state\n for p in key.split(\".\"):\n if p not in path:\n return False\n path = path[p]\n return True\n"
},
{
"path": "eda/src/autogluon/eda/utils/__init__.py",
"content": ""
},
{
"path": "eda/src/autogluon/eda/utils/common.py",
"content": "from typing import Any, Dict\n\n__all__ = [\"get_empty_dict_if_none\", \"expand_nested_args_into_nested_maps\"]\n\n\ndef get_empty_dict_if_none(value) -> dict:\n if value is None:\n value = {}\n return value\n\n\ndef expand_nested_args_into_nested_maps(args: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"\n Expands flat args with nested keys in dot notation into nested map (`{a.b.c: value} -> {'a': {'b': {'c': value}}}`)\n\n Parameters\n ----------\n args: Dict[str, Any]\n args to expand\n\n Returns\n -------\n nested expanded map\n\n \"\"\"\n result: Dict[str, Any] = {}\n for k, v in args.items():\n sub_keys = k.split(\".\")\n curr_pointer = result\n if len(sub_keys) > 1:\n for subkey in sub_keys[:-1]:\n if subkey not in curr_pointer:\n curr_pointer[subkey] = {}\n curr_pointer = curr_pointer[subkey]\n if type(curr_pointer) is not dict:\n raise ValueError(f\"{k} cannot be added - the key is already present\")\n curr_pointer[sub_keys[-1]] = v\n return result\n"
},
{
"path": "eda/src/autogluon/eda/utils/defaults.py",
"content": "class QuickFitDefaults:\n DEFAULT_RF_CONFIG = {\n \"RF\": [\n {\n \"criterion\": \"entropy\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]},\n },\n {\n \"criterion\": \"squared_error\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]},\n },\n ],\n }\n DEFAULT_LGBM_CONFIG = {\n \"GBM\": [\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n ]\n }\n"
},
{
"path": "eda/src/autogluon/eda/visualization/__init__.py",
"content": "from yellowbrick.style.rcmod import reset_orig\n\nfrom .anomaly import AnomalyScoresVisualization\nfrom .dataset import DatasetStatistics, DatasetTypeMismatch, LabelInsightsVisualization\nfrom .explain import ExplainForcePlot, ExplainWaterfallPlot\nfrom .interaction import (\n CorrelationSignificanceVisualization,\n CorrelationVisualization,\n FeatureDistanceAnalysisVisualization,\n FeatureInteractionVisualization,\n PDPInteractions,\n)\nfrom .layouts import (\n MarkdownSectionComponent,\n PropertyRendererComponent,\n SimpleHorizontalLayout,\n SimpleVerticalLinearLayout,\n TabLayout,\n)\nfrom .missing import MissingValues\nfrom .model import ConfusionMatrix, FeatureImportance, ModelLeaderboard, RegressionEvaluation\nfrom .shift import XShiftSummary\n\n# Reset plotting styles back to original style; this is to prevent issues with missing fonts\nreset_orig()\n"
},
{
"path": "eda/src/autogluon/eda/visualization/anomaly.py",
"content": "from typing import Any, Dict, Optional\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom .. import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\"AnomalyScoresVisualization\"]\n\nfrom ..utils.common import expand_nested_args_into_nested_maps, get_empty_dict_if_none\n\n\nclass AnomalyScoresVisualization(AbstractVisualization, JupyterMixin):\n \"\"\"\n Visualize anomaly scores across datasets.\n\n The report depends on :py:class:`~autogluon.eda.analysis.anomaly.AnomalyDetectorAnalysis`,\n\n Parameters\n ----------\n threshold_stds: float = 3,\n defines how many standard deviations from mean the scores will be marked as anomalies\n headers: bool, default = False\n if `True` then render headers\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into visualization component.\n chart_args\n kwargs to pass into visualization component\n The chart contains two scatterpolots combined: normal and anomaly data points; both can be customized via passing\n additional arguments in `chart_args` - see the example below.\n\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> import pandas as pd\n >>> import numpy as np\n >>>\n >>> df_train = pd.DataFrame(...)\n >>> df_test = pd.DataFrame(...)\n >>> label = 'target'\n >>> threshold_stds = 3 # mark 3 standard deviations score values as anomalies\n >>>\n >>> chart_args={\n >>> 'normal.color': 'lightgrey',\n >>> 'anomaly.color': 'orange',\n >>> }\n >>>\n >>> state = auto.analyze(\n >>> train_data=df_train,\n >>> test_data=df_test,\n >>> label=label,\n >>> return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.ProblemTypeControl(),\n >>> eda.transform.ApplyFeatureGenerator(category_to_numbers=True, children=[\n >>> eda.anomaly.AnomalyDetectorAnalysis(\n >>> store_explainability_data=True # Store additional functions for explainability\n >>> ),\n >>> ])\n >>> ],\n >>> viz_facets=[\n >>> viz.anomaly.AnomalyScoresVisualization(\n >>> threshold_stds=threshold_stds,\n >>> headers=True,\n >>> fig_args=dict(figsize=(8, 4)),\n >>> **chart_args, # pass chart args customizations\n >>> )\n >>> ]\n >>> )\n >>>\n >>> # explain top anomalies\n >>> train_anomaly_scores = state.anomaly_detection.scores.train_data\n >>> anomaly_idx = train_anomaly_scores[train_anomaly_scores >= train_anomaly_scores.std() * threshold_stds]\n >>> anomaly_idx = anomaly_idx.sort_values(ascending=False).index\n >>>\n >>> auto.explain_rows(\n >>> # Use helper function stored via `store_explainability_data=True`\n >>> **state.anomaly_detection.explain_rows_fns.train_data(anomaly_idx[:3]),\n >>> plot='waterfall',\n >>> )\n\n \"\"\"\n\n def __init__(\n self,\n threshold_stds: float = 3,\n headers: bool = False,\n namespace: Optional[str] = None,\n fig_args: Optional[Dict[str, Any]] = None,\n **chart_args,\n ) -> None:\n super().__init__(namespace, **chart_args)\n self.threshold_stds = threshold_stds\n self.headers = headers\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n self.chart_args = expand_nested_args_into_nested_maps(get_empty_dict_if_none(chart_args))\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"anomaly_detection\")\n\n def _render(self, state: AnalysisState) -> None:\n scores = state.anomaly_detection.scores\n threshold = scores.train_data.std() * self.threshold_stds\n for ds, ds_scores in scores.items():\n self.render_header_if_needed(\n state, f\"`{ds}` anomalies for {self.threshold_stds}-sigma outlier scores\", ds=ds\n )\n data = ds_scores.reset_index(drop=True).reset_index()\n\n fig, ax = plt.subplots(**self.fig_args)\n\n common_chart_args = dict(ax=ax, x=\"index\", y=\"score\")\n\n sns.scatterplot(\n data=data[data.score < threshold], **common_chart_args, **self.chart_args.get(\"normal\", {})\n )\n sns.scatterplot(\n data=data[data.score >= threshold], **common_chart_args, **self.chart_args.get(\"anomaly\", {})\n )\n\n ax.axhline(\n y=threshold,\n color=\"r\",\n linestyle=\"--\",\n )\n ax.text(\n x=0,\n y=threshold,\n s=f\"{threshold:.4f}\",\n color=\"red\",\n rotation=\"vertical\",\n horizontalalignment=\"right\",\n verticalalignment=\"top\",\n )\n plt.tight_layout(h_pad=0.3, w_pad=0.5)\n plt.show(fig)\n"
},
{
"path": "eda/src/autogluon/eda/visualization/base.py",
"content": "import logging\nfrom abc import ABC, abstractmethod\nfrom typing import List, Optional\n\nfrom ..state import AnalysisState, StateCheckMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass AbstractVisualization(ABC, StateCheckMixin):\n \"\"\"\n Base class for visualization functionality.\n\n Provides basic functionality for namespace management and helper method to access frequently-used methods.\n\n Specifying namespace would narrow visibility scope to specific subspace of `state`. Namespaces\n can be specified in a nested form: `ns_a.ns_b.ns_c`. Please see :py:class:`~autogluon.eda.analysis.base.Namespace`\n wrapper on how to create namespaces.\n\n The main entry method of analysis is `render` function. When called, the execution flow is the following:\n - narrow `state` scope to specified `namespace`\n - call `_render` function for each component that returned `True` from `can_handle` call\n\n Parameters\n ----------\n namespace: str\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n kwargs\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.base.Namespace`\n \"\"\"\n\n def __init__(self, namespace: Optional[str] = None, **kwargs) -> None:\n super().__init__()\n self.namespace: List[str] = []\n self._kwargs = kwargs\n if namespace is not None:\n self.namespace = namespace.split(\".\")\n\n def _get_namespace_state(self, state):\n if self.namespace is not None:\n for k in self.namespace:\n state = state[k]\n return state\n\n @abstractmethod\n def can_handle(self, state: AnalysisState) -> bool:\n \"\"\"\n Checks if state has all the required parameters for visualization.\n See also :func:`at_least_one_key_must_be_present` and :func:`all_keys_must_be_present` helpers\n to construct more complex logic.\n\n Parameters\n ----------\n state: AnalysisState\n fitted state\n\n Returns\n -------\n `True` if all the pre-requisites for rendering are present\n\n \"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def _render(self, state: AnalysisState) -> None:\n \"\"\"\n @override\n Component-specific rendering logic.\n This method is designed to be overridden by the component developer.\n\n Parameters\n ----------\n state\n\n Returns\n -------\n\n \"\"\"\n raise NotImplementedError\n\n def render(self, state: AnalysisState) -> None:\n \"\"\"\n Render component.\n\n Parameters\n ----------\n state: AnalysisState\n state to render\n\n \"\"\"\n state = self._get_namespace_state(state)\n if self.can_handle(state):\n self._render(state)\n"
},
{
"path": "eda/src/autogluon/eda/visualization/dataset.py",
"content": "from typing import Any, Dict, List, Optional\n\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom ..state import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\"DatasetStatistics\", \"DatasetTypeMismatch\", \"LabelInsightsVisualization\"]\n\n\nclass DatasetStatistics(AbstractVisualization, JupyterMixin):\n \"\"\"\n Display aggregate dataset statistics and dataset-level information.\n\n The report is a composite view of combination of performed analyses: :py:class:`~autogluon.eda.analysis.dataset.DatasetSummary`,\n :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis`, :py:class:`~autogluon.eda.analysis.dataset.VariableTypeAnalysis`,\n :py:class:`~autogluon.eda.analysis.dataset.SpecialTypesAnalysis`, :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis`.\n The components can be present in any combination (assuming their dependencies are satisfied).\n\n The report requires at least one of the analyses present to be rendered.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n sort_by: Optional[str], default = None\n column to sort the resulting table\n sort_asc: bool, default = True\n if `sort_by` provided, then if sorting should ascending or descending\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> state = auto.analyze(\n >>> train_data=..., label=..., return_state=True,\n >>> anlz_facets=[\n >>> eda.dataset.DatasetSummary(),\n >>> eda.dataset.RawTypesAnalysis(),\n >>> eda.dataset.VariableTypeAnalysis(),\n >>> eda.dataset.SpecialTypesAnalysis(),\n >>> eda.missing.MissingValuesAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetStatistics()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.DatasetSummary`\n :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis`\n :py:class:`~autogluon.eda.analysis.dataset.VariableTypeAnalysis`\n :py:class:`~autogluon.eda.analysis.dataset.SpecialTypesAnalysis`\n :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis`\n \"\"\"\n\n def __init__(\n self,\n headers: bool = False,\n namespace: Optional[str] = None,\n sort_by: Optional[str] = None,\n sort_asc: bool = True,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n self.sort_by = sort_by\n self.sort_asc = sort_asc\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.at_least_one_key_must_be_present(\n state, \"dataset_stats\", \"missing_statistics\", \"raw_type\", \"special_types\"\n )\n\n def _render(self, state: AnalysisState) -> None:\n datasets = []\n for k in [\"dataset_stats\", \"missing_statistics\", \"raw_type\", \"variable_type\", \"special_types\"]:\n if k in state:\n datasets = state[k].keys()\n break\n\n for ds in datasets:\n # Merge different metrics\n stats = self._merge_analysis_facets(ds, state)\n # Fix counts\n df = pd.DataFrame(stats)\n if \"dataset_stats\" in state:\n df = self._fix_counts(df, [\"unique\", \"freq\"])\n if \"missing_statistics\" in state:\n df = self._fix_counts(df, [\"missing_count\"])\n df = df.fillna(\"\")\n\n self.render_header_if_needed(state, f\"`{ds}` dataset summary\", ds=ds)\n if self.sort_by in df.columns:\n df = df.sort_values(by=self.sort_by, ascending=self.sort_asc)\n with pd.option_context(\"display.max_rows\", 100 if len(df) <= 100 else 20):\n self.display_obj(df)\n\n @staticmethod\n def _merge_analysis_facets(ds: str, state: AnalysisState):\n stats: Dict[str, Any] = {}\n if \"dataset_stats\" in state:\n stats = state.dataset_stats[ds].copy()\n if \"missing_statistics\" in state:\n stats = {\n **stats,\n **{f\"missing_{k}\": v for k, v in state.missing_statistics[ds].items() if k in [\"count\", \"ratio\"]},\n }\n if \"raw_type\" in state:\n stats[\"raw_type\"] = state.raw_type[ds]\n if \"variable_type\" in state:\n stats[\"variable_type\"] = state.variable_type[ds]\n if \"special_types\" in state:\n stats[\"special_types\"] = state.special_types[ds]\n return stats\n\n @staticmethod\n def _fix_counts(df: DataFrame, cols: List[str]) -> DataFrame:\n for k in cols:\n if k in df.columns:\n df[k] = df[k].fillna(-1).astype(int).replace({-1: \"\"})\n return df\n\n\nclass DatasetTypeMismatch(AbstractVisualization, JupyterMixin):\n \"\"\"\n Display mismatch between raw types between datasets provided. In case if mismatch found, mark the row with a warning.\n\n The report requires :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis` analysis present.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> auto.analyze(\n >>> train_data=..., test_data=...,\n >>> anlz_facets=[\n >>> eda.dataset.RawTypesAnalysis(),\n >>> ],\n >>> viz_facets=[\n >>> viz.dataset.DatasetTypeMismatch()\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.RawTypesAnalysis`\n \"\"\"\n\n def __init__(self, headers: bool = False, namespace: Optional[str] = None, **kwargs) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"raw_type\")\n\n def _render(self, state: AnalysisState) -> None:\n df = pd.DataFrame(state.raw_type).sort_index()\n warnings = df.eq(df.iloc[:, 0], axis=0)\n df[\"warnings\"] = warnings.all(axis=1).map({True: \"\", False: \"warning\"})\n df.fillna(\"--\", inplace=True)\n df = df[df[\"warnings\"] != \"\"]\n\n if len(df) > 0:\n self.render_header_if_needed(state, \"Types warnings summary\")\n with pd.option_context(\"display.max_rows\", 100 if len(df) <= 100 else 20):\n self.display_obj(df)\n\n\nclass LabelInsightsVisualization(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render label insights performed by :py:class:`~autogluon.eda.analysis.dataset.LabelInsightsAnalysis`.\n\n The following insights can be rendered:\n\n - classification: low cardinality classes detection\n - classification: classes present in test data, but not in the train data\n - regression: out-of-domain labels detection\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>> auto.analyze(\n >>> auto.analyze(train_data=..., test_data=..., label=..., anlz_facets=[\n >>> eda.dataset.ProblemTypeControl(),\n >>> eda.dataset.LabelInsightsAnalysis(low_cardinality_classes_threshold=50, regression_ood_threshold=0.01),\n >>> ], viz_facets=[\n >>> viz.dataset.LabelInsightsVisualization()\n >>> ])\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.dataset.ProblemTypeControl`\n :py:class:`~autogluon.eda.analysis.dataset.LabelInsightsAnalysis`\n \"\"\"\n\n def __init__(self, headers: bool = False, namespace: Optional[str] = None, **kwargs) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"label_insights\" in state\n\n def _render(self, state: AnalysisState) -> None:\n insights = state.label_insights\n\n md_lines: List[str] = []\n self._classification_add_low_cardinality_classes_insights(insights, md_lines)\n self._classification_add_minority_class_imbalance_insights(insights, md_lines)\n self._classification_add_missing_classes_insights(insights, md_lines)\n self._regression_add_out_of_domain_insights(insights, md_lines)\n\n if len(md_lines) > 0:\n self.render_header_if_needed(state, \"Label insights\")\n self.render_markdown(\"\\n\".join(md_lines))\n\n @staticmethod\n def _regression_add_out_of_domain_insights(insights: AnalysisState, md_lines: List[str]):\n if insights.ood is not None:\n md_lines.append(\n f\" - Rows with out-of-domain labels were found. Consider removing rows with labels outside of this range or expand training data since \"\n f\"some algorithms (i.e. trees) are unable to extrapolate beyond data present in the training data.\\n\"\n f\" - `{insights.ood.count}` rows\\n\"\n f\" - `train_data` values range `{insights.ood.train_range}`\\n\"\n f\" - `test_data` values range `{insights.ood.test_range}`\"\n )\n\n @staticmethod\n def _classification_add_missing_classes_insights(insights: AnalysisState, md_lines: List[str]):\n if insights.not_present_in_train is not None:\n md_lines.append(\n f\" - the following classes are found in `test_data`, but not present in `train_data`: \"\n f\"`{'`, `'.join(map(str, insights.not_present_in_train))}`. \"\n f\"Consider either removing the rows with classes not covered or adding more training data covering the classes.\"\n )\n\n @staticmethod\n def _classification_add_minority_class_imbalance_insights(insights: AnalysisState, md_lines: List[str]):\n if insights.minority_class_imbalance is not None:\n if insights.minority_class_imbalance.ratio < 0.01:\n severity = \"Extreme\"\n elif insights.minority_class_imbalance.ratio <= 0.2:\n severity = \"Moderate\"\n else:\n severity = \"Mild\"\n md_lines.append(\n f\" - {severity} minority class imbalance detected - imbalance ratio is `{insights.minority_class_imbalance.ratio:.2%}`. \"\n f\"Recommendations:\\n\"\n f\" - downsample majority class `{insights.minority_class_imbalance.majority_class}` to improve the balance\\n\"\n f\" - upweight downsampled class so that `sample_weight = original_weight x downsampling_factor`.\"\n f\"[TabularPredictor](https://auto.gluon.ai/stable/api/autogluon.predictor.html#module-0) \"\n f\"supports this via `sample_weight` parameter\"\n )\n\n @staticmethod\n def _classification_add_low_cardinality_classes_insights(insights: AnalysisState, md_lines: List[str]):\n if insights.low_cardinality_classes is not None:\n classes_info = \"\\n\".join(\n [f\" - class `{k}`: `{v}` instances\" for k, v in insights.low_cardinality_classes.instances.items()]\n )\n md_lines.append(\n f\" - Low-cardinality classes are detected. It is recommended to have at least `{insights.low_cardinality_classes.threshold}` \"\n f\"instances per class. Consider adding more data to cover the classes or remove such rows.\\n\"\n f\"{classes_info}\"\n )\n"
},
{
"path": "eda/src/autogluon/eda/visualization/explain.py",
"content": "from abc import ABC, abstractmethod\nfrom typing import Optional\n\nimport shap\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization.base import AbstractVisualization\n\n__all__ = [\"ExplainForcePlot\", \"ExplainWaterfallPlot\"]\n\nfrom autogluon.eda.visualization.jupyter import JupyterMixin\n\n\nclass _AbstractExplainPlot(AbstractVisualization, JupyterMixin, ABC):\n def __init__(self, display_rows: bool = False, namespace: Optional[str] = None, **kwargs) -> None:\n super().__init__(namespace, **kwargs)\n self.display_rows = display_rows\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"explain\") and self.all_keys_must_be_present(\n state.explain, \"shapley\"\n )\n\n def _render(self, state: AnalysisState) -> None:\n for s in state.explain.shapley:\n if self.display_rows:\n self.display_obj(s.row)\n self._render_internal(\n s.expected_value,\n s.shap_values,\n s.features,\n explainer=s.explainer,\n feature_names=s.feature_names,\n **self._kwargs,\n )\n\n @abstractmethod\n def _render_internal(self, expected_value, shap_values, features, explainer, feature_names, **kwargs):\n raise NotImplementedError # pragma: no cover\n\n\nclass ExplainForcePlot(_AbstractExplainPlot):\n \"\"\"\n Visualize the given SHAP values with an additive force layout\n\n Parameters\n ----------\n display_rows: bool, default = False\n if `True` then display the row before the explanation chart\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> rows_to_explain = ... # DataFrame\n >>>\n >>> auto.analyze(\n >>> train_data=..., model=...,\n >>> anlz_facets=[\n >>> eda.explain.ShapAnalysis(rows),\n >>> ],\n >>> viz_facets=[\n >>> viz.explain.ExplainForcePlot(text_rotation=45, matplotlib=True), # defaults used if not specified\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~shap.KernelExplainer`\n :py:class:`~autogluon.eda.analysis.explain.ShapAnalysis`\n \"\"\"\n\n def _render_internal(self, expected_value, shap_values, features, explainer, feature_names, **kwargs):\n _kwargs = {**dict(text_rotation=45, matplotlib=True), **kwargs}\n shap.force_plot(expected_value, shap_values, features, feature_names=feature_names, **_kwargs)\n\n\nclass ExplainWaterfallPlot(_AbstractExplainPlot):\n \"\"\"\n Visualize the given SHAP values with a waterfall layout\n\n Parameters\n ----------\n display_rows: bool, default = False\n if `True` then display the row before the explanation chart\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> rows_to_explain = ... # DataFrame\n >>>\n >>> auto.analyze(\n >>> train_data=..., model=...,\n >>> anlz_facets=[\n >>> eda.explain.ShapAnalysis(rows_to_explain),\n >>> ],\n >>> viz_facets=[\n >>> viz.explain.ExplainWaterfallPlot(),\n >>> ]\n >>> )\n\n See Also\n --------\n :py:class:`~shap.KernelExplainer`\n :py:class:`~autogluon.eda.analysis.explain.ShapAnalysis`\n \"\"\"\n\n def _render_internal(self, expected_value, shap_values, features, explainer, feature_names, **kwargs):\n shap.waterfall_plot(\n shap.Explanation(\n shap_values, base_values=expected_value, output_names=features, feature_names=features.index\n )\n )\n"
},
{
"path": "eda/src/autogluon/eda/visualization/interaction.py",
"content": "from abc import ABC, abstractmethod\nfrom typing import Any, Dict, List, Optional, Tuple, Type, Union\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom scipy import stats\nfrom scipy.cluster import hierarchy as hc\nfrom sklearn.inspection import PartialDependenceDisplay\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_DATETIME, R_FLOAT, R_INT, R_OBJECT\n\nfrom ..state import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\n \"CorrelationVisualization\",\n \"CorrelationSignificanceVisualization\",\n \"FeatureInteractionVisualization\",\n \"FeatureDistanceAnalysisVisualization\",\n \"PDPInteractions\",\n]\n\n\nclass _AbstractCorrelationChart(AbstractVisualization, JupyterMixin, ABC):\n def __init__(\n self,\n headers: bool = False,\n namespace: Optional[str] = None,\n fig_args: Optional[Dict[str, Any]] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n\n def _render_internal(self, state: AnalysisState, render_key: str, header: str, chart_args: Dict[str, Any]) -> None:\n for ds, corr in state[render_key].items():\n # Don't render single cell\n cells_num = len(state.correlations[ds])\n if cells_num <= 1:\n continue\n\n fig_args = self.fig_args.copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (cells_num, cells_num)\n\n if state.correlations_focus_field is not None:\n focus_field_header = f\"; focus: absolute correlation for `{state.correlations_focus_field}` >= `{state.correlations_focus_field_threshold}`\"\n else:\n focus_field_header = \"\"\n self.render_header_if_needed(\n state, f\"`{ds}` - `{state.correlations_method}` {header}{focus_field_header}\", ds=ds\n )\n\n fig, ax = plt.subplots(**fig_args)\n sns.heatmap(\n corr,\n annot=True,\n ax=ax,\n linewidths=0.5,\n linecolor=\"lightgrey\",\n fmt=\".2f\",\n square=True,\n cbar_kws={\"shrink\": 0.5},\n **chart_args,\n )\n plt.yticks(rotation=0)\n plt.show(fig)\n\n\nclass CorrelationVisualization(_AbstractCorrelationChart):\n \"\"\"\n Display feature correlations matrix.\n\n This report renders correlations between variable in a form of heatmap.\n The details of the report to be rendered depend on the configuration of\n :py:class:`~autogluon.eda.analysis.interaction.Correlation`\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]], default = None,\n kwargs to pass into chart figure\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.interaction.Correlation`\n \"\"\"\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"correlations\" in state\n\n def _render(self, state: AnalysisState) -> None:\n args = {\n **{\"vmin\": 0 if state.correlations_method == \"phik\" else -1, \"vmax\": 1, \"center\": 0, \"cmap\": \"Spectral\"},\n **self._kwargs,\n }\n self._render_internal(state, \"correlations\", \"correlation matrix\", args)\n\n\nclass _AbstractFeatureInteractionPlotRenderer(ABC):\n @abstractmethod\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n raise NotImplementedError # pragma: no cover\n\n def render(self, state, ds, params, param_types, data, fig_args, chart_args):\n fig, ax = plt.subplots(**fig_args)\n self._render(state, ds, params, param_types, ax, data, chart_args)\n plt.show(fig)\n\n\nclass CorrelationSignificanceVisualization(_AbstractCorrelationChart):\n \"\"\"\n Display feature correlations significance matrix.\n\n This report renders correlations significance matrix in a form of heatmap.\n The details of the report to be rendered depend on the configuration of\n :py:class:`~autogluon.eda.analysis.interaction.Correlation` and\n :py:class:`~autogluon.eda.analysis.interaction.CorrelationSignificance` analyses.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.interaction.Correlation`\n :py:class:`~autogluon.eda.analysis.interaction.CorrelationSignificance`\n \"\"\"\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"significance_matrix\" in state\n\n def _render(self, state: AnalysisState) -> None:\n args = {\"center\": 3, \"vmax\": 5, \"cmap\": \"Spectral\", \"robust\": True}\n self._render_internal(state, \"significance_matrix\", \"correlation significance matrix\", args)\n\n\nclass FeatureDistanceAnalysisVisualization(AbstractVisualization, JupyterMixin):\n \"\"\"\n Feature distance visualization.\n\n This component renders graphical representations of distances between features to highlight features that can be\n either simplified or completely removed.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n kwargs\n \"\"\"\n\n def __init__(\n self,\n namespace: Optional[str] = None,\n fig_args: Optional[Dict[str, Any]] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"feature_distance\")\n\n def _render(self, state: AnalysisState) -> None:\n fig_args = self.fig_args.copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (12, len(state.feature_distance.columns) / 4)\n\n fig, ax = plt.subplots(**fig_args)\n default_args = dict(orientation=\"left\")\n ax.grid(False)\n hc.dendrogram(\n ax=ax,\n Z=state.feature_distance.linkage,\n labels=state.feature_distance.columns,\n leaf_font_size=10,\n **{**default_args, **self._kwargs},\n )\n plt.show(fig)\n if len(state.feature_distance.near_duplicates) > 0:\n message = (\n f\"**The following feature groups are considered as near-duplicates**:\\n\\n\"\n f\"Distance threshold: <= `{state.feature_distance.near_duplicates_threshold}`. \"\n f\"Consider keeping only some of the columns within each group:\\n\"\n )\n for group in state.feature_distance.near_duplicates:\n message += f'\\n - `{\"`, `\".join(sorted(group[\"nodes\"]))}` - distance `{group[\"distance\"]:.2f}`'\n self.render_markdown(message)\n\n\nclass FeatureInteractionVisualization(AbstractVisualization, JupyterMixin):\n \"\"\"\n Feature interaction visualization.\n\n This report renders feature interaction analysis results.\n The details of the report to be rendered depend on the variable types combination in `x`/`y`/`hue`.\n `key` is used to link analysis and visualization - this allows to have multiple analyses/visualizations in one composite analysis.\n\n Parameters\n ----------\n key: str\n key used to store the analysis in the state; the value is placed in the state by FeatureInteraction.\n If the key is not provided, then use one of theform: 'x:A|y:B|hue:C' (omit corresponding x/y/hue if the value not provided)\n See also :class:`autogluon.eda.analysis.interaction.FeatureInteraction`\n numeric_as_categorical_threshold\n headers: bool, default = False\n if `True` then render headers\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n kwargs\n parameters to pass as a chart args\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.interaction.FeatureInteraction`\n \"\"\"\n\n def __init__(\n self,\n key: str,\n numeric_as_categorical_threshold: int = 20,\n max_categories_to_consider_render: int = 30,\n headers: bool = False,\n namespace: Optional[str] = None,\n fig_args: Optional[Dict[str, Any]] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.key = key\n self.headers = headers\n self.numeric_as_categorical_threshold = numeric_as_categorical_threshold\n self.max_categories_to_consider_render = max_categories_to_consider_render\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"interactions\", \"raw_type\")\n\n def _render(self, state: AnalysisState) -> None:\n for ds in state.interactions.keys():\n if self.key not in state.interactions[ds]:\n continue\n interaction = state.interactions[ds][self.key]\n interaction_features = interaction[\"features\"]\n df = interaction[\"data\"].copy()\n x, x_type = self._get_value_and_type(ds, df, state, interaction_features, \"x\")\n y, y_type = self._get_value_and_type(ds, df, state, interaction_features, \"y\")\n hue, hue_type = self._get_value_and_type(ds, df, state, interaction_features, \"hue\")\n\n # Don't render high-cardinality category variables\n features = \"/\".join(\n [f\"`{interaction_features[k]}`\" for k in [\"x\", \"y\", \"hue\"] if k in interaction_features]\n )\n for f, t in [(x, x_type), (y, y_type), (hue, hue_type)]:\n if t == \"category\" and df[f].nunique() > self.max_categories_to_consider_render:\n self.render_markdown(\n f\"Interaction {features} is not rendered due to `{f}` \"\n f\"having too many categories (`{df[f].nunique()}` > `{self.max_categories_to_consider_render}`) \"\n f\"to place on plot axis.\"\n )\n return\n\n y, y_type, hue, hue_type = self._swap_y_and_hue_if_necessary(x_type, y, y_type, hue, hue_type)\n\n renderer_cls: Optional[Type[_AbstractFeatureInteractionPlotRenderer]] = self._get_chart_renderer(\n x_type, y_type, hue_type\n )\n if renderer_cls is None:\n return\n renderer: _AbstractFeatureInteractionPlotRenderer = renderer_cls() # Create instance\n\n df = self._convert_categoricals_to_objects(df, x, x_type, y, y_type, hue, hue_type)\n chart_args, data, is_single_var = self._prepare_chart_args(df, x, x_type, y, y_type, hue)\n\n if self.headers:\n prefix = \"\" if is_single_var else \"Feature interaction between \"\n self.render_header_if_needed(state, f\"{prefix}{features} in `{ds}`\", ds=ds)\n\n fig_args = self.fig_args.copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (12, 6)\n\n renderer.render(\n state=state,\n ds=ds,\n params=(x, y, hue),\n param_types=(x_type, y_type, hue_type),\n data=data,\n fig_args=fig_args,\n chart_args=chart_args,\n )\n\n def _prepare_chart_args(self, df, x, x_type, y, y_type, hue) -> Tuple[Dict[str, Any], pd.DataFrame, bool]:\n chart_args = {\"x\": x, \"y\": y, \"hue\": hue, **self._kwargs}\n chart_args = {k: v for k, v in chart_args.items() if v is not None}\n data = df\n is_single_var = False\n if x is not None and y is None and hue is None:\n is_single_var = True\n if x_type == \"numeric\":\n data = df[x]\n chart_args.pop(\"x\")\n elif y is not None and x is None and hue is None:\n is_single_var = True\n if y_type == \"numeric\":\n data = df[y]\n chart_args.pop(\"y\")\n return chart_args, data, is_single_var\n\n def _convert_categoricals_to_objects(self, df, x, x_type, y, y_type, hue, hue_type):\n # convert to categoricals for plots\n for col, typ in zip([x, y, hue], [x_type, y_type, hue_type]):\n if typ == \"category\":\n df[col] = df[col].astype(\"object\")\n return df\n\n def _swap_y_and_hue_if_necessary(self, x_type, y, y_type, hue, hue_type):\n # swap y <-> hue when category vs category is provided and no hue is specified\n if (x_type is not None) and y_type == \"category\" and hue_type is None:\n hue, hue_type = y, y_type\n y, y_type = None, None\n return y, y_type, hue, hue_type\n\n def _get_value_and_type(\n self, ds: str, df: pd.DataFrame, state: AnalysisState, interaction_features: Dict[str, Any], param: str\n ) -> Tuple[Any, Optional[str]]:\n col = interaction_features.get(param, None)\n value_type = self._map_raw_type_to_feature_type(\n col, state.raw_type[ds].get(col, None), df, self.numeric_as_categorical_threshold\n )\n return col, value_type\n\n def _get_chart_renderer(\n self, x_type: Optional[str], y_type: Optional[str], hue_type: Optional[str]\n ) -> Optional[Type[_AbstractFeatureInteractionPlotRenderer]]:\n types = {\n (\"numeric\", None, None): self._HistPlotRenderer,\n (\"category\", None, None): self._CountPlotRenderer,\n (None, \"category\", None): self._CountPlotRenderer,\n (\"category\", None, \"category\"): self._CountPlotRenderer,\n (None, \"category\", \"category\"): self._CountPlotRenderer,\n (\"numeric\", None, \"category\"): self._HistPlotRenderer,\n (None, \"numeric\", \"category\"): self._HistPlotRenderer,\n (\"category\", \"category\", None): self._BarPlotRenderer,\n (\"category\", \"category\", \"category\"): self._BarPlotRenderer,\n (\"category\", \"numeric\", None): self._BoxPlotRenderer,\n (\"numeric\", \"category\", None): self._KdePlotRenderer,\n (\"category\", \"numeric\", \"category\"): self._BoxPlotRenderer,\n (\"numeric\", \"category\", \"category\"): self._KdePlotRenderer,\n (\"numeric\", \"numeric\", None): self._RegPlotRenderer,\n (\"numeric\", \"numeric\", \"category\"): self._ScatterPlotRenderer,\n (\"numeric\", \"numeric\", \"numeric\"): self._ScatterPlotRenderer,\n (\"datetime\", \"numeric\", None): self._LinePlotRenderer,\n }\n return types.get((x_type, y_type, hue_type), None)\n\n def _map_raw_type_to_feature_type(\n self, col: str, raw_type: str, df: pd.DataFrame, numeric_as_categorical_threshold: int = 20\n ) -> Optional[str]:\n if col is None:\n return None\n elif df[col].nunique() <= numeric_as_categorical_threshold:\n return \"category\"\n elif raw_type in [R_INT, R_FLOAT]:\n return \"numeric\"\n elif raw_type in [R_DATETIME]:\n return \"datetime\"\n elif raw_type in [R_OBJECT, R_CATEGORY, R_BOOL]:\n return \"category\"\n else:\n return None\n\n class _HistPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args, num_point_to_fit=200):\n x = params[0]\n fitted_distributions_present = (\n (\"distributions_fit\" in state)\n and (param_types == (\"numeric\", None, None)) # (x, y, hue)\n and (state.distributions_fit[ds].get(x, None) is not None)\n )\n\n if \"stat\" not in chart_args:\n chart_args[\"stat\"] = \"density\"\n sns.histplot(ax=ax, data=data, **chart_args)\n\n if fitted_distributions_present: # types for x, y, hue\n dists = state.distributions_fit[ds][x]\n x_min, x_max = ax.get_xlim()\n xs = np.linspace(x_min, x_max, num_point_to_fit)\n for dist, v in dists.items():\n _dist = getattr(stats, dist)\n ax.plot(\n xs,\n _dist.pdf(xs, *v[\"param\"]),\n ls=\"--\",\n label=f'{dist}: pvalue {v[\"pvalue\"]:.2f}',\n )\n ax.set_xlim(x_min, x_max) # set the limits back to the ones of the distplot\n plt.legend()\n\n class _KdePlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n chart_args.pop(\"fill\", None)\n chart = sns.kdeplot(ax=ax, data=data, **chart_args)\n plt.setp(chart.get_xticklabels(), rotation=90)\n\n class _BoxPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n chart = sns.boxplot(ax=ax, data=data, **chart_args)\n plt.setp(chart.get_xticklabels(), rotation=90)\n\n class _CountPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n chart = sns.countplot(ax=ax, data=data, **chart_args)\n plt.setp(chart.get_xticklabels(), rotation=90)\n for container in ax.containers:\n ax.bar_label(container)\n\n class _BarPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n chart_args[\"errorbar\"] = None # Don't show ci ticks\n chart = sns.barplot(ax=ax, data=data, **chart_args)\n plt.setp(chart.get_xticklabels(), rotation=90)\n\n class _ScatterPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n sns.scatterplot(ax=ax, data=data, **chart_args)\n\n class _RegPlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n sns.regplot(ax=ax, data=data, **chart_args)\n\n class _LinePlotRenderer(_AbstractFeatureInteractionPlotRenderer):\n def _render(self, state, ds, params, param_types, ax, data, chart_args):\n sns.lineplot(ax=ax, data=data, **chart_args)\n\n\nclass PDPInteractions(AbstractVisualization, JupyterMixin):\n \"\"\"\n Display Partial Dependence Plots (PDP) with Individual Conditional Expectation (ICE)\n\n The visualizations have two modes:\n - regular PDP + ICE plots - this is the default mode of operation\n - two-way PDP plots - this mode can be selected via passing two `features` and setting `two_way = True`\n\n ICE plots complement PDP by showing the relationship between a feature and the model's output for each individual instance in the dataset.\n ICE lines (blue) can be overlaid on PDPs (red) to provide a more detailed view of how the model behaves for specific instances.\n\n Parameters\n ----------\n features: Union[str, List[str]]\n feature to display on the plots\n two_way: bool, default = False\n render two-way PDP; this mode works only when two `features` are specified\n target: Optional[Any], default = None\n In a multiclass setting, specifies the class for which the PDPs should be computed.\n Ignored in binary classification or classical regression settings\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n headers: bool, default = False\n if `True` then render headers\n sample: Union[None, int, float], default = None\n sample size; if `int`, then row number is used;\n `float` must be between 0.0 and 1.0 and represents fraction of dataset to sample;\n `None` means no sampling\n See also :func:`autogluon.eda.analysis.dataset.Sampler`\n kwargs\n \"\"\"\n\n MAX_CHARTS_PER_ROW = 2\n\n def __init__(\n self,\n features: Union[str, List[str]],\n two_way: bool = False,\n target: Optional[Any] = None,\n namespace: Optional[str] = None,\n fig_args: Optional[Dict[str, Dict[str, Any]]] = None,\n sample: Optional[Union[float, int]] = 300,\n headers: bool = False,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n\n if type(features) is not list:\n features = [features] # type: ignore\n self.features: list = features\n\n self.target = target\n self.fig_args = fig_args\n self.sample = sample\n self.headers = headers\n self.two_way = two_way\n\n if two_way:\n assert len(self.features) == 2, \"`two_way` can only be used if only 2 `features` are passed\"\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.all_keys_must_be_present(state, \"model\", \"pdp_data\")\n\n def _render(self, state: AnalysisState) -> None:\n additional_kwargs, fig_args, features = self._get_args()\n\n if self.headers:\n self.render_header_if_needed(state, \"Partial Dependence Plots\")\n\n fig, axs = plt.subplots(**fig_args)\n\n kwargs = {**additional_kwargs, **self._kwargs}\n data = state.pdp_data\n\n if self.two_way:\n for f in self.features[:-1]:\n data = data[data[f].notna()]\n\n PartialDependenceDisplay.from_estimator(\n _SklearnAutoGluonWrapper(state.model),\n data,\n self.features,\n ax=axs.ravel()[: len(features)],\n target=self.target,\n subsample=self.sample,\n **kwargs,\n )\n plt.tight_layout(h_pad=0.3, w_pad=0.5)\n plt.show()\n\n def _get_args(self):\n n = len(self.features)\n cols = self.MAX_CHARTS_PER_ROW if n > self.MAX_CHARTS_PER_ROW else n\n rows = int(np.ceil(n / cols))\n if self.fig_args is None:\n self.fig_args = {}\n kind = \"both\"\n additional_kwargs: Dict[str, Any] = {}\n features = self.features\n\n if self.two_way:\n fig_args = {**dict(nrows=1, ncols=3, figsize=(12, 3)), **self.fig_args}\n if len(self.features) == 2:\n kind = \"average\"\n features.append(features.copy())\n else:\n fig_args = {**dict(nrows=rows, ncols=cols, figsize=(12, 3 * rows)), **self.fig_args}\n additional_kwargs[\"pd_line_kw\"] = {\"color\": \"red\"}\n additional_kwargs[\"ice_lines_kw\"] = {\"color\": \"blue\"}\n additional_kwargs[\"kind\"] = kind\n\n return additional_kwargs, fig_args, features\n\n\nclass _SklearnAutoGluonWrapper:\n def __init__(self, estimator):\n self.estimator = estimator\n self.estimator_type = \"regressor\" if estimator.problem_type == \"regression\" else \"classifier\"\n\n @property\n def _estimator_type(self):\n return self.estimator_type\n\n def __sklearn_is_fitted__(self):\n return True\n\n def fit(self, X, Y=None):\n self.estimator.fit(X)\n\n def predict(self, X):\n return self.estimator.predict(X)\n\n def predict_proba(self, X):\n return self.estimator.predict_proba(X)\n\n @property\n def classes_(self):\n return self.estimator.class_labels\n"
},
{
"path": "eda/src/autogluon/eda/visualization/jupyter.py",
"content": "from IPython.display import HTML, Markdown, display\n\n\nclass JupyterMixin:\n def __init__(self) -> None:\n self.headers = False\n\n @staticmethod\n def display_obj(obj):\n display(obj)\n\n def render_header_if_needed(self, state, header_text, ds=\"\"):\n sample_size = state.get(\"sample_size\", {}).get(ds, None)\n if self.headers:\n sample_info = \"\" if sample_size is None else f\" (sample size: {sample_size})\"\n header = f\"**{header_text}{sample_info}**\"\n self.render_markdown(header)\n\n @staticmethod\n def render_markdown(md):\n display(Markdown(md))\n\n\nclass JupyterTools:\n def fix_tabs_scrolling(self):\n \"\"\"\n Helper utility to fix Jupyter styles for Tab widgets. This enforces the Tab element to fully expand and not use scrollbar.\n See the issue: https://github.com/jupyter-widgets/ipywidgets/issues/1791\n \"\"\"\n style = \"\"\"\n \n \"\"\"\n display(HTML(style))\n"
},
{
"path": "eda/src/autogluon/eda/visualization/layouts.py",
"content": "from typing import Callable, Dict, List, Optional, Union\n\nfrom IPython.display import display\nfrom ipywidgets import HBox, Layout, Output, Tab\n\nfrom .. import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\n \"MarkdownSectionComponent\",\n \"SimpleVerticalLinearLayout\",\n \"SimpleHorizontalLayout\",\n \"PropertyRendererComponent\",\n \"TabLayout\",\n]\n\nfrom ..state import is_key_present_in_state\n\n\nclass SimpleVerticalLinearLayout(AbstractVisualization):\n \"\"\"\n Renders facets in a sequential order (facets will appear in a vertical layout).\n \"\"\"\n\n def __init__(\n self,\n facets: Union[AbstractVisualization, List[AbstractVisualization]],\n namespace: Optional[str] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n if not isinstance(facets, list):\n facets = [facets]\n self.facets = facets\n self._kwargs = kwargs\n\n def can_handle(self, state: AnalysisState) -> bool:\n return True\n\n def _render(self, state: AnalysisState) -> None:\n for facet in self.facets:\n facet.render(state)\n\n\nclass SimpleHorizontalLayout(SimpleVerticalLinearLayout):\n \"\"\"\n Render components horizontally using `HBox` widget.\n See `HBox widget `_ documentation for details.\n \"\"\"\n\n def _render(self, state: AnalysisState) -> None:\n outs = [Output() for _ in range(len(self.facets))]\n for out, facet in zip(outs, self.facets):\n with out:\n facet.render(state)\n\n display(HBox(outs, layout=Layout(flex=\"row wrap\")))\n\n\nclass TabLayout(SimpleVerticalLinearLayout):\n \"\"\"\n Render components using `Tab` widget.\n See `HBox widget `_ documentation for details.\n\n \"\"\"\n\n def __init__(self, facets: Dict[str, AbstractVisualization], namespace: Optional[str] = None, **kwargs) -> None:\n self.facet_tab_names = list(facets.keys())\n super().__init__(list(facets.values()), namespace, **kwargs)\n\n def _render(self, state: AnalysisState) -> None:\n outs = [Output() for _ in self.facets]\n tab = Tab(children=outs, width=400)\n for i, name in enumerate(self.facet_tab_names):\n tab.set_title(i, name)\n display(tab)\n for out, facet in zip(outs, self.facets):\n with out:\n facet.render(state)\n\n\nclass MarkdownSectionComponent(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render provided string as a Markdown cell.\n See `Jupyter Markdown cell `_\n documentation for details.\n\n Parameters\n ----------\n markdown: str\n markdown text to render\n condition_fn: Optional[Callable], default = None\n if specified, call the provided function with `state` arg. The function expected to return bool value\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n \"\"\"\n\n def __init__(\n self, markdown: str, condition_fn: Optional[Callable] = None, namespace: Optional[str] = None, **kwargs\n ) -> None:\n \"\"\"\n\n Parameters\n ----------\n markdown\n condition_fn\n namespace\n kwargs\n \"\"\"\n super().__init__(namespace, **kwargs)\n self.markdown = markdown\n self.condition_fn = condition_fn\n\n def can_handle(self, state: AnalysisState) -> bool:\n return True if self.condition_fn is None else self.condition_fn(state)\n\n def _render(self, state: AnalysisState) -> None:\n self.render_markdown(self.markdown)\n\n\nclass PropertyRendererComponent(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render component stored in `state`'s dot-separated path to property (i.e. `a.b` results in `state.a.b`)\n\n Parameters\n ----------\n property: str\n dot-separated path to property (i.e. `a.b` results in `state.a.b`)\n transform_fn: Optional[Callable], default = None\n if specified, call the provided function with the object extracted from `state`'s `property`.\n Returned transformation is then passed into render function\n namespace: Optional[str], default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n \"\"\"\n\n def __init__(\n self, property: str, transform_fn: Optional[Callable] = None, namespace: Optional[str] = None, **kwargs\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.property = property\n self.transform_fn = transform_fn\n\n def can_handle(self, state: AnalysisState) -> bool:\n return is_key_present_in_state(state, self.property)\n\n def _render(self, state: AnalysisState) -> None:\n obj = state\n for p in self.property.split(\".\"):\n obj = obj[p]\n if self.transform_fn is not None:\n obj = self.transform_fn(obj)\n self.display_obj(obj)\n"
},
{
"path": "eda/src/autogluon/eda/visualization/missing.py",
"content": "import logging\nfrom typing import Optional\n\nimport matplotlib.pyplot as plt\nimport missingno as msno\n\nfrom .. import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\"MissingValues\"]\nlogger = logging.getLogger(__name__)\n\n\nclass MissingValues(AbstractVisualization, JupyterMixin):\n \"\"\"\n Renders visualization of missingness for datasets using one of the methods specified in `graph_type'.\n\n This visualization depends on :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis` analysis.\n\n See also `missingno `_ documentation\n\n Parameters\n ----------\n graph_type: str, default = 'matrix'\n One of the following visualization types:\n - matrix - nullity matrix is a data-dense display which lets you quickly visually pick out patterns in data completion\n This visualization will comfortably accommodate up to 50 labelled variables.\n Past that range labels begin to overlap or become unreadable, and by default large displays omit them.\n - bar - visualizes how many rows are non-null vs null in the column. Logarithmic scale can by specifying `log=True` in `kwargs`\n - heatmap - correlation heatmap measures nullity correlation: how strongly the presence or absence of one\n variable affects the presence of another. Nullity correlation ranges from -1\n (if one variable appears the other definitely does not) to 0 (variables appearing or not appearing have no effect on one another)\n to 1 (if one variable appears the other definitely also does).\n Entries marked <1 or >-1 have a correlation that is close to being exactingly negative or positive but is still not quite perfectly so.\n - dendrogram - the dendrogram allows to more fully correlate variable completion, revealing trends deeper than the pairwise ones\n visible in the correlation heatmap. The dendrogram uses a hierarchical clustering algorithm (courtesy of scipy) to bin variables\n against one another by their nullity correlation (measured in terms of binary distance).\n At each step of the tree the variables are split up based on which combination minimizes the distance of the remaining clusters.\n The more monotone the set of variables, the closer their total distance is to zero, and the closer their average distance (the y-axis) is to zero.\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.missing.MissingValuesAnalysis`\n \"\"\"\n\n __OPERATIONS_MAPPING = {\n \"matrix\": msno.matrix,\n \"bar\": msno.bar,\n \"heatmap\": msno.heatmap,\n \"dendrogram\": msno.dendrogram,\n }\n\n MAX_MATRIX_VARIABLES_NUMBER = 50\n\n def __init__(\n self, graph_type: str = \"matrix\", headers: bool = False, namespace: Optional[str] = None, **kwargs\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.graph_type = graph_type\n assert (\n self.graph_type in self.__OPERATIONS_MAPPING\n ), f\"{self.graph_type} must be one of {self.__OPERATIONS_MAPPING.keys()}\"\n self.headers = headers\n\n def can_handle(self, state: AnalysisState) -> bool:\n can_handle = self.all_keys_must_be_present(state, \"missing_statistics\")\n if can_handle and self.graph_type == \"matrix\" and self._has_too_many_variables_for_matrix(state):\n logging.warning(\n f\"The dataset has more than {self.MAX_MATRIX_VARIABLES_NUMBER} variables; \"\n f\"matrix visualization will comfortably accommodate up to {self.MAX_MATRIX_VARIABLES_NUMBER} labelled variables. \"\n f\"Past that range labels begin to overlap or become unreadable, and by default large displays omit them.\"\n )\n return can_handle\n\n def _render(self, state: AnalysisState) -> None:\n for ds, ds_state in state.missing_statistics.items():\n self.render_header_if_needed(state, f\"`{ds}` missing values analysis\", ds=ds)\n widget = self._get_operation(self.graph_type)\n self._internal_render(widget, ds_state.data, **self._kwargs)\n\n def _has_too_many_variables_for_matrix(self, state: AnalysisState):\n for _, ds_state in state.missing_statistics.items():\n if len(ds_state.data.columns) > self.MAX_MATRIX_VARIABLES_NUMBER:\n return True\n return False\n\n @staticmethod\n def _internal_render(widget, data, **kwargs):\n fig = widget(data, fontsize=10, **kwargs)\n plt.show(fig)\n\n def _get_operation(self, graph_type):\n return self.__OPERATIONS_MAPPING[graph_type]\n"
},
{
"path": "eda/src/autogluon/eda/visualization/model.py",
"content": "from typing import Any, Dict, Optional\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nfrom yellowbrick.contrib.wrapper import REGRESSOR, ContribEstimator\nfrom yellowbrick.regressor import residuals_plot\n\nfrom autogluon.core.constants import REGRESSION\n\nfrom ..state import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\"ConfusionMatrix\", \"FeatureImportance\", \"RegressionEvaluation\", \"ModelLeaderboard\"]\n\n\nclass ConfusionMatrix(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render confusion matrix for binary/multiclass classificator.\n\n This visualization depends on :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator` analysis.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = ...\n >>> df_test = ...\n >>> predictor = ...\n >>>\n >>> auto.analyze(model=predictor, val_data=df_test, anlz_facets=[\n >>> eda.model.AutoGluonModelEvaluator(),\n >>> ], viz_facets=[\n >>> viz.model.ConfusionMatrix(fig_args=dict(figsize=(3,3)), annot_kws={\"size\": 12}),\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator`\n \"\"\"\n\n def __init__(\n self,\n fig_args: Optional[Dict[str, Any]] = None,\n headers: bool = False,\n namespace: Optional[str] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"model_evaluation\" in state and \"confusion_matrix\" in state.model_evaluation\n\n def _render(self, state: AnalysisState) -> None:\n self.render_header_if_needed(state, \"Confusion Matrix\")\n labels = state.model_evaluation.labels\n cm = pd.DataFrame(state.model_evaluation.confusion_matrix, columns=labels, index=labels)\n cm.index.name = \"Actual\"\n cm.columns.name = \"Predicted\"\n normalized = state.model_evaluation.confusion_matrix_normalized\n fmt = \",.2%\" if normalized else \"d\"\n\n cells_num = len(cm)\n fig_args = self.fig_args.copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (cells_num, cells_num)\n\n fig, ax = plt.subplots(**fig_args)\n sns.heatmap(\n cm,\n ax=ax,\n cmap=\"Blues\",\n annot=True,\n linewidths=0.5,\n linecolor=\"lightgrey\",\n fmt=fmt,\n cbar=False,\n **self._kwargs,\n )\n plt.show(fig)\n\n\nclass _YellowbrickAutoGluonWrapper(ContribEstimator):\n _estimator_type = REGRESSOR\n\n def score(self, y_pred, y_true, **kwargs):\n # note: this is not conventional use of API: we pass y_pred since we already have predictions done\n return self.estimator.evaluate_predictions(y_pred, y_true)[\"r2\"]\n\n def predict(self, y_pred, **kwargs):\n # note: this is not conventional use of API: we pass y_pred since we already have predictions done\n return y_pred\n\n\nclass RegressionEvaluation(AbstractVisualization, JupyterMixin):\n \"\"\"\n This plot shows residuals on the vertical axis vs prediction on horizontal axis.\n\n This visualization depends on :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator` analysis.\n\n Parameters\n ----------\n residuals_plot_mode: Optional[str], default = 'qoq'\n Additional plot to render to the right of the main plot. The supported values:\n - `qoq` (default) - Q-Q plot, which is a common way to check that residuals are normally distributed. If the residuals are normally distributed,\n then their quantiles when plotted against quantiles of normal distribution should form a straight line.\n - `hist` - display histogram that our error is normally distributed around zero, which also generally indicates a well fitted model\n - any other value - don't render additional details\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = ...\n >>> df_test = ...\n >>> predictor = ...\n >>>\n >>> auto.analyze(model=predictor, val_data=df_test, anlz_facets=[\n >>> eda.model.AutoGluonModelEvaluator(),\n >>> ], viz_facets=[\n >>> viz.model.RegressionEvaluation(fig_args=dict(figsize=(6,6)), marker='o', scatter_kws={'s':5}),\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator`\n \"\"\"\n\n def __init__(\n self,\n residuals_plot_mode: Optional[str] = \"qoq\",\n fig_args: Optional[Dict[str, Any]] = None,\n headers: bool = False,\n namespace: Optional[str] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n self.residuals_analysis_mode = residuals_plot_mode\n\n if fig_args is None:\n fig_args = {}\n fig_args = {**{\"figsize\": (12, 6)}, **fig_args}\n self.fig_args = fig_args\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"model_evaluation\" in state and state.model_evaluation.problem_type == REGRESSION\n\n def _get_plot_mode(self):\n res_plot_kwargs = {\n \"hist\": dict(hist=True, qqplot=False),\n \"qoq\": dict(hist=False, qqplot=True),\n }.get(\n self.residuals_analysis_mode,\n dict(hist=False, qqplot=False), # type: ignore\n )\n return res_plot_kwargs\n\n def _render(self, state: AnalysisState) -> None:\n self.render_header_if_needed(state, \"Prediction vs Target\")\n res_plot_kwargs = self._get_plot_mode()\n fig, ax = plt.subplots(**self.fig_args)\n y_pred_train, y_true_train, y_pred_test, y_true_test = RegressionEvaluation._repack_parameters(\n state.model_evaluation\n )\n residuals_plot(\n _YellowbrickAutoGluonWrapper(state.model),\n y_pred_train,\n y_true_train,\n y_pred_test,\n y_true_test,\n show=False,\n ax=ax,\n **res_plot_kwargs,\n )\n plt.show(fig)\n\n @staticmethod\n def _repack_parameters(ev):\n y_pred_train = ev.y_pred_train if \"y_pred_train\" in ev else ev.y_pred_val\n y_true_train = ev.y_true_train if \"y_pred_train\" in ev else ev.y_true_val\n y_pred_test = ev.y_pred_test if \"y_true_test\" in ev else (ev.y_pred_val if \"y_pred_train\" in ev else None)\n y_true_test = ev.y_true_test if \"y_true_test\" in ev else (ev.y_true_val if \"y_pred_train\" in ev else None)\n return y_pred_train, y_true_train, y_pred_test, y_true_test\n\n\nclass FeatureImportance(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render feature importance for the model.\n\n This visualization depends on :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator` analysis.\n\n Parameters\n ----------\n show_barplots: bool, default = False\n render features barplots if True\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n fig_args: Optional[Dict[str, Any]] = None,\n kwargs to pass into chart figure\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = ...\n >>> df_test = ...\n >>> predictor = ...\n >>>\n >>> auto.analyze(model=predictor, val_data=df_test, anlz_facets=[\n >>> eda.model.AutoGluonModelEvaluator(),\n >>> ], viz_facets=[\n >>> viz.model.FeatureImportance(show_barplots=True)\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator`\n \"\"\"\n\n def __init__(\n self,\n show_barplots: bool = False,\n fig_args: Optional[Dict[str, Any]] = None,\n headers: bool = False,\n namespace: Optional[str] = None,\n **kwargs,\n ) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n if fig_args is None:\n fig_args = {}\n self.fig_args = fig_args\n\n self.show_barplots = show_barplots\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"model_evaluation\" in state and \"importance\" in state.model_evaluation\n\n def _render(self, state: AnalysisState) -> None:\n self.render_header_if_needed(state, \"Feature Importance\")\n importance = state.model_evaluation.importance\n with pd.option_context(\"display.max_rows\", 100 if len(importance) <= 100 else 20):\n self.display_obj(importance)\n if self.show_barplots:\n fig_args = self.fig_args.copy()\n if \"figsize\" not in fig_args:\n fig_args[\"figsize\"] = (12, len(importance) / 4)\n\n fig, ax = plt.subplots(**fig_args)\n sns.barplot(ax=ax, data=importance.reset_index(), y=\"index\", x=\"importance\", **self._kwargs)\n plt.show(fig)\n\n\nclass ModelLeaderboard(AbstractVisualization, JupyterMixin):\n \"\"\"\n Render model leaderboard for trained model ensemble.\n\n Parameters\n ----------\n headers: bool, default = False\n if `True` then render headers\n namespace: str, default = None\n namespace to use; can be nested like `ns_a.ns_b.ns_c`\n\n Examples\n --------\n >>> import autogluon.eda.analysis as eda\n >>> import autogluon.eda.visualization as viz\n >>> import autogluon.eda.auto as auto\n >>>\n >>> df_train = ...\n >>> df_test = ...\n >>> predictor = ...\n >>>\n >>> auto.analyze(model=predictor, val_data=df_test, anlz_facets=[\n >>> eda.model.AutoGluonModelEvaluator(),\n >>> ], viz_facets=[\n >>> viz.model.ModelLeaderboard(),\n >>> ])\n\n See Also\n --------\n :py:class:`~autogluon.eda.analysis.model.AutoGluonModelEvaluator`\n \"\"\"\n\n def __init__(self, namespace: Optional[str] = None, headers: bool = False, **kwargs) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n def can_handle(self, state: AnalysisState) -> bool:\n return \"model_evaluation\" in state and \"leaderboard\" in state.model_evaluation\n\n def _render(self, state: AnalysisState) -> None:\n self.render_header_if_needed(state, \"Model Leaderboard\")\n df = state.model_evaluation.leaderboard\n with pd.option_context(\"display.max_rows\", 100 if len(df) <= 100 else 20):\n self.display_obj(df)\n"
},
{
"path": "eda/src/autogluon/eda/visualization/shift.py",
"content": "from typing import Optional\n\nfrom .. import AnalysisState\nfrom .base import AbstractVisualization\nfrom .jupyter import JupyterMixin\n\n__all__ = [\"XShiftSummary\"]\n\n\nclass XShiftSummary(AbstractVisualization, JupyterMixin):\n \"\"\"\n Summarize the results of the XShiftDetector. It will render the results as markdown in jupyter.\n This will contain the detection status (True if detected), the details of the hypothesis test (test\n statistic, pvalue), and the feature importances for the detection.\n \"\"\"\n\n MAX_FEATURES_TO_DISPLAY = 5\n\n def __init__(self, headers: bool = False, namespace: Optional[str] = None, **kwargs) -> None:\n super().__init__(namespace, **kwargs)\n self.headers = headers\n\n def _summary(self, results: dict) -> str:\n \"\"\"Output the results of C2ST in a human readable format\"\"\"\n if not results[\"detection_status\"]:\n ret_md = \"We did not detect a substantial difference between the training and test X distributions.\"\n return ret_md\n else:\n ret_md = (\n f\"We detected a substantial difference between the training and test X distributions,\\n\"\n f\"a type of distribution shift.\\n\"\n f\"\\n\"\n f\"**Test results**: \"\n f\"We can predict whether a sample is in the test vs. training set with a `{results['eval_metric']}` of\\n\"\n f\"`{results['test_statistic']:.4f}` with a p-value of `{results['pvalue']:.4f}` \"\n f\"(smaller than the threshold of `{results['pvalue_threshold']:.4f})`.\\n\"\n f\"\\n\"\n )\n return ret_md\n\n def _render_feature_importance_if_needed(self, state):\n if \"feature_importance\" in state and state[\"detection_status\"]:\n fi = state[\"feature_importance\"]\n fi = fi[fi.p_value <= state[\"pvalue_threshold\"]]\n if len(fi) > 0:\n self.render_markdown(\n \"**Feature importances**: The variables that are the most responsible for this shift are those with high feature importance:\\n\\n\"\n )\n self.display_obj(fi[: self.MAX_FEATURES_TO_DISPLAY])\n\n def can_handle(self, state: AnalysisState) -> bool:\n return self.at_least_one_key_must_be_present(state, \"xshift_results\")\n\n def _render(self, state: AnalysisState) -> None:\n header_text = \"Detecting distribution shift\"\n self.render_header_if_needed(state, header_text)\n self.render_markdown(self._summary(state.xshift_results))\n self._render_feature_importance_if_needed(state.xshift_results)\n"
},
{
"path": "eda/tests/__init__.py",
"content": ""
},
{
"path": "eda/tests/conftest.py",
"content": "import pytest\n\n\ndef pytest_addoption(parser):\n parser.addoption(\"--runslow\", action=\"store_true\", default=False, help=\"run slow tests\")\n\n\ndef pytest_configure(config):\n config.addinivalue_line(\"markers\", \"slow: mark test as slow to run\")\n plugin = config.pluginmanager.getplugin(\"mypy\")\n if plugin is not None:\n plugin.mypy_argv.append(\"--ignore-missing-imports\")\n\n\ndef pytest_collection_modifyitems(config, items):\n if config.getoption(\"--runslow\"):\n # --runslow given in cli: do not skip slow tests\n return\n skip_slow = pytest.mark.skip(reason=\"need --runslow option to run\")\n for item in items:\n if \"slow\" in item.keywords:\n item.add_marker(skip_slow)\n"
},
{
"path": "eda/tests/test_check_style.py",
"content": "import logging\nimport warnings\nfrom subprocess import PIPE, Popen\n\n\ndef test_check_style():\n logging.getLogger().setLevel(logging.INFO)\n logging.info(\"PEP8 Style check\")\n flake8_proc = Popen([\"flake8\", \"--count\", \"--max-line-length\", \"300\"], stdout=PIPE)\n flake8_out = flake8_proc.communicate()[0]\n lines = flake8_out.splitlines()\n count = int(lines[-1].decode())\n if count > 0:\n warnings.warn(f\"{count} PEP8 warnings remaining\", stacklevel=2)\n assert count < 10, \"Too many PEP8 warnings found, improve code quality to pass test.\"\n"
},
{
"path": "eda/tests/unittests/__init__.py",
"content": ""
},
{
"path": "eda/tests/unittests/analysis/__init__.py",
"content": ""
},
{
"path": "eda/tests/unittests/analysis/test_anomaly.py",
"content": "from unittest.mock import MagicMock\n\nimport numpy as np\nimport pandas as pd\nimport pytest\nfrom pandas.testing import assert_frame_equal\nfrom pyod.models.lof import LOF\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import AnomalyDetector, AnomalyDetectorAnalysis\n\n\ndef test_AnomalyDetector():\n df, cols = _generate_dataset()\n idx_anomaly = [8, 12, 25, 22, 76, 48]\n df.loc[idx_anomaly, cols] = 5\n ad = AnomalyDetector(label=\"label\", n_folds=2, detector_list=[LOF(n_neighbors=5), LOF(n_neighbors=10)])\n anomaly_scores = ad.fit_transform(df)\n assert sorted(anomaly_scores.sort_values(ascending=False).index[: len(idx_anomaly)]) == sorted(idx_anomaly)\n assert ad.predict(pd.DataFrame({\"A\": [0], \"B\": [1]}))[0] < 1\n assert ad.predict(pd.DataFrame({\"A\": [5], \"B\": [10]}))[0] > 1\n\n\ndef _generate_dataset(seed=0):\n np.random.seed(seed)\n cols = list(\"AB\")\n df = pd.DataFrame(np.random.randint(0, 2, size=(1000, 2)), columns=cols)\n df[\"label\"] = [0, 1] * int(len(df) / 2)\n return df, cols\n\n\ndef test_AnomalyDetector__defaults_init():\n ad = AnomalyDetector(\"label\", some_kwarg=\"value\")\n\n assert len(ad.detector_list) == 7\n assert len(ad._suod_kwargs[\"base_estimators\"]) == 7\n ad._suod_kwargs.pop(\"base_estimators\")\n\n assert ad._suod_kwargs[\"n_jobs\"] > 0\n ad._suod_kwargs.pop(\"n_jobs\")\n assert ad._suod_kwargs == {\n \"bps_flag\": False,\n \"combination\": \"average\",\n \"some_kwarg\": \"value\",\n \"verbose\": False,\n }\n\n assert ad._detectors is None\n assert ad.original_features is None\n assert ad.problem_type == \"regression\"\n\n\ndef test_AnomalyDetectorAnalysis(monkeypatch):\n call_create_detector = MagicMock()\n call_create_detector.fit_transform.side_effect = lambda x: {\n \"train_data_df\": \"train_df_score\",\n }[x]\n\n call_create_detector.transform.side_effect = lambda x: {\n \"test_data_df\": \"test_df_score\",\n \"val_data_df\": \"val_df_score\",\n }[x]\n\n with monkeypatch.context() as m:\n m.setattr(AnomalyDetectorAnalysis, \"_create_detector\", lambda *args: call_create_detector)\n ad = AnomalyDetectorAnalysis(\n train_data=\"train_data_df\",\n test_data=\"test_data_df\",\n val_data=\"val_data_df\",\n label=\"label\",\n n_folds=3,\n )\n ad.can_handle = MagicMock(return_value=True)\n state = ad.fit()\n\n assert state == {\n \"anomaly_detection\": {\n \"scores\": {\"test_data\": \"test_df_score\", \"train_data\": \"train_df_score\", \"val_data\": \"val_df_score\"}\n }\n }\n\n\ndef test_AnomalyDetectorAnalysis__interpret(monkeypatch):\n call_create_detector = MagicMock()\n call_create_detector.fit_transform.side_effect = lambda x: {\n \"train_data_df\": \"train_df_score\",\n }[x]\n\n call_create_detector.transform.side_effect = lambda x: {\n \"test_data_df\": \"test_df_score\",\n }[x]\n\n with monkeypatch.context() as m:\n m.setattr(AnomalyDetectorAnalysis, \"_create_detector\", lambda *args: call_create_detector)\n ad = AnomalyDetectorAnalysis(\n train_data=\"train_data_df\",\n test_data=\"test_data_df\",\n label=\"label\",\n store_explainability_data=True,\n )\n ad.can_handle = MagicMock(return_value=True)\n state = ad.fit()\n\n explain_rows_fns = state.anomaly_detection.pop(\"explain_rows_fns\")\n for ds, fn in explain_rows_fns.items():\n assert fn.args == (\n {\"train_data\": \"train_data_df\", \"test_data\": \"test_data_df\", \"label\": \"label\"},\n call_create_detector,\n ds,\n )\n\n assert state == {\"anomaly_detection\": {\"scores\": {\"test_data\": \"test_df_score\", \"train_data\": \"train_df_score\"}}}\n\n\ndef test_AnomalyDetectorAnalysis__create_detector():\n a = AnomalyDetectorAnalysis(n_folds=3, some_arg=42)\n ad = a._create_detector(AnalysisState(label=\"label\"))\n assert ad.label == \"label\"\n assert ad.n_folds == 3\n assert ad._suod_kwargs[\"some_arg\"] == 42\n\n\ndef test_AnomalyDetectorAnalysis__can_handle():\n a = AnomalyDetectorAnalysis()\n assert a.can_handle(None, AnalysisState()) is False\n\n df_train, cols = _generate_dataset()\n df_test, cols = _generate_dataset()\n\n assert a.can_handle(None, AnalysisState(train_data=df_train, label=\"label\")) is True\n assert a.can_handle(None, AnalysisState(train_data=df_train, test_data=df_test, label=\"label\")) is True\n\n _df_train = df_train.copy()\n _df_train.loc[[0, 1, 2], cols] = np.NAN\n _df_test = df_test.copy()\n _df_test.loc[[3, 4, 5], cols] = np.NAN\n\n assert a.can_handle(None, AnalysisState(train_data=_df_train, test_data=_df_test, label=\"label\")) is False\n assert a.can_handle(None, AnalysisState(train_data=df_train, test_data=_df_test, label=\"label\")) is False\n\n\n@pytest.mark.parametrize(\"dataset\", [\"train_data\", \"test_data\"])\ndef test_AnomalyDetectorAnalysis__explain_rows_fn(dataset):\n a = AnomalyDetectorAnalysis()\n detector = MagicMock()\n train_data = pd.DataFrame({\"train_data\": np.arange(4)})\n ds_data = pd.DataFrame({dataset: np.arange(4)})\n\n kwargs = {} if dataset == \"train_data\" else {dataset: ds_data}\n args = AnalysisState(train_data=train_data, **kwargs)\n\n result = a.explain_rows_fn(args, detector=detector, dataset=dataset, dataset_row_ids=[1, 3])\n\n # np.int64 is required for running tests on Windows\n assert_frame_equal(result[\"rows\"].astype(np.int64), pd.DataFrame({dataset: [1, 3]}, index=[1, 3]).astype(np.int64))\n result.pop(\"rows\")\n\n assert result[\"train_data\"] is train_data\n result.pop(\"train_data\")\n\n assert result == {\n \"model\": detector,\n }\n"
},
{
"path": "eda/tests/unittests/analysis/test_base.py",
"content": "from unittest.mock import MagicMock\n\nimport pandas as pd\nimport pytest\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import Namespace\nfrom autogluon.eda.analysis.base import AbstractAnalysis, BaseAnalysis, SaveArgsToState\n\n\ndef test_abstractanalysis_parameter_shadowing():\n a: BaseAnalysis = BaseAnalysis(\n x=\"x\",\n y=\"y\",\n children=[\n BaseAnalysis(x=\"q\"),\n BaseAnalysis(\n x=\"w\",\n children=[\n BaseAnalysis(x=\"z\", q=\"q\"),\n ],\n ),\n ],\n )\n\n assert a._gather_args() == {\"x\": \"x\", \"y\": \"y\"}\n assert a.children[0]._gather_args() == {\"x\": \"q\", \"y\": \"y\"}\n assert a.children[1]._gather_args() == {\"x\": \"w\", \"y\": \"y\"}\n assert a.children[1].children[0]._gather_args() == {\"x\": \"z\", \"y\": \"y\", \"q\": \"q\"}\n\n\ndef test_abstractanalysis_available_datasets():\n state = AnalysisState(\n train_data=pd.DataFrame(),\n test_data=pd.DataFrame(),\n tuning_data=pd.DataFrame(),\n val_data=pd.DataFrame(),\n )\n\n keys = []\n for ds, df in BaseAnalysis().available_datasets(state):\n assert state[ds] is df\n assert ds in state\n keys.append(ds)\n assert list(keys) == list(state.keys())\n\n\ndef test_abstractanalysis_available_datasets_some_present():\n state = AnalysisState(\n train_data=pd.DataFrame(),\n )\n\n keys = []\n for ds, df in BaseAnalysis().available_datasets(state):\n assert state[ds] is df\n assert ds in state\n keys.append(ds)\n assert list(keys) == list(state.keys())\n\n\ndef test_abstractanalysis_fit_is_not_called_if_cannot_handle():\n a = BaseAnalysis()\n a.can_handle = MagicMock(return_value=False)\n a._fit = MagicMock()\n a.fit()\n a._fit.assert_not_called()\n\n\ndef test_abstractanalysis_fit_is_called_if_can_handle():\n a = BaseAnalysis()\n a.can_handle = MagicMock(return_value=True)\n a._fit = MagicMock()\n a.fit()\n a._fit.assert_called()\n\n\ndef test_abstractanalysis_fit_on_inner_before_outer_raises_exception():\n inner_analysis = BaseAnalysis(a=1, b=2)\n outer_analysis = BaseAnalysis(b=3, c=4, children=[inner_analysis])\n with pytest.raises(AssertionError):\n inner_analysis.fit()\n state = outer_analysis.fit()\n assert state is not None\n\n\ndef test_abstractanalysis_fit_gathers_args():\n inner_analysis = BaseAnalysis(a=1, b=2)\n outer_analysis = BaseAnalysis(b=3, c=4, children=[inner_analysis])\n inner_analysis._fit = MagicMock()\n outer_analysis._fit = MagicMock()\n\n state = outer_analysis.fit(arg_a=10, arg_b=11)\n assert inner_analysis.state is state\n assert outer_analysis.state is state\n\n outer_analysis._fit.assert_called_once_with({}, {\"b\": 3, \"c\": 4}, arg_a=10, arg_b=11)\n inner_analysis._fit.assert_called_once_with({}, {\"a\": 1, \"b\": 2, \"c\": 4}, arg_a=10, arg_b=11)\n\n\ndef test_namespaces():\n def side_effect(state: AnalysisState, args: AnalysisState, **fit_kwargs):\n state.upd = fit_kwargs\n\n inner_analysis: BaseAnalysis = BaseAnalysis(arg=1)\n inner_analysis._fit = MagicMock()\n inner_analysis._fit.side_effect = side_effect\n\n # write outputs into ns1\n state = BaseAnalysis(children=[Namespace(namespace=\"ns1\", children=[inner_analysis])]).fit(op=\"fit1\")\n\n # write outputs into ns2\n state = BaseAnalysis(state=state, children=[Namespace(namespace=\"ns2\", children=[inner_analysis])]).fit(op=\"fit2\")\n\n # update outputs in ns2\n state = BaseAnalysis(state=state, children=[Namespace(namespace=\"ns2\", children=[inner_analysis])]).fit(op=\"fit3\")\n\n assert state == {\"ns1\": {\"upd\": {\"op\": \"fit1\"}}, \"ns2\": {\"upd\": {\"op\": \"fit3\"}}}\n\n\ndef test_SaveArgsToState():\n train_data = list(\"abc\")\n\n class SomeTransform(AbstractAnalysis):\n def can_handle(self, state: AnalysisState, args: AnalysisState) -> bool:\n return True\n\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n print(args)\n self.args[\"train_data\"] = [c.upper() for c in args[\"train_data\"]]\n\n state = BaseAnalysis(\n train_data=train_data,\n children=[SomeTransform(children=[SaveArgsToState(params_mapping={\"train_data\": \"result_arg\"})])],\n ).fit()\n\n assert state.result_arg == [\"A\", \"B\", \"C\"]\n\n\ndef test_SaveArgsToState__no_key():\n state = BaseAnalysis(children=[SaveArgsToState(params_mapping={\"train_data\": \"result_arg\"})]).fit()\n assert state.result_arg is None\n"
},
{
"path": "eda/tests/unittests/analysis/test_dataset.py",
"content": "import numpy as np\nimport pandas as pd\nimport pytest\n\nimport autogluon.eda.auto as auto\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT, R_OBJECT\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import Namespace, ProblemTypeControl, Sampler, TrainValidationSplit\nfrom autogluon.eda.analysis.base import BaseAnalysis\nfrom autogluon.eda.analysis.dataset import (\n DatasetSummary,\n LabelInsightsAnalysis,\n RawTypesAnalysis,\n SpecialTypesAnalysis,\n VariableTypeAnalysis,\n)\n\n\nclass SomeAnalysis(BaseAnalysis):\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.args = args.copy()\n\n\ndef test_Sampler():\n df_train = pd.DataFrame(np.random.randint(0, 100, size=(10, 4)), columns=list(\"ABCD\"))\n df_test = pd.DataFrame(np.random.randint(0, 100, size=(20, 4)), columns=list(\"EFGH\"))\n assert df_train.shape == (10, 4)\n assert df_test.shape == (20, 4)\n\n analysis = BaseAnalysis(\n train_data=df_train,\n test_data=df_test,\n children=[\n Namespace(namespace=\"ns_sampler\", children=[Sampler(sample=5, children=[SomeAnalysis()])]),\n Namespace(namespace=\"ns_sampler_2\", children=[Sampler(sample=15, children=[SomeAnalysis()])]),\n Namespace(namespace=\"ns_sampler_none\", children=[Sampler(sample=None, children=[SomeAnalysis()])]),\n Namespace(namespace=\"ns_no_sampler\", children=[SomeAnalysis()]),\n ],\n )\n\n state = analysis.fit()\n assert state.ns_sampler.args.train_data.shape == (5, 4)\n assert state.ns_sampler.args.test_data.shape == (5, 4)\n assert state.ns_sampler.sample_size == {\"test_data\": 5, \"train_data\": 5}\n\n assert state.ns_sampler_2.args.train_data.shape == (10, 4)\n assert state.ns_sampler_2.args.test_data.shape == (15, 4)\n assert state.ns_sampler_2.sample_size == {\"test_data\": 15}\n\n assert state.ns_sampler_none.args.train_data.shape == (10, 4)\n assert state.ns_sampler_none.args.test_data.shape == (20, 4)\n assert state.ns_sampler_none.sample_size is None\n\n assert state.ns_no_sampler.args.train_data.shape == (10, 4)\n assert state.ns_no_sampler.args.test_data.shape == (20, 4)\n assert state.ns_no_sampler.sample_size is None\n\n\ndef test_Sampler__window_larger_than_dataset():\n df_train = pd.DataFrame(np.random.randint(0, 100, size=(10, 4)), columns=list(\"ABCD\"))\n df_test = pd.DataFrame(np.random.randint(0, 100, size=(20, 4)), columns=list(\"EFGH\"))\n assert df_train.shape == (10, 4)\n assert df_test.shape == (20, 4)\n\n analysis = BaseAnalysis(\n train_data=df_train,\n test_data=df_test,\n children=[\n Sampler(sample=10000, children=[SomeAnalysis()]),\n ],\n )\n\n state = analysis.fit()\n assert state.args.train_data.shape == (10, 4)\n assert state.args.test_data.shape == (20, 4)\n assert state.sample_size is None\n\n\ndef test_Sampler_frac():\n df_train = pd.DataFrame(np.random.randint(0, 100, size=(10, 4)), columns=list(\"ABCD\"))\n df_test = pd.DataFrame(np.random.randint(0, 100, size=(20, 4)), columns=list(\"EFGH\"))\n assert df_train.shape == (10, 4)\n assert df_test.shape == (20, 4)\n\n analysis = BaseAnalysis(\n train_data=df_train, test_data=df_test, children=[Sampler(sample=0.5, children=[SomeAnalysis()])]\n )\n\n state = analysis.fit()\n assert state.sample_size == {\"test_data\": 0.5, \"train_data\": 0.5}\n assert state.args.train_data.shape == (5, 4)\n assert state.args.test_data.shape == (10, 4)\n\n\ndef test_TrainValidationSplit():\n df_train, _ = __get_dataset_summary_test_datasets()\n analysis = BaseAnalysis(\n train_data=df_train,\n label=\"D\",\n children=[\n Namespace(\n namespace=\"ns_val_split_specified\",\n children=[ProblemTypeControl(), TrainValidationSplit(val_size=0.4, children=[SomeAnalysis()])],\n ),\n Namespace(\n namespace=\"ns_val_split_default\",\n children=[\n ProblemTypeControl(problem_type=REGRESSION),\n TrainValidationSplit(children=[SomeAnalysis()]),\n ],\n ),\n Namespace(namespace=\"ns_no_split\", children=[SomeAnalysis()]),\n ],\n )\n\n state = analysis.fit()\n assert state.ns_val_split_specified.args.train_data.shape == (60, 7)\n assert state.ns_val_split_specified.args.val_data.shape == (40, 7)\n assert state.ns_val_split_specified.problem_type == MULTICLASS\n\n assert state.ns_val_split_default.args.train_data.shape == (70, 7)\n assert state.ns_val_split_default.args.val_data.shape == (30, 7)\n assert state.ns_val_split_default.problem_type == REGRESSION\n\n assert state.ns_no_split.args.train_data.shape == (100, 7)\n assert state.ns_no_split.args.val_data is None\n assert state.ns_no_split.problem_type is None\n\n\ndef __get_dataset_summary_test_datasets():\n cols = list(\"ABCDEFG\")\n df_train = pd.DataFrame((np.arange(100))[:, None].repeat([len(cols)], axis=1), columns=cols)\n df_test = pd.DataFrame((np.arange(200))[:, None].repeat([len(cols)], axis=1), columns=cols)\n str_mappings = {i: \"Lorem ipsum \" * (i + 1) for i in range(10)}\n for df in [df_train, df_test]:\n df[\"A\"] = (df[\"A\"] % 4).map({0: \"a\", 1: \"b\", 2: \"c\", 3: \"d\"})\n df[\"B\"] = (df[\"B\"] % 2).map({0: False, 1: True})\n df[\"C\"] = df[\"C\"] % 2\n df[\"D\"] = df[\"D\"] % 3\n df[\"E\"] = (df[\"E\"] % len(str_mappings.keys())).map(str_mappings)\n df[\"F\"] = df[\"F\"] * 0.1\n return df_train, df_test\n\n\ndef test_DatasetSummary():\n df_train, df_test = __get_dataset_summary_test_datasets()\n state = auto.analyze(train_data=df_train, test_data=df_test, return_state=True, anlz_facets=[DatasetSummary()])\n expected_cols = [\"25%\", \"50%\", \"75%\", \"count\", \"dtypes\", \"freq\", \"max\", \"mean\", \"min\", \"std\", \"top\", \"unique\"]\n expected_fields = list(\"ABCDEFG\")\n assert sorted(state.dataset_stats.train_data.keys()) == expected_cols\n assert sorted(pd.DataFrame(state.dataset_stats.train_data).index) == expected_fields\n assert sorted(state.dataset_stats.test_data.keys()) == expected_cols\n assert sorted(pd.DataFrame(state.dataset_stats.test_data).index) == expected_fields\n\n\ndef test_RawTypesAnalysis():\n df_train, df_test = __get_dataset_summary_test_datasets()\n state = auto.analyze(train_data=df_train, test_data=df_test, return_state=True, anlz_facets=[RawTypesAnalysis()])\n expected_types = {\"A\": \"object\", \"B\": \"bool\", \"C\": \"int\", \"D\": \"int\", \"E\": \"object\", \"F\": \"float\", \"G\": \"int\"}\n assert state.raw_type.train_data == expected_types\n assert state.raw_type.test_data == expected_types\n\n\ndef test_VariableTypeAnalysis_can_handle():\n df_train, _ = __get_dataset_summary_test_datasets()\n assert VariableTypeAnalysis().can_handle(state=AnalysisState(), args=AnalysisState()) is False\n assert (\n VariableTypeAnalysis().can_handle(state=AnalysisState({\"raw_type\": \"some_state\"}), args=AnalysisState())\n is True\n )\n\n\ndef test_VariableTypeAnalysis__map_raw_type_to_feature_type__special_cases():\n df = pd.DataFrame({\"a\": np.arange(20) % 10})\n f = VariableTypeAnalysis().map_raw_type_to_feature_type\n assert f(col=None, raw_type=\"\", df=df) is None\n assert f(col=\"a\", df=df, raw_type=\"\", numeric_as_categorical_threshold=3) is None\n assert f(col=\"a\", df=df, raw_type=\"\", numeric_as_categorical_threshold=100) == \"category\"\n\n\n@pytest.mark.parametrize(\n \"test_type,expected\",\n [(R_INT, \"numeric\"), (R_FLOAT, \"numeric\"), (R_OBJECT, \"category\"), (R_CATEGORY, \"category\"), (R_BOOL, \"category\")],\n)\ndef test_VariableTypeAnalysis__map_raw_type_to_feature_type__regular_cases(test_type, expected):\n df = pd.DataFrame({\"a\": np.arange(20) % 10})\n f = VariableTypeAnalysis().map_raw_type_to_feature_type\n args = dict(col=\"a\", df=df, numeric_as_categorical_threshold=3)\n assert f(**{**args, **dict(raw_type=test_type)}) == expected\n\n\ndef test_VariableTypeAnalysis():\n df_train, df_test = __get_dataset_summary_test_datasets()\n state = auto.analyze(\n train_data=df_train,\n test_data=df_test,\n return_state=True,\n anlz_facets=[RawTypesAnalysis(), VariableTypeAnalysis()],\n )\n\n expected_types = {\n \"A\": \"category\",\n \"B\": \"category\",\n \"C\": \"category\",\n \"D\": \"category\",\n \"E\": \"category\",\n \"F\": \"numeric\",\n \"G\": \"numeric\",\n }\n assert state.variable_type.train_data == expected_types\n assert state.variable_type.test_data == expected_types\n\n\ndef test_SpecialTypesAnalysis():\n df_train, df_test = __get_dataset_summary_test_datasets()\n state = auto.analyze(\n train_data=df_train, test_data=df_test, return_state=True, anlz_facets=[SpecialTypesAnalysis()]\n )\n\n expected_types = {\"E\": \"text\"}\n assert state.special_types.train_data == expected_types\n assert state.special_types.test_data == expected_types\n\n\n@pytest.mark.parametrize(\"threshold, is_warning_expected\", [(None, False), (191, True)])\ndef test_LabelInsightsAnalysis__classification__low_cardinality_classes(threshold, is_warning_expected):\n df_train = pd.DataFrame({\"label\": [1] * 200 + [2] * 190})\n\n if threshold is None:\n label_insights = LabelInsightsAnalysis()\n else:\n label_insights = LabelInsightsAnalysis(low_cardinality_classes_threshold=threshold)\n\n state = auto.analyze(\n train_data=df_train,\n test_data=df_train, # same as train\n label=\"label\",\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(problem_type=BINARY),\n label_insights,\n ],\n )\n\n if is_warning_expected:\n assert state == {\n \"label_insights\": {\"low_cardinality_classes\": {\"instances\": {2: 190}, \"threshold\": 191}},\n \"problem_type\": \"binary\",\n }\n else:\n assert state == {\"problem_type\": \"binary\"}\n\n\n@pytest.mark.parametrize(\"n_c1, n_c2, is_warning_expected\", [(100, 100, False), (100, 39, True)])\ndef test_LabelInsightsAnalysis__classification__class_imbalance(n_c1, n_c2, is_warning_expected):\n df_train = pd.DataFrame({\"label\": [1] * n_c1 + [2] * n_c2})\n\n label_insights = LabelInsightsAnalysis(low_cardinality_classes_threshold=1)\n\n state = auto.analyze(\n train_data=df_train,\n test_data=df_train, # same as train\n label=\"label\",\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(problem_type=BINARY),\n label_insights,\n ],\n )\n\n if is_warning_expected:\n assert state == {\n \"label_insights\": {\n \"minority_class_imbalance\": {\"majority_class\": 1, \"minority_class\": 2, \"ratio\": 0.39},\n },\n \"problem_type\": \"binary\",\n }\n else:\n assert state == {\"problem_type\": \"binary\"}\n\n\ndef test_LabelInsightsAnalysis__classification__not_present_in_train():\n df_train = pd.DataFrame({\"label\": [1] * 200 + [2] * 100})\n df_test = pd.DataFrame( # classes 3 and 4 are not present in train\n {\"label\": [1] * 20 + [2] * 40 + [3] * 20 + [4] * 20}\n )\n\n state = auto.analyze(\n train_data=df_train,\n test_data=df_test,\n label=\"label\",\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(problem_type=BINARY),\n LabelInsightsAnalysis(),\n ],\n )\n\n assert state == {\n \"label_insights\": {\"not_present_in_train\": {3, 4}},\n \"problem_type\": \"binary\",\n }\n\n\ndef test_LabelInsightsAnalysis__regression__no_ood():\n df_train = pd.DataFrame({\"label\": np.arange(0, 100)})\n df_test = pd.DataFrame({\"label\": np.arange(0, 100)})\n\n state = auto.analyze(\n train_data=df_train,\n test_data=df_test,\n label=\"label\",\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(problem_type=REGRESSION),\n LabelInsightsAnalysis(),\n ],\n )\n\n assert state == {\"problem_type\": \"regression\"}\n\n\n@pytest.mark.parametrize(\"threshold, is_warning_expected\", [(None, True), (0.02, False)])\ndef test_LabelInsightsAnalysis__regression__ood(threshold, is_warning_expected):\n df_train = pd.DataFrame({\"label\": np.arange(0, 1000)})\n df_test = pd.DataFrame({\"label\": np.arange(-15, 1015)})\n\n if threshold is None:\n label_insights = LabelInsightsAnalysis()\n else:\n label_insights = LabelInsightsAnalysis(regression_ood_threshold=threshold)\n\n state = auto.analyze(\n train_data=df_train,\n test_data=df_test,\n label=\"label\",\n return_state=True,\n anlz_facets=[\n ProblemTypeControl(problem_type=REGRESSION),\n label_insights,\n ],\n )\n\n if is_warning_expected:\n assert state == {\n \"label_insights\": {\n \"ood\": {\"count\": 12, \"test_range\": [-15, 1014], \"threshold\": 0.01, \"train_range\": [0, 999]}\n },\n \"problem_type\": \"regression\",\n }\n else:\n assert state == {\"problem_type\": \"regression\"}\n"
},
{
"path": "eda/tests/unittests/analysis/test_explain.py",
"content": "import os\nimport tempfile\n\nimport numpy as np\nimport pandas as pd\nimport pytest\nfrom pandas.testing import assert_frame_equal\n\nfrom autogluon.eda.analysis import ShapAnalysis\nfrom autogluon.eda.auto import analyze, quick_fit\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"..\", \"resources\"))\n\n\n@pytest.mark.parametrize(\n \"label, expected_task_type\",\n [\n (\"class\", \"binary\"),\n (\"fnlwgt\", \"regression\"),\n ],\n)\ndef test_ShapAnalysis(label, expected_task_type, monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(30, random_state=0)\n with tempfile.TemporaryDirectory() as path:\n state = quick_fit(\n estimator_args=dict(path=path),\n train_data=df_train,\n label=label,\n return_state=True,\n render_analysis=False,\n hyperparameters={\"RF\": {}},\n )\n assert state.model.problem_type == expected_task_type\n\n rows = state.model_evaluation.highest_error[:2]\n s = analyze(train_data=df_train, model=state.model, return_state=True, anlz_facets=[ShapAnalysis(rows)])\n\n assert len(s.explain.shapley) == 2\n for i, (_, row) in enumerate(rows.iterrows()):\n shap_data = s.explain.shapley[i]\n assert_frame_equal(shap_data[\"row\"], pd.DataFrame([row]))\n assert shap_data[\"feature_names\"] is None\n np.array_equal(shap_data[\"features\"], row.values[: len(shap_data[\"features\"])])\n assert sorted(list(shap_data.keys())) == sorted(\n [\n \"row\",\n \"expected_value\",\n \"shap_values\",\n \"features\",\n \"feature_names\",\n ]\n )\n"
},
{
"path": "eda/tests/unittests/analysis/test_interaction.py",
"content": "import os\nfrom unittest.mock import MagicMock\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nimport autogluon.eda.auto as auto\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import (\n ApplyFeatureGenerator,\n Correlation,\n CorrelationSignificance,\n DistributionFit,\n FeatureInteraction,\n)\nfrom autogluon.eda.analysis.dataset import RawTypesAnalysis\nfrom autogluon.eda.analysis.interaction import FeatureDistanceAnalysis\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"..\", \"resources\"))\n\nSAMPLE_SIZE = 200\n\n\ndef load_adult_data():\n train_data = os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")\n test_data = os.path.join(RESOURCE_PATH, \"adult\", \"test_data.csv\")\n train = pd.read_csv(train_data).sample(SAMPLE_SIZE, random_state=0)\n test = pd.read_csv(test_data).sample(SAMPLE_SIZE, random_state=0)\n data = (train, test)\n return data\n\n\ndef test_Correlation_spearman():\n state = auto.analyze(train_data=__create_test_df(), return_state=True, anlz_facets=[Correlation()])\n state.correlations.train_data = state.correlations.train_data.round(2)\n expected = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"abcd\"),\n data={\n \"a\": [1.00, 0.0, 0.88, -0.88],\n \"b\": [0.00, 1.0, 0.40, -0.40],\n \"c\": [0.88, 0.4, 1.00, -1.00],\n \"d\": [-0.88, -0.4, -1.00, 1.00],\n },\n )\n },\n \"correlations_method\": \"spearman\",\n }\n )\n __compare_outputs(expected, state)\n\n\ndef test_Correlation_pearson():\n state = auto.analyze(train_data=__create_test_df(), return_state=True, anlz_facets=[Correlation(method=\"pearson\")])\n state.correlations.train_data = state.correlations.train_data.round(2)\n expected = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"abcd\"),\n data={\n \"a\": [1.00, 0.03, 0.88, -0.88],\n \"b\": [0.03, 1.00, 0.43, -0.43],\n \"c\": [0.88, 0.43, 1.00, -1.00],\n \"d\": [-0.88, -0.43, -1.00, 1.00],\n },\n )\n },\n \"correlations_method\": \"pearson\",\n }\n )\n __compare_outputs(expected, state)\n\n\ndef test_Correlation_kendall():\n state = auto.analyze(train_data=__create_test_df(), return_state=True, anlz_facets=[Correlation(method=\"kendall\")])\n state.correlations.train_data = state.correlations.train_data.round(2)\n expected = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"abcd\"),\n data={\n \"a\": [1.00, 0.00, 0.77, -0.77],\n \"b\": [0.00, 1.00, 0.36, -0.36],\n \"c\": [0.77, 0.36, 1.00, -1.00],\n \"d\": [-0.77, -0.36, -1.00, 1.00],\n },\n )\n },\n \"correlations_method\": \"kendall\",\n }\n )\n __compare_outputs(expected, state)\n\n\ndef test_Correlation_phik():\n state = auto.analyze(train_data=__create_test_df(), return_state=True, anlz_facets=[Correlation(method=\"phik\")])\n state.correlations.train_data = state.correlations.train_data.round(2)\n expected = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"abcdef\"),\n data={\n \"a\": [1.0, 1.0, 1.00, 1.00, 1.00, 1.00],\n \"b\": [1.0, 1.0, 0.00, 0.00, 0.00, 0.00],\n \"c\": [1.0, 0.0, 1.00, 0.79, 0.79, 0.79],\n \"d\": [1.0, 0.0, 0.79, 1.00, 0.79, 0.79],\n \"e\": [1.0, 0.0, 0.79, 0.79, 1.00, 0.79],\n \"f\": [1.0, 0.0, 0.79, 0.79, 0.79, 1.00],\n },\n )\n },\n \"correlations_method\": \"phik\",\n }\n )\n __compare_outputs(expected, state)\n\n\ndef test_Correlation_focus():\n actual = auto.analyze(\n train_data=(__create_test_df()),\n return_state=True,\n anlz_facets=[Correlation(focus_field=\"c\", focus_field_threshold=0.5)],\n )\n print(actual)\n actual.correlations.train_data = actual.correlations.train_data.round(2)\n actual.correlations_focus_high_corr.train_data = actual.correlations_focus_high_corr.train_data.round(2)\n expected = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"acd\"),\n data={\n \"a\": [1.00, 0.88, -0.88],\n \"c\": [0.88, 1.00, -1.00],\n \"d\": [-0.88, -1.00, 1.00],\n },\n )\n },\n \"correlations_method\": \"spearman\",\n \"correlations_focus_field\": \"c\",\n \"correlations_focus_field_threshold\": 0.5,\n \"correlations_focus_high_corr\": {\"train_data\": pd.DataFrame(index=list(\"ad\"), data={\"c\": [0.88, -1.00]})},\n }\n )\n assert actual.correlations_focus_high_corr.train_data.equals(expected.correlations_focus_high_corr.train_data) # noqa\n actual.correlations_focus_high_corr.train_data = \"--\"\n expected.correlations_focus_high_corr.train_data = \"--\"\n __compare_outputs(expected, actual)\n\n\ndef test_CorrelationSignificance__can_handle():\n args = AnalysisState()\n assert (\n CorrelationSignificance().can_handle(\n AnalysisState({\"correlations\": 123, \"correlations_method\": \"something\"}), args\n )\n is True\n )\n assert (\n CorrelationSignificance().can_handle(\n AnalysisState({\"something\": 123, \"correlations_method\": \"something\"}), args\n )\n is False\n )\n assert (\n CorrelationSignificance().can_handle(AnalysisState({\"correlations\": 123, \"something\": \"something\"}), args)\n is False\n )\n\n\ndef test_CorrelationSignificance():\n state = AnalysisState(\n {\n \"correlations\": {\n \"train_data\": pd.DataFrame(\n index=list(\"acd\"),\n data={\n \"a\": [1.00, 0.88, -0.88],\n \"c\": [0.88, 1.00, -1.00],\n \"d\": [-0.88, -1.00, 1.00],\n },\n )\n },\n \"correlations_method\": \"spearman\",\n \"correlations_focus_field\": \"c\",\n \"correlations_focus_field_threshold\": 0.5,\n \"correlations_focus_high_corr\": {\"train_data\": pd.DataFrame(index=list(\"ad\"), data={\"c\": [0.88, -1.00]})},\n }\n )\n\n result = auto.analyze(\n train_data=__create_test_df(), state=state, return_state=True, anlz_facets=[CorrelationSignificance()]\n )\n\n assert result.significance_matrix.train_data.shape == (3, 3)\n\n\ndef __create_test_df():\n data = pd.DataFrame(\n {\n \"a\": [1, 2, 3, 4, 5, 6],\n \"b\": [5, 3, 2, 6, 7, 2],\n \"c\": [0, 0, 0, 1, 1, 1],\n \"d\": [1, 1, 1, 0, 0, 0],\n \"e\": [\"a\", \"a\", \"a\", \"b\", \"b\", \"b\"],\n \"f\": [\"b\", \"b\", \"b\", \"a\", \"a\", \"a\"],\n }\n )\n return data\n\n\ndef __compare_outputs(expected: AnalysisState, actual: AnalysisState):\n assert actual.correlations.train_data.equals(expected.correlations.train_data) # noqa\n actual.correlations.train_data = \"--\"\n expected.correlations.train_data = \"--\"\n assert actual == expected\n\n\ndef test_FeatureInteraction():\n df = __create_test_df()\n state = auto.analyze(\n train_data=df,\n return_state=True,\n anlz_facets=[\n RawTypesAnalysis(),\n FeatureInteraction(x=\"a\", y=\"b\", hue=\"c\", key=\"abc\"),\n FeatureInteraction(x=\"a\", y=\"b\", hue=\"d\"),\n FeatureInteraction(x=\"a\", y=\"b\", hue=\"q\", key=\"missing_col\"),\n ],\n )\n\n assert sorted(state.interactions.train_data.keys()) == [\"abc\", \"x:a|y:b|hue:d\"]\n assert state.interactions.train_data.abc.data.equals(df[[\"a\", \"b\", \"c\"]])\n assert state.interactions.train_data.abc.features == {\"hue\": \"c\", \"x\": \"a\", \"y\": \"b\"}\n assert state.interactions.train_data[\"x:a|y:b|hue:d\"].data.equals(df[[\"a\", \"b\", \"d\"]])\n assert state.interactions.train_data[\"x:a|y:b|hue:d\"].features == {\"hue\": \"d\", \"x\": \"a\", \"y\": \"b\"}\n\n\ndef test_FeatureInteraction__key_provided():\n assert (\n FeatureInteraction()._generate_key_if_not_provided(key=\"abc\", cols={\"x\": \"a\", \"y\": \"b\", \"hue\": \"c\"}) == \"abc\"\n )\n\n\n@pytest.mark.parametrize(\n \"cols, expected\",\n [\n ({\"x\": \"a\"}, \"x:a\"),\n ({\"y\": \"b\"}, \"y:b\"),\n ({\"hue\": \"c\"}, \"hue:c\"),\n ({\"x\": \"a\", \"y\": \"b\"}, \"x:a|y:b\"),\n ({\"x\": \"a\", \"hue\": \"c\"}, \"x:a|hue:c\"),\n ({\"y\": \"b\", \"hue\": \"c\"}, \"y:b|hue:c\"),\n ({\"x\": \"a\", \"y\": \"b\", \"hue\": \"c\"}, \"x:a|y:b|hue:c\"),\n ({\"x\": \"a\", \"y\": None, \"hue\": None}, \"x:a\"),\n ({\"x\": None, \"y\": \"b\", \"hue\": None}, \"y:b\"),\n ],\n)\ndef test_FeatureInteraction__generate_key_if_not_provided(cols, expected):\n assert FeatureInteraction()._generate_key_if_not_provided(key=None, cols=cols) == expected\n\n\ndef test_FeatureInteraction__can_handle():\n args = AnalysisState()\n assert FeatureInteraction().can_handle(AnalysisState({\"raw_type\": \"something\"}), args) is True\n assert FeatureInteraction().can_handle(AnalysisState({\"something\": 123}), args) is False\n\n\ndef test_DistributionFit__happy_path():\n df = __create_test_df()\n df[\"e\"] = [1000, 100, 10, 1, 0.1, 0.001]\n state = auto.analyze(\n train_data=df,\n return_state=True,\n anlz_facets=[\n DistributionFit(\n columns=[\"a\", \"unknown\"],\n keep_top_n=3,\n distributions_to_fit=[\"dweibull\", \"dgamma\", \"logistic\", \"lognorm\"],\n )\n ],\n )\n assert state.distributions_fit.train_data.a == {\n \"dgamma\": {\n \"param\": (2.7071122240167984, 3.4999828973430622, 0.5541010748597517),\n \"pvalue\": 0.9997989026208239,\n \"shapes\": [\"a\", \"loc\", \"scale\"],\n \"statistic\": 0.12375823666401586,\n },\n \"dweibull\": {\n \"param\": (1.9212313611673846, 3.5000360875942134, 1.6939405342565321),\n \"pvalue\": 0.9998695812922455,\n \"shapes\": [\"c\", \"loc\", \"scale\"],\n \"statistic\": 0.12094344045455918,\n },\n \"logistic\": {\n \"param\": (3.5, 1.0421523470240495),\n \"pvalue\": 0.9981811820582718,\n \"shapes\": [\"loc\", \"scale\"],\n \"statistic\": 0.14168404087671216,\n },\n }\n\n a = DistributionFit(\n columns=[\"e\"],\n keep_top_n=3,\n pvalue_min=0.9,\n distributions_to_fit=[\"dweibull\", \"dgamma\", \"logistic\", \"lognorm\"],\n )\n\n state = auto.analyze(\n train_data=df,\n return_state=True,\n anlz_facets=[a],\n )\n assert state.distributions_fit.train_data.e is None\n\n\ndef test_DistributionFit__constructor_defaults():\n a = DistributionFit(\"a\")\n assert a.distributions_to_fit == a.AVAILABLE_DISTRIBUTIONS\n assert a.keep_top_n == 3\n assert a.columns == [\"a\"]\n\n a = DistributionFit([\"a\"], distributions_to_fit=\"lognorm\")\n assert a.columns == [\"a\"]\n assert a.distributions_to_fit == [\"lognorm\"]\n\n a = DistributionFit([\"a\"], distributions_to_fit=[\"lognorm\", \"gamma\"])\n assert a.columns == [\"a\"]\n assert a.distributions_to_fit == [\"lognorm\", \"gamma\"]\n\n\ndef test_DistributionFit__constructor_unsupported_dist():\n with pytest.raises(ValueError) as exc_info:\n DistributionFit(\"a\", distributions_to_fit=\"unknown\")\n assert exc_info.value.args[0].startswith(\n \"The following distributions are not supported: ['unknown']. Supported distributions are\"\n )\n\n\ndef test_DistributionFit__non_numeric_col():\n df = __create_test_df()\n a = DistributionFit(\n columns=[\"a\", \"f\"],\n distributions_to_fit=[\"dweibull\", \"dgamma\", \"logistic\", \"lognorm\"],\n )\n a.logger.warning = MagicMock()\n state = auto.analyze(\n train_data=df,\n return_state=True,\n anlz_facets=[a],\n )\n assert len(state.distributions_fit.train_data.a) == 4\n assert state.distributions_fit.train_data.f is None\n a.logger.warning.assert_called_with(\"f: distribution cannot be fit; only numeric columns are supported\")\n\n\ndef test_FeatureDistanceAnalysis__happy_path():\n df, _ = load_adult_data()\n label = \"class\"\n\n state = auto.analyze(\n train_data=df,\n label=label,\n return_state=True,\n anlz_facets=[\n ApplyFeatureGenerator(\n category_to_numbers=True, children=[FeatureDistanceAnalysis(near_duplicates_threshold=0.85)]\n ),\n ],\n )\n\n assert np.allclose(\n state.feature_distance.linkage,\n np.array(\n [\n [9.0, 11.0, 0.6113, 2.0],\n [3.0, 10.0, 0.7652, 2.0],\n [7.0, 15.0, 0.7905, 3.0],\n [2.0, 6.0, 0.8097, 2.0],\n [0.0, 4.0, 0.8306, 2.0],\n [17.0, 18.0, 0.86785, 4.0],\n [12.0, 13.0, 0.8872, 2.0],\n [8.0, 20.0, 0.9333, 3.0],\n [1.0, 5.0, 0.946, 2.0],\n [16.0, 19.0, 0.94793333, 7.0],\n [21.0, 23.0, 0.9676619, 10.0],\n [22.0, 24.0, 0.981165, 12.0],\n [14.0, 25.0, 1.13729167, 14.0],\n ]\n ),\n )\n\n state.feature_distance.pop(\"linkage\")\n\n assert state == {\n \"feature_distance\": {\n \"columns\": [\n \"age\",\n \"fnlwgt\",\n \"education-num\",\n \"sex\",\n \"capital-gain\",\n \"capital-loss\",\n \"hours-per-week\",\n \"workclass\",\n \"education\",\n \"marital-status\",\n \"occupation\",\n \"relationship\",\n \"race\",\n \"native-country\",\n ],\n \"near_duplicates\": [\n {\"distance\": 0.6113, \"nodes\": [\"marital-status\", \"relationship\"]},\n {\"distance\": 0.7905, \"nodes\": [\"occupation\", \"sex\", \"workclass\"]},\n {\"distance\": 0.8097, \"nodes\": [\"education-num\", \"hours-per-week\"]},\n {\"distance\": 0.8306, \"nodes\": [\"age\", \"capital-gain\"]},\n ],\n \"near_duplicates_threshold\": 0.85,\n },\n }\n"
},
{
"path": "eda/tests/unittests/analysis/test_missing.py",
"content": "import numpy as np\nimport pandas as pd\n\nimport autogluon.eda.auto as auto\nfrom autogluon.eda.analysis import MissingValuesAnalysis\n\n\ndef test_MissingValuesAnalysis():\n cols = list(\"AB\")\n df_train = pd.DataFrame((np.arange(100))[:, None].repeat([len(cols)], axis=1), columns=cols)\n df_test = pd.DataFrame((np.arange(200))[:, None].repeat([len(cols)], axis=1), columns=cols)\n for df in [df_train, df_test]:\n df[\"A\"] = (df[\"A\"] % 4).replace(2, np.NaN)\n\n state = auto.analyze(\n train_data=df_train, test_data=df_test, return_state=True, anlz_facets=[MissingValuesAnalysis()]\n )\n\n assert state.missing_statistics.train_data.count == {\"A\": 25}\n assert state.missing_statistics.train_data.ratio == {\"A\": 0.25}\n assert state.missing_statistics.train_data.data is df_train\n\n assert state.missing_statistics.test_data.count == {\"A\": 50}\n assert state.missing_statistics.test_data.ratio == {\"A\": 0.25}\n assert state.missing_statistics.test_data.data is df_test\n"
},
{
"path": "eda/tests/unittests/analysis/test_model.py",
"content": "import os.path\nimport tempfile\n\nimport pandas as pd\nimport pytest\n\nimport autogluon.eda.analysis as eda\nimport autogluon.eda.auto as auto\nfrom autogluon.core.constants import REGRESSION\nfrom autogluon.tabular import TabularPredictor\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"..\", \"resources\"))\n\n\ndef fit_model(path, df_train, target, fit_args=None):\n if fit_args is None:\n fit_args = {\"hyperparameters\": {\"RF\": {}}}\n\n predictor = TabularPredictor(path=path, label=target, verbosity=0).fit(\n df_train,\n **fit_args,\n )\n return predictor\n\n\ndef test_AutoGluonModelEvaluator_regression():\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"houses\", \"train_data.csv\")).sample(100, random_state=0)\n df_val = pd.read_csv(os.path.join(RESOURCE_PATH, \"houses\", \"test_data.csv\")).sample(50, random_state=0)\n target_col = \"SalePrice\"\n\n with tempfile.TemporaryDirectory() as path:\n predictor = fit_model(path, df_train, target_col)\n\n state = auto.analyze(\n model=predictor,\n train_data=df_train,\n val_data=df_val,\n return_state=True,\n anlz_facets=[eda.model.AutoGluonModelEvaluator(normalize=\"true\")],\n )\n\n assert state.model_evaluation.problem_type == REGRESSION\n assert len(state.model_evaluation.y_true_val) == len(df_val)\n assert len(state.model_evaluation.y_pred_val) == len(df_val)\n expected = [c for c in df_train.columns if c not in [\"Street\", \"Utilities\", \"SalePrice\", \"PoolQC\"]]\n assert sorted(state.model_evaluation.importance.index.to_list()) == sorted(expected)\n _assert_importance_is_present(state)\n assert state.model_evaluation.confusion_matrix is None\n assert state.model_evaluation.confusion_matrix_normalized is None\n assert state.model_evaluation.y_true_test is None\n assert state.model_evaluation.y_pred_test is None\n assert state.model_evaluation.y_true_train is None\n assert state.model_evaluation.y_pred_train is None\n\n\ndef test_AutoGluonModelEvaluator_regression__with_test_data():\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"houses\", \"train_data.csv\")).sample(100, random_state=0)\n df_val = pd.read_csv(os.path.join(RESOURCE_PATH, \"houses\", \"test_data.csv\"))[:50]\n df_test = pd.read_csv(os.path.join(RESOURCE_PATH, \"houses\", \"test_data.csv\"))[50:101]\n target_col = \"SalePrice\"\n\n with tempfile.TemporaryDirectory() as path:\n predictor = fit_model(path, df_train, target_col)\n\n state = auto.analyze(\n model=predictor,\n train_data=df_train,\n val_data=df_val,\n test_data=df_test,\n return_state=True,\n anlz_facets=[eda.model.AutoGluonModelEvaluator(normalize=\"true\")],\n )\n\n assert state.model_evaluation.problem_type == REGRESSION\n assert len(state.model_evaluation.y_true_test) == len(df_test)\n assert len(state.model_evaluation.y_pred_test) == len(df_test)\n assert len(state.model_evaluation.y_true_val) == len(df_val)\n assert len(state.model_evaluation.y_pred_val) == len(df_val)\n expected = [c for c in df_train.columns if c not in [\"Street\", \"Utilities\", \"SalePrice\", \"PoolQC\"]]\n assert sorted(state.model_evaluation.importance.index.to_list()) == sorted(expected)\n _assert_importance_is_present(state)\n assert state.model_evaluation.confusion_matrix is None\n assert state.model_evaluation.confusion_matrix_normalized is None\n\n\ndef test_AutoGluonModelEvaluator_classification():\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n df_val = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"test_data.csv\")).sample(50, random_state=0)\n target_col = \"class\"\n\n with tempfile.TemporaryDirectory() as path:\n predictor = fit_model(path, df_train, target_col)\n\n state = auto.analyze(\n model=predictor,\n train_data=df_train,\n val_data=df_val,\n return_state=True,\n anlz_facets=[eda.model.AutoGluonModelEvaluator(normalize=\"true\")],\n )\n\n assert state.model_evaluation.problem_type == \"binary\"\n assert len(state.model_evaluation.y_true_val) == len(df_val)\n assert len(state.model_evaluation.y_pred_val) == len(df_val)\n expected = [c for c in df_train.columns if c not in [\"class\"]]\n assert sorted(state.model_evaluation.importance.index.to_list()) == sorted(expected)\n _assert_importance_is_present(state)\n\n\n@pytest.mark.parametrize(\"save_model_to_state\", [True, False])\ndef test_AutoGluonModelQuickFit(save_model_to_state):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n target_col = \"class\"\n\n with tempfile.TemporaryDirectory() as path:\n state = auto.analyze(\n train_data=df_train,\n label=target_col,\n return_state=True,\n anlz_facets=[\n eda.dataset.ProblemTypeControl(),\n eda.dataset.TrainValidationSplit(\n children=[\n eda.model.AutoGluonModelQuickFit(\n save_model_to_state=save_model_to_state,\n estimator_args=dict(path=path),\n verbosity=0,\n hyperparameters={\n \"RF\": {\n \"criterion\": \"entropy\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]},\n }\n },\n children=[eda.model.AutoGluonModelEvaluator()],\n )\n ]\n ),\n ],\n )\n\n assert state.model_evaluation.problem_type == \"binary\"\n assert len(state.model_evaluation.y_true_val) == int(len(df_train) * 0.3)\n assert len(state.model_evaluation.y_pred_val) == int(len(df_train) * 0.3)\n expected = [c for c in df_train.columns if c not in [\"class\"]]\n assert sorted(state.model_evaluation.importance.index.to_list()) == sorted(expected)\n _assert_importance_is_present(state)\n if save_model_to_state:\n assert str(state.model.__class__) == \"\"\n else:\n assert \"model\" not in state\n\n\ndef test_AutoGluonModelQuickFit__constructor_defaults():\n assert eda.model.AutoGluonModelQuickFit().estimator_args == {}\n\n\ndef _assert_importance_is_present(state):\n assert state.model_evaluation.importance.columns.to_list() == [\n \"importance\",\n \"stddev\",\n \"p_value\",\n \"n\",\n \"p99_high\",\n \"p99_low\",\n ]\n"
},
{
"path": "eda/tests/unittests/analysis/test_transform.py",
"content": "from typing import List\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import Namespace\nfrom autogluon.eda.analysis.base import BaseAnalysis\nfrom autogluon.eda.analysis.transform import ApplyFeatureGenerator\n\n\nclass SomeAnalysis(BaseAnalysis):\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.args = args.copy()\n\n\ndef __replace_ints(values: List[str]) -> List[str]:\n # Normalize ints between windows and linux environments\n # https://stackoverflow.com/questions/36278590/numpy-array-dtype-is-coming-as-int32-by-default-in-a-windows-10-64-bit-machine\n return [\"int\" if v in [\"int32\", \"int64\"] else v for v in values]\n\n\ndef test_ApplyFeatureGenerator():\n df_train = pd.DataFrame((np.arange(10))[:, None].repeat([4], axis=1), columns=list(\"ABCD\"))\n df_test = pd.DataFrame((np.arange(20))[:, None].repeat([4], axis=1), columns=list(\"ABCD\"))\n for df in [df_train, df_test]:\n df[\"A\"] = (df[\"A\"] % 4).map({0: \"a\", 1: \"b\", 2: \"c\", 3: \"d\"})\n df[\"B\"] = df[\"B\"] % 5\n df[\"C\"] = (df[\"C\"] % 3).map({0: \"a\", 1: \"b\", 2: \"c\"})\n df[\"D\"] = df[\"D\"] % 5\n assert df_train.shape == (10, 4)\n assert df_test.shape == (20, 4)\n\n analysis = BaseAnalysis(\n train_data=df_train,\n test_data=df_test,\n label=\"D\",\n children=[\n Namespace(\n namespace=\"feature_generator_numbers\",\n children=[ApplyFeatureGenerator(category_to_numbers=True, children=[SomeAnalysis()])],\n ),\n Namespace(\n namespace=\"feature_generator_default\", children=[ApplyFeatureGenerator(children=[SomeAnalysis()])]\n ),\n Namespace(namespace=\"no_wrapper\", children=[SomeAnalysis()]),\n ],\n )\n\n state = analysis.fit()\n assert __replace_ints(list(df_train.dtypes.apply(str).to_numpy())) == [\"object\", \"int\", \"object\", \"int\"]\n assert __replace_ints(list(df_test.dtypes.apply(str).to_numpy())) == [\"object\", \"int\", \"object\", \"int\"]\n\n assert __replace_ints(list(state.no_wrapper.args.train_data[df_train.columns].dtypes.apply(str).to_numpy())) == [\n \"object\",\n \"int\",\n \"object\",\n \"int\",\n ]\n assert __replace_ints(list(state.no_wrapper.args.test_data[df_test.columns].dtypes.apply(str).to_numpy())) == [\n \"object\",\n \"int\",\n \"object\",\n \"int\",\n ]\n\n assert __replace_ints(\n list(state.feature_generator_default.args.train_data[df_train.columns].dtypes.apply(str).to_numpy())\n ) == [\n \"category\",\n \"int\",\n \"category\",\n \"int\",\n ]\n assert __replace_ints(\n list(state.feature_generator_default.args.test_data[df_test.columns].dtypes.apply(str).to_numpy())\n ) == [\n \"category\",\n \"int\",\n \"category\",\n \"int\",\n ]\n\n assert __replace_ints(\n list(state.feature_generator_numbers.args.train_data[df_train.columns].dtypes.apply(str).to_numpy())\n ) == [\n \"int8\",\n \"int\",\n \"int8\",\n \"int\",\n ]\n assert __replace_ints(\n list(state.feature_generator_numbers.args.test_data[df_test.columns].dtypes.apply(str).to_numpy())\n ) == [\n \"int8\",\n \"int\",\n \"int8\",\n \"int\",\n ]\n"
},
{
"path": "eda/tests/unittests/auto/__init__.py",
"content": ""
},
{
"path": "eda/tests/unittests/auto/test_simple.py",
"content": "import os\nimport re\nimport tempfile\nfrom sys import platform\nfrom unittest.mock import ANY, MagicMock, call\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nimport autogluon.eda as eda\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import AnomalyDetectorAnalysis, Namespace, ShapAnalysis\nfrom autogluon.eda.analysis.base import BaseAnalysis\nfrom autogluon.eda.auto import (\n analyze,\n analyze_interaction,\n covariate_shift_detection,\n dataset_overview,\n detect_anomalies,\n explain_rows,\n partial_dependence_plots,\n quick_fit,\n target_analysis,\n)\nfrom autogluon.eda.auto.simple import (\n _is_single_numeric_variable,\n _prepare_pdp_data,\n _validate_and_normalize_pdp_args,\n get_default_estimator_if_not_specified,\n get_empty_dict_if_none,\n)\nfrom autogluon.eda.visualization import (\n AnomalyScoresVisualization,\n ConfusionMatrix,\n CorrelationVisualization,\n DatasetStatistics,\n DatasetTypeMismatch,\n ExplainForcePlot,\n ExplainWaterfallPlot,\n FeatureImportance,\n FeatureInteractionVisualization,\n LabelInsightsVisualization,\n MarkdownSectionComponent,\n ModelLeaderboard,\n PropertyRendererComponent,\n RegressionEvaluation,\n XShiftSummary,\n)\nfrom autogluon.eda.visualization.base import AbstractVisualization\nfrom autogluon.eda.visualization.interaction import FeatureDistanceAnalysisVisualization, PDPInteractions\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"..\", \"resources\"))\n\n\nclass SomeAnalysis(BaseAnalysis):\n def _fit(self, state: AnalysisState, args: AnalysisState, **fit_kwargs) -> None:\n state.args = args.copy()\n\n\nclass SomeVisualization(AbstractVisualization):\n def can_handle(self, state: AnalysisState) -> bool:\n return \"required_key\" in state\n\n def _render(self, state: AnalysisState) -> None:\n pass\n\n\ndef test_analyze():\n df_train = pd.DataFrame(np.random.randint(0, 100, size=(10, 2)), columns=list(\"AB\"))\n df_test = pd.DataFrame(np.random.randint(0, 100, size=(11, 2)), columns=list(\"CD\"))\n df_val = pd.DataFrame(np.random.randint(0, 100, size=(12, 2)), columns=list(\"EF\"))\n state = {\"some_previous_state\": {\"arg\": 1}}\n\n anlz = SomeAnalysis()\n anlz.can_handle = MagicMock(return_value=True)\n\n viz = SomeVisualization(namespace=\"ns1\")\n viz._render = MagicMock()\n viz.can_handle = MagicMock(wraps=viz.can_handle)\n\n state = analyze(\n train_data=df_train,\n test_data=df_test,\n val_data=df_val,\n model=\"model\",\n label=\"label\",\n state=state,\n sample=5,\n return_state=True,\n anlz_facets=[Namespace(namespace=\"ns1\", children=[anlz])],\n viz_facets=[viz],\n )\n assert state.sample_size == {\"test_data\": 5, \"train_data\": 5, \"val_data\": 5}\n assert state.some_previous_state == {\"arg\": 1}\n assert state.ns1.args.train_data.shape == (5, 2)\n assert state.ns1.args.test_data.shape == (5, 2)\n assert state.ns1.args.val_data.shape == (5, 2)\n assert state.ns1.args.model == \"model\"\n assert state.ns1.args.label == \"label\"\n\n\ndef test_analyze_return_state():\n state = {\"some_previous_state\": {\"arg\": 1}}\n assert analyze(state=state) is None\n assert analyze(state=state, return_state=True) == state\n\n\ndef test_analyze_None_state():\n state = None\n assert analyze(state=state, return_state=True) == {}\n\n\ndef test_analyze_state_dict_convert():\n state = {\"some_previous_state\": {\"arg\": 1}}\n assert not isinstance(state, AnalysisState)\n _state = analyze(state=state, return_state=True)\n assert _state == state\n assert _state is not state\n assert isinstance(_state, AnalysisState)\n\n\ndef test_analyze_state_no_AnalysisState_convert():\n state = AnalysisState({\"some_previous_state\": {\"arg\": 1}})\n assert isinstance(state, AnalysisState)\n _state = analyze(state=state, return_state=True)\n assert _state == state\n assert _state is state\n assert isinstance(_state, AnalysisState)\n\n\ndef test_quick_fit(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_md_render = MagicMock()\n call_cm_render = MagicMock()\n call_reg_render = MagicMock()\n call_ldr_render = MagicMock()\n call_fi_render = MagicMock()\n call_prc_render = MagicMock()\n\n with monkeypatch.context() as m:\n m.setattr(MarkdownSectionComponent, \"render\", call_md_render)\n m.setattr(ConfusionMatrix, \"render\", call_cm_render)\n m.setattr(RegressionEvaluation, \"render\", call_reg_render)\n m.setattr(ModelLeaderboard, \"render\", call_ldr_render)\n m.setattr(FeatureImportance, \"render\", call_fi_render)\n m.setattr(PropertyRendererComponent, \"render\", call_prc_render)\n _force_using_rf_if_on_mac(m)\n\n with tempfile.TemporaryDirectory() as path:\n quick_fit(path=path, train_data=df_train, label=\"class\")\n\n assert call_md_render.call_count == 9\n assert call_prc_render.call_count == 2\n call_cm_render.assert_called_once()\n call_reg_render.assert_called_once()\n call_ldr_render.assert_called_once()\n call_fi_render.assert_called_once()\n\n\ndef test_dataset_overview(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n df_train[\"education-num\"] = df_train[\"education-num\"] + np.random.rand(len(df_train)) / 100\n df_train[\"near_duplicate\"] = df_train[\"education-num\"] + 0.1\n\n call_ds_render = MagicMock()\n call_dtm_render = MagicMock()\n call_md_render = MagicMock()\n call_fdav_render = MagicMock()\n call_fiv_render = MagicMock()\n\n with monkeypatch.context() as m:\n m.setattr(DatasetStatistics, \"render\", call_ds_render)\n m.setattr(DatasetTypeMismatch, \"render\", call_dtm_render)\n m.setattr(MarkdownSectionComponent, \"render_markdown\", call_md_render)\n m.setattr(FeatureDistanceAnalysisVisualization, \"render\", call_fdav_render)\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n\n dataset_overview(train_data=df_train, label=\"class\")\n\n call_md_render.assert_has_calls(\n [\n call(\"### Feature Distance\"),\n call(\"Feature interaction between `education-num`/`near_duplicate`\"),\n ]\n )\n call_ds_render.assert_called_once()\n call_dtm_render.assert_called_once()\n call_fdav_render.assert_called_once()\n call_fiv_render.assert_called_once()\n\n\ndef test_covariate_shift_detection(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n df_train[\"shift_col\"] = np.random.rand(len(df_train))\n df_test = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"test_data.csv\")).sample(100, random_state=0)\n df_test[\"shift_col\"] = np.random.rand(len(df_test)) + 2\n\n call_xss_render = MagicMock()\n call_md_render = MagicMock()\n call_fiv_render = MagicMock()\n with monkeypatch.context() as m:\n _force_using_rf_if_on_mac(m)\n with tempfile.TemporaryDirectory() as path:\n m.setattr(XShiftSummary, \"render\", call_xss_render)\n m.setattr(MarkdownSectionComponent, \"render_markdown\", call_md_render)\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n state = covariate_shift_detection(\n path=path, train_data=df_train, test_data=df_test, label=\"class\", return_state=True, verbosity=2\n )\n\n call_xss_render.assert_called_once()\n assert state.xshift_results.detection_status is True\n assert state.xshift_results.test_statistic > 0.99\n assert state.xshift_results.pvalue < 0.01\n assert state.xshift_results.feature_importance.iloc[0].name == \"shift_col\"\n call_fiv_render.assert_called_once()\n call_md_render.assert_called_once_with(\"**`shift_col` values distribution between datasets; p-value: `0.0000`**\")\n\n\ndef _force_using_rf_if_on_mac(m):\n if platform == \"darwin\":\n # Stability - use RF instead of LightGBM can on mac for tests\n call_is_lightgbm_available = MagicMock(return_value=False)\n m.setattr(eda.auto.simple, \"_is_lightgbm_available\", call_is_lightgbm_available)\n\n\ndef test_get_empty_dict_if_none():\n assert get_empty_dict_if_none(None) == {}\n assert get_empty_dict_if_none({\"q\"}) == {\"q\"}\n\n\n@pytest.mark.parametrize(\n \"hyperparameters_present, lgbm_present, presets_present\",\n [\n (True, False, True),\n (True, False, False),\n (False, False, True),\n (False, False, False),\n (True, True, True),\n (True, True, False),\n (False, True, True),\n (False, True, False),\n ],\n)\ndef test_get_default_estimator_if_not_specified(monkeypatch, hyperparameters_present, lgbm_present, presets_present):\n fit_args = {}\n if hyperparameters_present:\n fit_args[\"hyperparameters\"] = \"some_params\"\n if presets_present:\n fit_args[\"presets\"] = \"some_presets\"\n\n with monkeypatch.context() as m:\n m.setattr(eda.auto.simple, \"_is_lightgbm_available\", lambda: lgbm_present)\n\n if (not hyperparameters_present) and (not presets_present):\n if lgbm_present:\n assert \"GBM\" in get_default_estimator_if_not_specified(fit_args)[\"hyperparameters\"]\n else:\n assert \"RF\" in get_default_estimator_if_not_specified(fit_args)[\"hyperparameters\"]\n else:\n assert get_default_estimator_if_not_specified(fit_args) == fit_args\n\n\n@pytest.mark.parametrize(\n \"fit_distributions, expected_dist\",\n [\n ([\"exponpow\", \"nakagami\", \"beta\", \"gamma\", \"lognorm\"], [\"exponpow\", \"nakagami\", \"beta\", \"lognorm\", \"gamma\"]),\n (\"lognorm\", [\"lognorm\"]),\n (True, [\"exponpow\", \"nakagami\", \"gompertz\", \"foldnorm\", \"genpareto\"]),\n ],\n)\ndef test_analyze_interaction__with_distribution(monkeypatch, fit_distributions, expected_dist):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_fiv_render = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n state = analyze_interaction(\n train_data=df_train,\n x=\"age\",\n fit_distributions=fit_distributions,\n return_state=True,\n )\n assert list(state.distributions_fit.train_data.age.keys()) == expected_dist\n call_fiv_render.assert_called_once()\n\n\ndef test_analyze_interaction__do_not_fit(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_fiv_render = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n state = analyze_interaction(\n train_data=df_train,\n x=\"age\",\n return_state=True,\n )\n assert state.distributions_fit is None\n call_fiv_render.assert_called_once()\n\n\n@pytest.mark.parametrize(\n \"x, y, hue, x_type, expected\",\n [\n (\"x\", \"y\", \"hue\", \"numeric\", False),\n (\"x\", \"y\", \"hue\", \"category\", False),\n (\"x\", \"y\", None, \"numeric\", False),\n (\"x\", \"y\", None, \"category\", False),\n (\"x\", None, \"hue\", \"numeric\", False),\n (\"x\", None, \"hue\", \"category\", False),\n (\"x\", None, None, \"numeric\", True),\n (\"x\", None, None, \"category\", False),\n ],\n)\ndef test_analyze_interaction__is_single_numeric_variable(x, y, hue, x_type, expected):\n assert _is_single_numeric_variable(x, y, hue, x_type) is expected\n\n\ndef test_target_analysis__classification(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_md_render = MagicMock()\n call_ds_render = MagicMock()\n call_cv_render = MagicMock()\n call_fiv_render = MagicMock()\n call_liv_render = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(MarkdownSectionComponent, \"render_markdown\", call_md_render)\n m.setattr(DatasetStatistics, \"render\", call_ds_render)\n m.setattr(CorrelationVisualization, \"render\", call_cv_render)\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n m.setattr(LabelInsightsVisualization, \"render\", call_liv_render)\n\n state = target_analysis(train_data=df_train, label=\"class\", return_state=True)\n\n call_md_render.assert_has_calls(\n [\n call(\"## Target variable analysis\"),\n call(\"### Label Insights\"),\n ]\n )\n call_ds_render.assert_called_once()\n call_cv_render.assert_called_once()\n call_liv_render.assert_called_once()\n assert call_fiv_render.call_count == 2\n assert sorted(set(state.keys())) == [\n \"correlations\",\n \"correlations_focus_field\",\n \"correlations_focus_field_threshold\",\n \"correlations_focus_high_corr\",\n \"correlations_method\",\n \"dataset_stats\",\n \"interactions\",\n \"label_insights\",\n \"missing_statistics\",\n \"problem_type\",\n \"raw_type\",\n \"special_types\",\n \"variable_type\",\n ]\n\n assert sorted(set(state.interactions.train_data.keys())) == [\"__analysis__\", \"relationship:class\"]\n assert sorted(state.correlations.train_data.columns.tolist()) == [\"class\", \"relationship\"]\n assert state.correlations_focus_high_corr.train_data.index.tolist() == [\"relationship\"]\n\n\ndef test_target_analysis__regression(monkeypatch):\n df_train = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_md_render = MagicMock()\n call_ds_render = MagicMock()\n call_cv_render = MagicMock()\n call_fiv_render = MagicMock()\n call_liv_render = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(MarkdownSectionComponent, \"render_markdown\", call_md_render)\n m.setattr(DatasetStatistics, \"render\", call_ds_render)\n m.setattr(CorrelationVisualization, \"render\", call_cv_render)\n m.setattr(FeatureInteractionVisualization, \"render\", call_fiv_render)\n m.setattr(LabelInsightsVisualization, \"render\", call_liv_render)\n\n state = target_analysis(train_data=df_train, label=\"fnlwgt\", return_state=True)\n\n assert call_md_render.call_count == 3\n calls = [re.sub(r\"[.][0-9]{4,}\", \".xx\", c[0][0]) for c in call_md_render.call_args_list]\n assert calls == [\n \"## Target variable analysis\",\n \"\\n\".join(\n [\n \"### Distribution fits for target variable\",\n \" - [kstwobign](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.kstwobign.html)\",\n \" - p-value: 0.976\",\n \" - Parameters: (loc: -134163.xx, scale: 377621.xx)\",\n \" - [gumbel_r](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.gumbel_r.html)\",\n \" - p-value: 0.966\",\n \" - Parameters: (loc: 149399.xx, scale: 79111.xx)\",\n \" - [nakagami](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.nakagami.html)\",\n \" - p-value: 0.965\",\n \" - Parameters: (nu: 0.xx, loc: 28236.xx, scale: 192280.xx)\",\n \" - [skewnorm](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.skewnorm.html)\",\n \" - p-value: 0.963\",\n \" - Parameters: (a: 3.xx, loc: 78497.xx, scale: 150470.xx)\",\n \" - [weibull_min](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.weibull_min.html)\",\n \" - p-value: 0.963\",\n \" - Parameters: (c: 1.xx, loc: 16324.xx, scale: 200669.xx)\",\n ]\n ),\n \"### Target variable correlations\\n - â ī¸ no fields with absolute correlation greater than `0.5` found for target variable `fnlwgt`.\",\n ]\n\n call_ds_render.assert_called_once()\n call_cv_render.assert_called_once()\n call_fiv_render.assert_called_once()\n call_liv_render.assert_called_once()\n assert sorted(set(state.keys())) == [\n \"correlations\",\n \"correlations_focus_field\",\n \"correlations_focus_field_threshold\",\n \"correlations_focus_high_corr\",\n \"correlations_method\",\n \"dataset_stats\",\n \"distributions_fit\",\n \"distributions_fit_pvalue_min\",\n \"interactions\",\n \"missing_statistics\",\n \"problem_type\",\n \"raw_type\",\n \"special_types\",\n \"variable_type\",\n ]\n\n assert sorted(set(state.interactions.train_data.keys())) == [\"__analysis__\"]\n assert sorted(state.correlations.train_data.columns.tolist()) == [\"fnlwgt\"]\n assert state.correlations_focus_high_corr.train_data.index.tolist() == []\n assert list(state.distributions_fit.train_data.fnlwgt.keys()) == [\n \"kstwobign\",\n \"gumbel_r\",\n \"nakagami\",\n \"skewnorm\",\n \"weibull_min\",\n ]\n\n\n@pytest.mark.parametrize(\n \"plot\",\n [\n (\"force\"),\n (\"waterfall\"),\n (None),\n ],\n)\ndef test_explain_rows(plot, monkeypatch):\n train_data = MagicMock()\n model = MagicMock()\n rows = MagicMock()\n with monkeypatch.context() as m:\n call_analyze = MagicMock(return_value=\"result\")\n m.setattr(eda.auto.simple, \"analyze\", call_analyze)\n\n result = explain_rows(\n train_data=train_data,\n model=model,\n rows=rows,\n plot=plot,\n return_state=True,\n baseline_sample=300,\n other_arg=\"other_arg\",\n )\n assert result == \"result\"\n\n call_analyze.assert_called_with(\n train_data=train_data[model.original_features],\n model=model,\n return_state=True,\n anlz_facets=ANY,\n viz_facets=ANY,\n )\n\n anlz_facet = call_analyze.call_args.kwargs[\"anlz_facets\"][0]\n assert type(anlz_facet) is ShapAnalysis\n assert anlz_facet.rows == rows\n assert anlz_facet.baseline_sample == 300\n\n expected_plot = {\n \"force\": ExplainForcePlot,\n \"waterfall\": ExplainWaterfallPlot,\n }.get(plot, None)\n viz_facet = call_analyze.call_args.kwargs[\"viz_facets\"]\n if expected_plot is None:\n assert viz_facet is None\n else:\n assert type(viz_facet[0]) is expected_plot\n\n\ndef test_partial_dependence_plots__prepare_pdp_data():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n train_data = train_data.drop(\n columns=[\"marital-status\", \"education\", \"occupation\", \"relationship\", \"workclass\", \"race\"]\n )\n assert sorted(train_data[\"native-country\"].unique().tolist()) == [\n \" ?\",\n \" Canada\",\n \" Cuba\",\n \" El-Salvador\",\n \" Haiti\",\n \" Mexico\",\n \" Philippines\",\n \" Puerto-Rico\",\n \" United-States\",\n ]\n pdp_data, state, id_to_category_mappings = _prepare_pdp_data(train_data, label=\"class\", sample=1000)\n assert id_to_category_mappings == {\n \"native-country\": {0: \" ?\", 1: \" Mexico\", 2: \" United-States\"},\n }\n assert state.pdp_train_data is not None\n assert sorted(pdp_data[\"native-country\"].unique().tolist()) == [-1, 0, 1, 2]\n\n\ndef test_partial_dependence_plots__prepare_pdp_data__features_specified():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n pdp_data, state, id_to_category_mappings = _prepare_pdp_data(\n train_data, label=\"class\", features=[\"native-country\"], sample=1000\n )\n assert id_to_category_mappings == {\n \"native-country\": {0: \" ?\", 1: \" Mexico\", 2: \" United-States\"},\n }\n\n\n@pytest.mark.parametrize(\"col_number_warning\", [5, 20])\ndef test_partial_dependence_plots__validate_and_normalize_pdp_args__none_args(col_number_warning):\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\"))\n chart_args, fig_args, features = _validate_and_normalize_pdp_args(\n train_data, features=None, fig_args=None, chart_args=None, col_number_warning=col_number_warning\n )\n assert chart_args == {}\n assert fig_args == {}\n assert features is None\n\n\ndef test_partial_dependence_plots__validate_and_normalize_pdp_args__single_feature():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\"))\n chart_args, fig_args, features = _validate_and_normalize_pdp_args(train_data, features=\"education\")\n assert chart_args == {}\n assert fig_args == {}\n assert features == [\"education\"]\n\n\ndef test_partial_dependence_plots__validate_and_normalize_pdp_args__wrong_features():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\"))\n with pytest.raises(AssertionError):\n _validate_and_normalize_pdp_args(train_data, features=[\"not_present_1\", \"not_present_1\"])\n\n\ndef test_partial_dependence_plots__validate_and_normalize_pdp_args():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\"))\n chart_args, fig_args, features = _validate_and_normalize_pdp_args(\n train_data,\n features=[\"education\", \"occupation\"],\n fig_args={\"fig_arg\": 1},\n chart_args={\"chart_arg\": 2},\n )\n assert chart_args == {\"chart_arg\": 2}\n assert fig_args == {\"fig_arg\": 1}\n assert features == [\"education\", \"occupation\"]\n\n\ndef test_partial_dependence_plots(monkeypatch):\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")).sample(100, random_state=0)\n\n call_md_render = MagicMock()\n call_pdp_interaction_render = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(MarkdownSectionComponent, \"render\", call_md_render)\n m.setattr(PDPInteractions, \"render\", call_pdp_interaction_render)\n\n with tempfile.TemporaryDirectory() as path:\n state = partial_dependence_plots(\n train_data,\n label=\"class\",\n features=[\"education\", \"native-country\"],\n return_state=True,\n path=path,\n hyperparameters={\"RF\": {}},\n )\n\n assert state == {\n \"pdp_id_to_category_mappings\": {\n \"education\": {\n 0: \" 10th\",\n 1: \" 11th\",\n 2: \" 12th\",\n 3: \" 7th-8th\",\n 4: \" Assoc-acdm\",\n 5: \" Assoc-voc\",\n 6: \" Bachelors\",\n 7: \" HS-grad\",\n 8: \" Masters\",\n 9: \" Prof-school\",\n 10: \" Some-college\",\n },\n \"native-country\": {0: \" ?\", 1: \" Mexico\", 2: \" United-States\"},\n }\n }\n\n call_pdp_interaction_render.assert_called_once()\n call_md_render.assert_called()\n\n\n@pytest.mark.parametrize(\n \"add_explainability\",\n [\n True,\n False,\n ],\n)\ndef test_detect_anomalies(monkeypatch, add_explainability):\n # np.int64 is required for running tests on Windows\n df_train = pd.DataFrame({\"A\": np.arange(4), \"B\": np.arange(4)}).astype(np.int64)\n df_test = pd.DataFrame({\"A\": np.arange(5), \"B\": np.arange(5)}).astype(np.int64)\n\n train_data_scores = pd.Series([0.13, 0.01, 0.08, 0.76], name=\"score\")\n test_data_scores = pd.Series([0.60, 0.20, 0.91, 0.60, 0.23], name=\"score\")\n assert len(df_train) == len(train_data_scores)\n assert len(df_test) == len(test_data_scores)\n\n call_md_render = MagicMock()\n call_df_render = MagicMock()\n call_anomaly_viz_render = MagicMock()\n\n def verify_explain_fn(anomaly_idx):\n return {\"called_vals\": list(anomaly_idx)}\n\n def call_anomaly_anlz_fit_mock(state: AnalysisState, args: AnalysisState, **fit_kwargs):\n assert state == {\"problem_type\": \"regression\"}\n assert args.train_data.equals(df_train)\n assert args.test_data.equals(df_test)\n assert args.label == \"B\"\n assert args.feature_generator is True\n\n state.anomaly_detection = {\n \"scores\": {\"train_data\": train_data_scores, \"test_data\": test_data_scores},\n \"explain_rows_fns\": {\n \"train_data\": verify_explain_fn,\n \"test_data\": verify_explain_fn,\n },\n }\n\n call_anomaly_anlz_fit = MagicMock(side_effect=call_anomaly_anlz_fit_mock)\n call_explain_rows = MagicMock()\n\n with monkeypatch.context() as m:\n m.setattr(AnomalyDetectorAnalysis, \"_fit\", call_anomaly_anlz_fit)\n m.setattr(MarkdownSectionComponent, \"render_markdown\", call_md_render)\n m.setattr(PropertyRendererComponent, \"display_obj\", call_df_render)\n m.setattr(AnomalyScoresVisualization, \"render\", call_anomaly_viz_render)\n m.setattr(\"autogluon.eda.auto.simple.explain_rows\", call_explain_rows)\n state = detect_anomalies(\n train_data=df_train,\n test_data=df_test,\n label=\"B\",\n threshold_stds=2,\n explain_top_n_anomalies=1 if add_explainability else None,\n show_help_text=False,\n return_state=True,\n )\n\n assert list(state.anomaly_detection.scores.keys()) == [\"train_data\", \"test_data\"]\n assert state.anomaly_detection.scores.train_data.equals(train_data_scores)\n assert state.anomaly_detection.scores.test_data.equals(test_data_scores)\n state.anomaly_detection.pop(\"scores\")\n\n assert list(state.anomaly_detection.anomalies.keys()) == [\"train_data\", \"test_data\"]\n # np.int64 is required for running tests on Windows\n assert state.anomaly_detection.anomalies.train_data.equals(\n pd.DataFrame({\"A\": [3], \"B\": [3], \"score\": 0.76}, index=[3]).astype({\"A\": np.int64, \"B\": np.int64})\n )\n assert state.anomaly_detection.anomalies.test_data.equals(\n pd.DataFrame({\"A\": [2], \"B\": [2], \"score\": 0.91}, index=[2]).astype({\"A\": np.int64, \"B\": np.int64})\n )\n state.anomaly_detection.pop(\"anomalies\")\n\n assert state == {\n \"problem_type\": \"regression\",\n \"anomaly_detection\": {\n \"anomaly_score_threshold\": 0.693685807840985,\n \"explain_rows_fns\": {\n \"train_data\": verify_explain_fn,\n \"test_data\": verify_explain_fn,\n },\n },\n }\n\n # Markdown rendering\n calls = [c[0][0] for c in call_md_render.call_args_list]\n expected_calls = [\n \"### Anomaly Detection Report\",\n # train_data\n \"**Top-10 `train_data` anomalies (total: 1)**\",\n # test_data\n \"**Top-10 `test_data` anomalies (total: 1)**\",\n ]\n if add_explainability:\n expected_calls.insert(\n 2,\n \"â ī¸ Please note that the feature values shown on the charts below are transformed \"\n \"into an internal representation; they may be encoded or modified based on internal preprocessing. \"\n \"Refer to the original datasets for the actual feature values.\",\n )\n expected_calls.insert(\n 3,\n \"â ī¸ The detector has seen this dataset; the may result in overly optimistic estimates. \"\n \"Although the anomaly score in the explanation might not match, the magnitude of the feature scores \"\n \"can still be utilized to evaluate the impact of the feature on the anomaly score.\",\n )\n expected_calls.append(\n \"â ī¸ Please note that the feature values shown on the charts below are transformed \"\n \"into an internal representation; they may be encoded or modified based on internal preprocessing. \"\n \"Refer to the original datasets for the actual feature values.\"\n )\n assert calls == expected_calls\n\n # Tables rendering\n calls = [c[0][0].to_dict() for c in call_df_render.call_args_list]\n assert calls == [\n {\"A\": {3: 3}, \"B\": {3: 3}, \"score\": {3: 0.76}},\n {\"A\": {2: 2}, \"B\": {2: 2}, \"score\": {2: 0.91}},\n ]\n\n # Explainability data\n if add_explainability:\n calls = [c.kwargs for c in call_explain_rows.call_args_list]\n assert calls == [\n {\"called_vals\": [3], \"plot\": \"waterfall\"},\n {\"called_vals\": [2], \"plot\": \"waterfall\"},\n ]\n else:\n call_explain_rows.assert_not_called()\n"
},
{
"path": "eda/tests/unittests/resources/adult/test_data.csv",
"content": "age,workclass,fnlwgt,education,education-num,marital-status,occupation,relationship,race,sex,capital-gain,capital-loss,hours-per-week,native-country,class\n31, Private,169085, 11th,7, Married-civ-spouse, Sales, Wife, White, Female,0,0,20, United-States, <=50K\n17, Self-emp-not-inc,226203, 12th,8, Never-married, Sales, Own-child, White, Male,0,0,45, United-States, <=50K\n47, Private,54260, Assoc-voc,11, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,1887,60, United-States, >50K\n21, Private,176262, Some-college,10, Never-married, Exec-managerial, Own-child, White, Female,0,0,30, United-States, <=50K\n17, Private,241185, 12th,8, Never-married, Prof-specialty, Own-child, White, Male,0,0,20, United-States, <=50K\n58, Private,170480, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,60, United-States, >50K\n42, Private,171069, Bachelors,13, Married-civ-spouse, Craft-repair, Husband, Black, Male,15024,0,40, United-States, >50K\n60, Self-emp-not-inc,78913, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, United-States, <=50K\n20, Private,66917, 11th,7, Married-civ-spouse, Farming-fishing, Own-child, White, Male,0,0,40, Mexico, <=50K\n29, Private,212091, Some-college,10, Divorced, Protective-serv, Not-in-family, White, Male,0,0,40, United-States, <=50K\n36, Self-emp-not-inc,269509, HS-grad,9, Divorced, Sales, Not-in-family, White, Male,0,0,35, United-States, <=50K\n72, Private,171181, HS-grad,9, Widowed, Adm-clerical, Unmarried, White, Female,2329,0,20, United-States, <=50K\n37, Private,302604, Some-college,10, Separated, Other-service, Other-relative, White, Female,0,0,40, United-States, <=50K\n43, Private,184321, 10th,6, Married-civ-spouse, Machine-op-inspct, Husband, Black, Male,0,1887,40, United-States, >50K\n67, Private,162009, 10th,6, Married-civ-spouse, Protective-serv, Husband, White, Male,0,0,16, United-States, <=50K\n40, Private,227466, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, Black, Male,0,0,50, United-States, <=50K\n49, ?,111282, 7th-8th,4, Married-civ-spouse, ?, Husband, White, Male,4386,0,99, United-States, >50K\n52, Federal-gov,221532, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,45, United-States, >50K\n45, Self-emp-inc,40666, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,55, United-States, >50K\n19, ?,41609, Some-college,10, Never-married, ?, Own-child, White, Male,0,0,10, United-States, <=50K\n60, Self-emp-not-inc,36568, 9th,5, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,70, United-States, <=50K\n19, Local-gov,176831, Some-college,10, Never-married, Other-service, Own-child, Black, Female,0,0,35, United-States, <=50K\n60, Self-emp-not-inc,220342, 11th,7, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,35, United-States, <=50K\n69, Private,541737, Some-college,10, Widowed, Adm-clerical, Not-in-family, White, Female,2050,0,24, United-States, <=50K\n54, Private,378747, 10th,6, Separated, Transport-moving, Unmarried, Black, Male,0,0,45, United-States, >50K\n23, Private,205939, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,38, United-States, <=50K\n41, Private,282964, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n59, Private,426001, HS-grad,9, Married-spouse-absent, Adm-clerical, Unmarried, White, Female,0,0,20, Puerto-Rico, <=50K\n39, Private,433592, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, Black, Male,0,0,45, United-States, >50K\n26, Local-gov,197897, HS-grad,9, Never-married, Adm-clerical, Own-child, Black, Female,0,0,20, England, <=50K\n23, Private,325921, Assoc-voc,11, Never-married, Prof-specialty, Not-in-family, White, Female,0,0,36, United-States, <=50K\n21, ?,199177, HS-grad,9, Never-married, ?, Not-in-family, White, Female,0,0,40, United-States, <=50K\n30, Private,221043, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,35, United-States, <=50K\n33, Private,221762, Some-college,10, Never-married, Prof-specialty, Not-in-family, Black, Male,0,0,40, United-States, <=50K\n36, Private,122493, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, <=50K\n48, Private,94342, Bachelors,13, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,55, United-States, <=50K\n25, Private,149875, Bachelors,13, Widowed, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n20, Private,190916, HS-grad,9, Never-married, Other-service, Own-child, White, Female,0,1721,20, United-States, <=50K\n39, Self-emp-not-inc,193689, HS-grad,9, Never-married, Exec-managerial, Not-in-family, White, Male,0,0,65, United-States, <=50K\n35, Private,186815, HS-grad,9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, United-States, <=50K\n22, Private,216134, Some-college,10, Never-married, Sales, Own-child, Black, Female,0,0,40, United-States, <=50K\n35, Private,174503, HS-grad,9, Divorced, Craft-repair, Not-in-family, White, Female,0,0,40, United-States, <=50K\n65, Local-gov,240166, 11th,7, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,35, United-States, <=50K\n59, Private,113959, HS-grad,9, Married-civ-spouse, Adm-clerical, Husband, White, Male,0,0,45, United-States, >50K\n30, Private,111567, HS-grad,9, Never-married, Craft-repair, Own-child, White, Male,0,0,48, United-States, <=50K\n20, ?,41356, Assoc-voc,11, Never-married, ?, Not-in-family, White, Female,0,0,40, United-States, <=50K\n57, Private,32365, Some-college,10, Never-married, Protective-serv, Not-in-family, White, Male,0,0,40, United-States, <=50K\n25, ?,218948, 7th-8th,4, Never-married, ?, Not-in-family, White, Female,0,0,32, Mexico, <=50K\n34, Private,191291, Some-college,10, Never-married, Craft-repair, Not-in-family, White, Male,0,0,40, United-States, <=50K\n24, Self-emp-not-inc,172047, Assoc-acdm,12, Never-married, Craft-repair, Own-child, White, Male,0,0,40, United-States, <=50K\n30, Private,312667, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,40, United-States, <=50K\n19, Private,187570, HS-grad,9, Never-married, Craft-repair, Own-child, White, Male,0,0,20, United-States, <=50K\n36, Self-emp-inc,200220, HS-grad,9, Never-married, Exec-managerial, Not-in-family, White, Male,0,0,55, United-States, <=50K\n35, Private,54363, Assoc-acdm,12, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, United-States, >50K\n21, Private,213041, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Female,0,0,40, Cuba, <=50K\n42, Private,121352, Bachelors,13, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,80, ?, >50K\n41, Private,190786, Assoc-voc,11, Married-civ-spouse, Adm-clerical, Husband, White, Male,7298,0,40, United-States, >50K\n34, Private,340458, 12th,8, Never-married, Adm-clerical, Unmarried, White, Female,0,0,40, United-States, <=50K\n22, Private,200318, Assoc-acdm,12, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,15, United-States, <=50K\n19, Private,277695, 9th,5, Never-married, Farming-fishing, Other-relative, White, Male,0,0,16, Mexico, <=50K\n32, Private,312403, Bachelors,13, Never-married, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n59, Self-emp-not-inc,178353, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, United-States, >50K\n37, Local-gov,89491, Masters,14, Divorced, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n39, Private,98975, Bachelors,13, Never-married, Sales, Not-in-family, White, Female,8614,0,50, United-States, >50K\n34, Self-emp-not-inc,198068, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, <=50K\n45, Local-gov,144940, Masters,14, Divorced, Prof-specialty, Unmarried, Black, Female,0,0,40, United-States, <=50K\n52, Private,153155, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,15024,0,40, United-States, >50K\n30, Private,115963, 7th-8th,4, Never-married, Machine-op-inspct, Unmarried, White, Male,0,0,42, United-States, <=50K\n26, Self-emp-not-inc,221626, HS-grad,9, Married-civ-spouse, Other-service, Wife, White, Female,0,1579,20, United-States, <=50K\n41, Private,276289, Bachelors,13, Never-married, Prof-specialty, Not-in-family, Black, Male,0,0,60, ?, <=50K\n36, Private,127865, Bachelors,13, Never-married, Prof-specialty, Not-in-family, White, Female,4650,0,25, United-States, <=50K\n27, Local-gov,106179, Bachelors,13, Married-spouse-absent, Prof-specialty, Not-in-family, White, Female,0,0,50, United-States, <=50K\n61, Private,313170, HS-grad,9, Widowed, Craft-repair, Not-in-family, White, Male,0,0,40, United-States, <=50K\n53, Federal-gov,227836, Some-college,10, Divorced, Farming-fishing, Not-in-family, White, Male,0,0,40, United-States, <=50K\n21, Private,249727, HS-grad,9, Never-married, Other-service, Own-child, White, Male,0,0,22, United-States, <=50K\n72, ?,114761, 7th-8th,4, Widowed, ?, Unmarried, White, Female,0,0,20, United-States, <=50K\n23, Private,202989, HS-grad,9, Never-married, Prof-specialty, Not-in-family, White, Male,0,0,40, Canada, <=50K\n53, Private,151159, HS-grad,9, Divorced, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K\n69, Self-emp-not-inc,204645, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,9386,0,72, United-States, >50K\n18, Private,97963, HS-grad,9, Never-married, Other-service, Own-child, White, Female,0,0,30, United-States, <=50K\n43, Federal-gov,117022, Assoc-voc,11, Divorced, Handlers-cleaners, Unmarried, Black, Male,0,1726,40, United-States, <=50K\n45, Local-gov,159816, Masters,14, Married-civ-spouse, Prof-specialty, Wife, White, Female,0,1977,35, United-States, >50K\n38, Federal-gov,104236, Assoc-acdm,12, Divorced, Adm-clerical, Unmarried, White, Female,1471,0,40, United-States, <=50K\n27, Private,203776, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,7688,0,45, United-States, >50K\n29, Private,283760, Bachelors,13, Never-married, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n60, State-gov,165827, Doctorate,16, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,55, United-States, <=50K\n40, Private,170230, Bachelors,13, Married-spouse-absent, Other-service, Not-in-family, White, Female,0,0,40, ?, <=50K\n50, Private,104501, Masters,14, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,40, United-States, >50K\n61, Private,227332, Bachelors,13, Never-married, Prof-specialty, Own-child, White, Male,0,0,40, United-States, <=50K\n32, Private,86808, Some-college,10, Never-married, Other-service, Own-child, White, Female,0,0,38, United-States, <=50K\n21, Private,119039, Some-college,10, Never-married, Adm-clerical, Own-child, White, Female,0,0,18, United-States, <=50K\n37, Private,139180, 11th,7, Never-married, Other-service, Unmarried, Black, Female,0,0,40, United-States, <=50K\n59, Private,32552, Bachelors,13, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,4, United-States, <=50K\n26, Private,104045, Bachelors,13, Never-married, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n41, ?,77937, 12th,8, Divorced, ?, Not-in-family, White, Female,0,0,40, Canada, <=50K\n27, Self-emp-not-inc,208577, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,45, United-States, >50K\n80, Private,227210, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,9386,0,40, United-States, >50K\n25, Private,178421, Bachelors,13, Never-married, Other-service, Not-in-family, White, Female,0,0,20, United-States, <=50K\n49, Private,23074, Some-college,10, Widowed, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, >50K\n47, Private,481987, HS-grad,9, Married-civ-spouse, Other-service, Husband, White, Male,0,0,40, United-States, >50K\n17, Private,198830, 11th,7, Never-married, Adm-clerical, Other-relative, White, Female,0,0,10, United-States, <=50K\n21, Private,169699, HS-grad,9, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n46, Private,185847, HS-grad,9, Divorced, Transport-moving, Not-in-family, White, Male,0,0,54, United-States, <=50K\n44, Private,172032, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,40, United-States, <=50K\n55, Self-emp-not-inc,111625, HS-grad,9, Married-civ-spouse, Sales, Husband, White, Male,0,0,40, United-States, <=50K\n21, Private,136610, 12th,8, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,32, United-States, <=50K\n47, Private,162034, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,0,0,60, United-States, >50K\n32, Private,125279, HS-grad,9, Divorced, Craft-repair, Unmarried, White, Male,0,0,99, United-States, <=50K\n33, ?,139051, 11th,7, Separated, ?, Unmarried, Black, Female,0,0,53, United-States, <=50K\n23, Private,215115, Bachelors,13, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n47, Private,197836, HS-grad,9, Married-civ-spouse, Sales, Husband, White, Male,0,0,45, United-States, <=50K\n43, Local-gov,101563, Bachelors,13, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,40, United-States, <=50K\n43, State-gov,98989, HS-grad,9, Married-civ-spouse, Other-service, Husband, Amer-Indian-Eskimo, Male,0,0,40, United-States, <=50K\n17, Private,102726, 12th,8, Never-married, Other-service, Own-child, White, Male,0,0,16, United-States, <=50K\n19, ?,255117, Some-college,10, Never-married, ?, Not-in-family, White, Female,0,0,30, United-States, <=50K\n47, Private,213668, Some-college,10, Separated, Machine-op-inspct, Not-in-family, White, Male,8614,0,65, United-States, >50K\n33, ?,393376, 11th,7, Never-married, ?, Not-in-family, White, Female,0,0,40, United-States, <=50K\n39, Private,224531, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,7298,0,40, United-States, >50K\n26, Private,463194, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, <=50K\n34, Private,120461, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,50, United-States, <=50K\n21, Private,57711, HS-grad,9, Never-married, Priv-house-serv, Not-in-family, White, Female,0,0,40, United-States, <=50K\n33, Private,159187, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,54, United-States, >50K\n17, Private,113301, 11th,7, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,12, ?, <=50K\n76, Private,201240, HS-grad,9, Never-married, Adm-clerical, Other-relative, White, Female,0,0,40, United-States, <=50K\n53, Private,121441, Some-college,10, Married-civ-spouse, Sales, Husband, White, Male,0,0,55, United-States, <=50K\n35, Private,119272, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,55, United-States, >50K\n52, Private,137815, 9th,5, Divorced, Transport-moving, Not-in-family, White, Male,0,0,40, United-States, <=50K\n40, Private,86143, Assoc-voc,11, Married-civ-spouse, Handlers-cleaners, Own-child, Asian-Pac-Islander, Male,0,0,40, Philippines, <=50K\n22, Private,117363, Some-college,10, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,35, United-States, <=50K\n45, Private,182703, Masters,14, Divorced, Adm-clerical, Not-in-family, Amer-Indian-Eskimo, Female,0,0,36, United-States, <=50K\n32, Private,267458, Assoc-acdm,12, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,40, United-States, <=50K\n20, Private,279538, 11th,7, Married-civ-spouse, Handlers-cleaners, Other-relative, White, Male,2961,0,35, United-States, <=50K\n33, Private,214129, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n21, Private,164991, HS-grad,9, Divorced, Sales, Unmarried, Amer-Indian-Eskimo, Female,0,0,38, United-States, <=50K\n69, Self-emp-not-inc,505365, Some-college,10, Married-civ-spouse, Sales, Husband, White, Male,6514,0,45, United-States, >50K\n59, Federal-gov,43280, Some-college,10, Never-married, Exec-managerial, Own-child, Black, Female,0,0,40, United-States, <=50K\n27, Private,95465, Some-college,10, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,55, United-States, <=50K\n39, Private,230467, Bachelors,13, Never-married, Sales, Own-child, White, Male,0,1092,40, Germany, <=50K\n34, Private,107914, Bachelors,13, Married-civ-spouse, Tech-support, Husband, White, Male,0,0,47, United-States, >50K\n24, Private,182117, Bachelors,13, Never-married, Adm-clerical, Own-child, White, Male,0,0,28, United-States, <=50K\n47, Self-emp-not-inc,107231, Doctorate,16, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,35, United-States, >50K\n22, Private,116800, Assoc-acdm,12, Never-married, Protective-serv, Own-child, White, Male,0,0,60, United-States, <=50K\n82, ?,403910, HS-grad,9, Never-married, ?, Not-in-family, White, Male,0,0,3, United-States, <=50K\n39, Private,202683, Bachelors,13, Married-civ-spouse, Adm-clerical, Husband, White, Male,0,0,40, United-States, <=50K\n48, Self-emp-inc,250674, Assoc-voc,11, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,60, United-States, <=50K\n62, Self-emp-not-inc,265007, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,60, Ecuador, <=50K\n49, Private,93639, Bachelors,13, Married-civ-spouse, Exec-managerial, Wife, White, Female,0,0,43, United-States, <=50K\n39, Without-pay,334291, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K\n22, Private,275385, Some-college,10, Never-married, Other-service, Other-relative, White, Male,0,0,25, United-States, <=50K\n62, Self-emp-not-inc,158712, HS-grad,9, Never-married, Other-service, Unmarried, White, Female,0,0,6, United-States, <=50K\n42, Federal-gov,70240, Some-college,10, Divorced, Exec-managerial, Unmarried, Asian-Pac-Islander, Female,0,0,40, United-States, <=50K\n49, Federal-gov,175428, Some-college,10, Married-civ-spouse, Adm-clerical, Husband, White, Male,0,0,40, United-States, >50K\n39, Local-gov,344855, Masters,14, Married-civ-spouse, Prof-specialty, Wife, White, Female,0,1977,20, United-States, >50K\n33, Private,318982, Bachelors,13, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,38, United-States, <=50K\n25, Private,311124, Prof-school,15, Never-married, Prof-specialty, Not-in-family, White, Female,0,0,20, United-States, <=50K\n42, Private,171351, HS-grad,9, Married-civ-spouse, Handlers-cleaners, Husband, White, Male,0,0,40, United-States, <=50K\n28, Self-emp-not-inc,146735, Doctorate,16, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,50, United-States, >50K\n30, Private,337494, Assoc-acdm,12, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,48, United-States, <=50K\n46, Private,102569, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,15024,0,65, United-States, >50K\n17, Private,148345, 11th,7, Never-married, Protective-serv, Own-child, White, Female,0,0,40, United-States, <=50K\n34, Private,178841, Bachelors,13, Married-civ-spouse, Exec-managerial, Wife, White, Female,0,0,40, United-States, >50K\n32, Private,391874, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, <=50K\n36, Private,192704, 12th,8, Never-married, Exec-managerial, Not-in-family, White, Male,4650,0,50, United-States, <=50K\n64, Local-gov,202984, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,3137,0,40, United-States, <=50K\n33, Self-emp-inc,184245, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, Mexico, >50K\n40, Private,242619, Bachelors,13, Never-married, Prof-specialty, Not-in-family, White, Male,4650,0,40, United-States, <=50K\n61, Private,128230, HS-grad,9, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,30, United-States, <=50K\n45, Private,293628, Some-college,10, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,50, Philippines, >50K\n42, Local-gov,198028, Some-college,10, Married-civ-spouse, Adm-clerical, Wife, White, Female,0,0,40, United-States, >50K\n23, Private,106615, Assoc-acdm,12, Never-married, Prof-specialty, Own-child, White, Female,2176,0,25, United-States, <=50K\n21, Private,311570, Assoc-acdm,12, Never-married, Tech-support, Own-child, White, Female,0,0,35, United-States, <=50K\n20, ?,175431, Some-college,10, Never-married, ?, Own-child, White, Male,0,0,40, United-States, <=50K\n20, ?,189203, Assoc-acdm,12, Never-married, ?, Not-in-family, White, Male,0,0,40, United-States, <=50K\n22, Private,94662, Assoc-voc,11, Never-married, Craft-repair, Not-in-family, White, Male,0,0,44, United-States, <=50K\n43, Local-gov,161240, HS-grad,9, Married-civ-spouse, Protective-serv, Husband, White, Male,0,0,45, United-States, >50K\n19, Private,230238, HS-grad,9, Never-married, Transport-moving, Own-child, White, Male,0,0,46, United-States, <=50K\n20, Private,89991, Some-college,10, Never-married, Sales, Other-relative, White, Female,0,0,35, United-States, <=50K\n29, Private,112403, Some-college,10, Never-married, Handlers-cleaners, Not-in-family, White, Male,0,0,35, United-States, <=50K\n24, Self-emp-inc,242138, Bachelors,13, Never-married, Sales, Own-child, White, Female,0,0,20, United-States, <=50K\n26, Private,377754, HS-grad,9, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K\n54, Self-emp-not-inc,136224, Masters,14, Divorced, Prof-specialty, Not-in-family, White, Female,0,0,30, United-States, <=50K\n19, ?,168471, Some-college,10, Never-married, ?, Own-child, White, Male,0,0,15, United-States, <=50K\n43, Self-emp-not-inc,260696, Masters,14, Divorced, Prof-specialty, Not-in-family, White, Female,0,0,20, United-States, <=50K\n62, Local-gov,115763, Masters,14, Married-civ-spouse, Prof-specialty, Wife, White, Female,0,0,40, United-States, >50K\n40, Self-emp-not-inc,175943, Assoc-voc,11, Married-civ-spouse, Prof-specialty, Wife, White, Female,0,1977,15, United-States, >50K\n34, Private,105141, Some-college,10, Divorced, Adm-clerical, Own-child, White, Male,0,0,40, United-States, <=50K\n19, Private,211355, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Male,0,0,12, United-States, <=50K\n46, Self-emp-not-inc,148599, Masters,14, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,50, United-States, >50K\n22, Private,338162, HS-grad,9, Married-civ-spouse, Machine-op-inspct, Wife, Black, Female,0,0,40, United-States, <=50K\n19, Private,199480, 10th,6, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,25, United-States, <=50K\n22, Private,146540, 11th,7, Never-married, Craft-repair, Not-in-family, White, Male,0,0,50, United-States, <=50K\n51, Self-emp-not-inc,321865, Some-college,10, Married-civ-spouse, Other-service, Husband, White, Male,0,0,99, United-States, <=50K\n34, ?,330301, 7th-8th,4, Separated, ?, Unmarried, Black, Female,0,0,40, United-States, <=50K\n37, Private,255454, 7th-8th,4, Married-civ-spouse, Machine-op-inspct, Husband, Black, Male,7298,0,35, Haiti, >50K\n47, Private,161558, 10th,6, Married-spouse-absent, Transport-moving, Not-in-family, Black, Male,0,0,45, United-States, <=50K\n60, Self-emp-not-inc,52900, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,42, United-States, >50K\n46, Local-gov,375675, 12th,8, Married-civ-spouse, Other-service, Husband, White, Male,0,0,35, United-States, >50K\n42, Self-emp-not-inc,212847, HS-grad,9, Never-married, Farming-fishing, Own-child, White, Male,0,0,85, United-States, <=50K\n19, Private,286469, Some-college,10, Never-married, Sales, Own-child, White, Female,0,0,30, United-States, <=50K\n51, Private,110747, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,1887,40, United-States, >50K\n"
},
{
"path": "eda/tests/unittests/resources/adult/train_data.csv",
"content": "age,workclass,fnlwgt,education,education-num,marital-status,occupation,relationship,race,sex,capital-gain,capital-loss,hours-per-week,native-country,class\n25, Private,178478, Bachelors,13, Never-married, Tech-support, Own-child, White, Female,0,0,40, United-States, <=50K\n23, State-gov,61743, 5th-6th,3, Never-married, Transport-moving, Not-in-family, White, Male,0,0,35, United-States, <=50K\n46, Private,376789, HS-grad,9, Never-married, Other-service, Not-in-family, White, Male,0,0,15, United-States, <=50K\n55, ?,200235, HS-grad,9, Married-civ-spouse, ?, Husband, White, Male,0,0,50, United-States, >50K\n36, Private,224541, 7th-8th,4, Married-civ-spouse, Handlers-cleaners, Husband, White, Male,0,0,40, El-Salvador, <=50K\n51, Private,178054, Some-college,10, Married-civ-spouse, Sales, Husband, White, Male,0,0,40, ?, >50K\n33, Private,263561, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,60, United-States, >50K\n46, Private,173613, HS-grad,9, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n18, Private,214617, Some-college,10, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,30, United-States, <=50K\n43, Private,84661, Assoc-voc,11, Married-civ-spouse, Sales, Husband, White, Male,0,0,45, United-States, <=50K\n41, Private,225892, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,48, United-States, >50K\n41, Private,191547, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,99999,0,55, United-States, >50K\n22, Private,150175, Some-college,10, Never-married, Adm-clerical, Own-child, White, Female,0,0,20, United-States, <=50K\n41, Private,227890, Some-college,10, Divorced, Craft-repair, Not-in-family, White, Male,0,0,46, United-States, >50K\n39, Private,80479, Bachelors,13, Divorced, Transport-moving, Not-in-family, White, Male,0,0,55, United-States, <=50K\n44, Local-gov,100479, Assoc-acdm,12, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,48, United-States, <=50K\n37, Private,191524, Assoc-voc,11, Separated, Prof-specialty, Own-child, White, Female,0,0,38, United-States, <=50K\n39, Private,173175, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,15024,0,50, United-States, >50K\n40, Private,72887, Assoc-voc,11, Married-civ-spouse, Machine-op-inspct, Husband, Asian-Pac-Islander, Male,0,0,40, United-States, >50K\n24, Private,295763, Bachelors,13, Never-married, Exec-managerial, Own-child, White, Female,0,0,50, United-States, <=50K\n61, ?,451327, Bachelors,13, Married-civ-spouse, ?, Husband, Other, Male,0,0,24, United-States, >50K\n39, Private,188540, HS-grad,9, Divorced, Exec-managerial, Not-in-family, White, Male,0,0,45, United-States, <=50K\n19, Private,318061, Some-college,10, Never-married, Machine-op-inspct, Not-in-family, White, Female,0,0,80, United-States, <=50K\n27, Private,70034, Some-college,10, Never-married, Other-service, Not-in-family, White, Male,0,0,40, United-States, <=50K\n47, Private,269620, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, ?, <=50K\n47, Private,181758, HS-grad,9, Never-married, Transport-moving, Not-in-family, Black, Male,0,0,40, United-States, <=50K\n51, State-gov,103063, Bachelors,13, Married-civ-spouse, Protective-serv, Husband, White, Male,7298,0,40, United-States, >50K\n34, Private,154667, HS-grad,9, Married-civ-spouse, Machine-op-inspct, Wife, White, Female,0,0,40, United-States, <=50K\n29, Private,198704, Assoc-voc,11, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K\n23, Private,134045, Assoc-voc,11, Never-married, Adm-clerical, Unmarried, White, Female,0,0,40, United-States, <=50K\n45, Private,353083, Some-college,10, Separated, Other-service, Unmarried, White, Female,0,0,40, United-States, <=50K\n55, Private,160631, HS-grad,9, Married-civ-spouse, Sales, Husband, White, Male,0,0,56, United-States, >50K\n31, Private,80511, Assoc-acdm,12, Divorced, Tech-support, Not-in-family, White, Female,0,0,44, United-States, <=50K\n59, Private,96459, 11th,7, Never-married, Machine-op-inspct, Not-in-family, White, Male,0,0,40, United-States, <=50K\n24, Private,233280, Assoc-acdm,12, Never-married, Sales, Own-child, White, Female,0,0,37, United-States, <=50K\n35, Self-emp-not-inc,278557, HS-grad,9, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,30, United-States, <=50K\n63, Local-gov,80655, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n43, Private,334991, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,0,0,50, United-States, >50K\n49, Private,122206, Assoc-voc,11, Married-civ-spouse, Sales, Husband, White, Male,0,0,25, United-States, <=50K\n23, Private,436798, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, United-States, <=50K\n44, Private,374423, Masters,14, Married-civ-spouse, Exec-managerial, Husband, Black, Male,0,1902,40, United-States, >50K\n49, Private,50567, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,32, United-States, <=50K\n18, Private,90934, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,28, United-States, <=50K\n21, Private,117210, HS-grad,9, Never-married, Machine-op-inspct, Own-child, White, Male,0,0,40, United-States, <=50K\n34, Private,179877, Some-college,10, Married-civ-spouse, Machine-op-inspct, Husband, White, Male,0,0,40, United-States, >50K\n69, Private,361561, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,3, United-States, <=50K\n38, Private,247111, HS-grad,9, Married-civ-spouse, Tech-support, Husband, White, Male,0,0,40, United-States, >50K\n37, Private,156266, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, Amer-Indian-Eskimo, Male,0,0,40, United-States, <=50K\n42, Private,173938, HS-grad,9, Separated, Sales, Own-child, White, Female,0,0,40, United-States, <=50K\n43, Self-emp-inc,260960, Bachelors,13, Divorced, Farming-fishing, Not-in-family, White, Male,0,0,35, United-States, <=50K\n58, Private,49893, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,50, United-States, >50K\n29, Self-emp-not-inc,58744, Assoc-acdm,12, Never-married, Other-service, Own-child, White, Male,0,0,60, United-States, <=50K\n51, Local-gov,74784, Some-college,10, Widowed, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n51, Private,54465, Assoc-voc,11, Divorced, Prof-specialty, Not-in-family, White, Female,0,0,48, United-States, <=50K\n31, Private,169122, HS-grad,9, Divorced, Other-service, Not-in-family, White, Male,0,0,28, United-States, <=50K\n40, Private,131899, Some-college,10, Divorced, Exec-managerial, Not-in-family, White, Male,0,0,40, United-States, <=50K\n46, Private,423222, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,3103,0,60, United-States, >50K\n48, State-gov,327886, Doctorate,16, Divorced, Prof-specialty, Own-child, White, Male,0,0,50, United-States, >50K\n29, Private,342989, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Female,0,0,40, United-States, <=50K\n69, State-gov,170458, HS-grad,9, Married-civ-spouse, Other-service, Husband, White, Male,0,0,20, United-States, <=50K\n19, Private,239057, HS-grad,9, Never-married, Craft-repair, Own-child, White, Male,0,0,40, United-States, <=50K\n20, Private,221661, HS-grad,9, Married-civ-spouse, Adm-clerical, Wife, White, Female,0,0,40, Mexico, <=50K\n26, Private,292692, 12th,8, Never-married, Craft-repair, Not-in-family, White, Male,0,0,40, Mexico, <=50K\n45, Private,461725, HS-grad,9, Married-civ-spouse, Handlers-cleaners, Husband, White, Male,0,0,40, United-States, <=50K\n48, Local-gov,148121, Bachelors,13, Married-spouse-absent, Adm-clerical, Unmarried, Asian-Pac-Islander, Female,0,0,40, Philippines, <=50K\n56, Private,174209, 5th-6th,3, Married-civ-spouse, Machine-op-inspct, Husband, Black, Male,0,0,40, United-States, <=50K\n72, Self-emp-inc,149689, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,20051,0,48, United-States, >50K\n53, Private,164198, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, Black, Male,0,0,40, United-States, >50K\n50, Private,33931, HS-grad,9, Divorced, Craft-repair, Not-in-family, White, Male,0,0,40, United-States, <=50K\n42, Self-emp-not-inc,27821, Assoc-voc,11, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,60, United-States, <=50K\n19, Private,369463, HS-grad,9, Never-married, Adm-clerical, Own-child, White, Female,0,0,40, United-States, <=50K\n43, Private,152958, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,7298,0,40, United-States, >50K\n19, Self-emp-not-inc,194205, Some-college,10, Never-married, Other-service, Own-child, White, Female,0,0,40, Mexico, <=50K\n18, Private,201871, 12th,8, Never-married, Other-service, Own-child, White, Male,0,0,20, United-States, <=50K\n49, Self-emp-inc,34998, Some-college,10, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,60, United-States, <=50K\n41, Private,110562, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K\n49, Local-gov,159641, Bachelors,13, Divorced, Exec-managerial, Unmarried, White, Female,0,625,40, United-States, <=50K\n23, Private,278391, HS-grad,9, Never-married, Transport-moving, Own-child, White, Male,0,0,40, United-States, <=50K\n41, Private,175642, Some-college,10, Never-married, Craft-repair, Not-in-family, White, Female,0,0,40, United-States, <=50K\n35, Private,317153, HS-grad,9, Never-married, Sales, Not-in-family, White, Female,0,0,40, United-States, <=50K\n65, Private,90907, 5th-6th,3, Married-civ-spouse, Other-service, Husband, Asian-Pac-Islander, Male,0,0,40, United-States, <=50K\n61, Private,159822, 7th-8th,4, Married-civ-spouse, Craft-repair, Husband, White, Male,7688,0,40, Poland, >50K\n51, Private,205100, Some-college,10, Divorced, Sales, Not-in-family, White, Female,0,0,45, United-States, >50K\n64, Private,207188, Bachelors,13, Divorced, Exec-managerial, Not-in-family, White, Female,0,0,40, United-States, <=50K\n26, ?,102541, 10th,6, Never-married, ?, Unmarried, Amer-Indian-Eskimo, Female,0,0,40, United-States, <=50K\n33, Private,182401, 10th,6, Never-married, Adm-clerical, Not-in-family, Black, Male,0,0,40, United-States, <=50K\n23, Private,348092, HS-grad,9, Never-married, Transport-moving, Own-child, Black, Male,0,0,40, Haiti, <=50K\n39, Private,144169, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K\n27, Private,167031, Bachelors,13, Never-married, Prof-specialty, Unmarried, Other, Female,0,0,33, United-States, <=50K\n66, Private,216856, Masters,14, Married-civ-spouse, Sales, Husband, White, Male,0,0,50, United-States, <=50K\n23, Private,214236, HS-grad,9, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n22, Private,52596, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,8, United-States, <=50K\n22, Private,176131, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, Black, Male,0,0,40, United-States, <=50K\n61, Private,137733, Some-college,10, Divorced, Other-service, Not-in-family, White, Male,0,0,25, United-States, <=50K\n18, Private,400616, Some-college,10, Never-married, Adm-clerical, Own-child, White, Male,0,0,40, United-States, <=50K\n43, Private,26252, Some-college,10, Separated, Other-service, Unmarried, Amer-Indian-Eskimo, Female,0,0,36, United-States, <=50K\n29, Federal-gov,182344, HS-grad,9, Married-spouse-absent, Other-service, Unmarried, Black, Male,0,0,40, United-States, <=50K\n31, Private,91964, Some-college,10, Never-married, Adm-clerical, Other-relative, White, Male,0,0,40, United-States, <=50K\n31, Private,359249, Assoc-voc,11, Never-married, Protective-serv, Own-child, Black, Male,0,0,40, United-States, <=50K\n34, Private,24529, HS-grad,9, Widowed, Adm-clerical, Unmarried, White, Male,0,0,15, United-States, <=50K\n27, Self-emp-not-inc,65308, HS-grad,9, Married-civ-spouse, Farming-fishing, Husband, White, Male,0,0,50, United-States, <=50K\n64, Private,166843, HS-grad,9, Widowed, Other-service, Other-relative, White, Male,0,0,28, United-States, <=50K\n56, Private,81220, Some-college,10, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, Canada, <=50K\n35, Private,85799, HS-grad,9, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n56, Self-emp-inc,124137, Prof-school,15, Married-civ-spouse, Prof-specialty, Husband, White, Male,99999,0,45, United-States, >50K\n53, Local-gov,231166, HS-grad,9, Separated, Transport-moving, Not-in-family, White, Male,0,0,40, United-States, <=50K\n18, Private,186909, HS-grad,9, Never-married, Sales, Other-relative, White, Female,1055,0,30, United-States, <=50K\n35, Private,246829, HS-grad,9, Divorced, Exec-managerial, Unmarried, White, Female,0,0,40, United-States, <=50K\n21, Private,99970, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,15, United-States, <=50K\n40, Private,143582, Masters,14, Widowed, Sales, Own-child, Asian-Pac-Islander, Female,0,0,50, United-States, <=50K\n47, Local-gov,287320, Masters,14, Never-married, Prof-specialty, Not-in-family, White, Male,0,0,40, United-States, <=50K\n53, Private,194501, 11th,7, Widowed, Other-service, Own-child, White, Female,0,0,47, United-States, <=50K\n21, Private,91189, Some-college,10, Never-married, Sales, Unmarried, White, Male,0,0,60, United-States, <=50K\n30, Private,243165, HS-grad,9, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K\n39, Private,172186, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, >50K\n39, Private,230356, Bachelors,13, Never-married, Adm-clerical, Not-in-family, White, Female,0,0,40, United-States, <=50K\n27, Private,189775, HS-grad,9, Never-married, Adm-clerical, Not-in-family, Black, Female,0,0,40, United-States, <=50K\n25, Private,108683, Some-college,10, Never-married, Handlers-cleaners, Unmarried, White, Male,0,0,50, United-States, <=50K\n36, Private,138441, Assoc-acdm,12, Married-civ-spouse, Tech-support, Husband, White, Male,0,0,55, United-States, <=50K\n27, Local-gov,123773, Assoc-acdm,12, Never-married, Adm-clerical, Unmarried, Black, Female,0,0,40, United-States, <=50K\n27, Private,136448, Bachelors,13, Never-married, Adm-clerical, Not-in-family, Black, Female,0,0,40, United-States, <=50K\n36, Private,65624, Bachelors,13, Never-married, Prof-specialty, Not-in-family, White, Male,0,0,40, United-States, <=50K\n19, Private,71691, 11th,7, Never-married, Other-service, Own-child, White, Male,0,0,18, United-States, <=50K\n24, Private,326334, Bachelors,13, Never-married, Prof-specialty, Own-child, White, Male,0,0,20, United-States, <=50K\n22, Private,102684, Some-college,10, Never-married, Transport-moving, Not-in-family, White, Male,0,0,32, United-States, <=50K\n90, Local-gov,214594, 7th-8th,4, Married-civ-spouse, Protective-serv, Husband, White, Male,2653,0,40, United-States, <=50K\n23, Private,67311, Bachelors,13, Never-married, Exec-managerial, Own-child, White, Female,0,0,40, Canada, <=50K\n38, Private,455379, Masters,14, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,60, United-States, >50K\n61, Self-emp-not-inc,44983, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,20, United-States, <=50K\n24, ?,154373, HS-grad,9, Married-civ-spouse, ?, Wife, White, Female,0,0,25, United-States, <=50K\n46, Private,160647, Bachelors,13, Widowed, Tech-support, Unmarried, White, Female,0,0,38, United-States, <=50K\n22, Private,273675, HS-grad,9, Married-spouse-absent, Other-service, Other-relative, Black, Female,0,0,35, Puerto-Rico, <=50K\n33, Private,92462, Assoc-acdm,12, Never-married, Sales, Unmarried, Black, Male,0,0,32, United-States, <=50K\n18, Private,43272, Some-college,10, Never-married, Other-service, Own-child, White, Male,0,0,20, United-States, <=50K\n27, Private,420351, HS-grad,9, Never-married, Adm-clerical, Not-in-family, White, Male,0,0,45, United-States, <=50K\n44, Self-emp-not-inc,98806, 7th-8th,4, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,38, United-States, <=50K\n17, Private,129396, 11th,7, Never-married, Sales, Other-relative, White, Female,0,0,26, United-States, <=50K\n58, Private,306233, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,15024,0,40, United-States, >50K\n30, Private,110643, Some-college,10, Never-married, Exec-managerial, Not-in-family, White, Male,0,0,52, United-States, <=50K\n30, Private,327112, Bachelors,13, Divorced, Sales, Not-in-family, White, Male,0,1564,40, United-States, >50K\n50, Private,133963, HS-grad,9, Divorced, Adm-clerical, Unmarried, White, Female,0,0,35, United-States, <=50K\n46, Private,123598, Bachelors,13, Married-civ-spouse, Sales, Husband, White, Male,0,0,40, United-States, <=50K\n51, State-gov,243631, 10th,6, Married-civ-spouse, Craft-repair, Husband, Amer-Indian-Eskimo, Male,0,0,40, United-States, <=50K\n49, State-gov,185800, Masters,14, Divorced, Prof-specialty, Unmarried, Black, Female,7430,0,40, United-States, >50K\n34, Private,318641, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, Black, Male,0,0,45, United-States, >50K\n47, Private,235986, HS-grad,9, Never-married, Exec-managerial, Own-child, White, Female,0,0,50, Cuba, <=50K\n33, Private,188467, HS-grad,9, Never-married, Farming-fishing, Not-in-family, White, Male,0,0,40, United-States, <=50K\n27, Private,187981, HS-grad,9, Never-married, Handlers-cleaners, Own-child, White, Male,0,0,40, United-States, <=50K\n24, Private,437666, HS-grad,9, Married-civ-spouse, Craft-repair, Husband, White, Male,2885,0,50, United-States, <=50K\n61, Private,149648, 11th,7, Widowed, Machine-op-inspct, Not-in-family, White, Female,0,0,40, United-States, <=50K\n42, ?,51795, HS-grad,9, Divorced, ?, Unmarried, Black, Female,0,0,32, United-States, <=50K\n64, ?,208862, HS-grad,9, Married-civ-spouse, ?, Husband, White, Male,0,0,50, United-States, >50K\n79, ?,27457, Masters,14, Never-married, ?, Not-in-family, White, Female,0,0,23, United-States, <=50K\n34, State-gov,334422, Some-college,10, Divorced, Protective-serv, Unmarried, Black, Male,0,0,47, United-States, <=50K\n27, Private,113866, HS-grad,9, Never-married, Other-service, Not-in-family, White, Female,0,0,30, United-States, <=50K\n38, Federal-gov,115433, Assoc-acdm,12, Married-civ-spouse, Adm-clerical, Wife, White, Female,7688,0,33, United-States, >50K\n17, Private,193748, 11th,7, Never-married, Sales, Own-child, White, Male,0,0,15, United-States, <=50K\n54, Private,38795, 9th,5, Separated, Craft-repair, Unmarried, White, Male,0,0,40, United-States, <=50K\n59, Self-emp-inc,122390, Some-college,10, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,1977,48, United-States, >50K\n34, Private,143083, Assoc-voc,11, Married-civ-spouse, Adm-clerical, Wife, White, Female,0,0,18, United-States, <=50K\n24, Private,165107, Bachelors,13, Never-married, Exec-managerial, Own-child, White, Female,0,0,40, United-States, <=50K\n28, Private,252424, Assoc-voc,11, Never-married, Transport-moving, Own-child, Black, Male,0,0,40, Cambodia, <=50K\n53, State-gov,246820, Doctorate,16, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,50, United-States, >50K\n26, Private,222539, Some-college,10, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, >50K\n38, Private,225399, HS-grad,9, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,40, United-States, <=50K\n35, Private,37655, HS-grad,9, Divorced, Machine-op-inspct, Unmarried, White, Female,0,0,60, United-States, <=50K\n25, Private,113654, HS-grad,9, Separated, Exec-managerial, Unmarried, White, Female,0,0,37, United-States, <=50K\n35, Private,44780, Some-college,10, Married-civ-spouse, Adm-clerical, Wife, White, Female,7688,0,20, United-States, >50K\n41, Self-emp-not-inc,44006, Assoc-voc,11, Divorced, Farming-fishing, Not-in-family, White, Male,0,0,40, United-States, <=50K\n22, Private,199698, Some-college,10, Never-married, Sales, Own-child, White, Male,0,0,15, United-States, <=50K\n63, Private,172740, Some-college,10, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,40, United-States, <=50K\n50, Self-emp-not-inc,183915, Bachelors,13, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,45, United-States, >50K\n19, Private,198700, Some-college,10, Never-married, Machine-op-inspct, Own-child, Black, Male,0,0,20, United-States, <=50K\n44, Private,187629, Assoc-acdm,12, Never-married, Craft-repair, Not-in-family, White, Male,0,0,25, United-States, <=50K\n57, Private,191983, Some-college,10, Married-civ-spouse, Protective-serv, Husband, White, Male,0,0,50, United-States, <=50K\n50, Federal-gov,159670, HS-grad,9, Married-civ-spouse, Adm-clerical, Husband, White, Male,0,0,40, United-States, >50K\n37, Local-gov,347136, Some-college,10, Divorced, Adm-clerical, Not-in-family, White, Female,0,0,44, United-States, <=50K\n37, Self-emp-not-inc,348960, Assoc-acdm,12, Married-civ-spouse, Transport-moving, Husband, White, Male,0,0,50, United-States, >50K\n49, Private,121602, Bachelors,13, Married-civ-spouse, Exec-managerial, Husband, White, Male,0,0,40, United-States, <=50K\n44, Local-gov,145522, Masters,14, Never-married, Prof-specialty, Not-in-family, White, Female,0,0,40, United-States, <=50K\n28, Private,204516, 10th,6, Never-married, Transport-moving, Not-in-family, White, Male,0,0,45, United-States, <=50K\n54, State-gov,137815, 12th,8, Never-married, Other-service, Own-child, White, Male,4101,0,40, United-States, <=50K\n58, Private,160662, 10th,6, Married-civ-spouse, Craft-repair, Husband, White, Male,0,0,40, United-States, >50K\n22, Private,263398, Some-college,10, Never-married, Tech-support, Not-in-family, White, Male,0,0,50, United-States, <=50K\n25, Private,116358, HS-grad,9, Never-married, Adm-clerical, Own-child, Asian-Pac-Islander, Male,0,0,40, Philippines, <=50K\n34, Private,290763, HS-grad,9, Divorced, Handlers-cleaners, Own-child, White, Female,0,0,40, United-States, <=50K\n68, Private,104438, Bachelors,13, Married-civ-spouse, Prof-specialty, Husband, White, Male,0,0,40, Ireland, >50K\n29, Private,202878, HS-grad,9, Never-married, Craft-repair, Not-in-family, White, Male,0,0,50, United-States, <=50K\n39, Private,156897, HS-grad,9, Never-married, Craft-repair, Own-child, White, Male,0,2258,42, United-States, >50K\n62, Self-emp-not-inc,236247, HS-grad,9, Married-civ-spouse, Other-service, Husband, White, Male,0,0,20, United-States, <=50K\n23, Private,203203, Bachelors,13, Never-married, Prof-specialty, Not-in-family, White, Female,0,0,40, United-States, <=50K\n56, Self-emp-not-inc,19896, Some-college,10, Married-civ-spouse, Sales, Wife, White, Female,0,0,60, United-States, >50K\n36, Private,272950, Some-college,10, Never-married, Sales, Not-in-family, White, Male,0,0,50, United-States, <=50K\n33, Private,199248, Some-college,10, Married-civ-spouse, Prof-specialty, Wife, White, Female,0,0,40, United-States, <=50K\n56, Local-gov,38573, Assoc-acdm,12, Married-civ-spouse, Tech-support, Husband, White, Male,0,0,45, United-States, >50K\n49, Private,81973, Some-college,10, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male,0,0,40, United-States, >50K\n18, Private,211344, HS-grad,9, Never-married, Machine-op-inspct, Own-child, White, Male,0,0,40, United-States, <=50K\n18, Private,131033, 11th,7, Never-married, Other-service, Other-relative, Black, Male,0,0,15, United-States, <=50K\n24, Private,300008, HS-grad,9, Married-civ-spouse, Adm-clerical, Wife, White, Female,0,0,40, United-States, <=50K\n45, State-gov,81853, Some-college,10, Married-civ-spouse, Adm-clerical, Wife, Asian-Pac-Islander, Female,0,0,40, United-States, >50K\n"
},
{
"path": "eda/tests/unittests/resources/houses/test_data.csv",
"content": "Id,MSSubClass,MSZoning,LotFrontage,LotArea,Street,Alley,LotShape,LandContour,Utilities,LotConfig,LandSlope,Neighborhood,Condition1,Condition2,BldgType,HouseStyle,OverallQual,OverallCond,YearBuilt,YearRemodAdd,RoofStyle,RoofMatl,Exterior1st,Exterior2nd,MasVnrType,MasVnrArea,ExterQual,ExterCond,Foundation,BsmtQual,BsmtCond,BsmtExposure,BsmtFinType1,BsmtFinSF1,BsmtFinType2,BsmtFinSF2,BsmtUnfSF,TotalBsmtSF,Heating,HeatingQC,CentralAir,Electrical,1stFlrSF,2ndFlrSF,LowQualFinSF,GrLivArea,BsmtFullBath,BsmtHalfBath,FullBath,HalfBath,BedroomAbvGr,KitchenAbvGr,KitchenQual,TotRmsAbvGrd,Functional,Fireplaces,FireplaceQu,GarageType,GarageYrBlt,GarageFinish,GarageCars,GarageArea,GarageQual,GarageCond,PavedDrive,WoodDeckSF,OpenPorchSF,EnclosedPorch,3SsnPorch,ScreenPorch,PoolArea,PoolQC,Fence,MiscFeature,MiscVal,MoSold,YrSold,SaleType,SaleCondition,SalePrice\n201,20,RM,80,8546,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,4,5,2003,2004,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1121,1121,GasA,Ex,Y,SBrkr,1121,0,0,1121,0,0,2,0,2,1,TA,5,Typ,0,NA,Attchd,2003,RFn,2,440,TA,TA,Y,132,64,0,0,0,0,NA,NA,NA,0,3,2010,WD,Normal,140000\n202,20,RL,75,10125,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Mitchel,Norm,Norm,1Fam,1Story,6,6,1977,1977,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,TA,TA,No,ALQ,641,LwQ,279,276,1196,GasA,TA,Y,SBrkr,1279,0,0,1279,0,1,2,0,3,1,TA,6,Typ,2,Fa,Detchd,1980,Unf,2,473,TA,TA,Y,238,83,0,0,0,0,NA,MnPrv,NA,0,2,2008,WD,Normal,171500\n203,50,RL,50,7000,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Artery,Norm,1Fam,1.5Fin,6,6,1924,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Gd,BrkTil,Fa,TA,No,LwQ,617,Unf,0,0,617,GasA,Gd,Y,SBrkr,865,445,0,1310,0,0,2,0,2,1,TA,6,Min1,0,NA,Attchd,1924,Unf,1,398,TA,TA,Y,0,0,126,0,0,0,NA,NA,NA,0,5,2006,COD,Normal,112000\n204,120,RM,NA,4438,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,TwnhsE,1Story,6,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,205,Gd,TA,PConc,Gd,TA,Av,GLQ,662,Unf,0,186,848,GasA,Ex,Y,SBrkr,848,0,0,848,1,0,1,0,1,1,Gd,3,Typ,1,Gd,Attchd,2004,RFn,2,420,TA,TA,Y,149,0,0,0,0,0,NA,NA,NA,0,1,2008,WD,Normal,149000\n205,50,RM,50,3500,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,5,7,1947,1950,Gable,CompShg,AsbShng,AsbShng,None,0,TA,TA,CBlock,TA,TA,No,LwQ,312,Unf,0,408,720,GasA,TA,Y,SBrkr,720,564,0,1284,0,0,1,1,2,1,TA,5,Typ,0,NA,Detchd,1948,Unf,1,240,TA,TA,Y,0,35,0,0,0,0,NA,MnWw,NA,0,4,2009,WD,Normal,110000\n206,20,RL,99,11851,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,1Story,7,5,1990,1990,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1424,1424,GasA,Ex,Y,SBrkr,1442,0,0,1442,0,0,2,0,3,1,TA,5,Typ,0,NA,Attchd,1990,RFn,2,500,TA,TA,Y,0,34,0,508,0,0,NA,NA,NA,0,5,2009,WD,Normal,180500\n207,20,RL,40,13673,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Sawyer,RRAe,Norm,1Fam,1Story,5,5,1962,1962,Gable,CompShg,HdBoard,HdBoard,None,0,TA,Gd,CBlock,TA,TA,No,Unf,0,Unf,0,1140,1140,GasA,TA,Y,SBrkr,1696,0,0,1696,0,0,1,1,3,1,TA,8,Min2,1,TA,Attchd,1962,RFn,1,349,TA,TA,Y,0,30,0,0,0,0,NA,NA,NA,0,3,2007,WD,Normal,143900\n208,20,RL,NA,12493,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,5,1960,1960,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,TA,TA,No,ALQ,419,Rec,306,375,1100,GasA,TA,Y,SBrkr,1100,0,0,1100,1,0,1,0,3,1,TA,6,Typ,1,Po,Attchd,1960,RFn,1,312,TA,TA,Y,355,0,0,0,0,0,NA,GdWo,NA,0,4,2008,WD,Normal,141000\n209,60,RL,NA,14364,Pave,NA,IR1,Low,AllPub,Inside,Mod,SawyerW,Norm,Norm,1Fam,2Story,7,5,1988,1989,Gable,CompShg,Plywood,Plywood,BrkFace,128,Gd,TA,CBlock,Gd,TA,Gd,GLQ,1065,Unf,0,92,1157,GasA,Ex,Y,SBrkr,1180,882,0,2062,1,0,2,1,3,1,TA,7,Typ,1,Gd,Attchd,1988,Fin,2,454,TA,TA,Y,60,55,0,0,154,0,NA,NA,NA,0,4,2007,WD,Normal,277000\n210,20,RL,75,8250,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,7,1964,1964,Hip,CompShg,HdBoard,HdBoard,Stone,260,TA,TA,CBlock,Gd,TA,No,Rec,787,Unf,0,305,1092,GasA,Ex,Y,SBrkr,1092,0,0,1092,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1964,RFn,2,504,TA,Gd,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,7,2008,WD,Normal,145000\n211,30,RL,67,5604,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1Story,5,6,1925,1950,Gable,CompShg,Stucco,Stucco,None,0,TA,TA,CBlock,TA,TA,No,Rec,468,Unf,0,396,864,GasA,TA,N,FuseA,864,0,0,864,1,0,1,0,2,1,TA,5,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,96,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,98000\n212,20,RL,83,10420,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,6,5,2009,2009,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,Mn,GLQ,36,Unf,0,1176,1212,GasA,Ex,Y,SBrkr,1212,0,0,1212,0,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,2009,RFn,2,460,TA,TA,Y,100,22,0,0,0,0,NA,NA,NA,0,3,2010,WD,Normal,186000\n213,60,FV,72,8640,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,7,5,2009,2009,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,GLQ,822,Unf,0,78,900,GasA,Ex,Y,SBrkr,932,920,0,1852,1,0,2,1,3,1,Gd,7,Typ,1,TA,Attchd,2009,RFn,2,644,TA,TA,Y,168,108,0,0,0,0,NA,NA,NA,0,7,2009,New,Partial,252678\n214,20,RL,43,13568,Pave,NA,IR2,Lvl,AllPub,CulDSac,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,5,1995,1995,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,ALQ,716,Unf,0,274,990,GasA,Ex,Y,SBrkr,990,0,0,990,0,1,1,0,3,1,TA,5,Typ,0,NA,Attchd,1996,Unf,2,576,TA,TA,Y,224,0,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,156000\n215,60,RL,NA,10900,Pave,NA,IR1,Lvl,AllPub,FR2,Gtl,CollgCr,Norm,Norm,1Fam,2Story,6,7,1977,1977,Gable,CompShg,HdBoard,HdBoard,BrkFace,153,TA,TA,CBlock,Gd,TA,No,GLQ,378,Unf,0,311,689,GasA,Ex,Y,SBrkr,689,703,0,1392,0,0,1,1,3,1,TA,6,Typ,0,NA,Attchd,1977,Fin,1,299,TA,TA,Y,0,36,0,0,0,0,NA,MnPrv,Shed,450,3,2010,WD,Normal,161750\n216,20,RL,72,10011,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1957,1996,Gable,CompShg,HdBoard,HdBoard,BrkFace,64,TA,TA,CBlock,TA,TA,No,BLQ,360,Unf,0,710,1070,GasA,TA,Y,SBrkr,1236,0,0,1236,0,1,1,0,2,1,Gd,6,Min1,1,Fa,Attchd,1957,Unf,1,447,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,5,2006,WD,Normal,134450\n217,20,RL,65,8450,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,266,Gd,TA,PConc,Gd,TA,Mn,GLQ,946,Unf,0,490,1436,GasA,Ex,Y,SBrkr,1436,0,0,1436,1,0,2,0,3,1,Gd,8,Typ,0,NA,Attchd,2004,Unf,2,484,TA,TA,Y,139,98,0,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,210000\n218,70,RM,57,9906,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,2Story,4,4,1925,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,686,686,GasA,Fa,N,SBrkr,810,518,0,1328,0,0,1,0,3,1,TA,8,Typ,0,NA,Detchd,1940,Unf,1,210,TA,TA,Y,0,172,60,0,0,0,NA,NA,NA,0,9,2006,WD,Family,107000\n219,50,RL,NA,15660,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Crawfor,Norm,Norm,1Fam,1.5Fin,7,9,1939,2006,Gable,CompShg,VinylSd,VinylSd,BrkFace,312,Gd,Gd,CBlock,TA,TA,No,BLQ,341,Unf,0,457,798,GasA,Ex,Y,SBrkr,1137,817,0,1954,0,1,1,1,3,1,Gd,8,Typ,2,TA,Attchd,1939,Unf,2,431,TA,TA,Y,0,119,150,0,0,0,NA,NA,NA,0,5,2008,WD,Normal,311500\n220,120,RL,43,3010,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Blmngtn,Norm,Norm,TwnhsE,1Story,7,5,2005,2006,Gable,CompShg,VinylSd,VinylSd,BrkFace,16,Gd,TA,PConc,Gd,TA,Av,GLQ,16,Unf,0,1232,1248,GasA,Ex,Y,SBrkr,1248,0,0,1248,0,0,2,0,2,1,Gd,5,Typ,0,NA,Attchd,2005,Fin,2,438,TA,TA,Y,108,0,0,0,0,0,NA,NA,NA,0,3,2006,New,Partial,167240\n221,20,RL,73,8990,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Mn,Unf,0,Unf,0,1498,1498,GasA,Ex,Y,SBrkr,1498,0,0,1498,0,0,2,0,2,1,Gd,5,Typ,0,NA,Attchd,2006,RFn,2,675,TA,TA,Y,351,33,0,0,0,0,NA,NA,NA,0,4,2006,New,Partial,204900\n222,60,RL,NA,8068,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,2002,2002,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1010,1010,GasA,Ex,Y,SBrkr,1010,1257,0,2267,0,0,2,1,4,1,Gd,8,Typ,1,TA,BuiltIn,2002,RFn,2,390,TA,TA,Y,120,46,0,0,0,0,NA,NA,NA,0,12,2009,ConLI,Normal,200000\n223,60,RL,85,11475,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NWAmes,RRAn,Norm,1Fam,2Story,6,6,1975,1975,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,Gd,TA,No,ALQ,550,Unf,0,163,713,GasA,TA,Y,SBrkr,811,741,0,1552,1,0,2,1,3,1,TA,6,Typ,1,TA,Attchd,1975,RFn,2,434,TA,TA,Y,209,208,0,0,0,0,NA,MnPrv,NA,0,2,2006,WD,Normal,179900\n224,20,RL,70,10500,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,6,1971,1971,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,ALQ,524,LwQ,180,160,864,GasA,Gd,Y,SBrkr,864,0,0,864,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1989,Unf,2,576,TA,TA,Y,216,0,0,0,0,0,NA,NA,NA,0,3,2009,WD,Abnorml,97000\n225,20,RL,103,13472,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,10,5,2003,2003,Hip,CompShg,VinylSd,VinylSd,BrkFace,922,Ex,TA,PConc,Ex,TA,Gd,GLQ,56,Unf,0,2336,2392,GasA,Ex,Y,SBrkr,2392,0,0,2392,0,0,2,0,3,1,Ex,8,Typ,1,Ex,Attchd,2003,Fin,3,968,TA,TA,Y,248,105,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,386250\n226,160,RM,21,1680,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrDale,Norm,Norm,Twnhs,2Story,5,5,1971,1971,Gable,CompShg,HdBoard,HdBoard,BrkFace,142,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,630,630,GasA,TA,Y,SBrkr,630,672,0,1302,0,0,2,1,3,1,TA,6,Typ,0,NA,Detchd,1991,Unf,1,280,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,5,2009,COD,Abnorml,112000\n227,60,RL,82,9950,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NoRidge,Norm,Norm,1Fam,2Story,7,5,1995,1995,Gable,CompShg,VinylSd,VinylSd,BrkFace,290,Gd,TA,PConc,Gd,TA,No,GLQ,565,Unf,0,638,1203,GasA,Ex,Y,SBrkr,1214,1306,0,2520,0,0,2,1,4,1,Gd,9,Typ,1,TA,Attchd,1995,RFn,3,721,TA,TA,Y,224,114,0,0,0,0,NA,NA,NA,0,6,2007,WD,Abnorml,290000\n228,160,RM,21,1869,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrDale,Norm,Norm,Twnhs,2Story,6,6,1970,1970,Gable,CompShg,HdBoard,HdBoard,BrkFace,127,TA,TA,CBlock,TA,TA,No,Rec,321,Unf,0,162,483,GasA,TA,Y,SBrkr,483,504,0,987,0,0,1,1,2,1,TA,5,Typ,0,NA,Detchd,1987,Unf,1,280,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,9,2008,WD,Normal,106000\n229,20,RL,70,8521,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,Sawyer,Feedr,Norm,1Fam,1Story,5,5,1967,1967,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,ALQ,842,Unf,0,70,912,GasA,TA,Y,SBrkr,912,0,0,912,0,0,1,0,3,1,TA,5,Typ,1,Fa,Detchd,1974,Unf,1,336,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,5,2010,WD,Normal,125000\n230,120,RL,43,3182,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Blmngtn,Norm,Norm,TwnhsE,1Story,7,5,2005,2006,Gable,CompShg,VinylSd,VinylSd,BrkFace,16,Gd,TA,PConc,Gd,TA,Av,GLQ,16,Unf,0,1357,1373,GasA,Ex,Y,SBrkr,1555,0,0,1555,0,0,2,0,2,1,Gd,7,Typ,1,TA,Attchd,2005,Fin,2,430,TA,TA,Y,143,20,0,0,0,0,NA,NA,NA,0,5,2009,WD,Normal,192500\n231,20,RL,73,8760,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,6,1959,1959,Hip,CompShg,MetalSd,MetalSd,BrkFace,220,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1194,1194,GasA,TA,Y,SBrkr,1194,0,0,1194,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1959,RFn,1,312,TA,TA,Y,0,0,120,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,148000\n232,60,RL,174,15138,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NoRidge,Norm,Norm,1Fam,2Story,8,5,1995,1996,Gable,CompShg,VinylSd,VinylSd,BrkFace,506,Gd,TA,PConc,Gd,TA,No,GLQ,689,Unf,0,773,1462,GasA,Ex,Y,SBrkr,1490,1304,0,2794,1,0,2,1,4,1,Ex,9,Typ,1,TA,Attchd,1995,Fin,3,810,TA,TA,Y,0,146,202,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,403000\n233,160,RM,21,1680,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrDale,Norm,Norm,Twnhs,2Story,6,5,1972,1972,Gable,CompShg,HdBoard,HdBoard,BrkFace,297,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,483,483,GasA,TA,Y,SBrkr,483,504,0,987,0,0,1,1,2,1,TA,5,Typ,1,Po,Attchd,1972,Unf,1,288,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,94500\n234,20,RL,75,10650,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,6,1976,1976,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,Gd,Av,LwQ,182,ALQ,712,0,894,GasA,TA,Y,SBrkr,894,0,0,894,1,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1976,Unf,1,308,TA,TA,Y,365,0,0,0,0,0,NA,MnPrv,NA,0,2,2010,WD,Normal,128200\n235,60,RL,NA,7851,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,2002,2002,Gable,CompShg,VinylSd,VinylSd,NA,NA,Gd,TA,PConc,Gd,TA,No,GLQ,625,Unf,0,235,860,GasA,Ex,Y,SBrkr,860,1100,0,1960,1,0,2,1,4,1,Gd,8,Typ,2,TA,BuiltIn,2002,Fin,2,440,TA,TA,Y,288,48,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,216500\n236,160,RM,21,1680,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrDale,Norm,Norm,TwnhsE,2Story,6,3,1971,1971,Gable,CompShg,HdBoard,HdBoard,BrkFace,604,TA,TA,CBlock,TA,TA,No,ALQ,358,Unf,0,125,483,GasA,TA,Y,SBrkr,483,504,0,987,0,0,1,1,2,1,TA,5,Typ,0,NA,Detchd,1971,Unf,1,264,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,8,2008,WD,Normal,89500\n237,20,RL,65,8773,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,98,Gd,TA,PConc,Gd,TA,Av,GLQ,24,Unf,0,1390,1414,GasA,Ex,Y,SBrkr,1414,0,0,1414,0,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,2004,RFn,2,494,TA,TA,Y,132,105,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,185500\n238,60,RL,NA,9453,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,SawyerW,RRNe,Norm,1Fam,2Story,7,7,1993,2003,Gable,CompShg,HdBoard,HdBoard,None,0,Gd,TA,PConc,Gd,TA,No,BLQ,402,Unf,0,594,996,GasA,Ex,Y,SBrkr,1014,730,0,1744,0,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,1993,RFn,2,457,TA,TA,Y,370,70,0,238,0,0,NA,NA,NA,0,2,2010,WD,Normal,194500\n239,20,RL,93,12030,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2007,2007,Hip,CompShg,VinylSd,VinylSd,BrkFace,254,Ex,TA,PConc,Ex,TA,No,Unf,0,Unf,0,1694,1694,GasA,Ex,Y,SBrkr,1694,0,0,1694,0,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2007,Fin,3,818,TA,TA,Y,168,228,0,0,0,0,NA,NA,NA,0,12,2007,New,Partial,318000\n240,50,RL,52,8741,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1.5Fin,6,4,1945,1950,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,Fa,No,LwQ,94,Unf,0,641,735,GasA,TA,Y,FuseA,798,689,0,1487,0,0,1,1,3,1,TA,7,Typ,1,Gd,Detchd,1949,Unf,1,220,TA,TA,Y,0,140,0,0,0,0,NA,MnPrv,NA,0,4,2010,WD,Normal,113000\n241,20,FV,75,9000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,8,5,2008,2008,Gable,CompShg,VinylSd,VinylSd,Stone,36,Gd,TA,PConc,Gd,TA,Av,GLQ,1078,Unf,0,488,1566,GasA,Ex,Y,SBrkr,1566,0,0,1566,1,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2008,RFn,2,750,TA,TA,Y,144,168,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,262500\n242,30,RM,40,3880,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,5,9,1945,1997,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,TA,No,ALQ,329,Unf,0,357,686,GasA,Gd,Y,SBrkr,866,0,0,866,0,0,1,0,2,1,Gd,4,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,58,42,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,110500\n243,50,RM,63,5000,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,5,4,1900,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,540,540,GasA,Gd,N,FuseA,889,551,0,1440,0,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1940,Unf,1,352,Fa,TA,Y,0,0,77,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,79000\n244,160,RL,75,10762,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,SawyerW,Norm,Norm,TwnhsE,2Story,6,6,1980,1980,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,Gd,TA,No,Unf,0,Unf,0,626,626,GasA,TA,Y,SBrkr,626,591,0,1217,0,0,1,1,3,1,TA,6,Typ,1,TA,Attchd,1980,RFn,1,288,TA,TA,Y,0,28,0,0,0,0,NA,NA,NA,0,4,2009,WD,Normal,120000\n245,60,RL,NA,8880,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,2Story,7,5,1994,2002,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,695,Unf,0,253,948,GasA,Ex,Y,SBrkr,1222,888,0,2110,1,0,2,1,3,1,Gd,8,Typ,2,Fa,Attchd,1994,RFn,2,463,TA,TA,Y,0,130,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,205000\n246,20,RL,80,10400,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,1Story,7,5,1988,1988,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,102,TA,TA,CBlock,Gd,TA,Av,GLQ,929,Unf,0,916,1845,GasA,Gd,Y,SBrkr,1872,0,0,1872,0,1,2,0,3,1,TA,6,Typ,1,TA,Attchd,1988,Fin,2,604,TA,TA,Y,197,39,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,241500\n247,190,RM,69,9142,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,2fmCon,2Story,6,8,1910,1950,Gable,CompShg,AsbShng,AsbShng,None,0,TA,Fa,Stone,Fa,TA,No,Unf,0,Unf,0,1020,1020,GasA,Gd,N,FuseP,908,1020,0,1928,0,0,2,0,4,2,Fa,9,Typ,0,NA,Detchd,1910,Unf,1,440,Po,Po,Y,0,60,112,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,137000\n248,20,RL,75,11310,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1954,1954,Hip,CompShg,Wd Sdng,BrkFace,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1367,1367,GasA,Ex,Y,SBrkr,1375,0,0,1375,0,0,1,0,2,1,TA,5,Typ,1,TA,Attchd,1954,Unf,2,451,TA,TA,Y,0,30,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,140000\n249,60,RL,72,11317,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,BrkFace,101,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,840,840,GasA,Ex,Y,SBrkr,840,828,0,1668,0,0,2,1,3,1,Gd,8,Typ,0,NA,Attchd,2003,RFn,2,500,TA,TA,Y,144,68,0,0,0,0,NA,NA,NA,0,9,2007,WD,Normal,180000\n250,50,RL,NA,159000,Pave,NA,IR2,Low,AllPub,CulDSac,Sev,ClearCr,Norm,Norm,1Fam,1.5Fin,6,7,1958,2006,Gable,CompShg,Wd Sdng,HdBoard,BrkCmn,472,Gd,TA,CBlock,Gd,TA,Gd,Rec,697,Unf,0,747,1444,GasA,Gd,Y,SBrkr,1444,700,0,2144,0,1,2,0,4,1,Gd,7,Typ,2,TA,Attchd,1958,Fin,2,389,TA,TA,Y,0,98,0,0,0,0,NA,NA,Shed,500,6,2007,WD,Normal,277000\n251,30,RL,55,5350,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1Story,3,2,1940,1966,Gable,CompShg,Wd Sdng,Plywood,None,0,TA,Po,CBlock,TA,TA,No,Unf,0,Unf,0,728,728,GasA,Ex,Y,SBrkr,1306,0,0,1306,0,0,1,0,3,1,Fa,6,Mod,0,NA,NA,NA,NA,0,0,NA,NA,Y,263,0,0,0,0,0,NA,GdWo,Shed,450,5,2010,WD,Normal,76500\n252,120,RM,44,4750,Pave,NA,IR1,HLS,AllPub,Inside,Mod,Crawfor,Norm,Norm,TwnhsE,1Story,8,5,2006,2007,Hip,CompShg,VinylSd,VinylSd,Stone,481,Gd,TA,PConc,Gd,TA,Gd,GLQ,1573,Unf,0,0,1573,GasA,Ex,Y,SBrkr,1625,0,0,1625,1,1,2,0,2,1,Gd,5,Typ,1,Gd,Attchd,2006,Fin,2,538,TA,TA,Y,123,0,0,0,153,0,NA,NA,NA,0,12,2007,WD,Family,235000\n253,60,RL,65,8366,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,2Story,6,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,798,798,GasA,Ex,Y,SBrkr,798,842,0,1640,0,0,2,1,3,1,Gd,6,Typ,0,NA,Attchd,2004,RFn,2,520,TA,TA,Y,138,45,0,0,0,0,NA,NA,NA,0,12,2008,WD,Normal,173000\n254,80,RL,85,9350,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,SLvl,6,7,1964,1991,Hip,CompShg,HdBoard,HdBoard,BrkFace,108,TA,TA,CBlock,Gd,TA,Gd,LwQ,270,ALQ,580,452,1302,GasA,Ex,Y,SBrkr,1302,0,0,1302,0,1,2,0,3,1,Gd,7,Min1,0,NA,Attchd,1964,RFn,1,309,TA,TA,Y,333,0,0,0,0,0,NA,MnPrv,NA,0,10,2007,CWD,Normal,158000\n255,20,RL,70,8400,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1957,1957,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Gd,CBlock,TA,TA,No,Rec,922,Unf,0,392,1314,GasA,TA,Y,SBrkr,1314,0,0,1314,1,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1957,RFn,1,294,TA,TA,Y,250,0,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,145000\n256,60,RL,66,8738,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,1999,1999,Gable,CompShg,VinylSd,VinylSd,BrkFace,302,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,975,975,GasA,Ex,Y,SBrkr,1005,1286,0,2291,0,0,2,1,4,1,Gd,8,Typ,1,TA,BuiltIn,1999,Fin,2,429,TA,TA,Y,192,0,0,0,0,0,NA,NA,NA,0,2,2006,WD,Normal,230000\n257,60,FV,64,8791,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,6,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Rec,503,Unf,0,361,864,GasA,Ex,Y,SBrkr,864,864,0,1728,0,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,2003,RFn,2,673,TA,TA,Y,216,56,0,0,0,0,NA,NA,NA,0,5,2008,WD,Normal,207500\n258,20,RL,68,8814,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,Stone,180,Gd,TA,PConc,Gd,TA,No,GLQ,1334,Unf,0,270,1604,GasA,Ex,Y,SBrkr,1604,0,0,1604,1,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2006,RFn,2,660,TA,TA,Y,123,110,0,0,0,0,NA,NA,NA,0,3,2009,WD,Abnorml,220000\n259,60,RL,80,12435,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2001,2001,Gable,CompShg,VinylSd,VinylSd,BrkFace,172,Gd,TA,PConc,Gd,TA,No,GLQ,361,Unf,0,602,963,GasA,Ex,Y,SBrkr,963,829,0,1792,0,0,2,1,3,1,Gd,7,Typ,1,TA,Attchd,2001,RFn,2,564,TA,TA,Y,0,96,0,245,0,0,NA,NA,NA,0,5,2008,WD,Normal,231500\n260,20,RM,70,12702,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,5,5,1956,1956,Gable,CompShg,BrkFace,BrkFace,None,0,TA,TA,PConc,NA,NA,NA,NA,0,NA,0,0,0,GasA,Gd,Y,FuseA,882,0,0,882,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1956,Unf,1,308,TA,TA,Y,0,45,0,0,0,0,NA,NA,NA,0,12,2008,WD,Normal,97000\n261,80,RL,120,19296,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Artery,Norm,1Fam,SLvl,6,5,1962,1962,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,399,TA,TA,CBlock,TA,TA,Gd,Rec,672,ALQ,690,0,1362,GasA,TA,Y,SBrkr,1382,0,0,1382,1,0,1,0,3,1,TA,6,Typ,1,TA,Attchd,1991,Unf,2,884,TA,TA,Y,0,0,252,0,0,0,NA,GdWo,NA,0,5,2009,WD,Normal,176000\n262,60,RL,69,9588,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,8,5,2007,2007,Gable,CompShg,CemntBd,CmentBd,Stone,270,Gd,TA,PConc,Ex,TA,No,Unf,0,Unf,0,1482,1482,GasA,Ex,Y,SBrkr,1482,1092,0,2574,0,0,2,1,3,1,Ex,10,Typ,1,Gd,BuiltIn,2007,Fin,3,868,TA,TA,Y,0,148,0,0,0,0,NA,NA,NA,0,11,2007,New,Partial,276000\n263,80,RL,88,8471,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Sawyer,Norm,Norm,1Fam,SLvl,6,7,1977,1995,Gable,CompShg,HdBoard,Plywood,BrkFace,46,TA,TA,CBlock,Gd,Gd,Av,ALQ,506,Unf,0,0,506,GasA,TA,Y,SBrkr,1212,0,0,1212,1,0,1,0,3,1,TA,6,Typ,1,TA,Attchd,1978,Unf,2,492,TA,TA,Y,292,12,0,0,0,0,NA,GdWo,NA,0,7,2006,WD,Normal,151000\n264,50,RM,50,5500,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,5,7,1929,2001,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,LwQ,234,ALQ,692,0,926,GasA,TA,Y,SBrkr,926,0,390,1316,1,0,1,0,3,1,TA,6,Typ,0,NA,Detchd,1974,Unf,2,484,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,130000\n265,30,RM,30,5232,Pave,Grvl,IR3,Bnk,AllPub,Inside,Gtl,OldTown,Artery,Norm,1Fam,1Story,5,5,1925,2004,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,Fa,TA,No,Unf,0,Unf,0,680,680,GasA,Gd,N,FuseP,764,0,0,764,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1965,Unf,2,504,TA,TA,N,0,0,0,0,0,0,NA,NA,NA,0,6,2008,WD,Normal,73000\n266,20,RL,78,12090,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,1Story,6,6,1981,1981,Gable,CompShg,MetalSd,MetalSd,BrkFace,210,TA,Gd,CBlock,Gd,TA,No,GLQ,588,LwQ,228,606,1422,GasA,TA,Y,SBrkr,1422,0,0,1422,0,0,2,0,3,1,Gd,7,Typ,1,TA,Attchd,1981,Fin,2,576,TA,TA,Y,276,0,0,0,0,0,NA,GdPrv,NA,0,6,2008,WD,Normal,175500\n267,60,RL,70,11207,Pave,NA,IR1,HLS,AllPub,FR2,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,1997,1997,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,Av,GLQ,714,Unf,0,88,802,GasA,Gd,Y,SBrkr,802,709,0,1511,1,0,2,1,3,1,TA,8,Typ,1,TA,Attchd,1997,Fin,2,413,TA,TA,Y,95,75,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,185000\n268,75,RL,60,8400,Pave,NA,Reg,Bnk,AllPub,Inside,Mod,SWISU,Norm,Norm,1Fam,2.5Fin,5,8,1939,1997,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,TA,TA,No,LwQ,378,Unf,0,342,720,GasA,Ex,Y,SBrkr,1052,720,420,2192,0,0,2,1,4,1,Gd,8,Typ,1,Gd,Detchd,1939,Unf,1,240,TA,TA,Y,262,24,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,179500\n269,30,RM,71,6900,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,1Story,5,6,1940,1955,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,TA,No,ALQ,403,Rec,125,212,740,GasA,Ex,Y,SBrkr,778,0,0,778,0,0,1,0,2,1,TA,4,Typ,1,Gd,Detchd,1966,Fin,1,924,Ex,Ex,Y,0,25,0,0,0,0,NA,NA,NA,0,2,2008,WD,Normal,120500\n270,20,RL,NA,7917,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,6,7,1976,1976,Hip,CompShg,HdBoard,HdBoard,BrkFace,174,TA,Gd,CBlock,TA,Gd,No,BLQ,751,Unf,0,392,1143,GasA,TA,Y,SBrkr,1113,0,0,1113,1,0,1,1,3,1,TA,6,Typ,1,Fa,Attchd,1987,RFn,1,504,TA,Gd,Y,370,30,0,0,0,0,NA,GdPrv,NA,0,5,2007,WD,Normal,148000\n271,60,FV,84,10728,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,8,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Mn,Unf,0,Unf,0,1095,1095,GasA,Gd,Y,SBrkr,1095,844,0,1939,0,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2006,RFn,3,1053,TA,TA,Y,192,51,0,0,0,0,NA,NA,NA,0,8,2006,New,Partial,266000\n272,20,RL,73,39104,Pave,NA,IR1,Low,AllPub,CulDSac,Sev,ClearCr,Norm,Norm,1Fam,1Story,7,7,1954,2005,Flat,Membran,Plywood,Plywood,None,0,TA,TA,CBlock,Gd,TA,Gd,LwQ,226,GLQ,1063,96,1385,GasA,Ex,Y,SBrkr,1363,0,0,1363,1,0,1,0,2,1,TA,5,Mod,2,TA,Attchd,1954,Unf,2,439,TA,TA,Y,81,0,0,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,241500\n273,60,RL,92,11764,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NoRidge,Norm,Norm,1Fam,2Story,8,7,1999,2007,Gable,CompShg,VinylSd,VinylSd,BrkFace,348,Gd,TA,PConc,Gd,TA,No,GLQ,524,Unf,0,628,1152,GasA,Ex,Y,SBrkr,1164,1106,0,2270,0,0,2,1,4,1,Gd,9,Typ,1,Gd,Attchd,1999,Fin,3,671,TA,TA,Y,132,57,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,290000\n274,20,RL,80,9600,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Feedr,Norm,1Fam,1Story,6,6,1958,1988,Hip,CompShg,Wd Sdng,Wd Sdng,BrkCmn,183,TA,TA,CBlock,TA,TA,No,Rec,620,LwQ,620,0,1240,GasA,Gd,Y,SBrkr,1632,0,0,1632,1,0,2,0,3,1,TA,6,Min1,1,Gd,Attchd,1958,RFn,1,338,TA,TA,Y,289,0,0,0,0,0,NA,MnPrv,NA,0,4,2009,WD,Normal,139000\n275,20,RL,76,8314,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Mitchel,Norm,Norm,1Fam,1Story,5,7,1982,1982,Gable,CompShg,HdBoard,ImStucc,None,0,TA,TA,CBlock,TA,TA,Gd,ALQ,546,Unf,0,270,816,GasA,TA,Y,SBrkr,816,0,0,816,0,0,1,0,2,1,TA,5,Typ,0,NA,Attchd,1982,Unf,1,264,TA,TA,Y,168,0,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,124500\n276,50,RL,55,7264,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1.5Fin,7,7,1925,2007,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,Gd,Gd,BrkTil,TA,TA,No,Unf,0,Unf,0,952,952,GasW,Gd,N,SBrkr,952,596,0,1548,0,0,2,1,3,1,Ex,5,Typ,0,NA,Detchd,1978,Unf,2,672,TA,TA,Y,74,0,0,0,144,0,NA,NA,NA,0,10,2009,WD,Normal,205000\n277,20,RL,129,9196,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Mitchel,Norm,Norm,1Fam,1Story,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Ex,TA,No,Unf,0,Unf,0,1560,1560,GasA,Ex,Y,SBrkr,1560,0,0,1560,0,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2003,Fin,2,573,TA,TA,Y,100,150,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,201000\n278,20,RL,140,19138,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,1Story,4,5,1951,1951,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,TA,No,LwQ,120,Unf,0,744,864,GasA,Ex,Y,SBrkr,864,0,0,864,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1951,Unf,2,400,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,141000\n279,20,RL,107,14450,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,9,5,2006,2007,Gable,CompShg,CemntBd,CmentBd,BrkFace,315,Ex,TA,PConc,Ex,TA,Gd,Unf,0,Unf,0,2121,2121,GasA,Ex,Y,SBrkr,2121,0,0,2121,0,0,2,1,3,1,Ex,8,Typ,1,Ex,Attchd,2007,Fin,3,732,TA,TA,Y,124,98,0,0,142,0,NA,NA,NA,0,5,2007,New,Partial,415298\n280,60,RL,83,10005,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,ClearCr,Norm,Norm,1Fam,2Story,7,5,1977,1977,Hip,CompShg,Plywood,Plywood,BrkFace,299,TA,TA,CBlock,Gd,TA,No,BLQ,392,Unf,0,768,1160,GasA,Ex,Y,SBrkr,1156,866,0,2022,0,0,2,1,4,1,TA,8,Typ,1,TA,Attchd,1977,Fin,2,505,TA,TA,Y,288,117,0,0,0,0,NA,NA,NA,0,3,2008,WD,Normal,192000\n281,60,RL,82,11287,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,2Story,7,6,1989,1989,Gable,CompShg,Plywood,Plywood,BrkFace,340,Gd,TA,CBlock,Gd,TA,Av,GLQ,421,Unf,0,386,807,GasA,Gd,Y,SBrkr,1175,807,0,1982,0,0,2,1,3,1,Gd,7,Typ,1,TA,Attchd,1989,Fin,2,575,TA,TA,Y,0,84,0,196,0,0,NA,NA,NA,0,1,2007,WD,Normal,228500\n282,20,FV,60,7200,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,6,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,Stone,68,Gd,TA,PConc,Gd,TA,No,GLQ,905,Unf,0,357,1262,GasA,Gd,Y,SBrkr,1262,0,0,1262,0,0,2,0,2,1,Gd,5,Typ,0,NA,Attchd,2006,Fin,2,572,TA,TA,Y,0,120,0,0,0,0,NA,NA,NA,0,5,2006,New,Partial,185000\n283,120,RL,34,5063,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,Twnhs,1Story,7,5,2007,2008,Gable,CompShg,VinylSd,VinylSd,Stone,166,Gd,TA,PConc,Gd,TA,No,GLQ,904,Unf,0,410,1314,GasA,Ex,Y,SBrkr,1314,0,0,1314,1,0,2,0,2,1,Gd,6,Typ,1,Gd,Attchd,2008,RFn,2,626,TA,TA,Y,172,62,0,0,0,0,NA,NA,NA,0,4,2009,ConLw,Normal,207500\n284,20,RL,74,9612,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Feedr,Norm,1Fam,1Story,8,5,2008,2009,Gable,CompShg,VinylSd,VinylSd,Stone,72,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1468,1468,GasA,Ex,Y,SBrkr,1468,0,0,1468,0,0,2,0,3,1,Gd,6,Typ,1,Gd,Attchd,2008,Fin,3,898,TA,TA,Y,210,150,0,0,0,0,NA,NA,NA,0,12,2009,New,Partial,244600\n285,120,RL,50,8012,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,TwnhsE,1Story,6,5,1992,1992,Gable,CompShg,Plywood,ImStucc,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,430,Unf,0,1145,1575,GasA,Gd,Y,SBrkr,1575,0,0,1575,1,0,2,0,2,1,Gd,5,Typ,0,NA,Attchd,1992,RFn,2,529,TA,TA,Y,0,0,52,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,179200\n286,160,FV,35,4251,Pave,Pave,IR1,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,TwnhsE,2Story,7,5,2006,2007,Gable,CompShg,MetalSd,MetalSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,625,625,GasA,Ex,Y,SBrkr,625,625,0,1250,0,0,2,1,2,1,Gd,5,Typ,0,NA,Detchd,2006,RFn,2,528,TA,TA,Y,0,54,0,0,0,0,NA,NA,NA,0,6,2007,New,Partial,164700\n287,50,RL,77,9786,Pave,NA,IR1,Bnk,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1.5Fin,6,7,1962,1981,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,Rec,600,Unf,0,312,912,GasA,TA,Y,SBrkr,1085,649,0,1734,0,0,1,1,3,1,Gd,7,Typ,1,Gd,Attchd,1962,RFn,2,440,TA,TA,Y,0,0,0,0,128,0,NA,GdPrv,NA,0,6,2006,WD,Normal,159000\n288,20,RL,NA,8125,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,4,1971,1971,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,BLQ,614,Unf,0,244,858,GasA,TA,Y,SBrkr,858,0,0,858,0,0,1,0,3,1,TA,5,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,88000\n289,20,RL,NA,9819,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,5,1967,1967,Gable,CompShg,MetalSd,MetalSd,BrkFace,31,TA,Gd,CBlock,TA,TA,No,BLQ,450,Unf,0,432,882,GasA,TA,Y,SBrkr,900,0,0,900,0,0,1,0,3,1,TA,5,Typ,0,NA,Detchd,1970,Unf,1,280,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,2,2010,WD,Normal,122000\n290,70,RL,60,8730,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,RRAn,Norm,1Fam,2Story,6,7,1915,2003,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,698,698,GasA,Ex,Y,FuseA,698,698,0,1396,0,0,1,0,3,1,TA,7,Typ,0,NA,Detchd,2003,Unf,1,384,TA,TA,Y,0,0,0,0,259,0,NA,NA,NA,0,7,2007,WD,Normal,153575\n291,60,RL,120,15611,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,8,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,1079,1079,GasA,Ex,Y,SBrkr,1079,840,0,1919,0,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2006,RFn,2,685,Gd,TA,Y,0,51,0,0,0,0,NA,NA,NA,0,7,2006,New,Partial,233230\n292,190,RL,55,5687,Pave,Grvl,Reg,Bnk,AllPub,Inside,Gtl,SWISU,Norm,Norm,2fmCon,2Story,5,6,1912,2000,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Fa,PConc,TA,Fa,No,Rec,210,Unf,0,570,780,GasA,Ex,N,SBrkr,936,780,0,1716,1,0,2,0,6,1,Fa,9,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,184,0,0,0,0,NA,NA,NA,0,3,2008,WD,Normal,135900\n293,50,RL,60,11409,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1.5Fin,5,4,1949,2008,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,LwQ,292,Unf,0,476,768,GasA,Gd,Y,SBrkr,1148,568,0,1716,0,0,1,1,3,1,TA,8,Min2,1,Gd,Attchd,1949,Unf,1,281,TA,TA,Y,0,0,0,0,160,0,NA,NA,NA,0,1,2009,WD,Normal,131000\n294,60,RL,NA,16659,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NWAmes,PosA,Norm,1Fam,2Story,7,7,1977,1994,Gable,CompShg,Plywood,Plywood,BrkFace,34,TA,TA,CBlock,TA,TA,No,ALQ,795,Unf,0,0,795,GasA,Fa,Y,SBrkr,1468,795,0,2263,1,0,2,1,3,1,Gd,9,Typ,1,TA,Attchd,1977,Fin,2,539,TA,TA,Y,0,250,0,0,0,0,NA,NA,NA,0,3,2006,WD,Normal,235000\n295,20,RL,80,9600,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1953,1953,Hip,CompShg,HdBoard,HdBoard,Stone,238,TA,TA,CBlock,TA,TA,No,GLQ,1285,Unf,0,131,1416,GasA,TA,Y,SBrkr,1644,0,0,1644,1,0,1,0,3,1,TA,7,Typ,2,Gd,Attchd,1953,Fin,2,418,TA,TA,Y,110,0,0,0,0,0,NA,NA,NA,0,10,2009,WD,Normal,167000\n296,80,RL,37,7937,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Mitchel,Norm,Norm,1Fam,SLvl,6,6,1984,1984,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,Av,GLQ,819,Unf,0,184,1003,GasA,TA,Y,SBrkr,1003,0,0,1003,1,0,1,0,3,1,TA,6,Typ,0,NA,Detchd,1984,Unf,2,588,TA,TA,Y,120,0,0,0,0,0,NA,GdPrv,NA,0,3,2006,WD,Normal,142500\n297,50,RM,75,13710,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,1.5Fin,5,5,1950,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,BLQ,420,Unf,0,490,910,GasA,TA,Y,FuseA,910,648,0,1558,0,0,1,1,4,1,TA,6,Typ,0,NA,Attchd,1950,Unf,1,282,TA,TA,Y,289,0,0,0,0,0,NA,MnPrv,NA,0,6,2007,WD,Normal,152000\n298,60,FV,66,7399,Pave,Pave,IR1,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,7,5,1997,1998,Hip,CompShg,VinylSd,VinylSd,BrkFace,1600,Gd,TA,PConc,Gd,TA,No,BLQ,649,Unf,0,326,975,GasA,Ex,Y,SBrkr,975,975,0,1950,0,0,2,1,3,1,Gd,7,Typ,1,TA,Detchd,1997,RFn,2,576,TA,TA,Y,0,10,0,0,198,0,NA,NA,NA,0,6,2007,WD,Normal,239000\n299,60,RL,90,11700,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,2Story,6,6,1968,1968,Mansard,CompShg,HdBoard,AsphShn,BrkFace,365,Gd,TA,CBlock,TA,TA,No,ALQ,384,Rec,175,143,702,GasA,Gd,Y,SBrkr,1041,702,0,1743,0,1,1,2,3,1,TA,7,Typ,1,Gd,Attchd,1968,Unf,2,539,TA,TA,Y,224,0,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,175000\n300,20,RL,80,14000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Crawfor,Norm,Norm,1Fam,1Story,6,8,1950,2004,Gable,CompShg,HdBoard,HdBoard,None,0,TA,Gd,CBlock,TA,TA,No,Unf,0,Unf,0,1092,1092,GasA,Ex,Y,SBrkr,1152,0,0,1152,0,1,1,0,3,1,Gd,6,Typ,1,Gd,Attchd,1950,Unf,1,300,TA,TA,Y,0,36,0,0,0,0,NA,GdPrv,NA,0,8,2009,WD,Family,158500\n301,190,RL,90,15750,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Crawfor,Norm,Norm,2fmCon,1Story,5,5,1953,1953,Hip,CompShg,MetalSd,MetalSd,BrkFace,56,TA,TA,CBlock,TA,TA,Mn,BLQ,841,Unf,0,324,1165,GasA,TA,Y,SBrkr,1336,0,0,1336,1,0,1,0,2,1,TA,5,Typ,2,Gd,Attchd,1953,Unf,1,375,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,157000\n302,60,RL,66,16226,Pave,NA,IR3,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,8,5,1998,1999,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,281,Unf,0,747,1028,GasA,Ex,Y,SBrkr,1210,1242,0,2452,0,0,2,1,4,1,Gd,9,Typ,1,TA,BuiltIn,1998,Fin,2,683,TA,TA,Y,208,50,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,267000\n303,20,RL,118,13704,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2001,2002,Gable,CompShg,VinylSd,VinylSd,BrkFace,150,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1541,1541,GasA,Ex,Y,SBrkr,1541,0,0,1541,0,0,2,0,3,1,Gd,6,Typ,1,TA,Attchd,2001,RFn,3,843,TA,TA,Y,468,81,0,0,0,0,NA,NA,NA,0,1,2006,WD,Normal,205000\n304,20,RL,70,9800,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,7,1972,1972,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,TA,TA,No,ALQ,894,Unf,0,0,894,GasA,TA,Y,SBrkr,894,0,0,894,1,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1975,Unf,2,552,TA,TA,Y,256,0,0,0,0,0,NA,GdWo,NA,0,7,2006,WD,Abnorml,149900\n305,75,RM,87,18386,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,2.5Fin,7,9,1880,2002,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,1470,1470,GasA,Ex,Y,SBrkr,1675,1818,0,3493,0,0,3,0,3,1,Gd,10,Typ,1,Ex,Attchd,2003,Unf,3,870,TA,TA,Y,302,0,0,0,0,0,NA,NA,NA,0,5,2008,WD,Normal,295000\n306,20,RL,80,10386,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,8,5,2004,2005,Gable,CompShg,CemntBd,CmentBd,Stone,246,Gd,TA,PConc,Gd,TA,No,GLQ,1464,Unf,0,536,2000,GasA,Ex,Y,SBrkr,2000,0,0,2000,1,0,2,0,3,1,Gd,8,Typ,0,NA,Attchd,2004,Fin,3,888,TA,TA,Y,168,0,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,305900\n307,60,RL,116,13474,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Feedr,Norm,1Fam,2Story,7,5,1990,1991,Gable,CompShg,HdBoard,Plywood,BrkFace,246,Gd,TA,CBlock,Gd,TA,No,ALQ,700,Unf,0,0,700,GasA,Gd,Y,SBrkr,1122,1121,0,2243,1,0,2,1,4,1,Gd,8,Typ,1,TA,Attchd,1990,RFn,3,746,TA,TA,Y,127,44,224,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,225000\n308,50,RM,NA,7920,Pave,Grvl,IR1,Lvl,AllPub,Inside,Gtl,IDOTRR,Artery,Norm,1Fam,1.5Fin,6,7,1920,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Fa,CBlock,TA,TA,No,Unf,0,Unf,0,319,319,GasA,TA,Y,FuseA,1035,371,0,1406,0,0,1,0,3,1,Fa,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,144,0,0,0,0,NA,MnPrv,NA,0,3,2008,WD,Normal,89500\n309,30,RL,NA,12342,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1Story,4,5,1940,1950,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,TA,No,BLQ,262,Unf,0,599,861,GasA,Ex,Y,SBrkr,861,0,0,861,0,0,1,0,1,1,TA,4,Typ,0,NA,Detchd,1961,Unf,2,539,TA,TA,Y,158,0,0,0,0,0,NA,NA,NA,0,3,2009,WD,Normal,82500\n310,20,RL,90,12378,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,9,5,2003,2004,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Ex,TA,Gd,GLQ,1274,Unf,0,622,1896,GasA,Ex,Y,SBrkr,1944,0,0,1944,1,0,2,0,3,1,Ex,8,Typ,3,Ex,Attchd,2003,Fin,3,708,TA,TA,Y,208,175,0,0,0,0,NA,NA,NA,0,11,2006,WD,Normal,360000\n311,60,RL,NA,7685,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,1993,1994,Gable,CompShg,HdBoard,HdBoard,BrkFace,112,TA,TA,PConc,Gd,TA,No,ALQ,518,Unf,0,179,697,GasA,Gd,Y,SBrkr,697,804,0,1501,0,0,2,1,3,1,Gd,6,Typ,1,TA,Attchd,1993,Fin,2,420,TA,TA,Y,190,63,0,0,0,0,NA,NA,NA,0,5,2006,WD,Normal,165600\n312,20,RL,50,8000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,6,1948,2002,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,TA,No,ALQ,680,Unf,0,292,972,GasA,Ex,Y,SBrkr,972,0,0,972,1,0,1,0,2,1,TA,5,Typ,1,Gd,Detchd,1948,Unf,1,240,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,5,2009,WD,Normal,132000\n313,190,RM,65,7800,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Artery,Norm,2fmCon,1.5Fin,5,7,1939,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,Gd,TA,Mn,Rec,507,Unf,0,286,793,GasA,TA,Y,SBrkr,793,325,0,1118,1,0,1,0,3,1,TA,5,Typ,1,Gd,Detchd,1939,Unf,2,410,TA,TA,Y,0,0,0,0,271,0,NA,MnPrv,NA,0,5,2006,WD,Normal,119900\n314,20,RL,150,215245,Pave,NA,IR3,Low,AllPub,Inside,Sev,Timber,Norm,Norm,1Fam,1Story,7,5,1965,1965,Hip,CompShg,BrkFace,BrkFace,None,0,TA,TA,CBlock,Gd,TA,Gd,ALQ,1236,Rec,820,80,2136,GasW,TA,Y,SBrkr,2036,0,0,2036,2,0,2,0,3,1,TA,8,Typ,2,Gd,Attchd,1965,RFn,2,513,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,375000\n315,70,RM,60,9600,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,2Story,7,7,1925,1990,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,Gd,No,LwQ,16,Unf,0,712,728,GasA,Ex,Y,SBrkr,832,809,0,1641,0,1,1,1,3,1,Ex,6,Typ,1,Gd,Detchd,1925,Unf,2,546,Fa,TA,Y,0,0,234,0,0,0,NA,NA,NA,0,8,2006,WD,Normal,178000\n316,60,RL,71,7795,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,425,Unf,0,291,716,GasA,Ex,Y,SBrkr,716,716,0,1432,1,0,2,1,3,1,Gd,6,Typ,1,Gd,Attchd,2004,Fin,2,432,TA,TA,Y,100,51,0,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,188500\n317,60,RL,94,13005,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NWAmes,Norm,Norm,1Fam,2Story,7,7,1980,1980,Gable,CompShg,CemntBd,CmentBd,BrkFace,278,Gd,TA,CBlock,Gd,TA,No,GLQ,692,Unf,0,153,845,GasA,TA,Y,SBrkr,1153,1200,0,2353,1,0,2,1,4,1,Ex,10,Typ,1,TA,Attchd,1983,RFn,2,484,TA,TA,Y,288,195,0,0,0,0,NA,GdPrv,NA,0,8,2009,WD,Normal,260000\n318,60,FV,75,9000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,8,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,1088,1088,GasA,Ex,Y,SBrkr,1088,871,0,1959,0,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2006,RFn,3,1025,TA,TA,Y,208,46,0,0,0,0,NA,NA,NA,0,12,2007,WD,Normal,270000\n319,60,RL,90,9900,Pave,NA,Reg,Low,AllPub,Inside,Mod,NoRidge,Norm,Norm,1Fam,2Story,7,5,1993,1993,Gable,CompShg,HdBoard,HdBoard,BrkFace,256,Gd,TA,PConc,Gd,TA,Gd,GLQ,987,Unf,0,360,1347,GasA,Ex,Y,SBrkr,1372,1274,0,2646,1,0,2,1,4,1,Gd,9,Typ,1,TA,Attchd,1993,RFn,3,656,TA,TA,Y,340,60,144,0,0,0,NA,NA,NA,0,4,2009,WD,Normal,260000\n320,80,RL,NA,14115,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,SLvl,7,5,1980,1980,Gable,CompShg,Plywood,Plywood,BrkFace,225,TA,TA,CBlock,Gd,TA,Av,GLQ,1036,Unf,0,336,1372,GasA,TA,Y,SBrkr,1472,0,0,1472,1,0,2,0,3,1,TA,6,Typ,2,TA,Attchd,1980,Unf,2,588,TA,TA,Y,233,48,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,187500\n321,60,RL,111,16259,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NridgHt,Norm,Norm,1Fam,2Story,9,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,Stone,370,TA,TA,PConc,Ex,Gd,Av,Unf,0,Unf,0,1249,1249,GasA,Ex,Y,SBrkr,1249,1347,0,2596,0,0,3,1,4,1,Gd,9,Typ,0,NA,Attchd,2006,RFn,3,840,TA,TA,Y,240,154,0,0,0,0,NA,NA,NA,0,9,2006,New,Partial,342643\n322,60,RL,99,12099,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,8,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,388,Gd,TA,PConc,Ex,TA,Av,GLQ,970,Unf,0,166,1136,GasA,Ex,Y,SBrkr,1136,1332,0,2468,1,0,2,1,4,1,Gd,10,Typ,1,Gd,BuiltIn,2004,Fin,3,872,TA,TA,Y,184,154,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,354000\n323,60,RL,86,10380,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,2Story,7,5,1986,1987,Gable,CompShg,Plywood,Plywood,BrkFace,172,Gd,TA,CBlock,TA,TA,Gd,LwQ,28,ALQ,1474,0,1502,GasA,Ex,Y,SBrkr,1553,1177,0,2730,1,0,2,1,4,1,Gd,8,Typ,1,TA,Attchd,1987,Fin,2,576,TA,TA,Y,201,96,0,0,0,0,NA,MnPrv,NA,0,8,2007,WD,Normal,301000\n324,20,RM,49,5820,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,3,8,1955,2005,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,TA,No,ALQ,256,Unf,0,906,1162,GasA,Ex,Y,SBrkr,1163,0,0,1163,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1955,Unf,1,220,Fa,TA,Y,142,98,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,126175\n325,80,RL,96,11275,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,PosN,Norm,1Fam,SLvl,7,7,1967,2007,Mansard,WdShake,Wd Sdng,Wd Sdng,BrkFace,300,Gd,Gd,CBlock,Gd,TA,No,Unf,0,Unf,0,710,710,GasA,Ex,Y,SBrkr,1898,1080,0,2978,0,0,2,1,5,1,Gd,11,Typ,1,Gd,BuiltIn,1961,Fin,2,564,TA,TA,Y,240,0,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,242000\n326,45,RM,50,5000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,RRAe,Norm,1Fam,1.5Unf,5,6,1941,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,Av,BLQ,116,Unf,0,604,720,GasA,Po,N,FuseF,803,0,0,803,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1941,Unf,2,360,TA,TA,Y,0,0,244,0,0,0,NA,NA,NA,0,12,2007,WD,Normal,87000\n327,120,RL,32,10846,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Veenker,Norm,Norm,TwnhsE,1Story,8,5,1993,1993,Gable,CompShg,BrkFace,BrkFace,None,0,Gd,TA,PConc,Gd,TA,Gd,GLQ,1619,Unf,0,100,1719,GasA,Ex,Y,SBrkr,1719,0,0,1719,2,0,1,1,1,1,Gd,6,Typ,2,Gd,Attchd,1993,Fin,2,473,TA,TA,Y,122,30,0,0,0,0,NA,NA,NA,0,5,2008,Con,Normal,324000\n328,20,RL,80,11600,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1960,1960,Hip,CompShg,Wd Sdng,Wd Sdng,BrkFace,175,TA,TA,CBlock,TA,TA,No,Rec,565,Unf,0,818,1383,GasA,TA,Y,SBrkr,1383,0,0,1383,0,0,1,1,3,1,TA,7,Typ,0,NA,Attchd,1960,RFn,1,292,TA,TA,Y,0,45,0,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,145250\n329,75,RL,NA,11888,Pave,Pave,IR1,Bnk,AllPub,Inside,Gtl,BrkSide,PosN,Norm,1Fam,2.5Unf,6,6,1916,1994,Gable,CompShg,Wd Sdng,Wd Shng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,844,844,GasA,Gd,N,FuseA,1445,689,0,2134,0,0,2,0,5,1,Gd,10,Typ,0,NA,Detchd,1930,Unf,2,441,TA,TA,Y,0,60,268,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,214500\n330,70,RM,60,6402,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,IDOTRR,Norm,Norm,1Fam,2Story,5,5,1920,1950,Gable,CompShg,Wd Sdng,Wd Shng,None,0,TA,TA,PConc,TA,TA,Mn,Unf,0,Unf,0,596,596,GasA,TA,N,SBrkr,596,596,0,1192,0,0,1,0,3,1,TA,6,Typ,0,NA,Detchd,1920,Unf,1,189,Fa,Fa,N,0,0,137,0,0,0,NA,GdWo,NA,0,7,2009,WD,Normal,78000\n331,90,RL,NA,10624,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,Duplex,1Story,5,4,1964,1964,Gable,CompShg,HdBoard,HdBoard,BrkFace,84,TA,TA,CBlock,TA,TA,No,GLQ,40,Rec,264,1424,1728,GasA,TA,Y,SBrkr,1728,0,0,1728,0,1,2,0,6,2,TA,10,Typ,0,NA,Detchd,2002,Unf,1,352,TA,TA,Y,155,0,0,0,0,0,NA,NA,NA,0,11,2007,WD,Normal,119000\n332,20,RL,70,8176,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1958,1992,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,Rec,846,Unf,0,210,1056,GasA,Fa,Y,SBrkr,1056,0,0,1056,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1958,RFn,1,308,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,139000\n333,20,RL,85,10655,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2003,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,296,Gd,TA,PConc,Gd,TA,No,GLQ,1124,NA,479,1603,3206,GasA,Ex,Y,SBrkr,1629,0,0,1629,1,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2003,RFn,3,880,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,10,2009,WD,Normal,284000\n334,120,RM,59,8198,Pave,NA,Reg,Lvl,AllPub,FR3,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,Stone,146,Gd,TA,PConc,Gd,TA,Av,GLQ,720,Unf,0,638,1358,GasA,Ex,Y,SBrkr,1358,0,0,1358,1,0,2,0,2,1,Gd,6,Typ,1,Gd,Attchd,2004,RFn,2,484,TA,TA,Y,192,30,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,207000\n335,60,RL,59,9042,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,1998,1998,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,Gd,GLQ,828,Unf,0,115,943,GasA,Gd,Y,SBrkr,943,695,0,1638,1,0,2,1,3,1,TA,7,Typ,2,TA,Attchd,1998,Fin,2,472,TA,TA,Y,100,38,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,192000\n336,190,RL,NA,164660,Grvl,NA,IR1,HLS,AllPub,Corner,Sev,Timber,Norm,Norm,2fmCon,1.5Fin,5,6,1965,1965,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,TA,TA,Gd,ALQ,1249,BLQ,147,103,1499,GasA,Ex,Y,SBrkr,1619,167,0,1786,2,0,2,0,3,1,TA,7,Typ,2,Gd,Attchd,1965,Fin,2,529,TA,TA,Y,670,0,0,0,0,0,NA,NA,Shed,700,8,2008,WD,Normal,228950\n337,20,RL,86,14157,Pave,NA,IR1,HLS,AllPub,Corner,Gtl,StoneBr,Norm,Norm,1Fam,1Story,9,5,2005,2006,Hip,CompShg,VinylSd,VinylSd,Stone,200,Gd,TA,PConc,Ex,TA,Gd,GLQ,1249,Unf,0,673,1922,GasA,Ex,Y,SBrkr,1922,0,0,1922,1,0,2,0,3,1,Gd,8,Typ,1,Gd,Attchd,2005,Fin,3,676,TA,TA,Y,178,51,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,377426\n338,20,RL,70,9135,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2002,2003,Gable,CompShg,VinylSd,VinylSd,BrkFace,113,Gd,TA,PConc,Gd,TA,Av,GLQ,810,Unf,0,726,1536,GasA,Ex,Y,SBrkr,1536,0,0,1536,1,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2002,RFn,2,532,TA,TA,Y,192,74,0,0,0,0,NA,NA,NA,0,12,2008,WD,Normal,214000\n339,20,RL,91,14145,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NWAmes,Norm,Norm,1Fam,1Story,7,7,1984,1998,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,Gd,TA,CBlock,Gd,TA,Mn,ALQ,213,Unf,0,995,1208,GasA,Ex,Y,SBrkr,1621,0,0,1621,1,0,2,0,3,1,Gd,8,Typ,0,NA,Attchd,1984,RFn,2,440,TA,TA,Y,108,45,0,0,0,0,NA,NA,Shed,400,5,2006,WD,Normal,202500\n340,20,RL,66,12400,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Feedr,Norm,1Fam,1Story,6,7,1958,1998,Hip,CompShg,Wd Sdng,Wd Sdng,BrkFace,176,TA,TA,CBlock,TA,Fa,No,Rec,585,Unf,0,630,1215,GasA,TA,Y,FuseA,1215,0,0,1215,0,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1958,Unf,1,297,TA,TA,Y,0,0,0,0,234,0,NA,NA,NA,0,6,2009,WD,Normal,155000\n341,60,RL,85,14191,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Timber,Norm,Norm,1Fam,2Story,8,5,2002,2002,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,967,967,GasA,Ex,Y,SBrkr,993,915,0,1908,0,0,2,1,4,1,Gd,9,Typ,0,NA,Attchd,2002,Fin,2,431,TA,TA,Y,135,0,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,202900\n342,20,RH,60,8400,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Feedr,Norm,1Fam,1Story,4,4,1950,1950,Gable,CompShg,Wd Sdng,AsbShng,None,0,Fa,Fa,CBlock,TA,Fa,No,Unf,0,Unf,0,721,721,GasA,Gd,Y,SBrkr,841,0,0,841,0,0,1,0,2,1,TA,4,Typ,0,NA,CarPort,1950,Unf,1,294,TA,TA,N,250,0,24,0,0,0,NA,NA,NA,0,9,2009,WD,Normal,82000\n343,90,RL,NA,8544,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,Duplex,1Story,3,4,1949,1950,Gable,CompShg,Stucco,Stucco,BrkFace,340,TA,TA,Slab,NA,NA,NA,NA,0,NA,0,0,0,Wall,Fa,N,FuseA,1040,0,0,1040,0,0,2,0,2,2,TA,6,Typ,0,NA,Detchd,1949,Unf,2,400,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,5,2006,WD,Normal,87500\n344,120,RL,63,8849,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,9,5,2005,2005,Hip,CompShg,MetalSd,MetalSd,BrkFace,616,Ex,TA,PConc,Ex,TA,No,GLQ,28,Unf,0,1656,1684,GasA,Ex,Y,SBrkr,1684,0,0,1684,0,0,2,0,2,1,Ex,6,Typ,1,Ex,Attchd,2005,RFn,2,564,TA,TA,Y,495,72,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,266000\n345,160,RM,36,2592,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,MeadowV,Norm,Norm,TwnhsE,2Story,5,3,1976,1976,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,CBlock,Gd,TA,No,Rec,129,BLQ,232,175,536,GasA,TA,Y,SBrkr,536,576,0,1112,0,0,1,1,3,1,TA,4,Typ,0,NA,Attchd,1976,Unf,1,336,TA,TA,Y,182,0,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,85000\n346,50,RL,65,6435,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,RRAn,Norm,1Fam,1.5Fin,6,5,1939,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,972,972,GasA,Gd,Y,SBrkr,972,605,0,1577,0,0,1,0,3,1,Fa,6,Typ,1,Gd,Detchd,1939,Unf,1,312,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,10,2006,WD,Normal,140200\n347,20,RL,NA,12772,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,8,1960,1998,Hip,CompShg,MetalSd,MetalSd,None,0,TA,Gd,CBlock,TA,TA,Mn,BLQ,498,Unf,0,460,958,GasA,TA,Y,SBrkr,958,0,0,958,0,0,1,0,2,1,TA,5,Typ,0,NA,Attchd,1960,RFn,1,301,TA,TA,Y,0,0,0,0,0,0,NA,NA,Gar2,15500,4,2007,WD,Normal,151500\n348,20,RL,NA,17600,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1960,1960,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,30,TA,TA,CBlock,TA,TA,No,BLQ,1270,Unf,0,208,1478,GasA,Ex,Y,FuseA,1478,0,0,1478,1,0,2,0,3,1,TA,6,Typ,2,Gd,Attchd,1960,Unf,2,498,TA,TA,Y,0,40,0,0,0,0,NA,NA,NA,0,12,2009,WD,Normal,157500\n349,160,RL,36,2448,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,Twnhs,2Story,7,5,2003,2004,Gable,CompShg,VinylSd,Wd Shng,Stone,106,Gd,TA,PConc,Gd,TA,No,GLQ,573,Unf,0,191,764,GasA,Ex,Y,SBrkr,764,862,0,1626,1,0,2,1,2,1,Gd,6,Typ,0,NA,BuiltIn,2003,RFn,2,474,TA,TA,Y,0,27,0,0,0,0,NA,NA,NA,0,10,2008,WD,Normal,154000\n350,60,RL,56,20431,Pave,NA,IR2,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,9,5,2005,2006,Hip,CompShg,CemntBd,CmentBd,BrkFace,870,Ex,TA,PConc,Ex,TA,No,GLQ,1410,Unf,0,438,1848,GasA,Ex,Y,SBrkr,1848,880,0,2728,1,0,2,1,4,1,Ex,10,Typ,2,Ex,Attchd,2006,Fin,3,706,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2006,New,Partial,437154\n351,120,RL,68,7820,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,9,5,2007,2007,Hip,CompShg,MetalSd,MetalSd,BrkFace,362,Ex,TA,PConc,Ex,TA,No,Unf,0,Unf,0,1869,1869,GasA,Ex,Y,SBrkr,1869,0,0,1869,0,0,2,0,2,1,Ex,6,Typ,1,Gd,Attchd,2007,RFn,2,617,TA,TA,Y,210,54,0,0,0,0,NA,NA,NA,0,12,2007,New,Partial,318061\n352,120,RL,NA,5271,Pave,NA,IR1,Low,AllPub,Inside,Mod,ClearCr,Norm,Norm,1Fam,1Story,7,5,1986,1986,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,Gd,TA,Gd,GLQ,1082,Unf,0,371,1453,GasA,Gd,Y,SBrkr,1453,0,0,1453,1,0,1,1,2,1,Gd,6,Typ,1,TA,Attchd,1986,RFn,2,445,TA,TA,Y,0,80,0,0,184,0,NA,NA,NA,0,12,2006,WD,Abnorml,190000\n353,50,RL,60,9084,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Artery,Norm,1Fam,1.5Fin,5,6,1941,1950,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,Fa,Mn,LwQ,236,Rec,380,0,616,GasA,TA,N,SBrkr,616,495,0,1111,0,1,1,0,3,1,TA,5,Typ,0,NA,Detchd,1941,Unf,1,200,TA,Fa,Y,48,0,0,0,0,0,NA,NA,NA,0,3,2008,ConLw,Normal,95000\n354,30,RM,60,8520,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,6,8,1928,2003,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,BrkTil,TA,TA,No,Unf,0,Unf,0,624,624,GasA,Gd,Y,SBrkr,720,0,0,720,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,2005,Unf,2,484,TA,TA,Y,106,0,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,105900\n355,50,RL,60,8400,Pave,NA,Reg,Bnk,AllPub,Inside,Gtl,SWISU,Norm,Norm,1Fam,1.5Fin,6,5,1940,2000,Gable,CompShg,Wd Sdng,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,LwQ,388,Unf,0,552,940,GasA,Ex,Y,SBrkr,1192,403,0,1595,0,0,1,0,2,1,TA,6,Typ,2,Gd,Attchd,1940,Unf,1,240,TA,TA,Y,0,0,108,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,140000\n356,20,RL,105,11249,Pave,NA,IR2,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,6,5,1995,1995,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,Gd,PConc,Gd,Gd,No,ALQ,334,BLQ,544,322,1200,GasA,Ex,Y,SBrkr,1200,0,0,1200,1,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1995,RFn,2,521,TA,TA,Y,0,26,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,177500\n357,20,RL,NA,9248,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,1Story,6,6,1992,1992,Gable,CompShg,HdBoard,HdBoard,BrkFace,106,TA,TA,PConc,Gd,TA,No,GLQ,560,Unf,0,598,1158,GasA,Gd,Y,SBrkr,1167,0,0,1167,1,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1992,RFn,2,400,TA,TA,Y,120,26,0,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,173000\n358,120,RM,44,4224,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,MeadowV,Norm,Norm,TwnhsE,1Story,5,5,1976,1976,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,PConc,Gd,TA,No,ALQ,874,Unf,0,268,1142,GasA,TA,Y,SBrkr,1142,0,0,1142,1,0,1,1,3,1,TA,6,Typ,1,Po,Attchd,1976,Fin,2,528,TA,TA,Y,536,90,0,0,0,0,NA,MnPrv,NA,0,8,2007,WD,Normal,134000\n359,80,RL,92,6930,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,ClearCr,Norm,Norm,1Fam,SLvl,5,4,1958,1958,Hip,CompShg,Wd Sdng,ImStucc,BrkFace,120,TA,TA,CBlock,TA,TA,Av,BLQ,300,Rec,294,468,1062,GasA,Ex,Y,FuseF,1352,0,0,1352,0,1,1,0,3,1,Gd,6,Min2,0,NA,BuiltIn,1958,Unf,1,288,TA,TA,Y,168,0,294,0,0,0,NA,NA,NA,0,7,2006,WD,Abnorml,130000\n360,60,RL,78,12011,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NoRidge,Norm,Norm,1Fam,2Story,8,5,1998,1998,Gable,CompShg,VinylSd,VinylSd,BrkFace,530,Gd,TA,PConc,Gd,TA,Av,GLQ,956,Unf,0,130,1086,GasA,Ex,Y,SBrkr,1086,838,0,1924,1,0,2,1,3,1,Gd,7,Typ,1,TA,Attchd,1998,RFn,2,592,TA,TA,Y,208,75,0,0,374,0,NA,NA,NA,0,6,2006,WD,Normal,280000\n361,85,RL,NA,7540,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Mitchel,Norm,Norm,1Fam,SFoyer,6,6,1978,1978,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,Gd,TA,Av,GLQ,773,Unf,0,115,888,GasA,Ex,Y,SBrkr,912,0,0,912,1,0,1,0,2,1,TA,5,Typ,1,TA,Attchd,1978,RFn,2,470,TA,TA,Y,0,0,0,0,192,0,NA,MnPrv,NA,0,6,2007,WD,Normal,156000\n362,50,RL,NA,9144,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1.5Fin,5,5,1940,1982,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Rec,399,Unf,0,484,883,GasA,Gd,Y,SBrkr,988,517,0,1505,1,0,1,0,3,1,TA,8,Typ,0,NA,Detchd,1940,Unf,1,240,TA,TA,N,0,0,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,145000\n363,85,RL,64,7301,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,SFoyer,7,5,2003,2003,Gable,CompShg,HdBoard,HdBoard,BrkFace,500,Gd,TA,Slab,NA,NA,NA,NA,0,NA,0,0,0,GasA,Ex,Y,SBrkr,495,1427,0,1922,0,0,3,0,4,1,Gd,7,Typ,1,Ex,BuiltIn,2003,RFn,2,672,TA,TA,Y,0,0,177,0,0,0,NA,NA,NA,0,7,2009,ConLD,Normal,198500\n364,160,RM,21,1680,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrDale,Norm,Norm,Twnhs,2Story,6,8,1972,2007,Gable,CompShg,HdBoard,HdBoard,BrkFace,510,TA,TA,CBlock,TA,TA,No,ALQ,162,Unf,0,321,483,GasA,Gd,Y,SBrkr,483,504,0,987,0,0,1,1,2,1,Gd,5,Typ,0,NA,Detchd,1972,Unf,1,264,TA,TA,Y,250,0,0,0,0,0,NA,NA,NA,0,5,2009,WD,Normal,118000\n365,60,RL,NA,18800,Pave,NA,IR1,Lvl,AllPub,FR2,Gtl,NWAmes,Norm,Norm,1Fam,2Story,6,5,1976,1976,Gable,CompShg,HdBoard,HdBoard,BrkFace,120,TA,TA,PConc,Gd,TA,Mn,GLQ,712,Unf,0,84,796,GasA,TA,Y,SBrkr,790,784,0,1574,1,0,2,1,3,1,TA,6,Typ,1,TA,Attchd,1976,Fin,2,566,TA,TA,Y,306,111,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,190000\n366,70,RM,59,10690,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,2Story,5,7,1920,1997,Hip,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,Fa,No,Rec,456,Unf,0,216,672,GasA,Gd,Y,FuseA,672,672,0,1344,0,0,1,0,3,1,TA,6,Typ,0,NA,Detchd,1964,Unf,1,468,TA,Fa,Y,0,128,218,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,147000\n367,20,RL,NA,9500,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1963,1963,Gable,CompShg,Plywood,Plywood,BrkFace,247,TA,TA,CBlock,Gd,TA,No,BLQ,609,Unf,0,785,1394,GasA,Gd,Y,SBrkr,1394,0,0,1394,1,0,1,1,3,1,TA,6,Typ,2,Gd,Attchd,1963,RFn,2,514,TA,TA,Y,0,76,0,0,185,0,NA,NA,NA,0,7,2009,WD,Normal,159000\n368,80,RL,101,9150,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,SLvl,6,5,1962,1962,Gable,Tar&Grv,Plywood,Plywood,BrkFace,305,TA,TA,CBlock,Gd,TA,Gd,GLQ,371,Unf,0,728,1099,GasA,Gd,Y,SBrkr,1431,0,0,1431,0,1,1,0,3,1,TA,6,Typ,1,Gd,Basment,1962,RFn,1,296,TA,TA,Y,64,110,0,0,0,0,NA,NA,NA,0,12,2008,WD,Normal,165000\n369,20,RL,78,7800,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1954,1954,Gable,CompShg,HdBoard,HdBoard,BrkFace,200,TA,TA,PConc,TA,TA,No,LwQ,540,Unf,0,728,1268,GasA,Gd,Y,SBrkr,1268,0,0,1268,0,0,1,0,2,1,TA,7,Typ,1,Gd,Attchd,1954,Fin,1,244,TA,TA,Y,0,98,0,0,0,0,NA,NA,NA,0,3,2010,WD,Normal,132000\n370,20,RL,NA,9830,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1959,2006,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,Gd,CBlock,TA,TA,No,ALQ,72,Rec,258,733,1063,GasA,Ex,Y,SBrkr,1287,0,0,1287,1,0,1,0,3,1,Gd,7,Typ,1,Gd,Detchd,1997,Fin,2,576,TA,TA,Y,364,17,0,0,182,0,NA,NA,NA,0,3,2010,WD,Normal,162000\n371,60,RL,NA,8121,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,2000,2000,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,953,953,GasA,Ex,Y,SBrkr,953,711,0,1664,0,0,2,1,3,1,TA,7,Typ,1,TA,Attchd,2000,RFn,2,460,TA,TA,Y,100,40,0,0,0,0,NA,NA,NA,0,1,2006,WD,Normal,172400\n372,50,RL,80,17120,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,ClearCr,Feedr,Norm,1Fam,1.5Fin,4,4,1959,1959,Gable,CompShg,WdShing,Plywood,None,0,TA,TA,CBlock,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,SBrkr,1120,468,0,1588,0,0,2,0,4,1,TA,7,Min2,1,Gd,Detchd,1991,Fin,2,680,TA,TA,N,0,59,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,134432\n373,120,RL,50,7175,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,TwnhsE,1Story,6,5,1984,1984,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,Gd,TA,No,ALQ,623,LwQ,121,0,744,GasA,TA,Y,SBrkr,752,0,0,752,1,0,1,0,2,1,TA,4,Typ,0,NA,Attchd,1984,Unf,1,264,TA,TA,Y,353,0,0,0,90,0,NA,MnPrv,NA,0,2,2010,WD,Normal,125000\n374,20,RL,79,10634,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1953,1953,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,PConc,TA,TA,No,BLQ,428,LwQ,180,0,608,GasA,TA,Y,SBrkr,1319,0,0,1319,1,0,1,0,3,1,TA,5,Min2,0,NA,Attchd,1953,Unf,1,270,TA,TA,Y,66,0,0,0,0,0,NA,GdWo,NA,0,11,2009,WD,Normal,123000\n375,60,RL,65,8200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2003,2004,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,847,847,GasA,Ex,Y,SBrkr,847,1081,0,1928,0,0,2,1,4,1,Gd,8,Typ,1,Gd,BuiltIn,2003,Fin,2,434,TA,TA,Y,100,48,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,219500\n376,30,RL,NA,10020,Pave,NA,IR1,Low,AllPub,Inside,Sev,Edwards,Norm,Norm,1Fam,1Story,1,1,1922,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,Fa,Fa,BrkTil,Fa,Po,Gd,BLQ,350,Unf,0,333,683,GasA,Gd,N,FuseA,904,0,0,904,1,0,0,1,1,1,Fa,4,Maj1,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,3,2009,WD,Normal,61000\n377,85,RL,57,8846,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,CollgCr,Norm,Norm,1Fam,SFoyer,5,5,1996,1996,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,GLQ,298,Unf,0,572,870,GasA,Ex,Y,SBrkr,914,0,0,914,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1998,Unf,2,576,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,148000\n378,60,FV,102,11143,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Somerst,Norm,Norm,1Fam,2Story,8,5,2004,2005,Gable,CompShg,CemntBd,CmentBd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1580,1580,GasA,Ex,Y,SBrkr,1580,886,0,2466,0,0,3,0,4,1,Gd,8,Typ,1,Gd,Attchd,2004,RFn,2,610,TA,TA,Y,159,214,0,0,0,0,NA,NA,NA,0,12,2007,WD,Normal,340000\n379,20,RL,88,11394,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,StoneBr,Norm,Norm,1Fam,1Story,9,2,2010,2010,Hip,CompShg,VinylSd,VinylSd,Stone,350,Gd,TA,PConc,Ex,TA,Av,GLQ,1445,Unf,0,411,1856,GasA,Ex,Y,SBrkr,1856,0,0,1856,1,0,1,1,1,1,Ex,8,Typ,1,Ex,Attchd,2010,Fin,3,834,TA,TA,Y,113,0,0,0,0,0,NA,NA,NA,0,6,2010,New,Partial,394432\n380,60,RL,60,8123,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,RRAn,Norm,1Fam,2Story,6,5,2000,2000,Gable,CompShg,VinylSd,VinylSd,BrkFace,16,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,982,982,GasA,Ex,Y,SBrkr,1007,793,0,1800,0,0,2,1,3,1,TA,7,Typ,1,TA,Attchd,2000,Fin,2,463,TA,TA,Y,100,63,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,179000\n381,50,RL,50,5000,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,SWISU,Norm,Norm,1Fam,1.5Fin,5,6,1924,1950,Gable,CompShg,BrkFace,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,LwQ,218,Unf,0,808,1026,GasA,TA,Y,SBrkr,1026,665,0,1691,0,0,2,0,3,1,Gd,6,Typ,1,Gd,Detchd,1924,Unf,1,308,TA,TA,Y,0,0,242,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,127000\n382,20,FV,60,7200,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,7,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,Gd,No,Unf,0,Unf,0,1293,1293,GasA,Ex,Y,SBrkr,1301,0,0,1301,1,0,2,0,2,1,Gd,5,Typ,1,Gd,Attchd,2006,RFn,2,572,TA,TA,Y,216,121,0,0,0,0,NA,NA,NA,0,8,2006,New,Partial,187750\n383,60,RL,79,9245,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,939,939,GasA,Ex,Y,SBrkr,939,858,0,1797,0,0,2,1,3,1,Gd,8,Typ,0,NA,Attchd,2006,RFn,2,639,TA,TA,Y,144,53,0,0,0,0,NA,NA,NA,0,4,2007,WD,Normal,213500\n384,45,RH,60,9000,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,SawyerW,Norm,Norm,1Fam,1.5Unf,6,3,1928,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,Fa,Fa,No,Unf,0,Unf,0,784,784,GasA,TA,N,FuseA,784,0,0,784,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1950,Unf,2,360,Fa,Fa,N,0,0,91,0,0,0,NA,NA,NA,0,10,2009,WD,Normal,76000\n385,60,RL,NA,53107,Pave,NA,IR2,Low,AllPub,Corner,Mod,ClearCr,Feedr,Norm,1Fam,2Story,6,5,1992,1992,Gable,CompShg,HdBoard,HdBoard,None,0,Gd,TA,PConc,Gd,TA,Av,GLQ,985,Unf,0,595,1580,GasA,Ex,Y,SBrkr,1079,874,0,1953,1,0,2,1,3,1,Gd,9,Typ,2,Fa,Attchd,1992,Fin,2,501,TA,TA,Y,216,231,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,240000\n386,120,RL,43,3182,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Blmngtn,Norm,Norm,TwnhsE,1Story,8,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,BrkFace,16,Gd,TA,PConc,Gd,TA,No,GLQ,24,Unf,0,1232,1256,GasA,Ex,Y,SBrkr,1269,0,0,1269,0,0,2,0,2,1,Gd,6,Typ,1,TA,Attchd,2004,Fin,2,430,TA,TA,Y,146,20,0,0,144,0,NA,NA,NA,0,4,2010,WD,Normal,192000\n387,50,RL,58,8410,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,Edwards,Feedr,Norm,1Fam,1.5Fin,5,3,1910,1996,Gambrel,CompShg,Wd Sdng,VinylSd,None,0,TA,Fa,PConc,TA,TA,No,Unf,0,Unf,0,658,658,GasA,TA,Y,SBrkr,658,526,0,1184,0,0,1,0,5,1,TA,8,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,151,0,0,0,0,NA,NA,NA,0,5,2006,WD,AdjLand,81000\n388,80,RL,72,7200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,SLvl,6,6,1976,1976,Hip,CompShg,MetalSd,MetalSd,BrkFace,255,TA,TA,CBlock,TA,TA,Av,ALQ,631,Unf,0,410,1041,GasA,Ex,Y,SBrkr,1125,0,0,1125,1,0,1,0,3,1,TA,6,Typ,1,Fa,Detchd,1977,Unf,1,352,TA,TA,Y,296,0,0,0,0,0,NA,GdWo,NA,0,10,2009,WD,Abnorml,125000\n389,20,RL,93,9382,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,1999,2000,Gable,CompShg,VinylSd,VinylSd,BrkFace,125,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1468,1468,GasA,Ex,Y,SBrkr,1479,0,0,1479,0,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1999,RFn,2,577,TA,TA,Y,120,25,0,0,0,0,NA,NA,NA,0,7,2008,WD,Normal,191000\n390,60,RL,96,12474,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,10,5,2007,2008,Gable,CompShg,VinylSd,VinylSd,Stone,272,Ex,TA,PConc,Ex,TA,Av,GLQ,1280,Unf,0,402,1682,GasA,Ex,Y,SBrkr,1742,590,0,2332,1,0,2,1,3,1,Ex,9,Typ,1,Ex,BuiltIn,2008,Fin,3,846,TA,TA,Y,196,134,0,0,0,0,NA,NA,NA,0,8,2008,New,Partial,426000\n391,50,RL,50,8405,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1.5Fin,5,8,1900,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,Gd,No,Rec,241,BLQ,391,229,861,GasA,Ex,Y,SBrkr,961,406,0,1367,1,0,1,0,4,1,TA,7,Typ,0,NA,Detchd,1978,Unf,1,384,TA,TA,Y,0,130,112,0,0,0,NA,MnPrv,NA,0,4,2008,WD,Normal,119000\n392,60,RL,71,12209,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Mitchel,Norm,Norm,1Fam,2Story,6,5,2001,2002,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Ex,TA,No,ALQ,690,Unf,0,114,804,GasA,Ex,Y,SBrkr,804,1157,0,1961,1,0,2,1,3,1,Gd,7,Typ,1,TA,BuiltIn,2001,Fin,2,560,TA,TA,Y,125,192,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,215000\n393,20,RL,NA,8339,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1959,1959,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,Slab,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,SBrkr,882,0,0,882,0,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1959,RFn,1,294,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,Shed,1200,7,2007,WD,Normal,106500\n394,30,RL,NA,7446,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,BrkSide,Feedr,Norm,1Fam,1Story,4,5,1941,1950,Gable,CompShg,WdShing,Wd Shng,None,0,TA,TA,CBlock,TA,TA,No,Rec,266,Unf,0,522,788,GasA,TA,Y,FuseA,788,0,0,788,0,0,1,0,2,1,TA,4,Typ,2,TA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,GdWo,NA,0,4,2006,WD,Abnorml,100000\n395,50,RL,60,10134,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,5,6,1940,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,735,735,GasA,Gd,Y,FuseA,735,299,0,1034,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1940,Unf,1,240,TA,TA,Y,0,39,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,109000\n396,20,RL,68,9571,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1Story,5,6,1956,1956,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,Av,BLQ,739,Unf,0,405,1144,GasA,TA,Y,SBrkr,1144,0,0,1144,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1956,Unf,1,596,TA,TA,Y,44,0,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,129000\n397,20,RL,60,7200,Pave,NA,Reg,Low,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,5,1972,1972,Hip,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,Av,Rec,777,Unf,0,117,894,GasA,TA,Y,SBrkr,894,0,0,894,0,0,1,0,2,1,TA,6,Typ,0,NA,Detchd,1985,RFn,2,600,TA,TA,Y,215,0,0,0,0,0,NA,NA,NA,0,9,2009,WD,Normal,123000\n398,60,RL,69,7590,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,PosN,Norm,1Fam,2Story,5,5,1962,1962,Gable,CompShg,VinylSd,VinylSd,BrkFace,288,TA,TA,CBlock,TA,TA,No,ALQ,540,Unf,0,324,864,GasA,TA,Y,SBrkr,876,936,0,1812,0,0,2,0,4,1,TA,8,Typ,1,TA,Attchd,1962,RFn,1,264,TA,TA,Y,0,168,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,169500\n399,30,RM,60,8967,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,IDOTRR,Norm,Norm,1Fam,1Story,5,2,1920,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Fa,BrkTil,Fa,Po,No,Unf,0,Unf,0,961,961,GasA,Gd,Y,Mix,1077,0,0,1077,0,0,1,0,2,1,TA,6,Maj2,0,NA,Detchd,1920,Unf,1,338,Po,Po,N,0,0,0,0,0,0,NA,NA,NA,0,11,2007,WD,Abnorml,67000\n400,60,FV,65,8125,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,7,5,2006,2007,Gable,CompShg,CemntBd,CmentBd,Stone,100,Gd,TA,PConc,Gd,TA,No,GLQ,812,Unf,0,280,1092,GasA,Ex,Y,SBrkr,1112,438,0,1550,1,0,2,0,2,1,Gd,7,Typ,0,NA,Attchd,2007,Fin,2,438,TA,TA,Y,0,168,0,0,0,0,NA,NA,NA,0,10,2009,WD,Normal,241000\n"
},
{
"path": "eda/tests/unittests/resources/houses/train_data.csv",
"content": "Id,MSSubClass,MSZoning,LotFrontage,LotArea,Street,Alley,LotShape,LandContour,Utilities,LotConfig,LandSlope,Neighborhood,Condition1,Condition2,BldgType,HouseStyle,OverallQual,OverallCond,YearBuilt,YearRemodAdd,RoofStyle,RoofMatl,Exterior1st,Exterior2nd,MasVnrType,MasVnrArea,ExterQual,ExterCond,Foundation,BsmtQual,BsmtCond,BsmtExposure,BsmtFinType1,BsmtFinSF1,BsmtFinType2,BsmtFinSF2,BsmtUnfSF,TotalBsmtSF,Heating,HeatingQC,CentralAir,Electrical,1stFlrSF,2ndFlrSF,LowQualFinSF,GrLivArea,BsmtFullBath,BsmtHalfBath,FullBath,HalfBath,BedroomAbvGr,KitchenAbvGr,KitchenQual,TotRmsAbvGrd,Functional,Fireplaces,FireplaceQu,GarageType,GarageYrBlt,GarageFinish,GarageCars,GarageArea,GarageQual,GarageCond,PavedDrive,WoodDeckSF,OpenPorchSF,EnclosedPorch,3SsnPorch,ScreenPorch,PoolArea,PoolQC,Fence,MiscFeature,MiscVal,MoSold,YrSold,SaleType,SaleCondition,SalePrice\n1,60,RL,65,8450,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,BrkFace,196,Gd,TA,PConc,Gd,TA,No,GLQ,706,Unf,0,150,856,GasA,Ex,Y,SBrkr,856,854,0,1710,1,0,2,1,3,1,Gd,8,Typ,0,NA,Attchd,2003,RFn,2,548,TA,TA,Y,0,61,0,0,0,0,NA,NA,NA,0,2,2008,WD,Normal,208500\n2,20,RL,80,9600,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,Veenker,Feedr,Norm,1Fam,1Story,6,8,1976,1976,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,Gd,TA,Gd,ALQ,978,Unf,0,284,1262,GasA,Ex,Y,SBrkr,1262,0,0,1262,0,1,2,0,3,1,TA,6,Typ,1,TA,Attchd,1976,RFn,2,460,TA,TA,Y,298,0,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,181500\n3,60,RL,68,11250,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2001,2002,Gable,CompShg,VinylSd,VinylSd,BrkFace,162,Gd,TA,PConc,Gd,TA,Mn,GLQ,486,Unf,0,434,920,GasA,Ex,Y,SBrkr,920,866,0,1786,1,0,2,1,3,1,Gd,6,Typ,1,TA,Attchd,2001,RFn,2,608,TA,TA,Y,0,42,0,0,0,0,NA,NA,NA,0,9,2008,WD,Normal,223500\n4,70,RL,60,9550,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Crawfor,Norm,Norm,1Fam,2Story,7,5,1915,1970,Gable,CompShg,Wd Sdng,Wd Shng,None,0,TA,TA,BrkTil,TA,Gd,No,ALQ,216,Unf,0,540,756,GasA,Gd,Y,SBrkr,961,756,0,1717,1,0,1,0,3,1,Gd,7,Typ,1,Gd,Detchd,1998,Unf,3,642,TA,TA,Y,0,35,272,0,0,0,NA,NA,NA,0,2,2006,WD,Abnorml,140000\n5,60,RL,84,14260,Pave,NA,IR1,Lvl,AllPub,FR2,Gtl,NoRidge,Norm,Norm,1Fam,2Story,8,5,2000,2000,Gable,CompShg,VinylSd,VinylSd,BrkFace,350,Gd,TA,PConc,Gd,TA,Av,GLQ,655,Unf,0,490,1145,GasA,Ex,Y,SBrkr,1145,1053,0,2198,1,0,2,1,4,1,Gd,9,Typ,1,TA,Attchd,2000,RFn,3,836,TA,TA,Y,192,84,0,0,0,0,NA,NA,NA,0,12,2008,WD,Normal,250000\n6,50,RL,85,14115,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Mitchel,Norm,Norm,1Fam,1.5Fin,5,5,1993,1995,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,Wood,Gd,TA,No,GLQ,732,Unf,0,64,796,GasA,Ex,Y,SBrkr,796,566,0,1362,1,0,1,1,1,1,TA,5,Typ,0,NA,Attchd,1993,Unf,2,480,TA,TA,Y,40,30,0,320,0,0,NA,MnPrv,Shed,700,10,2009,WD,Normal,143000\n7,20,RL,75,10084,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,8,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,Stone,186,Gd,TA,PConc,Ex,TA,Av,GLQ,1369,Unf,0,317,1686,GasA,Ex,Y,SBrkr,1694,0,0,1694,1,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2004,RFn,2,636,TA,TA,Y,255,57,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,307000\n8,60,RL,NA,10382,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NWAmes,PosN,Norm,1Fam,2Story,7,6,1973,1973,Gable,CompShg,HdBoard,HdBoard,Stone,240,TA,TA,CBlock,Gd,TA,Mn,ALQ,859,BLQ,32,216,1107,GasA,Ex,Y,SBrkr,1107,983,0,2090,1,0,2,1,3,1,TA,7,Typ,2,TA,Attchd,1973,RFn,2,484,TA,TA,Y,235,204,228,0,0,0,NA,NA,Shed,350,11,2009,WD,Normal,200000\n9,50,RM,51,6120,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Artery,Norm,1Fam,1.5Fin,7,5,1931,1950,Gable,CompShg,BrkFace,Wd Shng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,952,952,GasA,Gd,Y,FuseF,1022,752,0,1774,0,0,2,0,2,2,TA,8,Min1,2,TA,Detchd,1931,Unf,2,468,Fa,TA,Y,90,0,205,0,0,0,NA,NA,NA,0,4,2008,WD,Abnorml,129900\n10,190,RL,50,7420,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,BrkSide,Artery,Artery,2fmCon,1.5Unf,5,6,1939,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,TA,No,GLQ,851,Unf,0,140,991,GasA,Ex,Y,SBrkr,1077,0,0,1077,1,0,1,0,2,2,TA,5,Typ,2,TA,Attchd,1939,RFn,1,205,Gd,TA,Y,0,4,0,0,0,0,NA,NA,NA,0,1,2008,WD,Normal,118000\n11,20,RL,70,11200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,5,1965,1965,Hip,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,Rec,906,Unf,0,134,1040,GasA,Ex,Y,SBrkr,1040,0,0,1040,1,0,1,0,3,1,TA,5,Typ,0,NA,Detchd,1965,Unf,1,384,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,2,2008,WD,Normal,129500\n12,60,RL,85,11924,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,9,5,2005,2006,Hip,CompShg,WdShing,Wd Shng,Stone,286,Ex,TA,PConc,Ex,TA,No,GLQ,998,Unf,0,177,1175,GasA,Ex,Y,SBrkr,1182,1142,0,2324,1,0,3,0,4,1,Ex,11,Typ,2,Gd,BuiltIn,2005,Fin,3,736,TA,TA,Y,147,21,0,0,0,0,NA,NA,NA,0,7,2006,New,Partial,345000\n13,20,RL,NA,12968,Pave,NA,IR2,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,6,1962,1962,Hip,CompShg,HdBoard,Plywood,None,0,TA,TA,CBlock,TA,TA,No,ALQ,737,Unf,0,175,912,GasA,TA,Y,SBrkr,912,0,0,912,1,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1962,Unf,1,352,TA,TA,Y,140,0,0,0,176,0,NA,NA,NA,0,9,2008,WD,Normal,144000\n14,20,RL,91,10652,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2006,2007,Gable,CompShg,VinylSd,VinylSd,Stone,306,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,1494,1494,GasA,Ex,Y,SBrkr,1494,0,0,1494,0,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2006,RFn,3,840,TA,TA,Y,160,33,0,0,0,0,NA,NA,NA,0,8,2007,New,Partial,279500\n15,20,RL,NA,10920,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1960,1960,Hip,CompShg,MetalSd,MetalSd,BrkFace,212,TA,TA,CBlock,TA,TA,No,BLQ,733,Unf,0,520,1253,GasA,TA,Y,SBrkr,1253,0,0,1253,1,0,1,1,2,1,TA,5,Typ,1,Fa,Attchd,1960,RFn,1,352,TA,TA,Y,0,213,176,0,0,0,NA,GdWo,NA,0,5,2008,WD,Normal,157000\n16,45,RM,51,6120,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,BrkSide,Norm,Norm,1Fam,1.5Unf,7,8,1929,2001,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,832,832,GasA,Ex,Y,FuseA,854,0,0,854,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1991,Unf,2,576,TA,TA,Y,48,112,0,0,0,0,NA,GdPrv,NA,0,7,2007,WD,Normal,132000\n17,20,RL,NA,11241,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,7,1970,1970,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,180,TA,TA,CBlock,TA,TA,No,ALQ,578,Unf,0,426,1004,GasA,Ex,Y,SBrkr,1004,0,0,1004,1,0,1,0,2,1,TA,5,Typ,1,TA,Attchd,1970,Fin,2,480,TA,TA,Y,0,0,0,0,0,0,NA,NA,Shed,700,3,2010,WD,Normal,149000\n18,90,RL,72,10791,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,Duplex,1Story,4,5,1967,1967,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,Slab,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,SBrkr,1296,0,0,1296,0,0,2,0,2,2,TA,6,Typ,0,NA,CarPort,1967,Unf,2,516,TA,TA,Y,0,0,0,0,0,0,NA,NA,Shed,500,10,2006,WD,Normal,90000\n19,20,RL,66,13695,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,RRAe,Norm,1Fam,1Story,5,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,TA,TA,No,GLQ,646,Unf,0,468,1114,GasA,Ex,Y,SBrkr,1114,0,0,1114,1,0,1,1,3,1,Gd,6,Typ,0,NA,Detchd,2004,Unf,2,576,TA,TA,Y,0,102,0,0,0,0,NA,NA,NA,0,6,2008,WD,Normal,159000\n20,20,RL,70,7560,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1958,1965,Hip,CompShg,BrkFace,Plywood,None,0,TA,TA,CBlock,TA,TA,No,LwQ,504,Unf,0,525,1029,GasA,TA,Y,SBrkr,1339,0,0,1339,0,0,1,0,3,1,TA,6,Min1,0,NA,Attchd,1958,Unf,1,294,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,5,2009,COD,Abnorml,139000\n21,60,RL,101,14215,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NridgHt,Norm,Norm,1Fam,2Story,8,5,2005,2006,Gable,CompShg,VinylSd,VinylSd,BrkFace,380,Gd,TA,PConc,Ex,TA,Av,Unf,0,Unf,0,1158,1158,GasA,Ex,Y,SBrkr,1158,1218,0,2376,0,0,3,1,4,1,Gd,9,Typ,1,Gd,BuiltIn,2005,RFn,3,853,TA,TA,Y,240,154,0,0,0,0,NA,NA,NA,0,11,2006,New,Partial,325300\n22,45,RM,57,7449,Pave,Grvl,Reg,Bnk,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,1.5Unf,7,7,1930,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,TA,TA,No,Unf,0,Unf,0,637,637,GasA,Ex,Y,FuseF,1108,0,0,1108,0,0,1,0,3,1,Gd,6,Typ,1,Gd,Attchd,1930,Unf,1,280,TA,TA,N,0,0,205,0,0,0,NA,GdPrv,NA,0,6,2007,WD,Normal,139400\n23,20,RL,75,9742,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,8,5,2002,2002,Hip,CompShg,VinylSd,VinylSd,BrkFace,281,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1777,1777,GasA,Ex,Y,SBrkr,1795,0,0,1795,0,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2002,RFn,2,534,TA,TA,Y,171,159,0,0,0,0,NA,NA,NA,0,9,2008,WD,Normal,230000\n24,120,RM,44,4224,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,MeadowV,Norm,Norm,TwnhsE,1Story,5,7,1976,1976,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,PConc,Gd,TA,No,GLQ,840,Unf,0,200,1040,GasA,TA,Y,SBrkr,1060,0,0,1060,1,0,1,0,3,1,TA,6,Typ,1,TA,Attchd,1976,Unf,2,572,TA,TA,Y,100,110,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,129900\n25,20,RL,NA,8246,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,8,1968,2001,Gable,CompShg,Plywood,Plywood,None,0,TA,Gd,CBlock,TA,TA,Mn,Rec,188,ALQ,668,204,1060,GasA,Ex,Y,SBrkr,1060,0,0,1060,1,0,1,0,3,1,Gd,6,Typ,1,TA,Attchd,1968,Unf,1,270,TA,TA,Y,406,90,0,0,0,0,NA,MnPrv,NA,0,5,2010,WD,Normal,154000\n26,20,RL,110,14230,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2007,2007,Gable,CompShg,VinylSd,VinylSd,Stone,640,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1566,1566,GasA,Ex,Y,SBrkr,1600,0,0,1600,0,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2007,RFn,3,890,TA,TA,Y,0,56,0,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,256300\n27,20,RL,60,7200,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1951,2000,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,Mn,BLQ,234,Rec,486,180,900,GasA,TA,Y,SBrkr,900,0,0,900,0,1,1,0,3,1,Gd,5,Typ,0,NA,Detchd,2005,Unf,2,576,TA,TA,Y,222,32,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,134800\n28,20,RL,98,11478,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2007,2008,Gable,CompShg,VinylSd,VinylSd,Stone,200,Gd,TA,PConc,Ex,TA,No,GLQ,1218,Unf,0,486,1704,GasA,Ex,Y,SBrkr,1704,0,0,1704,1,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2008,RFn,3,772,TA,TA,Y,0,50,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,306000\n29,20,RL,47,16321,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1957,1997,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,Gd,BLQ,1277,Unf,0,207,1484,GasA,TA,Y,SBrkr,1600,0,0,1600,1,0,1,0,2,1,TA,6,Typ,2,Gd,Attchd,1957,RFn,1,319,TA,TA,Y,288,258,0,0,0,0,NA,NA,NA,0,12,2006,WD,Normal,207500\n30,30,RM,60,6324,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,BrkSide,Feedr,RRNn,1Fam,1Story,4,6,1927,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,520,520,GasA,Fa,N,SBrkr,520,0,0,520,0,0,1,0,1,1,Fa,4,Typ,0,NA,Detchd,1920,Unf,1,240,Fa,TA,Y,49,0,87,0,0,0,NA,NA,NA,0,5,2008,WD,Normal,68500\n31,70,C (all),50,8500,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Feedr,Norm,1Fam,2Story,4,4,1920,1950,Gambrel,CompShg,BrkFace,BrkFace,None,0,TA,Fa,BrkTil,TA,TA,No,Unf,0,Unf,0,649,649,GasA,TA,N,SBrkr,649,668,0,1317,0,0,1,0,3,1,TA,6,Typ,0,NA,Detchd,1920,Unf,1,250,TA,Fa,N,0,54,172,0,0,0,NA,MnPrv,NA,0,7,2008,WD,Normal,40000\n32,20,RL,NA,8544,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,6,1966,2006,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1228,1228,GasA,Gd,Y,SBrkr,1228,0,0,1228,0,0,1,1,3,1,Gd,6,Typ,0,NA,Attchd,1966,Unf,1,271,TA,TA,Y,0,65,0,0,0,0,NA,MnPrv,NA,0,6,2008,WD,Normal,149350\n33,20,RL,85,11049,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,8,5,2007,2007,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Ex,TA,Av,Unf,0,Unf,0,1234,1234,GasA,Ex,Y,SBrkr,1234,0,0,1234,0,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2007,RFn,2,484,TA,TA,Y,0,30,0,0,0,0,NA,NA,NA,0,1,2008,WD,Normal,179900\n34,20,RL,70,10552,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,5,1959,1959,Hip,CompShg,BrkFace,BrkFace,None,0,TA,TA,CBlock,TA,TA,No,Rec,1018,Unf,0,380,1398,GasA,Gd,Y,SBrkr,1700,0,0,1700,0,1,1,1,4,1,Gd,6,Typ,1,Gd,Attchd,1959,RFn,2,447,TA,TA,Y,0,38,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,165500\n35,120,RL,60,7313,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,9,5,2005,2005,Hip,CompShg,MetalSd,MetalSd,BrkFace,246,Ex,TA,PConc,Ex,TA,No,GLQ,1153,Unf,0,408,1561,GasA,Ex,Y,SBrkr,1561,0,0,1561,1,0,2,0,2,1,Ex,6,Typ,1,Gd,Attchd,2005,Fin,2,556,TA,TA,Y,203,47,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,277500\n36,60,RL,108,13418,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,8,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,Stone,132,Gd,TA,PConc,Ex,TA,Av,Unf,0,Unf,0,1117,1117,GasA,Ex,Y,SBrkr,1132,1320,0,2452,0,0,3,1,4,1,Gd,9,Typ,1,Gd,BuiltIn,2004,Fin,3,691,TA,TA,Y,113,32,0,0,0,0,NA,NA,NA,0,9,2006,WD,Normal,309000\n37,20,RL,112,10859,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,5,1994,1995,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1097,1097,GasA,Ex,Y,SBrkr,1097,0,0,1097,0,0,1,1,3,1,TA,6,Typ,0,NA,Attchd,1995,Unf,2,672,TA,TA,Y,392,64,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,145000\n38,20,RL,74,8532,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1954,1990,Hip,CompShg,Wd Sdng,Wd Sdng,BrkFace,650,TA,TA,CBlock,TA,TA,No,Rec,1213,Unf,0,84,1297,GasA,Gd,Y,SBrkr,1297,0,0,1297,0,1,1,0,3,1,TA,5,Typ,1,TA,Attchd,1954,Fin,2,498,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,10,2009,WD,Normal,153000\n39,20,RL,68,7922,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1953,2007,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,TA,No,GLQ,731,Unf,0,326,1057,GasA,TA,Y,SBrkr,1057,0,0,1057,1,0,1,0,3,1,Gd,5,Typ,0,NA,Detchd,1953,Unf,1,246,TA,TA,Y,0,52,0,0,0,0,NA,NA,NA,0,1,2010,WD,Abnorml,109000\n40,90,RL,65,6040,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,Duplex,1Story,4,5,1955,1955,Gable,CompShg,AsbShng,Plywood,None,0,TA,TA,PConc,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,N,FuseP,1152,0,0,1152,0,0,2,0,2,2,Fa,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,0,0,0,0,0,NA,NA,NA,0,6,2008,WD,AdjLand,82000\n41,20,RL,84,8658,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1965,1965,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,101,TA,TA,CBlock,TA,TA,No,Rec,643,Unf,0,445,1088,GasA,Ex,Y,SBrkr,1324,0,0,1324,0,0,2,0,3,1,TA,6,Typ,1,TA,Attchd,1965,RFn,2,440,TA,TA,Y,0,138,0,0,0,0,NA,GdWo,NA,0,12,2006,WD,Abnorml,160000\n42,20,RL,115,16905,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Timber,Norm,Norm,1Fam,1Story,5,6,1959,1959,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,CBlock,TA,TA,Gd,BLQ,967,Unf,0,383,1350,GasA,Gd,Y,SBrkr,1328,0,0,1328,0,1,1,1,2,1,TA,5,Typ,2,Gd,Attchd,1959,RFn,1,308,TA,TA,P,0,104,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,170000\n43,85,RL,NA,9180,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,SawyerW,Norm,Norm,1Fam,SFoyer,5,7,1983,1983,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,Gd,TA,Av,ALQ,747,LwQ,93,0,840,GasA,Gd,Y,SBrkr,884,0,0,884,1,0,1,0,2,1,Gd,5,Typ,0,NA,Attchd,1983,RFn,2,504,TA,Gd,Y,240,0,0,0,0,0,NA,MnPrv,NA,0,12,2007,WD,Normal,144000\n44,20,RL,NA,9200,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,6,1975,1980,Hip,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,Gd,TA,Av,LwQ,280,BLQ,491,167,938,GasA,TA,Y,SBrkr,938,0,0,938,1,0,1,0,3,1,TA,5,Typ,0,NA,Detchd,1977,Unf,1,308,TA,TA,Y,145,0,0,0,0,0,NA,MnPrv,NA,0,7,2008,WD,Normal,130250\n45,20,RL,70,7945,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1959,1959,Gable,CompShg,BrkFace,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,ALQ,179,BLQ,506,465,1150,GasA,Ex,Y,FuseA,1150,0,0,1150,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1959,RFn,1,300,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,5,2006,WD,Normal,141000\n46,120,RL,61,7658,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,9,5,2005,2005,Hip,CompShg,MetalSd,MetalSd,BrkFace,412,Ex,TA,PConc,Ex,TA,No,GLQ,456,Unf,0,1296,1752,GasA,Ex,Y,SBrkr,1752,0,0,1752,1,0,2,0,2,1,Ex,6,Typ,1,Gd,Attchd,2005,RFn,2,576,TA,TA,Y,196,82,0,0,0,0,NA,NA,NA,0,2,2010,WD,Normal,319900\n47,50,RL,48,12822,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Mitchel,Norm,Norm,1Fam,1.5Fin,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Ex,TA,No,GLQ,1351,Unf,0,83,1434,GasA,Ex,Y,SBrkr,1518,631,0,2149,1,0,1,1,1,1,Gd,6,Typ,1,Ex,Attchd,2003,RFn,2,670,TA,TA,Y,168,43,0,0,198,0,NA,NA,NA,0,8,2009,WD,Abnorml,239686\n48,20,FV,84,11096,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,8,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,GLQ,24,Unf,0,1632,1656,GasA,Ex,Y,SBrkr,1656,0,0,1656,0,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2006,RFn,3,826,TA,TA,Y,0,146,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,249700\n49,190,RM,33,4456,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,2fmCon,2Story,4,5,1920,2008,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,736,736,GasA,Gd,Y,SBrkr,736,716,0,1452,0,0,2,0,2,3,TA,8,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,0,102,0,0,0,NA,NA,NA,0,6,2009,New,Partial,113000\n50,20,RL,66,7742,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,7,1966,1966,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,BLQ,763,Unf,0,192,955,GasA,Ex,Y,SBrkr,955,0,0,955,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1966,Unf,1,386,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,1,2007,WD,Normal,127000\n51,60,RL,NA,13869,Pave,NA,IR2,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,6,1997,1997,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,Av,GLQ,182,Unf,0,612,794,GasA,Gd,Y,SBrkr,794,676,0,1470,0,1,2,0,3,1,TA,6,Typ,0,NA,Attchd,1997,Fin,2,388,TA,TA,Y,0,75,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,177000\n52,50,RM,52,6240,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1.5Fin,6,6,1934,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,TA,TA,No,Unf,0,Unf,0,816,816,GasA,TA,Y,SBrkr,816,0,360,1176,0,0,1,0,3,1,TA,6,Typ,1,Gd,Detchd,1985,Unf,2,528,TA,TA,Y,112,0,0,0,0,0,NA,MnPrv,Shed,400,9,2006,WD,Normal,114500\n53,90,RM,110,8472,Grvl,NA,IR2,Bnk,AllPub,Corner,Mod,IDOTRR,RRNn,Norm,Duplex,1Story,5,5,1963,1963,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,Fa,TA,CBlock,Gd,TA,Gd,LwQ,104,GLQ,712,0,816,GasA,TA,N,SBrkr,816,0,0,816,1,0,1,0,2,1,TA,5,Typ,0,NA,CarPort,1963,Unf,2,516,TA,TA,Y,106,0,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,110000\n54,20,RL,68,50271,Pave,NA,IR1,Low,AllPub,Inside,Gtl,Veenker,Norm,Norm,1Fam,1Story,9,5,1981,1987,Gable,WdShngl,WdShing,Wd Shng,None,0,Gd,TA,CBlock,Ex,TA,Gd,GLQ,1810,Unf,0,32,1842,GasA,Gd,Y,SBrkr,1842,0,0,1842,2,0,0,1,0,1,Gd,5,Typ,1,Gd,Attchd,1981,Fin,3,894,TA,TA,Y,857,72,0,0,0,0,NA,NA,NA,0,11,2006,WD,Normal,385000\n55,80,RL,60,7134,Pave,NA,Reg,Bnk,AllPub,Inside,Mod,NAmes,Norm,Norm,1Fam,SLvl,5,5,1955,1955,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,ALQ,384,Unf,0,0,384,GasA,TA,Y,SBrkr,1360,0,0,1360,0,0,1,0,3,1,TA,6,Min1,1,TA,Detchd,1962,Unf,2,572,TA,TA,Y,0,50,0,0,0,0,NA,MnPrv,NA,0,2,2007,WD,Normal,130000\n56,20,RL,100,10175,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1964,1964,Gable,CompShg,HdBoard,Plywood,BrkFace,272,TA,TA,CBlock,TA,TA,No,BLQ,490,Unf,0,935,1425,GasA,Gd,Y,SBrkr,1425,0,0,1425,0,0,2,0,3,1,TA,7,Typ,1,Gd,Attchd,1964,RFn,2,576,TA,TA,Y,0,0,0,407,0,0,NA,NA,NA,0,7,2008,WD,Normal,180500\n57,160,FV,24,2645,Pave,Pave,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,Twnhs,2Story,8,5,1999,2000,Gable,CompShg,MetalSd,MetalSd,BrkFace,456,Gd,TA,PConc,Gd,TA,No,GLQ,649,Unf,0,321,970,GasA,Ex,Y,SBrkr,983,756,0,1739,1,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,1999,Fin,2,480,TA,TA,Y,115,0,0,0,0,0,NA,NA,NA,0,8,2009,WD,Abnorml,172500\n58,60,RL,89,11645,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,860,860,GasA,Ex,Y,SBrkr,860,860,0,1720,0,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,2004,RFn,2,565,TA,TA,Y,0,70,0,0,0,0,NA,NA,NA,0,8,2006,WD,Normal,196500\n59,60,RL,66,13682,Pave,NA,IR2,HLS,AllPub,CulDSac,Gtl,StoneBr,Norm,Norm,1Fam,2Story,10,5,2006,2006,Hip,CompShg,VinylSd,VinylSd,BrkFace,1031,Ex,TA,PConc,Ex,TA,Gd,Unf,0,Unf,0,1410,1410,GasA,Ex,Y,SBrkr,1426,1519,0,2945,0,0,3,1,3,1,Gd,10,Typ,1,Gd,BuiltIn,2006,Fin,3,641,TA,TA,Y,192,0,37,0,0,0,NA,NA,NA,0,10,2006,New,Partial,438780\n60,20,RL,60,7200,Pave,NA,Reg,Bnk,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,7,1972,1972,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,Av,ALQ,632,Unf,0,148,780,GasA,Ex,Y,SBrkr,780,0,0,780,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1973,Unf,1,352,TA,TA,Y,196,0,0,0,0,0,NA,MnPrv,NA,0,1,2008,WD,Normal,124900\n61,20,RL,63,13072,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,RRAe,Norm,1Fam,1Story,6,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,ALQ,941,Unf,0,217,1158,GasA,Ex,Y,SBrkr,1158,0,0,1158,1,0,1,1,3,1,Gd,5,Typ,0,NA,Detchd,2006,Unf,2,576,TA,TA,Y,0,50,0,0,0,0,NA,NA,NA,0,5,2006,New,Partial,158000\n62,75,RM,60,7200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,2.5Unf,5,7,1920,1996,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,Fa,No,Unf,0,Unf,0,530,530,GasA,TA,N,SBrkr,581,530,0,1111,0,0,1,0,3,1,Fa,6,Typ,0,NA,Detchd,1935,Unf,1,288,TA,TA,N,0,0,144,0,0,0,NA,NA,NA,0,3,2007,WD,Normal,101000\n63,120,RL,44,6442,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,TwnhsE,1Story,8,5,2006,2006,Gable,CompShg,VinylSd,VinylSd,Stone,178,Gd,TA,PConc,Gd,Gd,Mn,GLQ,24,Unf,0,1346,1370,GasA,Ex,Y,SBrkr,1370,0,0,1370,0,0,2,0,2,1,Gd,6,Typ,1,Gd,Attchd,2006,RFn,2,484,TA,TA,Y,120,49,0,0,0,0,NA,NA,NA,0,10,2007,WD,Normal,202500\n64,70,RM,50,10300,Pave,NA,IR1,Bnk,AllPub,Inside,Gtl,OldTown,RRAn,Feedr,1Fam,2Story,7,6,1921,1950,Gable,CompShg,Stucco,Stucco,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,576,576,GasA,Gd,Y,SBrkr,902,808,0,1710,0,0,2,0,3,1,TA,9,Typ,0,NA,Detchd,1990,Unf,2,480,TA,TA,Y,12,11,64,0,0,0,NA,GdPrv,NA,0,4,2010,WD,Normal,140000\n65,60,RL,NA,9375,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,1997,1998,Gable,CompShg,VinylSd,VinylSd,BrkFace,573,TA,TA,PConc,Gd,TA,No,GLQ,739,Unf,0,318,1057,GasA,Ex,Y,SBrkr,1057,977,0,2034,1,0,2,1,3,1,Gd,8,Typ,0,NA,Attchd,1998,RFn,2,645,TA,TA,Y,576,36,0,0,0,0,NA,GdPrv,NA,0,2,2009,WD,Normal,219500\n66,60,RL,76,9591,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,8,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,BrkFace,344,Gd,TA,PConc,Ex,TA,Av,Unf,0,Unf,0,1143,1143,GasA,Ex,Y,SBrkr,1143,1330,0,2473,0,0,2,1,4,1,Gd,9,Typ,1,Gd,BuiltIn,2004,RFn,3,852,TA,TA,Y,192,151,0,0,0,0,NA,NA,NA,0,10,2007,WD,Normal,317000\n67,20,RL,NA,19900,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,PosA,Norm,1Fam,1Story,7,5,1970,1989,Gable,CompShg,Plywood,Plywood,BrkFace,287,TA,TA,CBlock,Gd,TA,Gd,GLQ,912,Unf,0,1035,1947,GasA,TA,Y,SBrkr,2207,0,0,2207,1,0,2,0,3,1,TA,7,Min1,1,Gd,Attchd,1970,RFn,2,576,TA,TA,Y,301,0,0,0,0,0,NA,NA,NA,0,7,2010,WD,Normal,180000\n68,20,RL,72,10665,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,BrkFace,167,Gd,TA,PConc,Gd,TA,Av,GLQ,1013,Unf,0,440,1453,GasA,Ex,Y,SBrkr,1479,0,0,1479,1,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2003,RFn,2,558,TA,TA,Y,144,29,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,226000\n69,30,RM,47,4608,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Artery,Norm,1Fam,1Story,4,6,1945,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Gd,CBlock,TA,TA,No,Unf,0,Unf,0,747,747,GasA,TA,Y,SBrkr,747,0,0,747,0,0,1,0,2,1,TA,4,Typ,0,NA,Attchd,1945,Unf,1,220,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,80000\n70,50,RL,81,15593,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,ClearCr,Norm,Norm,1Fam,1.5Fin,7,4,1953,1953,Gable,CompShg,BrkFace,AsbShng,None,0,Gd,TA,CBlock,TA,TA,No,BLQ,603,Unf,0,701,1304,GasW,TA,Y,SBrkr,1304,983,0,2287,0,0,2,0,3,1,TA,7,Typ,1,TA,Attchd,1953,Fin,2,667,TA,TA,Y,0,21,114,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,225000\n71,20,RL,95,13651,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,7,6,1973,1973,Gable,CompShg,Plywood,Plywood,BrkFace,1115,TA,Gd,CBlock,Gd,TA,Gd,ALQ,1880,Unf,0,343,2223,GasA,Ex,Y,SBrkr,2223,0,0,2223,1,0,2,0,3,1,TA,8,Typ,2,Gd,Attchd,1973,Fin,2,516,TA,TA,Y,300,0,0,0,0,0,NA,NA,NA,0,2,2007,WD,Normal,244000\n72,20,RL,69,7599,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Mitchel,Norm,Norm,1Fam,1Story,4,6,1982,2006,Gable,CompShg,HdBoard,Plywood,None,0,TA,TA,CBlock,TA,TA,No,ALQ,565,Unf,0,280,845,GasA,TA,Y,SBrkr,845,0,0,845,1,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1987,Unf,2,360,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,129500\n73,60,RL,74,10141,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,1998,1998,Gable,CompShg,VinylSd,VinylSd,BrkFace,40,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,832,832,GasA,Gd,Y,SBrkr,885,833,0,1718,0,0,2,1,3,1,TA,7,Typ,1,TA,Attchd,1998,Fin,2,427,TA,TA,Y,0,94,0,0,291,0,NA,NA,NA,0,12,2009,WD,Normal,185000\n74,20,RL,85,10200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1954,2003,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,104,TA,TA,CBlock,TA,TA,No,ALQ,320,BLQ,362,404,1086,GasA,Gd,Y,SBrkr,1086,0,0,1086,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1989,Unf,2,490,TA,TA,Y,0,0,0,0,0,0,NA,GdWo,NA,0,5,2010,WD,Normal,144900\n75,50,RM,60,5790,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,2Story,3,6,1915,1950,Gambrel,CompShg,VinylSd,VinylSd,None,0,Gd,Gd,CBlock,Fa,TA,No,Unf,0,Unf,0,840,840,GasA,Gd,N,SBrkr,840,765,0,1605,0,0,2,0,3,2,TA,8,Typ,0,NA,Detchd,1915,Unf,1,379,TA,TA,Y,0,0,202,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,107400\n76,180,RM,21,1596,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,MeadowV,Norm,Norm,Twnhs,SLvl,4,5,1973,1973,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,CBlock,Gd,TA,Gd,GLQ,462,Unf,0,0,462,GasA,TA,Y,SBrkr,526,462,0,988,1,0,1,0,2,1,TA,5,Typ,0,NA,BuiltIn,1973,Unf,1,297,TA,TA,Y,120,101,0,0,0,0,NA,GdWo,NA,0,11,2009,WD,Normal,91000\n77,20,RL,NA,8475,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,7,1956,1956,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,TA,No,ALQ,228,Unf,0,724,952,GasA,Ex,Y,FuseA,952,0,0,952,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1956,Unf,1,283,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,135750\n78,50,RM,50,8635,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1.5Fin,5,5,1948,2001,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,BLQ,336,GLQ,41,295,672,GasA,TA,Y,SBrkr,1072,213,0,1285,1,0,1,0,2,1,TA,6,Min1,0,NA,Detchd,1948,Unf,1,240,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,1,2008,WD,Normal,127000\n79,90,RL,72,10778,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,Duplex,1Story,4,5,1968,1968,Hip,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1768,1768,GasA,TA,N,SBrkr,1768,0,0,1768,0,0,2,0,4,2,TA,8,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,136500\n80,50,RM,60,10440,Pave,Grvl,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,2Story,5,6,1910,1981,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,PConc,TA,TA,No,Unf,0,Unf,0,440,440,GasA,Gd,Y,SBrkr,682,548,0,1230,0,0,1,1,2,1,TA,5,Typ,0,NA,Detchd,1966,Unf,2,440,TA,TA,Y,74,0,128,0,0,0,NA,MnPrv,NA,0,5,2009,WD,Normal,110000\n81,60,RL,100,13000,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,2Story,6,6,1968,1968,Gable,CompShg,VinylSd,VinylSd,BrkFace,576,TA,Gd,CBlock,Gd,TA,No,Rec,448,Unf,0,448,896,GasA,TA,Y,SBrkr,1182,960,0,2142,0,0,2,1,4,1,Gd,8,Typ,1,Gd,Attchd,1968,Fin,1,509,TA,TA,Y,0,72,0,0,252,0,NA,NA,NA,0,6,2009,WD,Normal,193500\n82,120,RM,32,4500,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,Mitchel,Norm,Norm,TwnhsE,1Story,6,5,1998,1998,Hip,CompShg,VinylSd,VinylSd,BrkFace,443,TA,Gd,PConc,Ex,Gd,No,GLQ,1201,Unf,0,36,1237,GasA,Ex,Y,SBrkr,1337,0,0,1337,1,0,2,0,2,1,TA,5,Typ,0,NA,Attchd,1998,Fin,2,405,TA,TA,Y,0,199,0,0,0,0,NA,NA,NA,0,3,2006,WD,Normal,153500\n83,20,RL,78,10206,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,8,5,2007,2007,Gable,CompShg,VinylSd,VinylSd,Stone,468,TA,TA,PConc,Gd,TA,No,GLQ,33,Unf,0,1530,1563,GasA,Ex,Y,SBrkr,1563,0,0,1563,0,0,2,0,3,1,Gd,6,Typ,1,Gd,Attchd,2007,RFn,3,758,TA,TA,Y,144,99,0,0,0,0,NA,NA,NA,0,10,2008,WD,Normal,245000\n84,20,RL,80,8892,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,5,1960,1960,Gable,CompShg,MetalSd,MetalSd,BrkCmn,66,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1065,1065,GasA,Gd,Y,SBrkr,1065,0,0,1065,0,0,1,1,3,1,TA,6,Typ,0,NA,Detchd,1974,Unf,2,461,TA,TA,Y,74,0,0,0,0,0,NA,NA,NA,0,7,2007,COD,Normal,126500\n85,80,RL,NA,8530,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,SLvl,7,5,1995,1996,Gable,CompShg,HdBoard,HdBoard,BrkFace,22,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,384,384,GasA,Gd,Y,SBrkr,804,670,0,1474,0,0,2,1,3,1,TA,7,Typ,1,TA,BuiltIn,1995,Fin,2,400,TA,TA,Y,120,72,0,0,0,0,NA,NA,Shed,700,5,2009,WD,Normal,168500\n86,60,RL,121,16059,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NoRidge,Norm,Norm,1Fam,2Story,8,5,1991,1992,Hip,CompShg,HdBoard,HdBoard,BrkFace,284,Gd,TA,CBlock,Gd,TA,No,Unf,0,Unf,0,1288,1288,GasA,Ex,Y,SBrkr,1301,1116,0,2417,0,0,2,1,4,1,Gd,9,Typ,1,TA,Attchd,1991,Unf,2,462,TA,TA,Y,127,82,0,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,260000\n87,60,RL,122,11911,Pave,NA,IR2,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,2005,2005,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,684,684,GasA,Ex,Y,SBrkr,684,876,0,1560,0,0,2,1,3,1,Gd,6,Typ,1,Gd,BuiltIn,2005,Fin,2,400,TA,TA,Y,100,38,0,0,0,0,NA,NA,NA,0,3,2009,WD,Normal,174000\n88,160,FV,40,3951,Pave,Pave,Reg,Lvl,AllPub,Corner,Gtl,Somerst,Norm,Norm,TwnhsE,2Story,6,5,2009,2009,Gable,CompShg,VinylSd,VinylSd,Stone,76,Gd,TA,PConc,Gd,TA,Av,Unf,0,Unf,0,612,612,GasA,Ex,Y,SBrkr,612,612,0,1224,0,0,2,1,2,1,Gd,4,Typ,0,NA,Detchd,2009,RFn,2,528,TA,TA,Y,0,234,0,0,0,0,NA,NA,NA,0,6,2009,New,Partial,164500\n89,50,C (all),105,8470,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,IDOTRR,Feedr,Feedr,1Fam,1.5Fin,3,2,1915,1982,Hip,CompShg,Plywood,Plywood,None,0,Fa,Fa,CBlock,TA,Fa,No,Unf,0,Unf,0,1013,1013,GasA,TA,N,SBrkr,1013,0,513,1526,0,0,1,0,2,1,Fa,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,0,156,0,0,0,NA,MnPrv,NA,0,10,2009,ConLD,Abnorml,85000\n90,20,RL,60,8070,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,4,5,1994,1995,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,GLQ,588,Unf,0,402,990,GasA,Ex,Y,SBrkr,990,0,0,990,1,0,1,0,3,1,TA,5,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,123600\n91,20,RL,60,7200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,5,1950,1950,Gable,CompShg,BrkFace,Wd Sdng,None,0,TA,TA,Slab,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,FuseA,1040,0,0,1040,0,0,1,0,2,1,TA,4,Typ,0,NA,Detchd,1950,Unf,2,420,TA,TA,Y,0,29,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,109900\n92,20,RL,85,8500,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,3,1961,1961,Hip,CompShg,HdBoard,HdBoard,BrkCmn,203,TA,TA,CBlock,TA,TA,No,Rec,600,Unf,0,635,1235,GasA,TA,Y,SBrkr,1235,0,0,1235,0,0,1,0,2,1,TA,6,Typ,0,NA,Attchd,1961,Unf,2,480,TA,TA,Y,0,0,0,0,0,0,NA,GdWo,NA,0,12,2006,WD,Abnorml,98600\n93,30,RL,80,13360,Pave,Grvl,IR1,HLS,AllPub,Inside,Gtl,Crawfor,Norm,Norm,1Fam,1Story,5,7,1921,2006,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,Gd,BrkTil,Gd,TA,No,ALQ,713,Unf,0,163,876,GasA,Ex,Y,SBrkr,964,0,0,964,1,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1921,Unf,2,432,TA,TA,Y,0,0,44,0,0,0,NA,NA,NA,0,8,2009,WD,Normal,163500\n94,190,C (all),60,7200,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,2fmCon,2.5Unf,6,6,1910,1998,Hip,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,TA,Fa,Mn,Rec,1046,Unf,0,168,1214,GasW,Ex,N,SBrkr,1260,1031,0,2291,0,1,2,0,4,2,TA,9,Typ,1,Gd,Detchd,1900,Unf,2,506,TA,TA,Y,0,0,0,0,99,0,NA,NA,NA,0,11,2007,WD,Normal,133900\n95,60,RL,69,9337,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,6,5,1997,1997,Gable,CompShg,VinylSd,VinylSd,None,0,TA,Gd,PConc,Gd,TA,No,GLQ,648,Unf,0,176,824,GasA,Ex,Y,SBrkr,905,881,0,1786,1,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,1997,RFn,2,684,TA,TA,Y,0,162,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,204750\n96,60,RL,NA,9765,Pave,NA,IR2,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,8,1993,1993,Gable,CompShg,VinylSd,VinylSd,BrkFace,68,Ex,Gd,PConc,Gd,Gd,No,ALQ,310,Unf,0,370,680,GasA,Gd,Y,SBrkr,680,790,0,1470,0,0,2,1,3,1,TA,6,Typ,1,TA,BuiltIn,1993,Fin,2,420,TA,TA,Y,232,63,0,0,0,0,NA,NA,Shed,480,4,2009,WD,Normal,185000\n97,20,RL,78,10264,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,1999,1999,Gable,CompShg,VinylSd,VinylSd,BrkFace,183,Gd,TA,PConc,Gd,TA,Av,ALQ,1162,Unf,0,426,1588,GasA,Ex,Y,SBrkr,1588,0,0,1588,0,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1999,RFn,2,472,TA,TA,Y,158,29,0,0,0,0,NA,NA,NA,0,8,2006,WD,Normal,214000\n98,20,RL,73,10921,Pave,NA,Reg,HLS,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1Story,4,5,1965,1965,Hip,CompShg,HdBoard,HdBoard,BrkFace,48,TA,TA,CBlock,TA,TA,No,Rec,520,Unf,0,440,960,GasA,TA,Y,FuseF,960,0,0,960,1,0,1,0,3,1,TA,6,Typ,0,NA,Attchd,1965,Fin,1,432,TA,TA,P,120,0,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,94750\n99,30,RL,85,10625,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,5,5,1920,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,ALQ,108,Unf,0,350,458,GasA,Fa,N,SBrkr,835,0,0,835,0,0,1,0,2,1,TA,5,Typ,0,NA,Basment,1920,Unf,1,366,Fa,TA,Y,0,0,77,0,0,0,NA,NA,Shed,400,5,2010,COD,Abnorml,83000\n100,20,RL,77,9320,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,5,1959,1959,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,TA,TA,No,ALQ,569,Unf,0,381,950,GasA,Fa,Y,SBrkr,1225,0,0,1225,1,0,1,1,3,1,TA,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,352,0,0,0,0,0,NA,NA,Shed,400,1,2010,WD,Normal,128950\n101,20,RL,NA,10603,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,1Story,6,7,1977,2001,Gable,CompShg,Plywood,Plywood,BrkFace,28,TA,TA,PConc,TA,TA,Mn,ALQ,1200,Unf,0,410,1610,GasA,Gd,Y,SBrkr,1610,0,0,1610,1,0,2,0,3,1,Gd,6,Typ,2,TA,Attchd,1977,RFn,2,480,TA,TA,Y,168,68,0,0,0,0,NA,NA,NA,0,2,2010,WD,Normal,205000\n102,60,RL,77,9206,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,2Story,6,5,1985,1985,Gable,CompShg,HdBoard,HdBoard,BrkFace,336,Gd,TA,CBlock,Gd,TA,No,Unf,0,Unf,0,741,741,GasA,TA,Y,SBrkr,977,755,0,1732,0,0,2,1,3,1,Gd,7,Typ,1,TA,Attchd,1985,Fin,2,476,TA,TA,Y,192,46,0,0,0,0,NA,NA,NA,0,6,2010,WD,Normal,178000\n103,90,RL,64,7018,Pave,NA,Reg,Bnk,AllPub,Inside,Gtl,SawyerW,Norm,Norm,Duplex,1Story,5,5,1979,1979,Gable,CompShg,HdBoard,HdBoard,None,0,TA,Fa,Slab,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,SBrkr,1535,0,0,1535,0,0,2,0,4,2,TA,8,Typ,0,NA,Attchd,1979,Unf,2,410,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2009,WD,Alloca,118964\n104,20,RL,94,10402,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2009,2009,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1226,1226,GasA,Ex,Y,SBrkr,1226,0,0,1226,0,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,2009,RFn,3,740,TA,TA,Y,0,36,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,198900\n105,50,RM,NA,7758,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,IDOTRR,Norm,Norm,1Fam,1.5Fin,7,4,1931,1950,Gable,CompShg,Stucco,Stucco,BrkFace,600,TA,Fa,PConc,TA,TA,No,LwQ,224,Unf,0,816,1040,GasA,Ex,Y,FuseF,1226,592,0,1818,0,0,1,1,4,1,TA,7,Typ,2,TA,Detchd,1951,Unf,1,240,TA,TA,Y,0,0,0,0,184,0,NA,NA,NA,0,6,2007,WD,Normal,169500\n106,60,FV,75,9375,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,2Story,8,5,2003,2004,Hip,CompShg,VinylSd,VinylSd,BrkFace,768,Gd,TA,PConc,Ex,TA,No,Unf,0,Unf,0,1053,1053,GasA,Ex,Y,SBrkr,1053,939,0,1992,0,0,2,1,3,1,Gd,9,Typ,1,Gd,Attchd,2003,RFn,2,648,TA,TA,Y,140,45,0,0,0,0,NA,NA,NA,0,8,2008,WD,Normal,250000\n107,30,RM,60,10800,Pave,Grvl,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,4,7,1885,1995,Mansard,CompShg,VinylSd,VinylSd,None,0,TA,TA,BrkTil,Fa,TA,No,Unf,0,Unf,0,641,641,GasA,Gd,Y,SBrkr,1047,0,0,1047,0,0,1,0,2,1,TA,6,Typ,0,NA,Detchd,1954,Unf,1,273,Fa,Fa,N,0,0,0,0,0,0,NA,NA,Shed,450,8,2007,WD,Normal,100000\n108,20,RM,50,6000,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,5,5,1948,1950,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,CBlock,TA,TA,No,ALQ,104,BLQ,169,516,789,GasA,Ex,Y,SBrkr,789,0,0,789,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1948,Unf,1,250,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2008,WD,Partial,115000\n109,50,RM,85,8500,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,IDOTRR,Artery,Norm,1Fam,1.5Fin,5,7,1919,2005,Gable,CompShg,CemntBd,CmentBd,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,793,793,GasW,TA,N,FuseF,997,520,0,1517,0,0,2,0,3,1,Fa,7,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,0,144,0,0,0,NA,NA,NA,0,8,2007,WD,Normal,115000\n110,20,RL,105,11751,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,1Story,6,6,1977,1977,Hip,CompShg,Plywood,Plywood,BrkFace,480,TA,TA,CBlock,Gd,TA,No,BLQ,705,Unf,0,1139,1844,GasA,Ex,Y,SBrkr,1844,0,0,1844,0,0,2,0,3,1,TA,7,Typ,1,TA,Attchd,1977,RFn,2,546,TA,TA,Y,0,122,0,0,0,0,NA,MnPrv,NA,0,1,2010,COD,Normal,190000\n111,50,RL,75,9525,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,1Fam,1.5Fin,6,4,1954,1972,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,Fa,No,Rec,444,Unf,0,550,994,GasA,Gd,Y,SBrkr,1216,639,0,1855,0,0,2,0,4,1,TA,7,Typ,0,NA,Attchd,1954,Unf,1,325,TA,TA,Y,182,0,0,0,0,0,NA,NA,NA,0,10,2006,WD,Normal,136900\n112,80,RL,NA,7750,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,SLvl,7,5,2000,2000,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,GLQ,250,Unf,0,134,384,GasA,Ex,Y,SBrkr,774,656,0,1430,0,0,2,1,3,1,TA,7,Typ,1,TA,BuiltIn,2000,Fin,2,400,TA,TA,Y,180,0,0,0,0,0,NA,NA,NA,0,4,2010,WD,Normal,180000\n113,60,RL,77,9965,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,7,5,2007,2007,Gable,CompShg,VinylSd,VinylSd,Stone,220,Gd,TA,PConc,Ex,TA,Av,GLQ,984,Unf,0,280,1264,GasA,Ex,Y,SBrkr,1282,1414,0,2696,1,0,2,1,4,1,Ex,10,Typ,1,Gd,BuiltIn,2007,Fin,3,792,TA,TA,Y,120,184,0,0,168,0,NA,NA,NA,0,10,2007,New,Partial,383970\n114,20,RL,NA,21000,Pave,NA,Reg,Bnk,AllPub,Corner,Gtl,Crawfor,Norm,Norm,1Fam,1Story,6,5,1953,1953,Hip,CompShg,Wd Sdng,Wd Sdng,BrkFace,184,TA,Gd,CBlock,Gd,TA,Mn,ALQ,35,Rec,869,905,1809,GasA,TA,Y,SBrkr,2259,0,0,2259,1,0,2,0,3,1,Gd,7,Typ,2,Gd,Basment,1953,Unf,2,450,TA,TA,Y,166,120,192,0,0,0,NA,MnPrv,NA,0,10,2007,COD,Abnorml,217000\n115,70,RL,61,7259,Pave,NA,IR1,Lvl,AllPub,Inside,Mod,Crawfor,Norm,Norm,1Fam,2Story,6,8,1945,2002,Gambrel,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,ALQ,774,LwQ,150,104,1028,GasA,Ex,Y,SBrkr,1436,884,0,2320,1,0,2,1,3,1,Gd,9,Typ,1,TA,Detchd,1945,Unf,1,180,TA,TA,Y,224,0,0,0,0,0,NA,MnPrv,NA,0,7,2007,WD,Normal,259500\n116,160,FV,34,3230,Pave,Pave,Reg,Lvl,AllPub,Corner,Gtl,Somerst,Norm,Norm,TwnhsE,2Story,6,5,1999,1999,Gable,CompShg,MetalSd,MetalSd,BrkFace,1129,TA,TA,PConc,Gd,TA,No,GLQ,419,Unf,0,310,729,GasA,Gd,Y,SBrkr,729,729,0,1458,0,0,2,1,2,1,TA,5,Typ,1,Fa,Detchd,1999,Unf,2,440,TA,TA,Y,0,32,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,176000\n117,20,RL,NA,11616,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,5,1962,1962,Gable,CompShg,Wd Sdng,Wd Sdng,BrkFace,116,TA,TA,CBlock,TA,TA,No,LwQ,170,BLQ,670,252,1092,GasA,TA,Y,SBrkr,1092,0,0,1092,0,1,1,0,3,1,TA,6,Typ,1,Po,Attchd,1962,Unf,1,288,TA,TA,Y,0,20,144,0,0,0,NA,NA,NA,0,9,2009,WD,Normal,139000\n118,20,RL,74,8536,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,5,5,2006,2007,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1125,1125,GasA,Gd,Y,SBrkr,1125,0,0,1125,0,0,1,1,2,1,TA,5,Typ,0,NA,Attchd,2007,Unf,2,430,TA,TA,Y,80,64,0,0,0,0,NA,NA,NA,0,4,2007,New,Partial,155000\n119,60,RL,90,12376,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,SawyerW,Norm,Norm,1Fam,2Story,7,5,1990,1990,Hip,CompShg,Plywood,Plywood,None,0,TA,TA,PConc,Gd,TA,Mn,GLQ,1470,Unf,0,203,1673,GasA,Gd,Y,SBrkr,1699,1523,0,3222,1,0,3,0,5,1,Gd,11,Typ,2,TA,Attchd,1990,Unf,3,594,TA,TA,Y,367,0,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,320000\n120,60,RL,65,8461,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,6,5,2005,2006,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,728,728,GasA,Ex,Y,SBrkr,728,728,0,1456,0,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2005,Fin,2,390,TA,TA,Y,0,24,0,0,0,0,NA,NA,NA,0,7,2006,New,Partial,163990\n121,80,RL,NA,21453,Pave,NA,IR1,Low,AllPub,CulDSac,Sev,ClearCr,Norm,Norm,1Fam,SLvl,6,5,1969,1969,Flat,Metal,Plywood,Plywood,None,0,TA,TA,CBlock,TA,TA,Gd,ALQ,938,Unf,0,0,938,GasA,Ex,Y,SBrkr,988,0,0,988,1,0,1,0,1,1,TA,4,Typ,2,TA,Attchd,1969,Unf,2,540,TA,TA,Y,0,130,0,130,0,0,NA,NA,NA,0,10,2006,WD,Normal,180000\n122,50,RM,50,6060,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,1Fam,1.5Fin,4,5,1939,1950,Gable,CompShg,AsbShng,AsbShng,None,0,TA,TA,PConc,TA,TA,No,Unf,0,Unf,0,732,732,GasA,Gd,Y,SBrkr,772,351,0,1123,0,0,1,0,3,1,TA,4,Typ,0,NA,Detchd,1979,Unf,1,264,TA,TA,P,0,0,140,0,0,0,NA,MnPrv,NA,0,6,2007,WD,Normal,100000\n123,20,RL,75,9464,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,7,1958,1958,Hip,CompShg,MetalSd,MetalSd,BrkFace,135,TA,Gd,CBlock,TA,TA,No,BLQ,570,Unf,0,510,1080,GasA,Gd,Y,SBrkr,1080,0,0,1080,0,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1958,Unf,1,288,TA,TA,Y,0,0,0,0,130,0,NA,NA,NA,0,6,2008,WD,Normal,136000\n124,120,RL,55,7892,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,TwnhsE,1Story,6,5,1993,1993,Gable,CompShg,Plywood,Plywood,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,300,Unf,0,899,1199,GasA,Ex,Y,SBrkr,1199,0,0,1199,0,0,2,0,2,1,Gd,5,Typ,0,NA,Attchd,1993,RFn,2,530,TA,TA,Y,0,63,0,0,0,0,NA,NA,NA,0,3,2008,WD,Normal,153900\n125,20,RL,48,17043,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NWAmes,Norm,Norm,1Fam,1Story,6,5,1979,1998,Gable,CompShg,HdBoard,HdBoard,None,0,TA,Gd,CBlock,Gd,Fa,No,Unf,0,Unf,0,1362,1362,GasA,TA,Y,SBrkr,1586,0,0,1586,0,0,2,0,3,1,TA,7,Typ,1,TA,Attchd,1979,Unf,2,435,TA,TA,Y,192,0,0,0,0,0,NA,NA,NA,0,1,2009,WD,Normal,181000\n126,190,RM,60,6780,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,IDOTRR,Norm,Norm,2fmCon,1.5Fin,6,8,1935,1982,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,Fa,CBlock,TA,TA,Av,GLQ,490,Unf,0,30,520,GasA,Gd,N,SBrkr,520,0,234,754,1,0,1,0,2,1,TA,5,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,53,0,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,84500\n127,120,RL,NA,4928,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NPkVill,Norm,Norm,TwnhsE,1Story,6,5,1976,1976,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,Gd,TA,No,ALQ,120,Unf,0,958,1078,GasA,TA,Y,SBrkr,958,0,0,958,0,0,2,0,2,1,TA,5,Typ,1,TA,Attchd,1977,RFn,2,440,TA,TA,Y,0,205,0,0,0,0,NA,NA,NA,0,2,2007,WD,Normal,128000\n128,45,RM,55,4388,Pave,NA,IR1,Bnk,AllPub,Inside,Gtl,OldTown,Feedr,Norm,1Fam,1.5Unf,5,7,1930,1950,Gable,CompShg,WdShing,Wd Sdng,None,0,TA,Gd,BrkTil,TA,TA,No,LwQ,116,Unf,0,556,672,GasA,Ex,Y,SBrkr,840,0,0,840,0,0,1,0,3,1,TA,5,Typ,1,TA,NA,NA,NA,0,0,NA,NA,N,0,0,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,87000\n129,60,RL,69,7590,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,PosN,Norm,1Fam,2Story,6,5,1966,1966,Gable,CompShg,VinylSd,VinylSd,BrkFace,266,TA,TA,CBlock,TA,TA,No,BLQ,512,Unf,0,148,660,GasA,TA,Y,SBrkr,660,688,0,1348,0,0,1,1,3,1,TA,6,Typ,1,Fa,Attchd,1966,RFn,2,453,TA,TA,Y,188,108,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,155000\n130,20,RL,69,8973,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1958,1991,Gable,CompShg,Plywood,Plywood,BrkFace,85,TA,TA,CBlock,TA,TA,No,Rec,567,BLQ,28,413,1008,GasA,TA,Y,FuseA,1053,0,0,1053,0,1,1,1,3,1,Ex,6,Typ,0,NA,2Types,1998,RFn,2,750,TA,TA,Y,0,80,0,180,0,0,NA,MnWw,NA,0,7,2006,WD,Abnorml,150000\n131,60,RL,88,14200,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,2Story,7,6,1966,1966,Gable,CompShg,MetalSd,MetalSd,BrkFace,309,TA,TA,CBlock,TA,TA,No,Rec,445,Unf,0,479,924,GasA,Ex,Y,SBrkr,1216,941,0,2157,0,0,2,1,4,1,Gd,8,Typ,2,Gd,Attchd,1966,Fin,2,487,TA,TA,Y,105,66,0,0,0,0,NA,GdPrv,NA,0,5,2006,WD,Normal,226000\n132,60,RL,NA,12224,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,2Story,6,5,2000,2000,Gable,CompShg,VinylSd,VinylSd,BrkFace,40,Gd,TA,PConc,Gd,TA,No,GLQ,695,Unf,0,297,992,GasA,Ex,Y,SBrkr,1022,1032,0,2054,1,0,2,1,3,1,Gd,7,Typ,1,TA,BuiltIn,2000,RFn,2,390,TA,TA,Y,24,48,0,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,244000\n133,20,RL,75,7388,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,6,1959,2002,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Rec,405,Unf,0,658,1063,GasA,Gd,Y,SBrkr,1327,0,0,1327,1,0,1,0,3,1,Gd,7,Typ,0,NA,Detchd,1974,Unf,2,624,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,150750\n134,20,RL,NA,6853,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Timber,Norm,Norm,1Fam,1Story,8,5,2001,2002,Gable,CompShg,VinylSd,VinylSd,BrkFace,136,Gd,TA,PConc,Ex,TA,No,GLQ,1005,Unf,0,262,1267,GasA,Ex,Y,SBrkr,1296,0,0,1296,1,0,2,0,2,1,Gd,6,Typ,0,NA,Attchd,2001,Fin,2,471,TA,TA,Y,192,25,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,220000\n135,20,RL,78,10335,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1Story,5,6,1968,1993,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,CBlock,TA,TA,No,Rec,570,Unf,0,891,1461,GasA,Gd,Y,SBrkr,1721,0,0,1721,0,0,2,1,3,1,TA,7,Min1,1,TA,Attchd,1968,RFn,2,440,TA,TA,Y,0,96,180,0,0,0,NA,MnPrv,NA,0,7,2006,WD,Normal,180000\n136,20,RL,80,10400,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NWAmes,Norm,Norm,1Fam,1Story,7,6,1970,1970,Hip,CompShg,Plywood,Plywood,BrkFace,288,TA,TA,PConc,TA,TA,No,Unf,0,Unf,0,1304,1304,GasA,Gd,Y,SBrkr,1682,0,0,1682,0,0,2,0,3,1,TA,7,Typ,1,Gd,Attchd,1970,Unf,2,530,TA,TA,Y,98,0,0,0,0,0,NA,MnPrv,NA,0,5,2008,WD,Normal,174000\n137,20,RL,NA,10355,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,5,1967,1967,Gable,CompShg,MetalSd,MetalSd,BrkFace,196,TA,TA,CBlock,TA,TA,No,BLQ,695,Unf,0,519,1214,GasA,TA,Y,SBrkr,1214,0,0,1214,0,0,2,0,3,1,TA,5,Typ,1,Fa,Attchd,1967,RFn,1,318,TA,TA,Y,0,111,0,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,143000\n138,90,RL,82,11070,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Mitchel,Norm,Norm,Duplex,1Story,7,5,1988,1989,Gable,CompShg,VinylSd,VinylSd,BrkFace,70,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,1907,1907,GasA,Gd,Y,SBrkr,1959,0,0,1959,0,0,3,0,5,2,TA,9,Typ,0,NA,2Types,1989,Unf,3,766,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,7,2006,WD,Family,171000\n139,60,RL,73,9066,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,8,5,1999,2000,Gable,CompShg,VinylSd,VinylSd,BrkFace,320,Gd,TA,PConc,Gd,TA,Mn,GLQ,668,Unf,0,336,1004,GasA,Ex,Y,SBrkr,1004,848,0,1852,0,0,2,1,3,1,Gd,7,Typ,2,TA,Attchd,1999,Fin,3,660,TA,TA,Y,224,106,0,0,0,0,NA,GdPrv,NA,0,12,2008,WD,Normal,230000\n140,60,RL,65,15426,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,2Story,6,5,1997,1997,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,GLQ,821,Unf,0,107,928,GasA,Ex,Y,SBrkr,928,836,0,1764,1,0,2,1,3,1,Gd,7,Typ,0,NA,Attchd,1997,RFn,2,470,TA,TA,Y,276,99,0,0,0,0,NA,MnPrv,NA,0,8,2009,WD,Normal,231500\n141,20,RL,70,10500,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,NAmes,Norm,Norm,1Fam,1Story,4,5,1971,1971,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,No,ALQ,432,Unf,0,432,864,GasA,TA,Y,SBrkr,864,0,0,864,0,0,1,0,3,1,TA,5,Typ,1,Po,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2010,ConLI,Normal,115000\n142,20,RL,78,11645,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,2005,2005,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,GLQ,1300,Unf,0,434,1734,GasA,Ex,Y,SBrkr,1734,0,0,1734,1,0,2,0,3,1,Gd,7,Typ,0,NA,Attchd,2005,Fin,2,660,TA,TA,Y,160,24,0,0,0,0,NA,NA,NA,0,1,2006,WD,Normal,260000\n143,50,RL,71,8520,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Artery,Norm,1Fam,1.5Fin,5,4,1952,1952,Gable,CompShg,BrkFace,Wd Sdng,None,0,TA,Fa,CBlock,TA,TA,No,Rec,507,Unf,0,403,910,GasA,Fa,Y,SBrkr,910,475,0,1385,0,0,2,0,4,1,TA,6,Typ,0,NA,Detchd,2000,Unf,2,720,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,6,2010,WD,Normal,166000\n144,20,RL,78,10335,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,1999,1999,Gable,CompShg,VinylSd,VinylSd,BrkFace,183,Gd,TA,PConc,Gd,TA,Gd,GLQ,679,Unf,0,811,1490,GasA,Ex,Y,SBrkr,1501,0,0,1501,1,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1999,RFn,2,577,TA,TA,Y,144,29,0,0,0,0,NA,NA,NA,0,6,2009,WD,Normal,204000\n145,90,RM,70,9100,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,RRAe,Norm,Duplex,1Story,5,5,1963,1963,Gable,CompShg,HdBoard,HdBoard,BrkFace,336,TA,TA,CBlock,TA,TA,No,Rec,1332,Unf,0,396,1728,GasA,TA,Y,SBrkr,1728,0,0,1728,1,0,2,0,6,2,TA,10,Typ,0,NA,Detchd,1963,Unf,2,504,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,11,2006,ConLI,Abnorml,125000\n146,160,RM,24,2522,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,Twnhs,2Story,6,5,2004,2006,Gable,CompShg,VinylSd,VinylSd,Stone,50,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,970,970,GasA,Ex,Y,SBrkr,970,739,0,1709,0,0,2,0,3,1,Gd,7,Maj1,0,NA,Detchd,2004,Unf,2,380,TA,TA,Y,0,40,0,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,130000\n147,30,RM,51,6120,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,BrkSide,Norm,Norm,1Fam,1Story,5,7,1931,1993,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,BLQ,209,Unf,0,506,715,GasA,TA,Y,FuseA,875,0,0,875,1,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1931,Unf,1,180,Fa,TA,Y,48,0,0,0,0,0,NA,NA,NA,0,11,2009,WD,Normal,105000\n148,60,RL,NA,9505,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,2001,2001,Gable,CompShg,VinylSd,VinylSd,BrkFace,180,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,884,884,GasA,Ex,Y,SBrkr,884,1151,0,2035,0,0,2,1,3,1,Gd,8,Typ,1,Gd,BuiltIn,2001,Fin,2,434,TA,TA,Y,144,48,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,222500\n149,20,RL,63,7500,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,SawyerW,Norm,Norm,1Fam,1Story,7,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,BrkFace,120,TA,TA,PConc,Gd,TA,No,GLQ,680,Unf,0,400,1080,GasA,Ex,Y,SBrkr,1080,0,0,1080,1,0,1,0,3,1,Gd,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,141000\n150,50,RM,NA,6240,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,BrkSide,Norm,Norm,1Fam,1.5Fin,5,4,1936,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,BrkTil,Gd,TA,No,Unf,0,Unf,0,896,896,GasA,Gd,Y,FuseA,896,448,0,1344,0,0,1,0,3,1,TA,7,Typ,0,NA,Detchd,1936,Unf,1,240,Fa,TA,Y,200,114,0,0,0,0,NA,NA,NA,0,4,2006,WD,Normal,115000\n151,20,RL,120,10356,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,6,1975,1975,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,Av,BLQ,716,Unf,0,253,969,GasA,TA,Y,SBrkr,969,0,0,969,0,0,1,1,3,1,TA,5,Typ,0,NA,Attchd,1975,Unf,2,440,TA,TA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,1,2007,WD,Normal,122000\n152,20,RL,107,13891,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2007,2008,Hip,CompShg,VinylSd,VinylSd,Stone,436,Gd,TA,PConc,Ex,TA,Gd,GLQ,1400,Unf,0,310,1710,GasA,Ex,Y,SBrkr,1710,0,0,1710,1,0,2,0,2,1,Gd,6,Typ,1,Gd,Attchd,2007,RFn,3,866,TA,TA,Y,0,102,0,0,0,0,NA,NA,NA,0,1,2008,New,Partial,372402\n153,60,RL,NA,14803,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NWAmes,Norm,Norm,1Fam,2Story,6,5,1971,1971,Gable,CompShg,HdBoard,HdBoard,BrkFace,252,TA,TA,CBlock,TA,TA,No,Rec,416,Unf,0,409,825,GasA,Gd,Y,SBrkr,1097,896,0,1993,0,0,2,1,4,1,TA,8,Typ,1,Gd,Attchd,1971,RFn,2,495,TA,TA,Y,0,66,0,0,0,0,NA,GdWo,NA,0,6,2006,WD,Normal,190000\n154,20,RL,NA,13500,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,ClearCr,Norm,Norm,1Fam,1Story,6,7,1960,1975,Flat,CompShg,BrkFace,Plywood,None,0,TA,TA,CBlock,Gd,TA,Gd,BLQ,429,ALQ,1080,93,1602,GasA,Gd,Y,SBrkr,1252,0,0,1252,1,0,1,0,1,1,TA,4,Typ,1,Gd,Attchd,1960,RFn,2,564,TA,TA,Y,409,0,0,0,0,0,NA,NA,NA,0,3,2008,WD,Normal,235000\n155,30,RM,84,11340,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,1Story,6,5,1923,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,1200,1200,GasA,TA,Y,FuseA,1200,0,0,1200,0,0,1,0,4,1,TA,7,Typ,0,NA,Detchd,1923,Unf,1,312,Fa,Fa,Y,0,0,228,0,0,0,NA,NA,NA,0,3,2006,WD,Family,125000\n156,50,RL,60,9600,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Artery,Norm,1Fam,1.5Fin,6,5,1924,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,BrkTil,TA,TA,No,Unf,0,Unf,0,572,572,Grav,Fa,N,FuseF,572,524,0,1096,0,0,1,0,2,1,TA,5,Typ,0,NA,NA,NA,NA,0,0,NA,NA,N,0,8,128,0,0,0,NA,NA,NA,0,4,2008,WD,Normal,79000\n157,20,RL,60,7200,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,5,7,1950,1950,Hip,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,NA,NA,NA,NA,0,NA,0,0,0,GasA,TA,Y,FuseF,1040,0,0,1040,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1950,Unf,2,625,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,109500\n158,60,RL,92,12003,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Timber,Norm,Norm,1Fam,2Story,8,5,2009,2010,Gable,CompShg,VinylSd,VinylSd,BrkFace,84,Gd,TA,PConc,Ex,TA,No,Unf,0,Unf,0,774,774,GasA,Ex,Y,SBrkr,774,1194,0,1968,0,0,2,1,4,1,Ex,8,Typ,1,Gd,BuiltIn,2009,Fin,3,680,TA,TA,Y,0,75,0,0,0,0,NA,NA,NA,0,5,2010,New,Partial,269500\n159,60,FV,100,12552,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Somerst,Norm,Norm,1Fam,2Story,7,5,2004,2005,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,GLQ,222,Unf,0,769,991,GasA,Ex,Y,SBrkr,991,956,0,1947,0,0,2,1,3,1,Gd,8,Typ,1,Gd,Attchd,2004,RFn,2,678,TA,TA,Y,0,136,0,0,0,0,NA,GdWo,NA,0,5,2010,WD,Normal,254900\n160,60,RL,134,19378,Pave,NA,IR1,HLS,AllPub,Corner,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,2005,2006,Gable,CompShg,VinylSd,VinylSd,BrkFace,456,Gd,TA,PConc,Gd,TA,Mn,GLQ,57,Unf,0,1335,1392,GasA,Ex,Y,SBrkr,1392,1070,0,2462,1,0,2,1,4,1,Gd,9,Typ,1,Gd,Attchd,2006,RFn,2,576,TA,TA,Y,239,132,0,168,0,0,NA,NA,NA,0,3,2006,New,Partial,320000\n161,20,RL,NA,11120,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Veenker,Norm,Norm,1Fam,1Story,6,6,1984,1984,Gable,CompShg,Plywood,Plywood,None,0,TA,TA,PConc,Gd,TA,No,BLQ,660,Unf,0,572,1232,GasA,TA,Y,SBrkr,1232,0,0,1232,0,0,2,0,3,1,TA,6,Typ,0,NA,Attchd,1984,Unf,2,516,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2008,WD,Normal,162500\n162,60,RL,110,13688,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,9,5,2003,2004,Gable,CompShg,VinylSd,VinylSd,BrkFace,664,Gd,TA,PConc,Ex,TA,Av,GLQ,1016,Unf,0,556,1572,GasA,Ex,Y,SBrkr,1572,1096,0,2668,1,0,2,1,3,1,Ex,10,Typ,2,Gd,BuiltIn,2003,Fin,3,726,TA,TA,Y,400,0,0,0,0,0,NA,NA,NA,0,3,2008,WD,Normal,412500\n163,20,RL,95,12182,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NridgHt,Norm,Norm,1Fam,1Story,7,5,2005,2005,Gable,CompShg,VinylSd,VinylSd,BrkFace,226,Gd,TA,PConc,Gd,TA,Mn,BLQ,1201,Unf,0,340,1541,GasA,Ex,Y,SBrkr,1541,0,0,1541,0,0,2,0,3,1,Gd,7,Typ,1,Gd,Attchd,2005,RFn,2,532,TA,TA,Y,0,70,0,0,0,0,NA,NA,NA,0,5,2010,New,Partial,220000\n164,45,RL,55,5500,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1.5Unf,4,6,1956,1956,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,882,882,GasA,Ex,Y,SBrkr,882,0,0,882,0,0,1,0,1,1,TA,4,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,0,0,0,0,0,NA,MnPrv,NA,0,4,2007,WD,Normal,103200\n165,40,RM,40,5400,Pave,Pave,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,1Story,6,7,1926,2004,Gable,CompShg,MetalSd,MetalSd,None,0,TA,Gd,BrkTil,TA,TA,Mn,LwQ,370,Unf,0,779,1149,GasA,Gd,Y,FuseA,1149,467,0,1616,0,0,2,0,3,1,Gd,5,Typ,0,NA,Detchd,1926,Unf,1,216,TA,TA,Y,0,0,183,0,0,0,NA,NA,NA,0,10,2007,WD,Normal,152000\n166,190,RL,62,10106,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,2fmCon,1.5Fin,5,7,1940,1999,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,Gd,BrkTil,TA,TA,No,ALQ,351,Rec,181,112,644,GasA,Gd,Y,SBrkr,808,547,0,1355,1,0,2,0,4,2,TA,6,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,140,0,0,0,0,0,NA,NA,NA,0,9,2008,WD,Normal,127500\n167,20,RL,NA,10708,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,ClearCr,Norm,Norm,1Fam,1Story,5,5,1955,1993,Hip,CompShg,Wd Sdng,Wd Sdng,None,0,Gd,TA,CBlock,TA,TA,No,LwQ,379,BLQ,768,470,1617,GasA,Ex,Y,FuseA,1867,0,0,1867,1,0,1,0,2,1,TA,7,Typ,3,Gd,Attchd,1955,Fin,1,303,TA,TA,Y,476,0,0,0,142,0,NA,GdWo,NA,0,11,2009,COD,Normal,190000\n168,60,RL,86,10562,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,2Story,8,5,2007,2007,Gable,CompShg,VinylSd,VinylSd,Stone,300,Gd,TA,PConc,Ex,TA,No,GLQ,1288,Unf,0,294,1582,GasA,Ex,Y,SBrkr,1610,551,0,2161,1,0,1,1,3,1,Ex,8,Typ,1,Gd,Attchd,2007,Fin,3,789,TA,TA,Y,178,120,0,0,0,0,NA,NA,NA,0,11,2007,New,Partial,325624\n169,60,RL,62,8244,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Gilbert,Norm,Norm,1Fam,2Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,840,840,GasA,Ex,Y,SBrkr,840,880,0,1720,0,0,2,1,3,1,Gd,7,Typ,1,Gd,Attchd,2004,Fin,2,440,TA,TA,Y,100,48,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,183500\n170,20,RL,NA,16669,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,Timber,Norm,Norm,1Fam,1Story,8,6,1981,1981,Hip,WdShake,Plywood,Plywood,BrkFace,653,Gd,TA,CBlock,Gd,TA,No,Unf,0,Unf,0,1686,1686,GasA,TA,Y,SBrkr,1707,0,0,1707,0,0,2,1,2,1,TA,6,Typ,1,TA,Attchd,1981,RFn,2,511,TA,TA,Y,574,64,0,0,0,0,NA,NA,NA,0,1,2006,WD,Normal,228000\n171,50,RM,NA,12358,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,OldTown,Feedr,Norm,1Fam,1.5Fin,5,6,1941,1950,Gable,CompShg,MetalSd,MetalSd,None,0,TA,TA,CBlock,TA,TA,No,Rec,360,Unf,0,360,720,GasA,TA,Y,SBrkr,854,0,528,1382,0,0,1,1,2,1,TA,7,Typ,0,NA,Detchd,1991,Unf,2,660,TA,TA,Y,237,0,0,0,0,0,NA,NA,NA,0,5,2007,WD,Normal,128500\n172,20,RL,141,31770,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1960,1960,Hip,CompShg,BrkFace,Plywood,Stone,112,TA,TA,CBlock,TA,Gd,Gd,BLQ,639,Unf,0,441,1080,GasA,Fa,Y,SBrkr,1656,0,0,1656,1,0,1,0,3,1,TA,7,Typ,2,Gd,Attchd,1960,Fin,2,528,TA,TA,P,210,62,0,0,0,0,NA,NA,NA,0,5,2010,WD,Normal,215000\n173,160,RL,44,5306,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,StoneBr,Norm,Norm,TwnhsE,2Story,7,7,1987,1987,Gable,CompShg,HdBoard,HdBoard,None,0,Gd,Gd,PConc,Gd,Gd,No,GLQ,495,Rec,215,354,1064,GasA,Gd,Y,SBrkr,1064,703,0,1767,1,0,2,0,2,1,Gd,5,Typ,1,TA,Attchd,1987,RFn,2,504,Gd,TA,Y,441,35,0,0,0,0,NA,NA,NA,0,6,2006,WD,Normal,239000\n174,20,RL,80,10197,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,NAmes,Norm,Norm,1Fam,1Story,6,5,1961,1961,Gable,CompShg,WdShing,Wd Shng,BrkCmn,491,TA,TA,CBlock,TA,TA,No,ALQ,288,Rec,374,700,1362,GasA,TA,Y,SBrkr,1362,0,0,1362,1,0,1,1,3,1,TA,6,Typ,1,TA,Attchd,1961,Unf,2,504,TA,TA,Y,0,20,0,0,0,0,NA,NA,NA,0,6,2008,COD,Normal,163000\n175,20,RL,47,12416,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,Timber,Norm,Norm,1Fam,1Story,6,5,1986,1986,Gable,CompShg,VinylSd,Plywood,Stone,132,TA,TA,CBlock,Gd,Fa,No,ALQ,1398,LwQ,208,0,1606,GasA,TA,Y,SBrkr,1651,0,0,1651,1,0,2,0,3,1,TA,7,Min2,1,TA,Attchd,1986,Fin,2,616,TA,TA,Y,192,0,0,0,0,0,NA,NA,NA,0,11,2008,WD,Normal,184000\n176,20,RL,84,12615,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,Edwards,Norm,Norm,1Fam,1Story,6,7,1950,2001,Gable,CompShg,WdShing,Wd Shng,None,0,TA,TA,CBlock,TA,Gd,Av,ALQ,477,Unf,0,725,1202,GasA,TA,Y,SBrkr,2158,0,0,2158,1,0,2,0,4,1,Gd,7,Typ,1,Gd,Attchd,1950,Unf,2,576,TA,TA,Y,0,29,39,0,0,0,NA,MnPrv,NA,0,6,2007,WD,Normal,243000\n177,60,RL,97,10029,Pave,NA,IR1,Lvl,AllPub,Corner,Gtl,ClearCr,Norm,Norm,1Fam,2Story,6,5,1988,1989,Gable,CompShg,Plywood,Plywood,BrkFace,268,Gd,TA,PConc,Gd,TA,No,GLQ,831,Unf,0,320,1151,GasA,TA,Y,SBrkr,1164,896,0,2060,0,1,2,1,4,1,TA,8,Typ,1,TA,Attchd,1988,Unf,2,521,TA,TA,Y,0,228,0,0,192,0,NA,NA,NA,0,9,2007,WD,Normal,211000\n178,50,RL,NA,13650,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Sawyer,Norm,Norm,1Fam,1.5Fin,5,5,1958,1958,Gable,CompShg,MetalSd,MetalSd,None,0,Gd,Gd,CBlock,TA,TA,No,ALQ,57,BLQ,441,554,1052,GasA,Ex,Y,SBrkr,1252,668,0,1920,1,0,2,0,4,1,Gd,8,Typ,1,Gd,Attchd,1958,Unf,2,451,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,172500\n179,20,RL,63,17423,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,StoneBr,Norm,Norm,1Fam,1Story,9,5,2008,2009,Hip,CompShg,VinylSd,VinylSd,Stone,748,Ex,TA,PConc,Ex,TA,No,GLQ,1904,Unf,0,312,2216,GasA,Ex,Y,SBrkr,2234,0,0,2234,1,0,2,0,1,1,Ex,9,Typ,1,Gd,Attchd,2009,Fin,3,1166,TA,TA,Y,0,60,0,0,0,0,NA,NA,NA,0,7,2009,New,Partial,501837\n180,30,RM,60,8520,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1Story,5,6,1923,2006,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,Gd,TA,CBlock,TA,TA,No,Unf,0,Unf,0,968,968,GasA,TA,Y,SBrkr,968,0,0,968,0,0,1,0,2,1,TA,5,Typ,0,NA,Detchd,1935,Unf,2,480,Fa,TA,N,0,0,184,0,0,0,NA,NA,NA,0,7,2007,WD,Normal,100000\n181,160,FV,NA,2117,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,Twnhs,2Story,6,5,2000,2000,Gable,CompShg,MetalSd,MetalSd,BrkFace,456,Gd,TA,PConc,Gd,TA,No,GLQ,436,Unf,0,320,756,GasA,Ex,Y,SBrkr,769,756,0,1525,0,0,2,1,3,1,Gd,5,Typ,1,TA,Detchd,2000,Unf,2,440,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,177000\n182,70,RL,54,7588,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Crawfor,Norm,Norm,1Fam,2Story,7,6,1920,1950,Gable,CompShg,Stucco,Stucco,None,0,TA,TA,BrkTil,Fa,TA,No,LwQ,352,Unf,0,441,793,GasA,Gd,Y,SBrkr,901,901,0,1802,0,0,1,1,4,1,TA,9,Typ,1,Gd,Detchd,1920,Unf,1,216,Fa,TA,Y,0,0,40,0,0,0,NA,NA,NA,0,7,2006,WD,Normal,200100\n183,20,RL,60,9060,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Artery,Norm,1Fam,1Story,5,6,1957,2006,Hip,CompShg,Wd Sdng,Wd Sdng,BrkFace,98,TA,TA,PConc,NA,NA,NA,NA,0,NA,0,0,0,GasA,Ex,Y,SBrkr,1340,0,0,1340,0,0,1,0,3,1,TA,7,Typ,1,Gd,Attchd,1957,RFn,1,252,TA,TA,Y,116,0,0,180,0,0,NA,MnPrv,NA,0,6,2007,WD,Normal,120000\n184,50,RM,63,11426,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,7,5,2003,2003,Gable,CompShg,VinylSd,VinylSd,None,0,TA,TA,PConc,Gd,TA,No,Unf,0,Unf,0,1362,1362,GasA,Ex,Y,SBrkr,1362,720,0,2082,0,0,2,1,3,1,Gd,6,Mod,0,NA,Detchd,2003,Unf,2,484,TA,TA,N,280,238,0,0,0,0,NA,NA,NA,0,6,2008,WD,Normal,200000\n185,50,RL,92,7438,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,BrkSide,RRAn,Feedr,1Fam,1.5Fin,5,8,1908,1991,Gable,CompShg,AsbShng,Plywood,None,0,TA,TA,PConc,Fa,TA,No,Unf,0,Unf,0,504,504,GasA,Gd,Y,SBrkr,936,316,0,1252,0,0,1,0,3,1,TA,5,Typ,0,NA,Attchd,1986,Unf,2,576,TA,TA,Y,104,0,0,0,0,0,NA,MnPrv,NA,0,6,2006,WD,Normal,127000\n186,75,RM,90,22950,Pave,NA,IR2,Lvl,AllPub,Inside,Gtl,OldTown,Artery,Norm,1Fam,2.5Fin,10,9,1892,1993,Gable,WdShngl,Wd Sdng,Wd Sdng,None,0,Gd,Gd,BrkTil,TA,TA,Mn,Unf,0,Unf,0,1107,1107,GasA,Ex,Y,SBrkr,1518,1518,572,3608,0,0,2,1,4,1,Ex,12,Typ,2,TA,Detchd,1993,Unf,3,840,Ex,TA,Y,0,260,0,0,410,0,NA,GdPrv,NA,0,6,2006,WD,Normal,475000\n187,80,RL,NA,9947,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,Mitchel,Norm,Norm,1Fam,SLvl,7,5,1990,1991,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,PConc,Gd,TA,Av,GLQ,611,Unf,0,577,1188,GasA,Ex,Y,SBrkr,1217,0,0,1217,1,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1990,Unf,2,497,TA,TA,Y,168,27,0,0,0,0,NA,GdPrv,NA,0,6,2009,WD,Normal,173000\n188,50,RL,60,10410,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,OldTown,Norm,Norm,1Fam,1.5Fin,5,7,1916,1987,Gable,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,Fa,TA,No,Unf,0,Unf,0,660,660,GasA,Ex,Y,SBrkr,808,704,144,1656,0,0,2,1,3,1,TA,8,Min2,0,NA,Detchd,1916,Unf,1,180,Fa,Fa,N,0,0,0,140,0,0,NA,MnPrv,NA,0,8,2009,WD,Normal,135000\n189,90,RL,64,7018,Pave,NA,Reg,Bnk,AllPub,Inside,Gtl,SawyerW,Feedr,Norm,Duplex,SFoyer,5,5,1979,1979,Gable,CompShg,Plywood,Plywood,Stone,275,TA,TA,CBlock,Gd,TA,Av,GLQ,1086,Unf,0,0,1086,GasA,TA,Y,SBrkr,1224,0,0,1224,2,0,0,2,2,2,TA,6,Typ,2,TA,Detchd,1979,Unf,2,528,TA,TA,Y,120,0,0,0,0,0,NA,NA,NA,0,6,2009,WD,Alloca,153337\n190,120,RL,41,4923,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,StoneBr,Norm,Norm,TwnhsE,1Story,8,5,2001,2002,Gable,CompShg,CemntBd,CmentBd,None,0,Gd,TA,PConc,Ex,TA,Av,GLQ,1153,Unf,0,440,1593,GasA,Ex,Y,SBrkr,1593,0,0,1593,1,0,1,1,0,1,Ex,5,Typ,1,Gd,Attchd,2001,Fin,2,682,TA,TA,Y,0,120,0,0,224,0,NA,NA,NA,0,8,2008,WD,Normal,286000\n191,70,RL,70,10570,Pave,NA,Reg,Bnk,AllPub,Inside,Mod,Crawfor,Norm,Norm,1Fam,2Story,8,8,1932,1994,Hip,CompShg,BrkFace,BrkFace,None,0,Gd,TA,CBlock,Gd,Gd,No,Rec,297,Unf,0,556,853,GasA,TA,Y,SBrkr,1549,1178,0,2727,0,0,2,1,3,1,Gd,10,Maj1,2,TA,Detchd,1932,Unf,2,440,TA,TA,Y,0,74,0,0,0,0,NA,NA,NA,0,12,2007,WD,Normal,315000\n192,60,RL,NA,7472,Pave,NA,IR1,Lvl,AllPub,CulDSac,Gtl,NAmes,Norm,Norm,1Fam,2Story,7,9,1972,2004,Gable,CompShg,HdBoard,HdBoard,BrkFace,138,TA,TA,CBlock,TA,TA,No,ALQ,626,Unf,0,99,725,GasA,Gd,Y,SBrkr,725,754,0,1479,1,0,1,1,4,1,Gd,7,Typ,0,NA,Attchd,1972,Fin,2,484,TA,TA,Y,0,32,0,0,0,0,NA,NA,NA,0,6,2007,WD,Normal,184000\n193,20,RL,68,9017,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,7,5,1999,1999,Gable,CompShg,VinylSd,VinylSd,None,0,Gd,TA,PConc,Gd,TA,Av,GLQ,560,Unf,0,871,1431,GasA,Ex,Y,SBrkr,1431,0,0,1431,1,0,2,0,3,1,Gd,6,Typ,0,NA,Attchd,1999,Fin,2,666,TA,TA,Y,0,35,0,0,0,0,NA,NA,NA,0,9,2009,WD,Normal,192000\n194,160,RM,24,2522,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Edwards,Norm,Norm,Twnhs,2Story,7,5,2004,2004,Gable,CompShg,VinylSd,VinylSd,Stone,50,Gd,TA,PConc,Gd,TA,No,Unf,0,Unf,0,970,970,GasA,Ex,Y,SBrkr,970,739,0,1709,0,0,2,0,3,1,Gd,7,Maj1,0,NA,Detchd,2004,Unf,2,380,TA,TA,Y,0,40,0,0,0,0,NA,NA,NA,0,5,2006,WD,Normal,130000\n195,20,RL,60,7180,Pave,NA,IR1,Lvl,AllPub,Inside,Gtl,CollgCr,Norm,Norm,1Fam,1Story,5,7,1972,1972,Hip,CompShg,HdBoard,HdBoard,None,0,TA,TA,CBlock,TA,TA,Av,ALQ,390,Unf,0,474,864,GasA,TA,Y,SBrkr,864,0,0,864,0,0,1,0,3,1,TA,5,Typ,0,NA,Detchd,1989,Unf,1,352,TA,TA,Y,0,0,0,0,0,0,NA,NA,NA,0,5,2008,WD,Normal,127000\n196,160,RL,24,2280,Pave,NA,Reg,Lvl,AllPub,FR2,Gtl,NPkVill,Norm,Norm,Twnhs,2Story,6,6,1976,1976,Gable,CompShg,Plywood,Brk Cmn,None,0,TA,TA,CBlock,Gd,TA,No,ALQ,566,Unf,0,289,855,GasA,TA,Y,SBrkr,855,601,0,1456,0,0,2,1,3,1,TA,7,Typ,1,TA,Attchd,1976,Unf,2,440,TA,TA,Y,87,0,0,0,0,0,NA,NA,NA,0,7,2009,WD,Normal,148500\n197,20,RL,79,9416,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,Somerst,Norm,Norm,1Fam,1Story,7,5,2007,2007,Hip,CompShg,CemntBd,CmentBd,Stone,205,Ex,TA,PConc,Ex,TA,No,GLQ,1126,Unf,0,600,1726,GasA,Ex,Y,SBrkr,1726,0,0,1726,1,0,2,0,3,1,Ex,8,Typ,1,Gd,Attchd,2007,Fin,3,786,TA,TA,Y,171,138,0,0,266,0,NA,NA,NA,0,9,2007,New,Partial,311872\n198,75,RL,174,25419,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,NAmes,Artery,Norm,1Fam,2Story,8,4,1918,1990,Gable,CompShg,Stucco,Stucco,None,0,Gd,Gd,PConc,TA,TA,No,GLQ,1036,LwQ,184,140,1360,GasA,Gd,Y,SBrkr,1360,1360,392,3112,1,1,2,0,4,1,Gd,8,Typ,1,Ex,Detchd,1918,Unf,2,795,TA,TA,Y,0,16,552,0,0,512,Ex,GdPrv,NA,0,3,2006,WD,Abnorml,235000\n199,75,RM,92,5520,Pave,NA,Reg,Lvl,AllPub,Corner,Gtl,OldTown,Norm,Norm,1Fam,2.5Fin,6,6,1912,1950,Gable,CompShg,Wd Sdng,Wd Sdng,None,0,TA,TA,CBlock,TA,TA,No,Unf,0,Unf,0,755,755,GasA,Ex,Y,SBrkr,929,929,371,2229,0,0,1,0,5,1,TA,8,Typ,0,NA,NA,NA,NA,0,0,NA,NA,Y,0,198,30,0,0,0,NA,MnPrv,NA,0,7,2009,WD,Abnorml,104000\n200,20,RL,76,9591,Pave,NA,Reg,Lvl,AllPub,Inside,Gtl,NridgHt,Norm,Norm,1Fam,1Story,8,5,2004,2005,Hip,CompShg,VinylSd,VinylSd,BrkFace,262,Gd,TA,PConc,Ex,TA,Av,GLQ,1088,Unf,0,625,1713,GasA,Ex,Y,SBrkr,1713,0,0,1713,1,0,2,0,3,1,Ex,7,Typ,1,Gd,Attchd,2004,Fin,3,856,TA,TA,Y,0,26,0,0,170,0,NA,NA,NA,0,1,2009,WD,Normal,274900\n"
},
{
"path": "eda/tests/unittests/test_shift.py",
"content": "import os\nimport tempfile\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nimport autogluon.eda.analysis as eda\nimport autogluon.eda.visualization as viz\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources\"))\n\nSAMPLE_SIZE = 200\n\n\ndef load_adult_data():\n train_data = os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\")\n test_data = os.path.join(RESOURCE_PATH, \"adult\", \"test_data.csv\")\n train = pd.read_csv(train_data).sample(SAMPLE_SIZE, random_state=0)\n test = pd.read_csv(test_data).sample(SAMPLE_SIZE, random_state=0)\n data = (train, test)\n return data\n\n\ndef sim_cov_shift(train, test, p_nonmarr=0.75, val=False):\n \"\"\"Simulate covariate shift by biasing training set toward married\"\"\"\n data = pd.concat((train, test))\n data.loc[:, \"race\"] = data[\"race\"].str.strip()\n data.loc[:, \"sex\"] = data[\"sex\"].str.strip()\n data.loc[:, \"marital-status\"] = data[\"marital-status\"].str.strip()\n data.index = pd.Index(range(data.shape[0]))\n data_married = data[\"marital-status\"] == \"Married-civ-spouse\"\n p_married = (0.5 + data_married.mean() - p_nonmarr) / data_married.mean()\n train_p = data_married * p_married + (1 - data_married) * p_nonmarr\n train_ind = np.random.binomial(1, train_p) == 1\n train_cs = data[train_ind]\n test_cs = data[~train_ind]\n if val:\n train_cs, val_cs = train_test_split(train_cs)\n return train_cs, val_cs, test_cs\n else:\n return train_cs, test_cs\n\n\ndef test_shift():\n train, test = load_adult_data()\n train, test = sim_cov_shift(train, test)\n expected_threshold = 0.5\n with tempfile.TemporaryDirectory() as path:\n shft_ana = eda.shift.XShiftDetector(\n train_data=train,\n test_data=test,\n label=\"class\",\n classifier_kwargs={\"path\": os.path.join(path, \"AutogluonModels\")},\n classifier_fit_kwargs={\"hyperparameters\": {\"RF\": {}}},\n pvalue_thresh=expected_threshold,\n )\n shft_ana.fit()\n shft_viz = viz.shift.XShiftSummary(headers=True)\n shft_viz.render(shft_ana.state)\n result = shft_ana.state.xshift_results\n assert result.pop(\"feature_importance\", None).shape == (14, 6)\n assert result.pop(\"pvalue\", -100) > 0\n assert result.pop(\"test_statistic\", -100) > 0\n assert len(result.pop(\"shift_features\")) > 0\n assert result == {\n \"detection_status\": True,\n \"eval_metric\": \"roc_auc\",\n \"pvalue_threshold\": expected_threshold,\n }\n"
},
{
"path": "eda/tests/unittests/test_state.py",
"content": "import pytest\n\nfrom autogluon.eda.state import AnalysisState, StateCheckMixin\nfrom autogluon.eda.utils.common import expand_nested_args_into_nested_maps\n\n\ndef test_analysis_state():\n state: AnalysisState = AnalysisState(dict(a=1, b=2, c=3), {\"b\": 3}, c=4, d=5)\n assert state.a == 1\n assert state.b == 3\n assert state.c == 4\n assert state.d == 5\n\n\ndef test_analysis_state_ignore_non_dict_args():\n state: AnalysisState = AnalysisState(1, 2, 3, q=5)\n assert state.__dict__ == {\"q\": 5}\n\n\ndef test_analysis_state_nested():\n state: AnalysisState = AnalysisState(a={\"b\": 4})\n assert state.a.b == 4\n\n\ndef test_analysis_state_nested_missing():\n state: AnalysisState = AnalysisState(a={\"b\": 4})\n assert state.missing is None\n\n\ndef test_analysis_state_mutation():\n state: AnalysisState = AnalysisState()\n assert state.a is None\n state.a = 42\n assert state.a == 42\n\n\ndef test_analysis_state_nested_mutation():\n state: AnalysisState = AnalysisState()\n assert state.a is None\n state.a = {}\n state.a.b = 42\n assert state.a.b == 42\n\n\ndef test_statecheckmixin_at_least_one_key_must_be_present():\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=42, w=43), \"missing\") is False\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=42, w=43), \"q\") is True\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=42, w=43), \"w\") is True\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=42, w=43), \"q\", \"w\") is True\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=42, w=43), \"q\", \"missing\") is True\n assert StateCheckMixin().at_least_one_key_must_be_present(AnalysisState(q=None, w=43), \"q\", \"missing\") is False\n\n\ndef test_statecheckmixin_all_keys_must_be_present():\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=42, w=43), \"missing\") is False\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=42, w=43), \"q\") is True\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=None), \"q\") is False\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=42, w=43), \"w\") is True\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=42, w=43), \"q\", \"w\") is True\n assert StateCheckMixin().all_keys_must_be_present(AnalysisState(q=42, w=43), \"q\", \"missing\") is False\n\n\ndef test_expand_nested_args_into_nested_maps():\n assert expand_nested_args_into_nested_maps({\"a\": 1, \"b.a\": 3, \"c.a.b\": 4}) == {\n \"a\": 1,\n \"b\": {\"a\": 3},\n \"c\": {\"a\": {\"b\": 4}},\n }\n\n\ndef test_expand_nested_args_into_nested_maps__namespaces_overlap():\n with pytest.raises(ValueError):\n expand_nested_args_into_nested_maps({\"a\": 1, \"a.b\": 2})\n with pytest.raises(ValueError):\n expand_nested_args_into_nested_maps({\"a.b\": 1, \"a.b.c\": 2})\n"
},
{
"path": "eda/tests/unittests/visualization/__init__.py",
"content": ""
},
{
"path": "eda/tests/unittests/visualization/test_anomaly.py",
"content": "from typing import List\nfrom unittest.mock import MagicMock\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import AnomalyScoresVisualization\n\n\ndef test_AnomalyScoresVisualization__init():\n chart_args = {\n \"normal.color\": \"grey\",\n \"anomaly.color\": \"orange\",\n }\n viz = AnomalyScoresVisualization(**chart_args)\n assert viz.threshold_stds == 3\n assert viz.headers is False\n assert viz.fig_args == {}\n assert viz.chart_args == {\"anomaly\": {\"color\": \"orange\"}, \"normal\": {\"color\": \"grey\"}}\n\n\ndef test_AnomalyScoresVisualization(monkeypatch):\n train_data = [0.13, 0.01, 0.08, 0.76]\n test_data = [0.60, 0.20, 0.91, 0.60]\n state = AnalysisState(\n {\n \"anomaly_detection\": {\n \"scores\": {\n \"train_data\": pd.Series(train_data, name=\"score\"),\n \"test_data\": pd.Series(test_data, name=\"score\"),\n }\n }\n }\n )\n\n chart_args = {\n \"normal.color\": \"grey\",\n \"anomaly.color\": \"orange\",\n }\n\n call_ax = MagicMock()\n call_plt_subplots = MagicMock(return_value=(\"fig\", call_ax))\n call_plt_show = MagicMock()\n call_plt_tight_layout = MagicMock()\n call_sns_scatterplot = MagicMock()\n call_render_markdown = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_plt_subplots)\n m.setattr(plt, \"show\", call_plt_show)\n m.setattr(plt, \"tight_layout\", call_plt_tight_layout)\n m.setattr(sns, \"scatterplot\", call_sns_scatterplot)\n\n viz = AnomalyScoresVisualization(threshold_stds=2, headers=True, **chart_args)\n viz.render_markdown = call_render_markdown\n\n viz.render(state)\n\n call_plt_tight_layout.assert_called_with(h_pad=0.3, w_pad=0.5)\n call_plt_show.assert_called_with(\"fig\")\n call_ax.axhline.assert_called_with(y=0.693685807840985, color=\"r\", linestyle=\"--\")\n call_ax.text.assert_called_with(\n x=0,\n y=0.693685807840985,\n s=\"0.6937\",\n color=\"red\",\n rotation=\"vertical\",\n horizontalalignment=\"right\",\n verticalalignment=\"top\",\n )\n\n calls: List[dict] = [call.kwargs for call in call_sns_scatterplot.call_args_list]\n for call in calls:\n call[\"data\"] = call[\"data\"].to_dict()\n\n assert calls == [\n {\n \"data\": {\"index\": {0: 0, 1: 1, 2: 2}, \"score\": {0: 0.13, 1: 0.01, 2: 0.08}},\n \"ax\": call_ax,\n \"color\": \"grey\",\n \"x\": \"index\",\n \"y\": \"score\",\n },\n {\"data\": {\"index\": {3: 3}, \"score\": {3: 0.76}}, \"ax\": call_ax, \"color\": \"orange\", \"x\": \"index\", \"y\": \"score\"},\n {\n \"data\": {\"index\": {0: 0, 1: 1, 3: 3}, \"score\": {0: 0.6, 1: 0.2, 3: 0.6}},\n \"ax\": call_ax,\n \"color\": \"grey\",\n \"x\": \"index\",\n \"y\": \"score\",\n },\n {\"data\": {\"index\": {2: 2}, \"score\": {2: 0.91}}, \"ax\": call_ax, \"color\": \"orange\", \"x\": \"index\", \"y\": \"score\"},\n ]\n"
},
{
"path": "eda/tests/unittests/visualization/test_base.py",
"content": "from unittest.mock import MagicMock\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization.base import AbstractVisualization\n\n\nclass SomeVisualization(AbstractVisualization):\n def can_handle(self, state: AnalysisState) -> bool:\n return \"required_key\" in state\n\n def _render(self, state: AnalysisState) -> None:\n pass\n\n\ndef test_AbstractVisualization_cannot_render():\n viz = SomeVisualization()\n viz._render = MagicMock()\n viz.can_handle = MagicMock(wraps=viz.can_handle)\n viz.render(AnalysisState({\"ns1\": {\"required_key\": True, \"data\": 1}, \"ns2\": {}}))\n viz.can_handle.assert_called_once()\n viz._render.assert_not_called()\n\n\ndef test_AbstractVisualization_can_render():\n viz = SomeVisualization(namespace=\"ns1\")\n viz._render = MagicMock()\n viz.can_handle = MagicMock(wraps=viz.can_handle)\n viz.render(AnalysisState({\"ns1\": {\"required_key\": True, \"data\": 1}, \"ns2\": {}}))\n viz.can_handle.assert_called_once()\n viz._render.assert_called_with({\"required_key\": True, \"data\": 1})\n"
},
{
"path": "eda/tests/unittests/visualization/test_dataset.py",
"content": "import os\nfrom unittest.mock import ANY, MagicMock\n\nimport numpy as np\nimport pandas as pd\nimport pytest\nfrom numpy import dtype\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.analysis import MissingValuesAnalysis, RawTypesAnalysis, SpecialTypesAnalysis, VariableTypeAnalysis\nfrom autogluon.eda.analysis.dataset import DatasetSummary\nfrom autogluon.eda.auto import analyze\nfrom autogluon.eda.visualization import DatasetStatistics, DatasetTypeMismatch, LabelInsightsVisualization\n\nRESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), \"..\", \"resources\"))\n\n\ndef test_DatasetStatistics():\n train_data = pd.read_csv(os.path.join(RESOURCE_PATH, \"adult\", \"train_data.csv\"))[[\"education\", \"class\"]]\n\n viz = DatasetStatistics()\n viz.display_obj = MagicMock()\n\n analyze(\n train_data=train_data,\n label=\"class\",\n anlz_facets=[\n DatasetSummary(),\n RawTypesAnalysis(),\n VariableTypeAnalysis(),\n SpecialTypesAnalysis(),\n MissingValuesAnalysis(),\n ],\n viz_facets=[viz],\n )\n\n assert viz.display_obj.call_args.args[0].to_dict() == {\n \"count\": {\"class\": 200, \"education\": 200},\n \"dtypes\": {\"class\": dtype(\"O\"), \"education\": dtype(\"O\")},\n \"freq\": {\"class\": 154, \"education\": 59},\n \"missing_count\": {\"class\": \"\", \"education\": \"\"},\n \"missing_ratio\": {\"class\": \"\", \"education\": \"\"},\n \"raw_type\": {\"class\": \"object\", \"education\": \"object\"},\n \"special_types\": {\"class\": \"\", \"education\": \"\"},\n \"top\": {\"class\": \" <=50K\", \"education\": \" HS-grad\"},\n \"unique\": {\"class\": 2, \"education\": 14},\n \"variable_type\": {\"class\": \"category\", \"education\": \"category\"},\n }\n\n\n@pytest.mark.parametrize(\n \"state_present,expected\",\n [\n (\"dataset_stats\", True),\n (\"missing_statistics\", True),\n (\"raw_type\", True),\n (\"special_types\", True),\n (\"unknown_type\", False),\n ],\n)\ndef test_DatasetStatistics__can_handle(state_present, expected):\n assert DatasetStatistics().can_handle(AnalysisState({state_present: \"\"})) is expected\n\n\n@pytest.mark.parametrize(\"field\", [\"dataset_stats\", \"raw_type\", \"variable_type\", \"special_types\"])\ndef test__merge_analysis_facets__single_values(field):\n expected_result = {\"some_stat\": \"value\"}\n state = AnalysisState({field: {\"ds\": expected_result}})\n if field == \"dataset_stats\":\n assert DatasetStatistics._merge_analysis_facets(\"ds\", state) == expected_result\n else:\n assert DatasetStatistics._merge_analysis_facets(\"ds\", state) == {field: expected_result}\n\n\ndef test__merge_analysis_facets__single_values__missing_statistics():\n state = AnalysisState(\n {\"missing_statistics\": {\"ds\": {\"count\": [1, 2], \"ratio\": [0.1, 0.2], \"some_field\": [\"a\", \"b\"]}}}\n )\n assert DatasetStatistics._merge_analysis_facets(\"ds\", state) == {\n \"missing_count\": [1, 2],\n \"missing_ratio\": [0.1, 0.2],\n }\n\n\ndef test__merge_analysis_facets__multiple_values():\n state = AnalysisState(\n {\n \"dataset_stats\": {\"ds\": {\"dataset_stats\": \"value\"}},\n \"missing_statistics\": {\"ds\": {\"count\": [1, 2], \"ratio\": [0.1, 0.2], \"some_field\": [\"a\", \"b\"]}},\n \"raw_type\": {\"ds\": {\"raw_type\": \"value\"}},\n \"variable_type\": {\"ds\": {\"variable_type\": \"value\"}},\n \"special_types\": {\"ds\": {\"special_types\": \"value\"}},\n }\n )\n assert DatasetStatistics._merge_analysis_facets(\"ds\", state) == {\n \"dataset_stats\": \"value\",\n \"missing_count\": [1, 2],\n \"missing_ratio\": [0.1, 0.2],\n \"raw_type\": {\"raw_type\": \"value\"},\n \"special_types\": {\"special_types\": \"value\"},\n \"variable_type\": {\"variable_type\": \"value\"},\n }\n\n\ndef test__fix_counts():\n df = pd.DataFrame(\n {\n \"a\": [1.0, np.NaN],\n \"b\": [1.0, np.NaN],\n \"c\": [1, np.NaN],\n \"d\": [1, 2],\n }\n )\n expected_out = {\"a\": {0: 1, 1: \"\"}, \"b\": {0: 1.0, 1: \"--NA--\"}, \"c\": {0: 1, 1: \"\"}, \"d\": {0: 1, 1: 2}}\n assert DatasetStatistics._fix_counts(df, cols=[\"a\", \"c\", \"d\"]).fillna(\"--NA--\").to_dict() == expected_out\n\n\ndef test_DatasetTypeMismatch():\n df_train = pd.DataFrame({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\n df_test = pd.DataFrame({\"x\": [\"1\", \"2\", \"3\"], \"y\": [4, 5, 6]})\n\n viz = DatasetTypeMismatch()\n viz.render_header_if_needed = MagicMock()\n viz.display_obj = MagicMock()\n\n analyze(train_data=df_train, test_data=df_test, anlz_facets=[RawTypesAnalysis()], viz_facets=[viz])\n\n viz.render_header_if_needed.assert_called_with(ANY, \"Types warnings summary\")\n assert viz.display_obj.call_args.args[0].to_dict() == {\n \"test_data\": {\"x\": \"object\"},\n \"train_data\": {\"x\": \"int\"},\n \"warnings\": {\"x\": \"warning\"},\n }\n\n\ndef test_DatasetTypeMismatch__no_warnings():\n df_train = pd.DataFrame({\"x\": [1, 2, 3], \"y\": [4, 5, 6]})\n\n viz = DatasetTypeMismatch()\n viz.render_header_if_needed = MagicMock()\n viz.display_obj = MagicMock()\n analyze(train_data=df_train, test_data=df_train, anlz_facets=[RawTypesAnalysis()], viz_facets=[viz])\n\n viz.render_header_if_needed.assert_not_called()\n viz.display_obj.assert_not_called()\n\n\ndef test_LabelInsightsVisualization():\n state = AnalysisState(\n {\n \"label_insights\": {\n \"low_cardinality_classes\": {\n \"instances\": {\"A\": 10, \"B\": 20},\n \"threshold\": 50,\n },\n \"not_present_in_train\": [1, \"2\", True],\n \"ood\": {\n \"count\": 100,\n \"train_range\": [10, 100],\n \"test_range\": [10, 50],\n },\n }\n }\n )\n viz = LabelInsightsVisualization()\n viz.render_header_if_needed = MagicMock()\n viz.render_markdown = MagicMock()\n\n viz.render(state)\n\n viz.render_header_if_needed.assert_called_with(state, \"Label insights\")\n viz.render_markdown.assert_called_with(\n \" - Low-cardinality classes are detected. It is recommended to have at least \"\n \"`50` instances per class. Consider adding more data to cover the classes or \"\n \"remove such rows.\\n\"\n \" - class `A`: `10` instances\\n\"\n \" - class `B`: `20` instances\\n\"\n \" - the following classes are found in `test_data`, but not present in \"\n \"`train_data`: `1`, `2`, `True`. Consider either removing the rows with \"\n \"classes not covered or adding more training data covering the classes.\\n\"\n \" - Rows with out-of-domain labels were found. Consider removing rows with \"\n \"labels outside of this range or expand training data since some algorithms \"\n \"(i.e. trees) are unable to extrapolate beyond data present in the training \"\n \"data.\\n\"\n \" - `100` rows\\n\"\n \" - `train_data` values range `[10, 100]`\\n\"\n \" - `test_data` values range `[10, 50]`\"\n )\n\n\ndef test_LabelInsightsVisualization__no_state():\n state = AnalysisState()\n\n viz = LabelInsightsVisualization()\n viz.render_header_if_needed = MagicMock()\n viz.render_markdown = MagicMock()\n\n viz.render(state)\n\n viz.render_header_if_needed.assert_not_called()\n viz.render_markdown.assert_not_called()\n"
},
{
"path": "eda/tests/unittests/visualization/test_explain.py",
"content": "from unittest.mock import MagicMock, call\n\nimport numpy as np\nimport pandas as pd\nimport pytest\nfrom pandas import RangeIndex\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import ExplainForcePlot, ExplainWaterfallPlot\nfrom autogluon.eda.visualization.jupyter import JupyterMixin\n\n\n@pytest.mark.parametrize(\n \"display_rows\",\n [\n (True),\n (False),\n ],\n)\ndef test_ExplainForcePlot(display_rows, monkeypatch):\n state = AnalysisState()\n state.explain = {\n \"shapley\": [\n AnalysisState(\n row=\"row\",\n expected_value=\"expected_value\",\n shap_values=\"shap_values\",\n features=\"features\",\n feature_names=\"feature_names\",\n )\n ]\n }\n with monkeypatch.context() as m:\n call_shap_force_plot = MagicMock()\n call_display_obj = MagicMock()\n m.setattr(\"shap.force_plot\", call_shap_force_plot)\n m.setattr(JupyterMixin, \"display_obj\", call_display_obj)\n\n ExplainForcePlot(display_rows=display_rows, text_rotation=40, extra_arg=\"extra_arg\").render(state)\n\n if display_rows:\n call_display_obj.assert_called_with(\"row\")\n else:\n call_display_obj.assert_not_called()\n\n call_shap_force_plot.assert_called_with(\n \"expected_value\",\n \"shap_values\",\n \"features\",\n feature_names=\"feature_names\",\n text_rotation=40,\n matplotlib=True,\n extra_arg=\"extra_arg\",\n )\n\n\n@pytest.mark.parametrize(\n \"display_rows\",\n [\n (True),\n (False),\n ],\n)\ndef test_ExplainWaterfallPlot(display_rows, monkeypatch):\n state = AnalysisState()\n state.explain = {\n \"shapley\": [\n AnalysisState(\n row=\"row\",\n expected_value=\"expected_value\",\n shap_values=\"shap_values\",\n features=pd.DataFrame({\"feature_names\": [\"features\"]}),\n feature_names=\"feature_names\",\n )\n ]\n }\n with monkeypatch.context() as m:\n call_shap_waterfall_plot = MagicMock()\n call_shap_explanation = MagicMock(return_value=\"explanation_mock\")\n call_display_obj = MagicMock()\n m.setattr(\"shap.waterfall_plot\", call_shap_waterfall_plot)\n m.setattr(\"shap.Explanation\", call_shap_explanation)\n m.setattr(JupyterMixin, \"display_obj\", call_display_obj)\n\n ExplainWaterfallPlot(display_rows=display_rows, extra_arg=\"extra_arg\").render(state)\n\n if display_rows:\n call_display_obj.assert_called_with(\"row\")\n else:\n call_display_obj.assert_not_called()\n\n call_shap_waterfall_plot.assert_called_with(\"explanation_mock\")\n\n assert call_shap_explanation.call_args.args == (\"shap_values\",)\n kwargs = call_shap_explanation.call_args.kwargs\n output_names = kwargs.pop(\"output_names\")\n assert output_names.equals(pd.DataFrame({\"feature_names\": [\"features\"]}))\n assert kwargs == dict(base_values=\"expected_value\", feature_names=RangeIndex(start=0, stop=1, step=1))\n"
},
{
"path": "eda/tests/unittests/visualization/test_interaction.py",
"content": "from unittest.mock import ANY, MagicMock\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pytest\nimport scipy\nimport seaborn as sns\nfrom hamcrest.library.integration import match_equality\nfrom pandas import DataFrame\nfrom sklearn.inspection import PartialDependenceDisplay\n\nimport autogluon.eda.auto as auto\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT, R_OBJECT\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import (\n CorrelationSignificanceVisualization,\n CorrelationVisualization,\n FeatureInteractionVisualization,\n PDPInteractions,\n)\nfrom autogluon.eda.visualization.interaction import FeatureDistanceAnalysisVisualization\n\n\ndef test_CorrelationVisualization_single_value(monkeypatch):\n state = AnalysisState(\n {\n \"correlations\": {\"train_data\": pd.DataFrame(index=[\"a\"], data={\"a\": [1.00]})},\n \"correlations_method\": \"kendall\",\n }\n )\n call_subplots = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n auto.analyze(state=state, viz_facets=[(CorrelationVisualization())])\n call_subplots.assert_not_called()\n\n\ndef test_CorrelationVisualization(monkeypatch):\n state = AnalysisState({\"correlations\": {\"train_data\": __get_train_data()}, \"correlations_method\": \"kendall\"})\n heatmap_args = dict(vmin=-1, vmax=1, center=0, cmap=\"Spectral\")\n __test_internal(monkeypatch, \"correlations\", state, heatmap_args, CorrelationVisualization)\n\n\ndef test_CorrelationVisualization_phik(monkeypatch):\n state = AnalysisState({\"correlations\": {\"train_data\": __get_train_data()}, \"correlations_method\": \"phik\"})\n heatmap_args = dict(vmin=0, vmax=1, center=0, cmap=\"Spectral\")\n __test_internal(\n monkeypatch,\n \"correlations\",\n state,\n heatmap_args,\n CorrelationVisualization,\n \"**`train_data` - `phik` correlation matrix**\",\n headers=True,\n )\n\n\ndef test_CorrelationVisualization_focus(monkeypatch):\n state = AnalysisState(\n {\n \"correlations\": {\"train_data\": __get_train_data()},\n \"correlations_method\": \"spearman\",\n \"correlations_focus_field\": \"c\",\n \"correlations_focus_field_threshold\": 0.6,\n \"correlations_focus_high_corr\": {\"train_data\": pd.DataFrame(index=list(\"ad\"), data={\"c\": [0.88, -1.00]})},\n }\n )\n heatmap_args = dict(vmin=-1, vmax=1, center=0, cmap=\"Spectral\")\n __test_internal(\n monkeypatch,\n \"correlations\",\n state,\n heatmap_args,\n CorrelationVisualization,\n \"**`train_data` - `spearman` correlation matrix; focus: absolute correlation for `c` >= `0.6`**\",\n headers=True,\n )\n\n\ndef test_CorrelationVisualization__can_handle():\n assert (\n CorrelationVisualization().can_handle(AnalysisState({\"correlations\": 123, \"something\": \"something\"})) is True\n )\n assert CorrelationVisualization().can_handle(AnalysisState({\"something\": \"something\"})) is False\n\n\ndef test_CorrelationSignificanceVisualization__can_handle():\n assert (\n CorrelationSignificanceVisualization().can_handle(\n AnalysisState({\"significance_matrix\": 123, \"something\": \"something\"})\n )\n is True\n )\n assert CorrelationSignificanceVisualization().can_handle(AnalysisState({\"something\": \"something\"})) is False\n\n\ndef test_CorrelationSignificanceVisualization(monkeypatch):\n state = AnalysisState(\n {\n \"correlations\": {\"train_data\": __get_train_data()},\n \"significance_matrix\": {\n \"train_data\": __get_train_data(),\n },\n \"correlations_method\": \"spearman\",\n \"correlations_focus_field\": \"c\",\n \"correlations_focus_field_threshold\": 0.6,\n \"correlations_focus_high_corr\": {\"train_data\": pd.DataFrame(index=list(\"ad\"), data={\"c\": [0.88, -1.00]})},\n }\n )\n heatmap_args = dict(center=3, vmax=5, cmap=\"Spectral\", robust=True)\n __test_internal(monkeypatch, \"significance_matrix\", state, heatmap_args, CorrelationSignificanceVisualization)\n\n\ndef __get_train_data():\n return pd.DataFrame(\n index=list(\"abcd\"),\n data={\n \"a\": [1.00, 0.00, 0.77, -0.77],\n \"b\": [0.00, 1.00, 0.36, -0.36],\n \"c\": [0.77, 0.36, 1.00, -1.00],\n \"d\": [-0.77, -0.36, -1.00, 1.00],\n },\n )\n\n\ndef __test_internal(monkeypatch, render_key, state, heatmap_args, facet_cls, text_render=None, **kwargs):\n call_heatmap = MagicMock()\n call_show = MagicMock()\n call_yticks = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n call_render_markdown = MagicMock()\n\n analysis = facet_cls(fig_args=dict(some_arg=123), **kwargs)\n analysis.render_markdown = call_render_markdown\n\n with monkeypatch.context() as m:\n m.setattr(sns, \"heatmap\", call_heatmap)\n m.setattr(plt, \"show\", call_show)\n m.setattr(plt, \"yticks\", call_yticks)\n m.setattr(plt, \"subplots\", call_subplots)\n auto.analyze(state=state, viz_facets=[analysis])\n\n call_yticks.assert_called_with(rotation=0)\n call_subplots.assert_called_with(some_arg=123, figsize=(4, 4))\n call_show.assert_called_with(\"fig\")\n\n call_heatmap.assert_called_with(\n state[render_key].train_data,\n annot=True,\n ax=\"ax\",\n linewidths=0.5,\n linecolor=\"lightgrey\",\n fmt=\".2f\",\n square=True,\n cbar_kws={\"shrink\": 0.5},\n **heatmap_args,\n )\n if text_render is not None:\n call_render_markdown.assert_called_with(text_render)\n else:\n call_render_markdown.assert_not_called()\n\n\ndef test_FeatureInteractionVisualization__happy_path(monkeypatch):\n state = __get_feature_interaction_state()\n call_render = MagicMock()\n call_show = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n\n viz = FeatureInteractionVisualization(\n key=\"abc\", numeric_as_categorical_threshold=2, some_chart_arg=123, fig_args=dict(key=\"value\")\n )\n\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"show\", call_show)\n m.setattr(viz._RegPlotRenderer, \"_render\", call_render)\n auto.analyze(state=state, viz_facets=[viz])\n\n call_render.assert_called_with(\n match_equality(state), # noqa\n \"train_data\",\n (\"a\", \"b\", None),\n (\"numeric\", \"numeric\", None),\n \"ax\",\n match_equality(state.interactions.train_data.abc.data),\n dict(x=\"a\", y=\"b\", some_chart_arg=123),\n )\n call_show.assert_called_with(\"fig\")\n call_subplots.assert_called_with(key=\"value\", figsize=(12, 6))\n\n\n@pytest.mark.parametrize(\"is_single, header\", [(True, True), (False, True), (True, False), (False, False)])\ndef test_FeatureInteractionVisualization__headers(monkeypatch, is_single, header):\n state = __get_feature_interaction_state()\n if is_single:\n state.interactions.train_data.abc.features.pop(\"y\")\n call_render = MagicMock()\n call_show = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n\n viz = FeatureInteractionVisualization(key=\"abc\", numeric_as_categorical_threshold=2, headers=header)\n viz.render_markdown = MagicMock()\n\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"show\", call_show)\n if is_single:\n m.setattr(viz._HistPlotRenderer, \"_render\", call_render)\n else:\n m.setattr(viz._RegPlotRenderer, \"_render\", call_render)\n auto.analyze(state=state, viz_facets=[viz])\n\n if not header:\n viz.render_markdown.assert_not_called()\n elif is_single:\n viz.render_markdown.assert_called_with(\"**`a` in `train_data`**\")\n else:\n viz.render_markdown.assert_called_with(\"**Feature interaction between `a`/`b` in `train_data`**\")\n\n\ndef test_FeatureInteractionVisualization__state_different_key(monkeypatch):\n state = __get_feature_interaction_state()\n call_render = MagicMock()\n call_show = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n\n viz = FeatureInteractionVisualization(\n key=\"not_present\", numeric_as_categorical_threshold=2, some_chart_arg=123, fig_args=dict(key=\"value\")\n )\n\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"show\", call_show)\n m.setattr(viz._RegPlotRenderer, \"_render\", call_render)\n auto.analyze(state=state, viz_facets=[viz])\n\n call_render.assert_not_called()\n call_show.assert_not_called()\n call_subplots.assert_not_called()\n\n\ndef test_FeatureInteractionVisualization__no_renderer(monkeypatch):\n state = __get_feature_interaction_state()\n state.interactions.train_data.abc.features = {}\n call_show = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n\n viz = FeatureInteractionVisualization(key=\"abc\")\n\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"show\", call_show)\n auto.analyze(state=state, viz_facets=[viz])\n\n call_show.assert_not_called()\n call_subplots.assert_not_called()\n\n\ndef test_FeatureInteractionVisualization__convert_categoricals_to_objects():\n df = pd.DataFrame(\n {\n \"a\": [1, 2, 3],\n \"b\": [True, False, True],\n \"c\": [0.1, 0.2, 0.1],\n }\n )\n assert [str(t).replace(\"64\", \"\").replace(\"32\", \"\") for t in df.dtypes.to_list()] == [\"int\", \"bool\", \"float\"]\n FeatureInteractionVisualization(key=\"abc\")._convert_categoricals_to_objects(\n df, \"a\", \"category\", \"b\", \"category\", \"c\", \"category\"\n )\n assert [str(t).replace(\"64\", \"\").replace(\"32\", \"\") for t in df.dtypes.to_list()] == [\"object\", \"object\", \"object\"]\n\n\n@pytest.mark.parametrize(\n \"x, x_type, y, y_type, hue, expected_is_single, restrict_to_col\",\n [\n (\"aa\", \"numeric\", \"bb\", \"numeric\", None, False, None),\n (None, None, \"bb\", \"numeric\", None, True, \"bb\"),\n (\"aa\", \"category\", \"bb\", \"numeric\", None, False, None),\n (None, None, \"bb\", \"numeric\", None, True, \"bb\"),\n (\"aa\", \"numeric\", None, None, None, True, \"aa\"),\n (None, None, None, None, None, False, None),\n (\"aa\", \"category\", None, None, None, True, None),\n (None, None, None, None, None, False, None),\n (\"aa\", \"numeric\", \"bb\", \"category\", None, False, None),\n (None, None, \"bb\", \"category\", None, True, None),\n (\"aa\", \"category\", \"bb\", \"category\", None, False, None),\n (None, None, \"bb\", \"category\", None, True, None),\n (\"aa\", \"numeric\", None, None, None, True, \"aa\"),\n (None, None, None, None, None, False, None),\n (\"aa\", \"category\", None, None, None, True, None),\n (None, None, None, None, None, False, None),\n (\"aa\", \"numeric\", \"bb\", \"numeric\", \"cc\", False, None),\n (None, None, \"bb\", \"numeric\", \"cc\", False, None),\n (\"aa\", \"category\", \"bb\", \"numeric\", \"cc\", False, None),\n (None, None, \"bb\", \"numeric\", \"cc\", False, None),\n (\"aa\", \"numeric\", None, None, \"cc\", False, None),\n (None, None, None, None, \"cc\", False, None),\n (\"aa\", \"category\", None, None, \"cc\", False, None),\n (None, None, None, None, \"cc\", False, None),\n (\"aa\", \"numeric\", \"bb\", \"category\", \"cc\", False, None),\n (None, None, \"bb\", \"category\", \"cc\", False, None),\n (\"aa\", \"category\", \"bb\", \"category\", \"cc\", False, None),\n (None, None, \"bb\", \"category\", \"cc\", False, None),\n (\"aa\", \"numeric\", None, None, \"cc\", False, None),\n (None, None, None, None, \"cc\", False, None),\n (\"aa\", \"category\", None, None, \"cc\", False, None),\n (None, None, None, None, \"cc\", False, None),\n ],\n)\ndef test_FeatureInteractionVisualization__prepare_chart_args__single_var(\n x, x_type, y, y_type, hue, expected_is_single, restrict_to_col\n):\n df = pd.DataFrame(\n {\n \"aa\": [1, 2, 3],\n \"bb\": [0.1, 0.2, 0.1],\n }\n )\n\n viz = FeatureInteractionVisualization(key=\"abc\", some_kwarg=123)\n chart_args, data, is_single_var = viz._prepare_chart_args(df, x=x, x_type=x_type, y=y, y_type=y_type, hue=hue)\n\n assert is_single_var is expected_is_single\n assert chart_args[\"some_kwarg\"] == 123\n if restrict_to_col is None:\n assert data.equals(df)\n for var, col, val in [(x, \"x\", \"aa\"), (y, \"y\", \"bb\"), (hue, \"hue\", \"cc\")]:\n if var is not None:\n assert chart_args[col] == val\n else:\n assert col not in chart_args\n else:\n assert data.equals(df[restrict_to_col])\n\n\n@pytest.mark.parametrize(\n \"x_type, y_type, hue_type, swapped\",\n [\n (None, \"category\", \"~~\", False),\n (None, \"other~~~\", \"~~\", False),\n (None, \"category\", None, False),\n (None, \"other~~~\", None, False),\n (\"~~\", \"other~~~\", \"~~\", False),\n (\"~~\", \"other~~~\", None, False),\n (\"~~\", \"category\", \"~~\", False),\n (\"~~\", \"category\", None, True),\n ],\n)\ndef test_FeatureInteractionVisualization__fig_args(x_type, y_type, hue_type, swapped):\n y = 2\n hue = 3\n _y, _y_type, _hue, _hue_type = FeatureInteractionVisualization(key=\"abc\")._swap_y_and_hue_if_necessary(\n x_type, y, y_type, hue, hue_type\n )\n if swapped:\n assert _hue == y\n assert _hue_type == y_type\n assert _y is None\n assert _y_type is None\n else:\n assert _y == y\n assert _y_type == y_type\n assert _hue == hue\n assert _hue_type == hue_type\n\n\n@pytest.mark.parametrize(\n \"col, raw_type, numeric_as_categorical_threshold, expected_type\",\n [\n (None, R_INT, 20, None),\n (\"a\", R_INT, 20, \"category\"),\n (\"a\", R_INT, 2, \"numeric\"),\n (\"a\", R_FLOAT, 2, \"numeric\"),\n (\"a\", R_OBJECT, 2, \"category\"),\n (\"a\", R_CATEGORY, 2, \"category\"),\n (\"a\", R_BOOL, 2, \"category\"),\n (\"a\", \"some_type\", 2, None),\n ],\n)\ndef test_FeatureInteractionVisualization__map_raw_type_to_feature_type(\n monkeypatch, col, raw_type, numeric_as_categorical_threshold, expected_type\n):\n df = pd.DataFrame({\"a\": [1, 2, 3]})\n v = FeatureInteractionVisualization(key=\"abc\")\n actual_type = v._map_raw_type_to_feature_type(col, raw_type, df, numeric_as_categorical_threshold)\n if expected_type is None:\n assert actual_type is None\n else:\n assert actual_type == expected_type\n\n\ndef __get_feature_interaction_state():\n class EqualDataFrames(DataFrame):\n def __eq__(self, other):\n return self.equals(other)\n\n state = AnalysisState(\n {\n \"interactions\": {\n \"train_data\": {\n \"abc\": {\n \"data\": EqualDataFrames(data={\"a\": [1, 2, 3], \"b\": [\"a\", \"b\", \"c\"], \"c\": [0.1, 0.2, 0.3]}),\n \"features\": {\"x\": \"a\", \"y\": \"b\"},\n }\n }\n },\n \"raw_type\": {\"train_data\": {\"a\": \"int\", \"b\": \"int\", \"c\": \"int\", \"d\": \"int\", \"e\": \"object\", \"f\": \"object\"}},\n }\n )\n return state\n\n\n@pytest.mark.parametrize(\"has_dist_fit\", [True, False])\ndef test_FeatureInteractionVisualization__HistPlotRenderer(monkeypatch, has_dist_fit):\n r = FeatureInteractionVisualization._HistPlotRenderer()\n\n state = AnalysisState(\n {\n \"distributions_fit\": {\n \"train_data\": {\n \"aaa\": {\n \"fisk\": {\"param\": (11, -65, 94), \"statistic\": 0.04, \"pvalue\": 0.17},\n \"lognorm\": {\"param\": (0.12, -82, 111), \"statistic\": 0.05, \"pvalue\": 0.021},\n }\n }\n }\n }\n )\n if not has_dist_fit:\n state.distributions_fit.train_data.aaa = None\n\n params = (\"aaa\", None, None)\n param_types = (\"numeric\", None, None)\n chart_args = dict(some_chart_arg=123)\n\n ax = MagicMock()\n ax.get_xlim = MagicMock(return_value=(0, 1))\n call_sns_histplot = MagicMock()\n call_plt_legend = MagicMock()\n\n with monkeypatch.context() as m:\n m.setattr(sns, \"histplot\", call_sns_histplot)\n m.setattr(plt, \"legend\", call_plt_legend)\n r._render(state, \"train_data\", params, param_types, ax, \"data\", chart_args, num_point_to_fit=3)\n\n if has_dist_fit:\n call_sns_histplot.assert_called_with(ax=ax, data=\"data\", some_chart_arg=123, stat=\"density\")\n ax.get_xlim.assert_called()\n ax.plot.assert_called_with(\n ANY,\n ANY,\n ls=\"--\",\n label=\"lognorm: pvalue 0.02\",\n )\n ax.set_xlim.assert_called()\n call_plt_legend.assert_called()\n else:\n call_sns_histplot.assert_called_with(ax=ax, data=\"data\", stat=\"density\", some_chart_arg=123)\n ax.get_xlim.assert_not_called()\n ax.plot.assert_not_called()\n ax.set_xlim.assert_not_called()\n call_plt_legend.assert_not_called()\n\n\n@pytest.mark.parametrize(\"has_near_duplicates\", [True, False])\ndef test_FeatureDistanceAnalysisVisualization__happy_path(monkeypatch, has_near_duplicates):\n state = AnalysisState(\n {\n \"feature_distance\": {\n \"columns\": [\n \"age\",\n \"fnlwgt\",\n \"education-num\",\n \"sex\",\n \"capital-gain\",\n \"capital-loss\",\n \"hours-per-week\",\n \"workclass\",\n \"education\",\n \"marital-status\",\n \"occupation\",\n \"relationship\",\n \"race\",\n \"native-country\",\n ],\n \"linkage\": np.array(\n [\n [9.0, 11.0, 0.6113, 2.0],\n [3.0, 10.0, 0.7652, 2.0],\n [7.0, 15.0, 0.7905, 3.0],\n [2.0, 6.0, 0.8097, 2.0],\n [0.0, 4.0, 0.8306, 2.0],\n [17.0, 18.0, 0.86785, 4.0],\n [12.0, 13.0, 0.8872, 2.0],\n [8.0, 20.0, 0.9333, 3.0],\n [1.0, 5.0, 0.946, 2.0],\n [16.0, 19.0, 0.94793333, 7.0],\n [21.0, 23.0, 0.9676619, 10.0],\n [22.0, 24.0, 0.981165, 12.0],\n [14.0, 25.0, 1.13729167, 14.0],\n ]\n ),\n \"near_duplicates\": [],\n \"near_duplicates_threshold\": 0.85,\n }\n }\n )\n\n if has_near_duplicates:\n state.feature_distance[\"near_duplicates\"] = [\n {\"distance\": 0.6113, \"nodes\": [\"marital-status\", \"relationship\"]},\n {\"distance\": 0.7905, \"nodes\": [\"occupation\", \"sex\", \"workclass\"]},\n {\"distance\": 0.8097, \"nodes\": [\"education-num\", \"hours-per-week\"]},\n {\"distance\": 0.8306, \"nodes\": [\"age\", \"capital-gain\"]},\n ]\n\n ax = MagicMock()\n call_subplots = MagicMock(return_value=(\"fig\", ax))\n call_dendrogram = MagicMock()\n call_show = MagicMock()\n call_render_markdown = MagicMock()\n\n viz = FeatureDistanceAnalysisVisualization(some_chart_arg=123, fig_args=dict(key=\"value\"))\n viz.render_markdown = call_render_markdown\n\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"show\", call_show)\n m.setattr(scipy.cluster.hierarchy, \"dendrogram\", call_dendrogram)\n auto.analyze(state=state, viz_facets=[viz])\n\n call_dendrogram.assert_called_with(\n ax=ax, Z=ANY, labels=state.feature_distance.columns, leaf_font_size=10, orientation=\"left\", some_chart_arg=123\n )\n np.testing.assert_array_equal(call_dendrogram.call_args[1][\"Z\"], state.feature_distance.linkage)\n call_show.assert_called_with(\"fig\")\n call_subplots.assert_called_with(key=\"value\", figsize=(12, 3.5))\n if has_near_duplicates:\n call_render_markdown.assert_called_with(\n \"**The following feature groups are considered as near-duplicates**:\\n\\nDistance threshold: <= `0.85`. \"\n \"Consider keeping only some of the columns within each group:\\n\\n\"\n \" - `marital-status`, `relationship` - distance `0.61`\\n\"\n \" - `occupation`, `sex`, `workclass` - distance `0.79`\\n\"\n \" - `education-num`, `hours-per-week` - distance `0.81`\\n\"\n \" - `age`, `capital-gain` - distance `0.83`\"\n )\n else:\n call_render_markdown.assert_not_called()\n\n\ndef test_FeatureInteractionVisualization__no_fig_args():\n assert FeatureDistanceAnalysisVisualization().fig_args == {}\n\n\n__get_args_RESULTS_KWARGS_1 = {\"ice_lines_kw\": {\"color\": \"blue\"}, \"kind\": \"both\", \"pd_line_kw\": {\"color\": \"red\"}}\n\n\ndef __get_args__figargs(figsize, ncols, nrows):\n return {\"figsize\": figsize, \"ncols\": ncols, \"nrows\": nrows}\n\n\n@pytest.mark.parametrize(\n \"features, two_way, expected_result\",\n [\n (\"A\", False, (__get_args_RESULTS_KWARGS_1, __get_args__figargs((12, 3), 1, 1), [\"A\"])),\n ([\"A\"], False, (__get_args_RESULTS_KWARGS_1, __get_args__figargs((12, 3), 1, 1), [\"A\"])),\n ([\"A\", \"B\"], False, (__get_args_RESULTS_KWARGS_1, __get_args__figargs((12, 3), 2, 1), [\"A\", \"B\"])),\n ([\"A\", \"B\", \"C\"], False, (__get_args_RESULTS_KWARGS_1, __get_args__figargs((12, 6), 2, 2), [\"A\", \"B\", \"C\"])),\n (\"A\", True, AssertionError),\n ([\"A\"], True, AssertionError),\n ([\"A\", \"B\"], True, ({\"kind\": \"average\"}, __get_args__figargs((12, 3), 3, 1), [\"A\", \"B\", [\"A\", \"B\"]])),\n ([\"A\", \"B\", \"C\"], True, AssertionError),\n ],\n)\ndef test_PDPInteractions__get_args(features, two_way, expected_result):\n if expected_result is AssertionError:\n with pytest.raises(AssertionError):\n viz = PDPInteractions(features=features, two_way=two_way)\n viz._get_args()\n else:\n viz = PDPInteractions(features=features, two_way=two_way)\n assert viz._get_args() == expected_result\n\n\ndef test_PDPInteractions(monkeypatch):\n ax = MagicMock()\n ax.ravel = MagicMock(return_value=[\"ax1, ax2\"])\n call_subplots = MagicMock(return_value=(\"fig\", ax))\n call_from_estimator = MagicMock()\n model = MagicMock()\n model.problem_type = \"regression\"\n state = AnalysisState(pdp_data=\"data\", model=model)\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"tight_layout\", MagicMock())\n m.setattr(plt, \"show\", MagicMock())\n m.setattr(PartialDependenceDisplay, \"from_estimator\", call_from_estimator)\n viz = PDPInteractions(features=[\"A\", \"B\"], target=\"target\", sample=123)\n viz.render(state)\n\n ax.ravel.assert_called_once()\n call_subplots.assert_called_with(nrows=1, ncols=2, figsize=(12, 3))\n call_from_estimator.assert_called_with(\n ANY,\n \"data\",\n [\"A\", \"B\"],\n ax=[\"ax1, ax2\"],\n target=\"target\",\n subsample=123,\n pd_line_kw={\"color\": \"red\"},\n ice_lines_kw={\"color\": \"blue\"},\n kind=\"both\",\n )\n wrapper = call_from_estimator.call_args.args[0]\n assert wrapper.estimator == model\n assert wrapper.estimator_type == \"regressor\"\n\n\ndef test_PDPInteractions__two_way(monkeypatch):\n data = pd.DataFrame({\"A\": [\"1\", \"2\", \"3\", None, None, \"6\"], \"B\": [\"6\", \"5\", None, \"3\", None, \"1\"]})\n\n ax = MagicMock()\n ax.ravel = MagicMock(return_value=[\"ax1, ax2\"])\n call_subplots = MagicMock(return_value=(\"fig\", ax))\n call_from_estimator = MagicMock()\n model = MagicMock()\n model.problem_type = \"binary\"\n state = AnalysisState(pdp_data=data, model=model)\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_subplots)\n m.setattr(plt, \"tight_layout\", MagicMock())\n m.setattr(plt, \"show\", MagicMock())\n m.setattr(PartialDependenceDisplay, \"from_estimator\", call_from_estimator)\n viz = PDPInteractions(features=[\"A\", \"B\"], target=\"target\", two_way=True, sample=123)\n viz.render(state)\n\n ax.ravel.assert_called_once()\n call_subplots.assert_called_with(nrows=1, ncols=3, figsize=(12, 3))\n call_from_estimator.assert_called_with(\n ANY,\n ANY,\n [\"A\", \"B\", [\"A\", \"B\"]],\n ax=[\"ax1, ax2\"],\n target=\"target\",\n subsample=123,\n kind=\"average\",\n )\n wrapper = call_from_estimator.call_args.args[0]\n called_data = call_from_estimator.call_args.args[1]\n assert wrapper.estimator == model\n assert wrapper.estimator_type == \"classifier\"\n assert called_data.to_dict() == {\"A\": {0: \"1\", 1: \"2\", 5: \"6\"}, \"B\": {0: \"6\", 1: \"5\", 5: \"1\"}}\n"
},
{
"path": "eda/tests/unittests/visualization/test_layouts.py",
"content": "from unittest.mock import MagicMock\n\nimport pytest\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import PropertyRendererComponent\n\n\n@pytest.mark.parametrize(\"with_transform_fn, expected\", [(True, \"VALUE\"), (False, \"value\")])\ndef test_PropertyRendererComponent(with_transform_fn, expected):\n state = AnalysisState({\"some\": {\"prop\": \"value\"}})\n call_display_obj = MagicMock()\n\n transform_fn = (lambda v: v.upper()) if with_transform_fn else None\n\n viz = PropertyRendererComponent(\"some.prop\", transform_fn=transform_fn)\n viz.display_obj = call_display_obj\n\n viz.render(state)\n\n call_display_obj.assert_called_with(expected)\n"
},
{
"path": "eda/tests/unittests/visualization/test_missing.py",
"content": "from unittest.mock import MagicMock, call\n\nimport missingno as msno\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import MissingValues\n\n\ndef test_MissingValues():\n state = __prepare_test_data()\n viz = MissingValues(graph_type=\"matrix\", headers=True, abc=123)\n viz._internal_render = MagicMock()\n viz.render_markdown = MagicMock()\n\n viz.render(state)\n\n assert viz._internal_render.call_count == 2\n\n assert viz.render_markdown.call_count == 2\n viz.render_markdown.assert_has_calls(\n calls=[\n call(\"**`train_data` missing values analysis**\"),\n call(\"**`test_data` missing values analysis**\"),\n ]\n )\n\n viz._internal_render.assert_has_calls(\n calls=[\n call(msno.matrix, state.missing_statistics.train_data.data, abc=123),\n call(msno.matrix, state.missing_statistics.test_data.data, abc=123),\n ]\n )\n\n\ndef test_MissingValues__no_headers():\n state = __prepare_test_data()\n viz = MissingValues(headers=False)\n viz._internal_render = MagicMock()\n viz.render_markdown = MagicMock()\n\n viz.render(state)\n\n assert viz._internal_render.call_count == 2\n assert viz.render_markdown.call_count == 0\n\n\n@pytest.mark.parametrize(\n \"input_type,expected\",\n [(\"matrix\", msno.matrix), (\"bar\", msno.bar), (\"heatmap\", msno.heatmap), (\"dendrogram\", msno.dendrogram)],\n)\ndef test_get_operation(input_type, expected):\n assert MissingValues()._get_operation(input_type) is expected\n\n\n@pytest.mark.parametrize(\n \"cols_number,expected\",\n [\n (0, False),\n (1, False),\n (50, False),\n (51, True),\n ],\n)\ndef test_has_too_many_variables_for_matrix(cols_number, expected):\n cols = [f\"col{i}\" for i in range(cols_number)]\n df_test = pd.DataFrame((np.arange(100))[:, None].repeat([len(cols)], axis=1), columns=cols)\n s = {\n \"missing_statistics\": {\n \"test_data\": {\n \"data\": df_test,\n }\n }\n }\n assert MissingValues()._has_too_many_variables_for_matrix(AnalysisState(s)) is expected\n\n\ndef __prepare_test_data():\n cols = list(\"AB\")\n df_train = pd.DataFrame((np.arange(100))[:, None].repeat([len(cols)], axis=1), columns=cols)\n df_test = pd.DataFrame((np.arange(200))[:, None].repeat([len(cols)], axis=1), columns=cols)\n for df in [df_train, df_test]:\n df[\"A\"] = (df[\"A\"] % 4).replace(2, np.NaN)\n s = {\n \"missing_statistics\": {\n \"train_data\": {\n \"data\": df_train,\n },\n \"test_data\": {\n \"data\": df_test,\n },\n }\n }\n state = AnalysisState(s)\n return state\n"
},
{
"path": "eda/tests/unittests/visualization/test_model.py",
"content": "from unittest.mock import ANY, MagicMock\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pytest\nimport seaborn as sns\n\nfrom autogluon.eda import AnalysisState\nfrom autogluon.eda.visualization import ConfusionMatrix, FeatureImportance, ModelLeaderboard, RegressionEvaluation\n\n\n@pytest.mark.parametrize(\"confusion_matrix_normalized,expected_fmt\", [(True, \",.2%\"), (False, \"d\")])\ndef test_ConfusionMatrix(monkeypatch, confusion_matrix_normalized, expected_fmt):\n state = AnalysisState(\n {\n \"model_evaluation\": {\n \"problem_type\": \"binary\",\n \"y_true\": pd.Series([0, 1, 0, 1]),\n \"y_pred\": pd.Series([1, 0, 1, 0]),\n \"importance\": pd.DataFrame(\n {\n \"importance\": [0.1, 0.2, 0.3],\n \"stddev\": [0.01, 0.02, 0.03],\n \"p_value\": [0.1, 0.2, 0.3],\n \"n\": [5, 5, 5],\n \"p99_high\": [0.02, 0.03, 0.04],\n \"p99_low\": [-0.02, -0.03, -0.04],\n }\n ),\n \"confusion_matrix_normalized\": confusion_matrix_normalized,\n \"confusion_matrix\": np.array([[148, 20], [32, 68]]),\n }\n }\n )\n\n call_plt_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n call_plt_show = MagicMock()\n call_sns_heatmap = MagicMock()\n call_render_markdown = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_plt_subplots)\n m.setattr(plt, \"show\", call_plt_show)\n m.setattr(sns, \"heatmap\", call_sns_heatmap)\n viz = ConfusionMatrix(fig_args=dict(a=1, b=2), headers=True, some_kwarg=123)\n viz.render_markdown = call_render_markdown\n viz.render(state)\n call_plt_subplots.assert_called_with(a=1, b=2, figsize=(2, 2))\n call_plt_show.assert_called_with(\"fig\")\n call_sns_heatmap.assert_called_with(\n ANY,\n ax=\"ax\",\n cmap=\"Blues\",\n annot=True,\n linewidths=0.5,\n linecolor=\"lightgrey\",\n fmt=expected_fmt,\n cbar=False,\n some_kwarg=123,\n )\n call_render_markdown.assert_called_with(\"**Confusion Matrix**\")\n\n\ndef test_ConfusionMatrix__can_handle():\n state = AnalysisState({\"model_evaluation\": {\"confusion_matrix\": np.array([])}})\n assert ConfusionMatrix().can_handle(state) is True\n assert ConfusionMatrix().can_handle(AnalysisState({\"some_args\": {}})) is False\n\n\ndef test_RegressionEvaluation(monkeypatch):\n state = AnalysisState(\n {\n \"model_evaluation\": {\n \"problem_type\": \"regression\",\n \"y_true_train\": pd.Series([0, 1, 0, 0]),\n \"y_pred_train\": pd.Series([1, 0, 1, 1]),\n \"y_true_val\": pd.Series([0, 1, 0, 1]),\n \"y_pred_val\": pd.Series([1, 0, 1, 0]),\n }\n }\n )\n\n call_plt_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n call_plt_show = MagicMock()\n call_residuals_plot = MagicMock()\n call_render_markdown = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_plt_subplots)\n m.setattr(plt, \"show\", call_plt_show)\n m.setattr(\"autogluon.eda.visualization.model.residuals_plot\", call_residuals_plot)\n viz = RegressionEvaluation(headers=True, fig_args=dict(a=1, b=2), some_kwarg=123)\n viz.residuals_plot = call_residuals_plot\n viz.render_markdown = call_render_markdown\n viz.render(state)\n call_plt_subplots.assert_called_with(figsize=(12, 6), a=1, b=2)\n call_plt_show.assert_called_with(\"fig\")\n call_residuals_plot.assert_called_with(\n ANY,\n state.model_evaluation.y_pred_train,\n state.model_evaluation.y_true_train,\n state.model_evaluation.y_pred_val,\n state.model_evaluation.y_true_val,\n show=False,\n ax=\"ax\",\n hist=False,\n qqplot=True,\n )\n call_render_markdown.assert_called_with(\"**Prediction vs Target**\")\n\n\n@pytest.mark.parametrize(\n \"train_present, val_present, test_present, expected\",\n [\n (True, True, True, (\"pred_train\", \"true_train\", \"pred_test\", \"true_test\")),\n (False, True, True, (\"pred_val\", \"true_val\", \"pred_test\", \"true_test\")),\n (True, False, True, (\"pred_train\", \"true_train\", \"pred_test\", \"true_test\")),\n (False, False, True, (None, None, \"pred_test\", \"true_test\")),\n (True, True, False, (\"pred_train\", \"true_train\", \"pred_val\", \"true_val\")),\n (False, True, False, (\"pred_val\", \"true_val\", None, None)),\n (True, False, False, (\"pred_train\", \"true_train\", None, None)),\n (False, False, False, (None, None, None, None)),\n ],\n)\ndef test_RegressionEvaluation__repack_parameters(train_present, val_present, test_present, expected):\n s = {}\n if train_present:\n s[\"y_true_train\"] = \"true_train\"\n s[\"y_pred_train\"] = \"pred_train\"\n if val_present:\n s[\"y_true_val\"] = \"true_val\"\n s[\"y_pred_val\"] = \"pred_val\"\n if test_present:\n s[\"y_true_test\"] = \"true_test\"\n s[\"y_pred_test\"] = \"pred_test\"\n\n state = AnalysisState(model_evaluation=s)\n assert RegressionEvaluation._repack_parameters(state.model_evaluation) == expected\n\n\ndef test_RegressionEvaluation__can_handle():\n assert (\n RegressionEvaluation().can_handle(\n AnalysisState(\n {\n \"model_evaluation\": {\n \"problem_type\": \"regression\",\n }\n }\n )\n )\n is True\n )\n assert (\n RegressionEvaluation().can_handle(\n AnalysisState(\n {\n \"model_evaluation\": {\n \"problem_type\": \"binary\",\n }\n }\n )\n )\n is False\n )\n assert RegressionEvaluation().can_handle(AnalysisState({\"some_args\": {}})) is False\n\n\ndef test_RegressionEvaluation__handle_None_fig_args():\n assert RegressionEvaluation(fig_args={\"abc\": 1}).fig_args == {\"abc\": 1, \"figsize\": (12, 6)}\n assert RegressionEvaluation(fig_args=None).fig_args == {\"figsize\": (12, 6)}\n assert RegressionEvaluation(fig_args={\"figsize\": (6, 6)}).fig_args == {\"figsize\": (6, 6)}\n\n\ndef test_FeatureImportance__can_handle():\n assert (\n FeatureImportance().can_handle(\n AnalysisState(\n {\n \"model_evaluation\": {\n \"importance\": \"something\",\n }\n }\n )\n )\n is True\n )\n assert FeatureImportance().can_handle(AnalysisState({\"model_evaluation\": {}})) is False\n assert FeatureImportance().can_handle(AnalysisState({\"some_args\": {}})) is False\n\n\ndef test_FeatureImportance__handle_None_fig_args():\n assert FeatureImportance(fig_args={\"abc\": 1}).fig_args == {\"abc\": 1}\n assert FeatureImportance(fig_args=None).fig_args == {}\n\n\n@pytest.mark.parametrize(\n \"show_barplots\",\n [\n True,\n False,\n ],\n)\ndef test_FeatureImportance(monkeypatch, show_barplots):\n state = AnalysisState(\n {\n \"model_evaluation\": {\n \"importance\": pd.DataFrame(\n {\n \"importance\": [0.1, 0.2, 0.3],\n \"stddev\": [0.01, 0.02, 0.03],\n \"p_value\": [0.1, 0.2, 0.3],\n \"n\": [5, 5, 5],\n \"p99_high\": [0.02, 0.03, 0.04],\n \"p99_low\": [-0.02, -0.03, -0.04],\n }\n ),\n }\n }\n )\n\n call_plt_subplots = MagicMock(return_value=(\"fig\", \"ax\"))\n call_plt_show = MagicMock()\n call_sns_barplot = MagicMock()\n call_display_obj = MagicMock()\n call_render_markdown = MagicMock()\n with monkeypatch.context() as m:\n m.setattr(plt, \"subplots\", call_plt_subplots)\n m.setattr(plt, \"show\", call_plt_show)\n m.setattr(sns, \"barplot\", call_sns_barplot)\n viz = FeatureImportance(headers=True, show_barplots=show_barplots, fig_args=dict(a=1, b=2), some_kwarg=123)\n viz.display_obj = call_display_obj\n viz.render_markdown = call_render_markdown\n viz.render(state)\n call_display_obj.assert_called_once()\n call_render_markdown.assert_called_with(\"**Feature Importance**\")\n if show_barplots:\n call_plt_subplots.assert_called_with(a=1, b=2, figsize=(12, 0.75))\n call_plt_show.assert_called_with(\"fig\")\n call_sns_barplot.assert_called_with(ax=\"ax\", data=ANY, y=\"index\", x=\"importance\", some_kwarg=123)\n else:\n call_plt_subplots.assert_not_called()\n call_plt_show.assert_not_called()\n call_sns_barplot.assert_not_called()\n\n\ndef test_ModelLeaderboard():\n assert ModelLeaderboard().can_handle(state=AnalysisState(model_evaluation={})) is False\n\n state = AnalysisState(model_evaluation={\"leaderboard\": \"some_leaderboard\"})\n\n viz = ModelLeaderboard(headers=True)\n assert viz.can_handle(state=state) is True\n\n viz.render_markdown = MagicMock()\n viz.display_obj = MagicMock()\n viz.render(state)\n viz.render_markdown.assert_called_with(\"**Model Leaderboard**\")\n viz.display_obj.assert_called_with(\"some_leaderboard\")\n"
},
{
"path": "examples/automm/Conv-LoRA/README.md",
"content": "# Conv-LoRA: Convolution Meets LoRA: Parameter Efficient Finetuning for Segment Anything Model (ICLR 2024)\n\nExamples showing how to use `Conv-LoRA` for parameter efficient fine-tuning SAM.\n\n## 1. Installation\nThe installation may take a while since AutoGluon Multimodal has multiple dependencies.\n```shell\n conda create -n conv-lora python=3.10\n conda activate conv-lora\n pip install -U pip\n pip install -U setuptools wheel\n git clone https://github.com/autogluon/autogluon\n cd autogluon && pip install -e multimodal/[tests]\n ```\n\n## 2. Dataset\n\nEnter the `autogluon/examples/automm/Conv-LoRA` directory and run the following script to download the datasets.\n\n`python prepare_semantic_segmentation_datasets.py`\n\n## 3. Training\n\n`python run_semantic_segmentation.py -- `\n\n- `task` refers to the dataset name, i.e., one of the datasets we have downloaded. Options are `polyp, leaf_disease_segmentation, camo_sem_seg, isic2017, road_segmentation, or SBU-shadow`.\n- `seed` determines the random seed.\n- `rank` determines the rank of Conv-LoRA. Default is 3.\n- `expert_num` determines the used expert number of Conv-LoRA. Default is 8.\n- `num_gpus` determines the number of gpu used for training. Default is 1.\n- `output_dir` determines the path of output directory. Default is \"outputs\" folder.\n- `ckpt_path` determines the path of model for evaluation. Default is \"outputs\" folder.\n- `per_gpu_batch_size` is the batch size for each GPU. Default is 1.\n- `batch_size` effective batch size. If batch_size > per_gpu_batch_size * num_gpus, gradient accumulation would be used. Default is 4.\n\n## 4. Evaluation\n\nAfter running the benchmark, the evaluation results of test set are stored in \"{output_dir}/metrics.txt\".\n\nYou can also run the following command to evaluate a checkpoint:\n\n`python3 run_semantic_segmentation.py --task {dataset_name} --output_dir {output_dir} --ckpt_path {ckpt_path} --eval`\n\n\n### Citation\n\n```\n@article{zhong2024convolution,\n title={Convolution Meets LoRA: Parameter Efficient Finetuning for Segment Anything Model},\n author={Zhong, Zihan and Tang, Zhiqiang and He, Tong and Fang, Haoyang and Yuan, Chun},\n journal={arXiv preprint arXiv:2401.17868},\n year={2024}\n}\n```"
},
{
"path": "examples/automm/Conv-LoRA/prepare_semantic_segmentation_datasets.py",
"content": "import os\n\nfrom autogluon.common.loaders import load_zip\n\n\ndef get_data_home_dir():\n return os.path.join(os.getcwd().replace(\"\\\\\", \"/\"), \"datasets\")\n\n\nif __name__ == \"__main__\":\n base_dir = get_data_home_dir()\n for name in [\"polyp\", \"leaf_disease_segmentation\", \"camo_sem_seg\", \"isic2017\", \"road_segmentation\", \"SBU-shadow\"]:\n url = f\"s3://automl-mm-bench/semantic_segmentation/{name}.zip\"\n\n dataset_dir = os.path.join(base_dir, name)\n load_zip.unzip(url, unzip_dir=dataset_dir)\n"
},
{
"path": "examples/automm/Conv-LoRA/run_semantic_segmentation.py",
"content": "import argparse\nimport os\n\nimport pandas as pd\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef get_default_training_setting(dataset_name):\n validation_metric = \"iou\"\n loss = \"structure_loss\"\n max_epoch = 30\n lr = 1e-4\n\n if dataset_name == \"SBU-shadow\":\n validation_metric = \"ber\"\n loss = \"balanced_bce\"\n max_epoch = 10\n\n elif dataset_name == \"polyp\":\n validation_metric = \"sm\"\n\n elif dataset_name == \"camo_sem_seg\":\n validation_metric = \"sm\"\n max_epoch = 20\n\n elif dataset_name == \"road_segmentation\":\n validation_metric = \"iou\"\n max_epoch = 20\n lr = 3e-4\n\n elif dataset_name == \"leaf_disease_segmentation\":\n validation_metric = \"iou\"\n lr = 3e-4\n\n return validation_metric, loss, max_epoch, lr\n\n\ndef expand_path(df, dataset_dir):\n for col in [\"image\", \"label\"]:\n df[col] = df[col].apply(lambda ele: os.path.join(dataset_dir, ele))\n return df\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(description=\"This script support converting voc format xmls to coco format json\")\n parser.add_argument(\n \"--task\",\n type=str,\n default=\"leaf_disease_segmentation\",\n choices=[\"polyp\", \"leaf_disease_segmentation\", \"camo_sem_seg\", \"isic2017\", \"road_segmentation\", \"SBU-shadow\"],\n )\n parser.add_argument(\"--seed\", type=int, default=42686693)\n parser.add_argument(\"--rank\", type=int, default=3)\n parser.add_argument(\"--expert_num\", type=int, default=8)\n parser.add_argument(\"--num_gpus\", type=int, default=1)\n parser.add_argument(\"--output_dir\", type=str, default=\"outputs\")\n parser.add_argument(\"--ckpt_path\", type=str, default=\"outputs\", help=\"Checkpoint path.\")\n parser.add_argument(\"--per_gpu_batch_size\", type=int, default=1, help=\"The batch size for each GPU.\")\n parser.add_argument(\n \"--batch_size\",\n type=int,\n default=4,\n help=\"The effective batch size. If batch_size > per_gpu_batch_size * num_gpus, gradient accumulation would be used.\",\n )\n parser.add_argument(\"--eval\", action=\"store_true\")\n args = parser.parse_args()\n\n dataset_name = args.task\n dataset_dir = os.path.join(f\"datasets/{dataset_name}\", dataset_name)\n os.makedirs(args.output_dir, exist_ok=True)\n\n # prepare dataframes\n train_df = expand_path(pd.read_csv(os.path.join(dataset_dir, f\"train.csv\")), dataset_dir)\n val_df = expand_path(pd.read_csv(os.path.join(dataset_dir, f\"val.csv\")), dataset_dir)\n\n # get the validation metric\n validation_metric, loss, max_epoch, lr = get_default_training_setting(dataset_name)\n\n hyperparameters = {}\n hyperparameters.update(\n {\n \"optim.lora.r\": args.rank,\n \"optim.peft\": \"conv_lora\",\n \"optim.lora.conv_lora_expert_num\": args.expert_num,\n \"env.num_gpus\": args.num_gpus,\n \"optim.loss_func\": loss,\n \"optim.max_epochs\": max_epoch,\n \"optim.lr\": lr,\n \"env.per_gpu_batch_size\": args.per_gpu_batch_size,\n \"env.batch_size\": args.batch_size,\n }\n )\n\n if args.eval: # load a checkpoint for evaluation\n predictor = MultiModalPredictor.load(args.ckpt_path)\n else: # training\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters=hyperparameters,\n label=\"label\",\n )\n predictor.fit(train_data=train_df, tuning_data=val_df, seed=args.seed)\n\n # evaluation\n metric_file = os.path.join(args.output_dir, \"metrics.txt\")\n f = open(metric_file, \"a\")\n if dataset_name in [\"isic2017\", \"SBU-shadow\", \"road_segmentation\", \"leaf_disease_segmentation\"]:\n test_df = expand_path(pd.read_csv(os.path.join(dataset_dir, f\"test.csv\")), dataset_dir)\n if dataset_name == \"SBU-shadow\":\n eval_metrics = [\"ber\"]\n else:\n eval_metrics = [\"iou\"]\n\n res = predictor.evaluate(test_df, metrics=eval_metrics)\n print(f\"Evaluation results for test dataset {dataset_name}: \", res)\n f.write(f\"Evaluation results for test dataset {dataset_name}: {res} \\n\")\n elif dataset_name in [\"polyp\", \"camo_sem_seg\"]:\n if dataset_name == \"polyp\":\n test_datasets = [\"CVC-ClinicDB\", \"Kvasir\"]\n elif dataset_name == \"camo_sem_seg\":\n test_datasets = [\"CAMO\"]\n else:\n raise ValueError(f\"Unknown dataset name: {dataset_name}.\")\n for per_dataset in test_datasets:\n test_df = expand_path(pd.read_csv(os.path.join(dataset_dir, f\"test_{per_dataset}.csv\")), dataset_dir)\n res = predictor.evaluate(test_df, metrics=[\"sm\", \"fm\", \"em\", \"mae\"])\n print(f\"Evaluation results for test dataset {per_dataset}: \", res)\n f.write(f\"Evaluation results for test dataset {per_dataset}: {res} \\n\")\n else:\n raise ValueError(f\"Unknown dataset name: {dataset_name}.\")\n\n f.close()\n"
},
{
"path": "examples/automm/TCGA_cancer_survival/README.md",
"content": "# Predict Vital Status from Patients with Cancer\n\n### 1. Task and dataset\n\nThis is a simple example of using AutoGluon for classification tasks on [the Cancer Genome Atlas (TCGA) portal](https://portal.gdc.cancer.gov/). In this example, we leveraged the clinical information to predict whether patients have survived from Head and Neck Squamous Cell Carcinoma or not. Features include patients' age, gender, cancer stage, therapy received, and many others. More details about this dataset and the task are available at [TCGA-HNSC](https://portal.gdc.cancer.gov/projects/TCGA-HNSC). We used AutoGluon TabularPredictor for this task, with a special focus on the FT_Transformer model.\n\n### 2. Run experiments:\n\n```bash\n# Benchmark on multiple AG models\npython3 example_cancer_survival.py --task TCGA_HNSC --mode all_models\n# Just on FT_Transformer \npython3 example_cancer_survival.py --task TCGA_HNSC --mode FT_Transformer\n```\nAll models were run for a maximum time of 900 seconds. Use ```--path``` for intended path to save TCGA dataset, and ```--num_gpus, --num_workers``` to configure the computing resource.\n\n### 3. Results:\n\nThe performance of AutoGluon on the TCGA-HNSC dataset is as follows:\n\nModel(TCGA-HNSC) | Test accuracy | Validation accuracy | Train time | Test time \n---- | ---- | ---- | ---- | ---- \nNeuralNetTorch | 0.943218 | 0.925676 | 2.700217 | 0.027071\nRandomForestGini | 0.940063 | 0.891892 | 0.603893 | 0.108412\nLightGBMLarge | 0.908517 | 0.939189 | 1.351058 | 0.014151 \nCatBoost | 0.905363 | 0.939189 | 4.804489 | 0.025413\nXGBoost | 0.905363 | 0.925676 | 0.416847 | 0.027664\nWeightedEnsemble_L2 | 0.873817 | 0.945946 | 1.636808 | 0.028049\nFTTransformer | 0.864353 | 0.891892 | 51.305847 | 0.384669\n\nFull leaderboard details are available at ```./results/leaderboard.csv```. Note that the TCGA-HNSC is a very small tabular dataset with only 1k rows and 29 columns. To test the performance on a larger dataset, we have also included the [adult dataset](https://archive.ics.uci.edu/ml/datasets/adult) with 49k instances. Simply change the task from ```TCGA_HNSC``` to ```adult```. For example, ``` python3 example_cancer_survival.py --task adult --mode all_models ```. The results are as follows:\n\nModel(adults) | Test accuracy | Validation accuracy | Train time | Test time \n---- | ---- | ---- | ---- | ---- \nXGBoost | 0.877162 | 0.8872 | 0.697971 | 0.038446\nWeightedEnsemble_L2 | 0.876548 | 0.8908 | 42.201000 | 0.316964\nCatBoost | 0.874808 | 0.8828 | 4.263231 | 0.016138 \nFTTransformer | 0.862217 | 0.8696 | 820.485878 | 2.732730\nRandomForestEntr | 0.857918 | 0.8620 | 0.979820 | 0.249948\nNeuralNetFastAI | 0.857304 | 0.8620 | 32.106148 | 0.137593\nNeuralNetTorch | 0.856382 | 0.8588 | 40.264039 | 0.177079 \n\nWhile Decision Tree-based models are still the top-performing approaches, FT_Transformer beats other deep-learning approaches.\n"
},
{
"path": "examples/automm/TCGA_cancer_survival/example_cancer_survival.py",
"content": "\"\"\"\nExample script to predict the vital status of patients with Head-Neck Squamous Cell Carcinoma.\nDataset is originally from https://portal.gdc.cancer.gov/projects/TCGA-HNSC.\nPaper working on similar task: https://bmcbioinformatics.biomedcentral.com/track/pdf/10.1186/s12859-019-2929-8.pdf\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport argparse\nimport os\nimport random\nfrom autogluon.tabular import TabularPredictor, TabularDataset\nfrom autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config\nimport torch as th\nfrom sklearn.model_selection import train_test_split\nfrom autogluon.multimodal.utils import download\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# Dataset information for TCGA dataset\nINFO = {\n \"name\": \"cancer_survival.tsv\",\n \"url\": \"s3://automl-mm-bench/life-science/clinical.tsv\",\n \"sha1sum\": \"6d19609c2a8492f767efd9f2c0b7687bcd3845a3\"\n}\n\n\ndef get_parser():\n parser = argparse.ArgumentParser(\n description='The Basic Example of AutoGluon for TCGA dataset.')\n parser.add_argument('--path', default='./dataset')\n parser.add_argument('--test_size', default=0.3)\n parser.add_argument('--shuffle', default=True)\n parser.add_argument('--seed', type=int, default=123)\n parser.add_argument('--task', choices=['TCGA_HNSC', 'adult'],\n default='adult')\n parser.add_argument('--mode', choices=['FT_Transformer', 'all_models'],\n default='all_models')\n parser.add_argument('--num_gpus', type=int, default=-1)\n parser.add_argument('--num_workers', type=int, default=2)\n return parser\n\n\ndef data_loader(path=\"./dataset/\", ):\n name = INFO[\"name\"]\n full_path = os.path.join(path, name)\n if os.path.exists(full_path):\n print(f\"Existing dataset: {name}\")\n else:\n print(f\"Dataset not exist. Start downloading: {name}\")\n download(INFO[\"url\"], path=full_path, sha1_hash=INFO[\"sha1sum\"])\n df = pd.read_csv(full_path, sep='\\t')\n return df\n\n\n# Preprocessing steps include:\n# (1) Remove \"id\"-related columns and columns with the same values;\n# (2) Some column shared common information with the target label. Those \"shortcuts\" were removed.\n# e.g. We aim to predict whether patents are alive or not. The \"death_date\" is an invalid feature.\n# (3) Split data into train/test sets, by specifying \"test_size\" and \"shuffle\".\ndef preprocess(df, test_size, shuffle):\n N, _ = df.shape\n\n df = df[df != \"'--\"] # Replace missing entries with nan\n n_unique = df.nunique()\n\n for col, n in n_unique.items():\n if \"id\" in col or n <= 1:\n df.drop(col, axis=1, inplace=True)\n\n shortcut_col = [\"days_to_death\", \"year_of_death\"] # Shortcut columns should be removed\n for col in shortcut_col:\n df.drop(col, axis=1, inplace=True)\n\n df_train, df_test = train_test_split(df, test_size=test_size, shuffle=shuffle)\n return df_train, df_test\n\n\ndef train(args):\n if args.task == \"adult\":\n df_train = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv')\n df_test = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv')\n label = \"class\"\n elif args.task == \"TCGA_HNSC\":\n df = data_loader(args.path)\n df_train, df_test = preprocess(df, args.test_size, args.shuffle)\n label = \"vital_status\"\n else:\n raise NotImplementedError\n\n metric = \"accuracy\"\n hyperparameters = {} if args.mode == \"FT_Transformer\" else get_hyperparameter_config('default')\n hyperparameters['FT_TRANSFORMER'] = {\"env.num_gpus\": args.num_gpus, \"env.num_workers\": args.num_workers}\n predictor = TabularPredictor(label=label,\n eval_metric=metric,\n ).fit(\n train_data=df_train,\n hyperparameters=hyperparameters,\n time_limit=900,\n )\n leaderboard = predictor.leaderboard(df_test)\n leaderboard.to_csv(\"./leaderboard.csv\")\n return\n\n\ndef set_seed(seed):\n th.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n\n\nif __name__ == '__main__':\n parser = get_parser()\n args = parser.parse_args()\n set_seed(args.seed)\n train(args)\n\n"
},
{
"path": "examples/automm/distillation/README.md",
"content": "# Model Distillation in AutoMM\n\nExamples showing how to use `MultiModalPredictor` for model distillation.\n\n## 1. Distillation on GLUE and PAWS-X tasks\n\n### 1.1 Example\n\n[`automm_distillation_glue.py`](./automm_distillation_glue.py) : This example provides a use case for GLUE tasks\n\n[`automm_distillation_pawsx.py`](./automm_distillation_pawsx.py) : This example provides a use case for PAWS_X\n\nTo run the example:\n\n`python automm_distillation_.py -- `\n\n- `glue_task` (only for GLUE example) determines to run the experiments on which GLUE task, refers to [Dataset Section](###1.2-Datasets).\n- `pawsx_teacher_tasks` (only for PAWS-X example) determines which language data to use while training the teacher model\n- `pawsx_student_tasks` (only for PAWS-X example) determines which language data to use while training the student model\n- `teacher_model` is the name of teacher model\n- `student_model` is the name of the student model. It is recommended to have the same backbone as teacher model (e.g. both ELECTRA or both BERT).\n- `seed` determines the random seed. Default is 123.\n- `precision` determines precision of the GPU operations\n- `max_epochs` determines the max epoch for training student model.\n- `time_limit` determines the max time to train each model. The unit is second.\n- `num_gpu` determines num of gpu used to train the models. Default is -1 which means using all.\n- `temperature` determines the temperature for soft cross entropy.\n- `hard_label_weight` determines the weight of hard label loss (ground truth logits).\n- `soft_label_weight` determines the weight of soft label loss (teacher model's output).\n- `softmax_regression_weight` determines the weight of softmax regression loss\n- `output_feature_loss_weight` determines the weight of output_feature loss\n- `rkd_distance_loss_weight` determines the weight of RKD distance loss\n- `rkd_angle_loss_weight` determines the weight of RKD angle loss\n- `soft_label_loss_type` determines the loss function of soft label loss\n- `softmax_regression_loss_type` determines the loss function of softmax regression loss\n- `output_feature_loss_type` determines the loss function of output_feature loss, currently support \"mean_square_error\" and \"cosine_distance\"\n- `finetuned_model_cache_folder` is the path to cache models trained without distillation.\n- `resume` if True, models without distillation will be loaded from cache.\n\n### 1.2 Dataset\n\n#### GLUE\n\nWe borrow 7 NLP tasks in GLUE [1], and use identically the same abbreviation as in [1] to name each tasks,\ni.e. \"mnli(m/mm)\", \"qqp\", \"qnli\", \"sst2\", \"stsb\", \"mrpc\", and \"rte\".\nThe dataset are loaded using huggingface's API: https://huggingface.co/datasets/glue.\nAll Data will be automatically downloaded from HuggingFace (thus online connection is necessary) if it does not exist with the given dataset path.\n\nIn GLUE, labels in test sets are private and thus we use training set for training/validation and use validation set for testing.\n\n#### PAWS-X\n\nCross-lingual PAWS (PAWS-X) [3] is an extension of the Wikipedia portion\nof the PAWS evaluation and test examples to six languages:\nSpanish, French, German, Chinese, Japanese, and Korean.\nThis new corpus consists of 23,659 human translated example pairs\nwith paraphrase judgments in each target language.\nLike another work XNLI [4] on multilingual corpus creation,\nthe training set is from machine translate the original PAWS English training set (49,401 pairs).\n\nWe use PAWS-X to test the distillation performance in a multilingual setting.\nThe dataset are loaded using huggingface's API: https://huggingface.co/datasets/paws-x.\nAll Data will be automatically downloaded from HuggingFace (thus online connection is necessary) if it does not exist with the given dataset path.\n\n### 1.3 Distillation Loss\n\nThe student model's loss is the weighted sum of hard_label_loss, soft_label_loss,\nsoftmax_regression_loss, output_feature_loss, rkd_distance_loss, and rkd_angle_loss.\n\n#### Output Feature Loss\n\nOutput Feature Loss is the distance (Euclidean/Cosine/...) between output feature\nof teacher and student.\n\n#### Softmax Regression Loss\n\n[Knowledge Distillation via Softmax Regression Representation Learning](https://www.adrianbulat.com/downloads/ICLR2021/knowledge_distillation_via_softmax_regression_representation_learning.pdf) [5]\n\n#### RKD Distance/Angle Loss\n\n[Relational Knowledge Distillation](https://arxiv.org/abs/1904.05068?context=cs.LG) [6]\n\n### 1.4 Performance\n\n#### GLUE\n\nHere we show the importance of output feature loss.\n\n```bash\nglue_task=qnli\nteacher_model=google/bert_uncased_L-12_H-768_A-12\nstudent_model=google/bert_uncased_L-6_H-768_A-12\nseed=123\nmax_epoch=12\nmetric=\"accuracy\"\ntemperature=5\nhard_label_weight=0.1\nsoft_label_weight=1\n\npython3 automm_distillation_glue.py --teacher_model ${teacher_model} \\\n --student_model ${student_model} \\\n --seed ${seed} \\\n --max_epoch ${max_epoch} \\\n --hard_label_weight ${hard_label_weight} \\\n --soft_label_weight ${soft_label_weight} \\\n --glue_task ${glue_task}\n```\n\nTo reproduce the best performance\n```bash\n`python3 automm_distillation.py --teacher_model google/bert_uncased_L-12_H-768_A-12 \\\n --student_model google/bert_uncased_L-6_H-768_A-12 \\\n --seed 123 \\\n --max_epoch 8 \\\n --hard_label_weight 0.5 \\\n --soft_label_weight 5\n```\n\n| output_feature_loss_weight | Teacher Model Acc | Pretrained Model Acc | Student Model Acc | Distillation Ratio [2] | Speed Up |\n| -------------------------- | ----------------- | -------------------- | ----------------- | ---------------------- | -------- |\n| 0 | 0.91726 | 0.89401 | 0.89713 | 0.13 | 3.52x |\n| 0.01 | 0.91726 | 0.89401 | 0.90298 | 0.39 | 3.52x |\n| 0.1 | 0.91726 | 0.89401 | 0.89365 | -0.02 | 3.52x |\n\n#### PAWS-X\n\nHere we show the importance of softmax regression loss and rkd loss. Some common settings:\n\n```bash\n# pawsx_teacher_tasks = [\"en\",\"de\",\"es\",\"fr\", \"ja\", \"ko\", \"zh\"]\n# pawsx_student_tasks = [\"en\", \"de\", \"es\", \"fr\", \"ja\", \"ko\", \"zh\"]\nteacher_model=microsoft/mdeberta-v3-base\nstudent_model=nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large\nseed=123\nprecision=16\nmax_epochs=10\nnum_gpu=-1\ntemperature=5\nhard_label_weight=0.1\nsoft_label_weight=1\nsoftmax_regression_loss_type=mse\noutput_feature_loss_type=mse\n```\n\nTo reproduce the best model, run\n```bash\npython3 automm_distillation_pawsx.py --precision 16 \\\n --output_feature_loss_weight 0.01 \\\n --softmax_regression_weight 0.1 \\\n --rkd_distance_loss_weight 1 \\\n --rkd_angle_loss_weight 2\n```\n\n| Teacher Model | Student Model | Softmax Regression Weight | RKD Distance Weight | RKD Angle Weight | Teacher Performance | No Distill Performance | Student Performance | Distillation Ratio | Speed Up |\n| :------------------------: | :--------------------------------------------------: | :-----------------------: | :-----------------: | :--------------: | :-----------------: | :--------------------: | :-----------------: | :----------------: | :------: |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0 | 0 | 0 | 0.91293 | 0.88493 | 0.88857 | 0.1301 | ~3.3x |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0.01 | 0 | 0 | 0.91293 | 0.88493 | 0.8855 | 0.02041 | ~3.3x |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0.1 | 0 | 0 | 0.91293 | 0.88493 | 0.89093 | 0.21429 | ~3.3x |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0.1 | 0.1 | 0.2 | 0.91293 | 0.88493 | 0.89264 | 0.27551 | ~3.3x |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0.1 | 1 | 2 | 0.91293 | 0.88493 | 0.891 | 0.21684 | ~3.3x |\n| microsoft/mdeberta-v3-base | nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large | 0.1 | 10 | 20 | 0.91293 | 0.88493 | 0.88971 | 0.17092 | ~3.3x |\n\n### Reference\n\n[1] Wang A, Singh A, Michael J, et al.\nGLUE: A multi-task benchmark and analysis platform for natural language understanding[J].\narXiv preprint arXiv:1804.07461, 2018.\n\n[2] H He, X Shi, J Mueller, S Zha, M Li, G Karypis\n[Towards Automated Distillation: A Systematic Study of Knowledge Distillation in Natural Language Processing.](https://automl.cc/wp-content/uploads/2022/07/towards_automated_distillation.pdf)\nInternational Conference on Automated Machine Learning: Late-Breaking Workshop, 2022\n\n[3] Yang Y, Zhang Y, Tar C, et al.\nPAWS-X: A cross-lingual adversarial dataset for paraphrase identification[J].\narXiv preprint arXiv:1908.11828, 2019.\n\n[4] Conneau A, Lample G, Rinott R, et al.\nXNLI: Evaluating cross-lingual sentence representations[J].\narXiv preprint arXiv:1809.05053, 2018.\n\n[5] Yang J, Martinez B, Bulat A, et al. Knowledge distillation via softmax regression representation learning[C].\nInternational Conference on Learning Representations (ICLR), 2021.\n\n[6] Park W, Kim D, Lu Y, et al. Relational knowledge distillation[C].\nProceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition.\n2019: 3967-3976.\n"
},
{
"path": "examples/automm/distillation/automm_distillation_glue.py",
"content": "import argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import load_dataset\n\nfrom time import time\nimport os\n\nGLUE_METRICS = {\n \"mnli\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\"]},\n \"qqp\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\", \"f1\"]},\n \"qnli\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\"]},\n \"sst2\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\"]},\n \"stsb\": {\n \"val\": \"pearsonr\",\n \"eval\": [\"pearsonr\", \"spearmanr\"],\n }, # Current default soft label loss func is for classification, should automatically select loss_func\n \"mrpc\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\"]},\n \"rte\": {\"val\": \"accuracy\", \"eval\": [\"accuracy\"]},\n # \"cola\": \"\", #phi coefficient is not implemented\n}\n\n\ndef get_parser():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--glue_task\", default=\"qnli\", type=str)\n parser.add_argument(\"--teacher_model\", default=\"google/bert_uncased_L-12_H-768_A-12\", type=str)\n parser.add_argument(\"--student_model\", default=\"google/bert_uncased_L-6_H-768_A-12\", type=str)\n parser.add_argument(\"--seed\", default=123, type=int)\n parser.add_argument(\"--max_epochs\", default=1000, type=int)\n parser.add_argument(\"--time_limit\", default=None, type=int)\n parser.add_argument(\"--num_gpu\", default=-1, type=int)\n parser.add_argument(\"--temperature\", default=5.0, type=float)\n parser.add_argument(\"--hard_label_weight\", default=0.1, type=float)\n parser.add_argument(\"--soft_label_weight\", default=1.0, type=float)\n parser.add_argument(\"--train_nodistill\", default=True, type=bool,\n help=\"Whether to train the student model without distillation.\")\n parser.add_argument(\"--softmax_regression_weight\", default=0.1, type=float)\n parser.add_argument(\"--output_feature_loss_weight\", default=0.01, type=float)\n parser.add_argument(\"--rkd_distance_loss_weight\", default=0.0, type=float)\n parser.add_argument(\"--rkd_angle_loss_weight\", default=0.0, type=float)\n parser.add_argument(\"--soft_label_loss_type\", default=\"\", type=str)\n parser.add_argument(\"--softmax_regression_loss_type\", default=\"mse\", type=str)\n parser.add_argument(\"--output_feature_loss_type\", default=\"mse\", type=str)\n parser.add_argument(\n \"--save_path\",\n default=\"./AutogluonModels/cache_finetuned\",\n type=str,\n )\n parser.add_argument(\"--resume\", action=\"store_true\")\n return parser\n\n\ndef main(args):\n assert args.glue_task in (list(GLUE_METRICS.keys()) + [\"mnlim\", \"mnlimm\"]), \"Unsupported dataset name.\"\n\n ### Dataset Loading\n if args.glue_task == \"mnlimm\":\n glue_task = \"mnli\"\n mnli_mismatched = True\n elif args.glue_task in [\"mnli\" or \"mnlim\"]:\n glue_task = \"mnli\"\n mnli_mismatched = False\n else:\n glue_task = args.glue_task\n dataset = load_dataset(\"glue\", glue_task)\n train_df = dataset[\"train\"].to_pandas().drop(\"idx\", axis=1)\n if args.glue_task == \"mnli\":\n if mnli_mismatched:\n valid_df = dataset[\"validation_matched\"].to_pandas()\n else:\n valid_df = dataset[\"validation_mismatched\"].to_pandas()\n else:\n valid_df = dataset[\"validation\"].to_pandas()\n\n teacher_predictor_name = f\"{args.glue_task}-{args.teacher_model.replace('/', '-')}\"\n teacher_predictor_path = os.path.join(args.save_path, teacher_predictor_name)\n nodistill_predictor_name = f\"{args.glue_task}-{args.student_model.replace('/', '-')}\"\n nodistill_predictor_path = os.path.join(args.save_path, nodistill_predictor_name)\n\n ### Train and evaluate the teacher model\n resume_teacher = args.resume\n try:\n teacher_predictor = MultiModalPredictor.load(teacher_predictor_path)\n print(\"Using pretrained teacher model: %s\" % teacher_predictor_path)\n except:\n resume_teacher = False\n print(\"No pretrained model at: %s\" % teacher_predictor_path)\n if not resume_teacher:\n teacher_predictor = MultiModalPredictor(label=\"label\", eval_metric=GLUE_METRICS[glue_task][\"val\"])\n teacher_predictor.fit(\n train_df,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"model.hf_text.checkpoint_name\": args.teacher_model,\n \"optim.lr\": 1.0e-4,\n \"optim.weight_decay\": 1.0e-3,\n },\n time_limit=args.time_limit,\n seed=args.seed,\n )\n teacher_predictor.save(teacher_predictor_path)\n start = time()\n teacher_result = teacher_predictor.evaluate(data=valid_df, metrics=GLUE_METRICS[glue_task][\"eval\"])\n teacher_usedtime = time() - start\n\n ### Train and evaluate a smaller pretrained model\n resume_nodistill = args.resume\n try:\n nodistill_predictor = MultiModalPredictor.load(nodistill_predictor_path)\n print(\"Using pretrained nodistill model: %s\" % nodistill_predictor_path)\n except:\n print(\"No pretrained model at: %s\" % nodistill_predictor_path)\n resume_nodistill = False\n if not resume_nodistill and args.train_nodistill:\n nodistill_predictor = MultiModalPredictor(label=\"label\", eval_metric=GLUE_METRICS[glue_task][\"val\"])\n nodistill_predictor.fit(\n train_df,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"optim.max_epochs\": args.max_epochs,\n \"model.hf_text.checkpoint_name\": args.student_model,\n \"optim.lr\": 1.0e-4,\n \"optim.weight_decay\": 1.0e-3,\n },\n time_limit=args.time_limit,\n seed=args.seed,\n )\n nodistill_predictor.save(nodistill_predictor_path)\n nodistill_result = nodistill_predictor.evaluate(data=valid_df, metrics=GLUE_METRICS[glue_task][\"eval\"])\n\n ### Distill and evaluate a student model\n\n student_predictor = MultiModalPredictor(label=\"label\", eval_metric=GLUE_METRICS[glue_task][\"val\"])\n student_predictor.fit(\n train_df,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"optim.max_epochs\": args.max_epochs,\n \"model.hf_text.checkpoint_name\": args.student_model,\n \"optim.lr\": 1.0e-4,\n \"optim.weight_decay\": 1.0e-3,\n \"distiller.temperature\": args.temperature,\n \"distiller.hard_label_weight\": args.hard_label_weight,\n \"distiller.soft_label_weight\": args.soft_label_weight,\n \"distiller.softmax_regression_weight\": args.softmax_regression_weight,\n \"distiller.output_feature_loss_weight\": args.output_feature_loss_weight,\n \"distiller.rkd_distance_loss_weight\": args.rkd_distance_loss_weight,\n \"distiller.rkd_angle_loss_weight\": args.rkd_angle_loss_weight,\n \"distiller.soft_label_loss_type\": args.soft_label_loss_type,\n \"distiller.softmax_regression_loss_type\": args.softmax_regression_loss_type,\n \"distiller.output_feature_loss_type\": args.output_feature_loss_type,\n \"model.hf_text.text_trivial_aug_maxscale\": 0.0,\n },\n teacher_predictor=teacher_predictor,\n time_limit=args.time_limit,\n seed=args.seed,\n )\n start = time()\n student_result = student_predictor.evaluate(data=valid_df, metrics=GLUE_METRICS[glue_task][\"eval\"])\n student_usedtime = time() - start\n\n ### Print distillation's performance\n print(\"Distillation Result:\")\n print(\"Teacher Model: %s\" % args.teacher_model)\n print(\"Student Model: %s\" % args.student_model)\n for k in teacher_result:\n print(f\"For metric {k}:\")\n print(\"Teacher Model's %s: %.6f\" % (k, teacher_result[k]))\n print(\"Pretrained Model's %s: %.6f\" % (k, nodistill_result[k]))\n print(\"Student Model's %s: %.6f\" % (k, student_result[k]))\n print(\n \"Distillation Ratio (the fraction of the teacher's performance achieved by the student): %.6f\"\n % (float(student_result[k] - nodistill_result[k]) / float(teacher_result[k] - nodistill_result[k]))\n )\n print(\"Teacher Model's time: %.6f\" % teacher_usedtime)\n print(\"Student Model's time: %.6f\" % student_usedtime)\n print(\"speed up: %.6fx\" % (teacher_usedtime / student_usedtime))\n\n\nif __name__ == \"__main__\":\n parser = get_parser()\n args = parser.parse_args()\n main(args)\n"
},
{
"path": "examples/automm/distillation/automm_distillation_pawsx.py",
"content": "import argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import load_dataset\n\nfrom time import time\nimport os\nimport pandas as pd\n\nPAWS_TASKS = [\"en\", \"de\", \"es\", \"fr\", \"ja\", \"ko\", \"zh\"]\n\n\ndef tasks_to_id(pawsx_tasks):\n id = \"\"\n for task in PAWS_TASKS:\n if task in pawsx_tasks:\n id += task\n return id\n\ndef getDatasetSplits(pawsx_tasks):\n datasets = {}\n train_dfs = {}\n val_dfs = {}\n test_dfs = {}\n for task in pawsx_tasks:\n datasets[task] = load_dataset(\"paws-x\", task)\n train_dfs[task] = datasets[task][\"train\"].to_pandas()\n val_dfs[task] = datasets[task][\"validation\"].to_pandas()\n test_dfs[task] = datasets[task][\"test\"].to_pandas()\n print(\n \"task %s: train %d, val %d, test %d\"\n % (task, len(train_dfs[task]), len(val_dfs[task]), len(test_dfs[task]))\n )\n train_df = pd.concat(train_dfs)\n val_df = pd.concat(val_dfs)\n test_dfs[\"all\"] = pd.concat(test_dfs)\n\n return train_df, val_df, test_dfs\n\ndef main(args):\n pawsx_teacher_tasks = args.pawsx_teacher_tasks\n assert all(task in PAWS_TASKS for task in pawsx_teacher_tasks)\n teacher_tasks_id = tasks_to_id(pawsx_teacher_tasks)\n pawsx_student_tasks = args.pawsx_student_tasks\n assert all(task in PAWS_TASKS for task in pawsx_student_tasks)\n student_tasks_id = tasks_to_id(pawsx_student_tasks)\n\n teacher_train_df, teacher_val_df, teacher_test_dfs = getDatasetSplits(pawsx_teacher_tasks)\n student_train_df, student_val_df, student_test_dfs = getDatasetSplits(pawsx_student_tasks)\n\n teacher_predictor_name = f\"pawsx-{teacher_tasks_id}-{args.teacher_model.replace('/', '-')}\"\n teacher_predictor_path = os.path.join(args.save_path, teacher_predictor_name)\n nodistill_predictor_name = f\"pawsx-{student_tasks_id}-{args.student_model.replace('/', '-')}\"\n nodistill_predictor_path = os.path.join(args.save_path, nodistill_predictor_name)\n\n teacher_result = {}\n nodistill_result = {}\n student_result = {}\n\n ### Train and evaluate the teacher model\n resume_teacher = args.resume\n try:\n teacher_predictor = MultiModalPredictor.load(teacher_predictor_path)\n print(\"Using pretrained teacher model: %s\" % teacher_predictor_path)\n except:\n resume_teacher = False\n print(\"No pretrained model at: %s\" % teacher_predictor_path)\n if not resume_teacher:\n teacher_predictor = MultiModalPredictor(label=\"label\", eval_metric=\"accuracy\")\n teacher_predictor.fit(\n teacher_train_df,\n tuning_data=teacher_val_df,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"env.precision\": args.precision,\n \"env.per_gpu_batch_size\": 6,\n \"model.hf_text.checkpoint_name\": args.teacher_model,\n \"optim.lr\": 1.0e-4,\n \"optim.weight_decay\": 1.0e-3,\n },\n time_limit=args.time_limit,\n seed=args.seed,\n )\n teacher_predictor.save(teacher_predictor_path)\n for test_name, test_df in teacher_test_dfs.items():\n teacher_result[test_name] = teacher_predictor.evaluate(data=test_df, metrics=\"accuracy\")\n #use same dataset to measure computation time\n start = time()\n for test_name, test_df in student_test_dfs.items():\n teacher_predictor.evaluate(data=test_df, metrics=\"accuracy\")\n teacher_usedtime = time() - start\n\n ### Train and evaluate a smaller pretrained model\n resume_nodistill = args.resume\n try:\n nodistill_predictor = MultiModalPredictor.load(nodistill_predictor_path)\n print(\"Using pretrained nodistill model: %s\" % nodistill_predictor_path)\n except:\n print(\"No pretrained model at: %s\" % nodistill_predictor_path)\n resume_nodistill = False\n if not resume_nodistill:\n nodistill_predictor = MultiModalPredictor(label=\"label\", eval_metric=\"accuracy\")\n nodistill_predictor.fit(\n student_train_df,\n tuning_data=student_val_df,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"env.precision\": args.precision,\n \"optim.max_epochs\": args.max_epochs,\n \"model.hf_text.checkpoint_name\": args.student_model,\n \"optim.lr\": 2.0e-4,\n \"optim.weight_decay\": 2.0e-3,\n },\n time_limit=args.time_limit,\n seed=args.seed,\n )\n nodistill_predictor.save(nodistill_predictor_path)\n for test_name, test_df in student_test_dfs.items():\n nodistill_result[test_name] = nodistill_predictor.evaluate(data=test_df, metrics=\"accuracy\")\n\n ### Distill and evaluate a student model\n from autogluon.multimodal.constants import MODEL, DATA, OPTIM, ENV, DISTILLER\n\n config = {\n MODEL: f\"default\",\n DATA: \"default\",\n DISTILLER: \"default\",\n OPTIM: \"default\",\n ENV: \"default\",\n }\n student_predictor = MultiModalPredictor(label=\"label\", eval_metric=\"accuracy\")\n student_predictor.fit(\n student_train_df,\n tuning_data=student_val_df,\n config=config,\n hyperparameters={\n \"env.num_gpus\": args.num_gpu,\n \"env.precision\": args.precision,\n \"optim.max_epochs\": args.max_epochs,\n \"model.hf_text.checkpoint_name\": args.student_model,\n \"model.hf_text.text_trivial_aug_maxscale\": args.aug_scale,\n \"optim.lr\": 2.0e-4,\n \"optim.weight_decay\": 2.0e-3,\n \"distiller.temperature\": args.temperature,\n \"distiller.hard_label_weight\": args.hard_label_weight,\n \"distiller.soft_label_weight\": args.soft_label_weight,\n \"distiller.softmax_regression_weight\": args.softmax_regression_weight,\n \"distiller.output_feature_loss_weight\": args.output_feature_loss_weight,\n \"distiller.rkd_distance_loss_weight\": args.rkd_distance_loss_weight,\n \"distiller.rkd_angle_loss_weight\": args.rkd_angle_loss_weight,\n \"distiller.soft_label_loss_type\": args.soft_label_loss_type,\n \"distiller.softmax_regression_loss_type\": args.softmax_regression_loss_type,\n \"distiller.output_feature_loss_type\": args.output_feature_loss_type,\n \"model.hf_text.text_trivial_aug_maxscale\": args.aug_scale,\n # \"optim.top_k\": 1,\n # \"optim.top_k_average_method\": \"best\",\n },\n teacher_predictor=teacher_predictor,\n time_limit=args.time_limit,\n seed=args.seed,\n )\n start = time()\n for test_name, test_df in student_test_dfs.items():\n student_result[test_name] = student_predictor.evaluate(data=test_df, metrics=\"accuracy\")\n student_usedtime = time() - start\n\n ### Print distillation's performance\n for test_name in student_test_dfs.keys():\n print(\"Distillation Result (%s):\" % test_name)\n print(\"Teacher Model: %s\" % args.teacher_model)\n print(\"Student Model: %s\" % args.student_model)\n for k in teacher_result[test_name]:\n print(f\"For metric {k}:\")\n print(\"Teacher Model's %s: %.6f\" % (k, teacher_result[test_name][k]))\n print(\"Pretrained Model's %s: %.6f\" % (k, nodistill_result[test_name][k]))\n print(\"Student Model's %s: %.6f\" % (k, student_result[test_name][k]))\n print(\n \"Distillation Ratio (the fraction of the teacher's performance achieved by the student): %.6f\"\n % (\n float(student_result[test_name][k] - nodistill_result[test_name][k])\n / float(teacher_result[test_name][k] - nodistill_result[test_name][k])\n )\n )\n print(\"Teacher Model's time: %.6f\" % teacher_usedtime)\n print(\"Student Model's time: %.6f\" % student_usedtime)\n print(\"speed up: %.6fx\" % (teacher_usedtime / student_usedtime))\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--pawsx_teacher_tasks\", nargs=\"+\", default=[\"en\", \"de\", \"es\", \"fr\", \"ja\", \"ko\", \"zh\"])\n parser.add_argument(\"--pawsx_student_tasks\", nargs=\"+\", default=[\"en\", \"de\", \"es\", \"fr\", \"ja\", \"ko\", \"zh\"])\n parser.add_argument(\"--teacher_model\", default=\"microsoft/mdeberta-v3-base\", type=str)\n parser.add_argument(\"--student_model\", default=\"nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large\", type=str)\n parser.add_argument(\"--seed\", default=123, type=int)\n parser.add_argument(\"--precision\", default=\"bf16\", type=str)\n parser.add_argument(\"--max_epochs\", default=10, type=int)\n parser.add_argument(\"--time_limit\", default=None, type=int)\n parser.add_argument(\"--num_gpu\", default=-1, type=int)\n parser.add_argument(\"--temperature\", default=5.0, type=float)\n parser.add_argument(\"--hard_label_weight\", default=0.1, type=float)\n parser.add_argument(\"--soft_label_weight\", default=1.0, type=float)\n parser.add_argument(\"--softmax_regression_weight\", default=0, type=float)\n parser.add_argument(\"--output_feature_loss_weight\", default=0.01, type=float)\n parser.add_argument(\"--rkd_distance_loss_weight\", default=0.0, type=float)\n parser.add_argument(\"--rkd_angle_loss_weight\", default=0.0, type=float)\n parser.add_argument(\"--soft_label_loss_type\", default=\"\", type=str)\n parser.add_argument(\"--softmax_regression_loss_type\", default=\"mse\", type=str)\n parser.add_argument(\"--output_feature_loss_type\", default=\"mse\", type=str)\n parser.add_argument(\n \"--save_path\",\n default=\"./AutogluonModels/cache_finetuned\",\n type=str,\n )\n parser.add_argument(\"--resume\", action=\"store_true\")\n parser.add_argument(\"--aug_scale\", default=0., type=float)\n args = parser.parse_args()\n main(args)\n"
},
{
"path": "examples/automm/distillation/eval_pawsx.py",
"content": "import argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import load_dataset\n\nfrom time import time\nimport os\nimport pandas as pd\n\nPAWS_TASKS = [\"en\", \"de\", \"es\", \"fr\", \"ja\", \"ko\", \"zh\"]\n\n\ndef tasks_to_id(pawsx_tasks):\n id = \"\"\n for task in PAWS_TASKS:\n if task in pawsx_tasks:\n id += task\n return id\n\n\ndef main(args):\n model_path = args.model_path\n pawsx_tasks = args.pawsx_tasks\n assert all(task in PAWS_TASKS for task in pawsx_tasks)\n\n datasets = {}\n val_dfs = {}\n test_dfs = {}\n for task in args.pawsx_tasks:\n datasets[task] = load_dataset(\"paws-x\", task)\n val_dfs[task] = datasets[task][\"validation\"].to_pandas()\n test_dfs[task] = datasets[task][\"test\"].to_pandas()\n print(\n \"task %s: val %d, test %d\"\n % (task, len(val_dfs[task]), len(test_dfs[task]))\n )\n val_df = pd.concat(val_dfs)\n test_dfs[\"all\"] = pd.concat(test_dfs)\n test_dfs[\"val\"] = val_df\n\n result = {}\n predictor = MultiModalPredictor.load(model_path)\n start = time()\n for test_name, test_df in test_dfs.items():\n result[test_name] = predictor.evaluate(data=test_df, metrics=\"accuracy\")\n usedtime = time() - start\n\n for test_name in test_dfs.keys():\n print(\"Distillation Result (%s):\" % test_name)\n print(\"Model: %s\" % model_path)\n for k in result[test_name]:\n print(f\"For metric {k}:\")\n print(\"Model's %s: %.6f\" % (k, result[test_name][k]))\n print(\"Teacher Model's time: %.6f\" % usedtime)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--pawsx_tasks\", default=[\"en\"], type=list)\n parser.add_argument(\"--model_path\", type=str)\n args = parser.parse_args()\n\n main(args)\n"
},
{
"path": "examples/automm/kaggle_california_house_price/README.md",
"content": "# Use AutoMM to Predict California House Prices\n\nExample shows how to combine the tabular deep learning (DL) model in AutoMM and other tree models via the auto-ensembling logic in AutoGluon-Tabular \nto train a good model on the [Kaggle: California House Price Competition](https://www.kaggle.com/c/california-house-prices).\n\nThe task is to predict house sale prices based on the house information, such as # of bedrooms, living areas, locations, \nnear-by schools, and the seller summary. The data consist of houses sold in California on 2020, with houses in the \ntest dataset sold after the ones in the training dataset.\n\n```bash\nkaggle competitions download -c california-house-prices\nunzip california-house-prices.zip -d california-house-prices\n```\n\nRun experiments:\n\n```bash\n# Single MultiModalPredictor (MLP)\npython3 example_kaggle_house.py --automm-mode mlp --mode single 2>&1 | tee -a logs/automm_single_mlp.txt\n\n# Single MultiModalPredictor (FT-Transformer For Tabular)\npython3 example_kaggle_house.py --automm-mode ft-transformer --mode single 2>&1 | tee -a logs/automm_single_ft.txt\n\n# MultiModalPredictor + 5-Fold Bagging\npython3 example_kaggle_house.py --automm-mode ft-transformer --mode automm_bag5 2>&1 | tee -a logs/automm_ft_bag5.txt\n\n# MultiModalPredictor + other Tree Models (Weighted Ensemble) \npython3 example_kaggle_house.py --automm-mode ft-transformer --mode weighted 2>&1 | tee -a logs/automm_ft_weighted.txt\n\n# MultiModalPredictor + other Tree Models (5-fold Stack Ensemble) \npython3 example_kaggle_house.py --automm-mode ft-transformer --mode stack5 2>&1 | tee -a logs/automm_ft_stack5.txt\n```\n\nFor more details about the advanced tabular DL models in AutoMM, you may check [tabular_dl](../tabular_dl).\n"
},
{
"path": "examples/automm/kaggle_california_house_price/example_kaggle_house.py",
"content": "import pandas as pd\nimport numpy as np\nimport argparse\nimport os\nimport random\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.multimodal import MultiModalPredictor\nimport torch as th\n\n\ndef get_parser():\n parser = argparse.ArgumentParser(\n description='The Basic Example of AutoGluon for House Price Prediction.')\n parser.add_argument('--mode',\n choices=['stack5',\n 'weighted',\n 'single',\n 'single_bag5'],\n default='weighted',\n help='\"stack5\" means 5-fold stacking. \"weighted\" means weighted ensemble.'\n ' \"single\" means use a single model.'\n ' \"single_bag5\" means 5-fold bagging via the AutoMM model.')\n parser.add_argument('--automm-mode', choices=['ft-transformer', 'mlp'],\n default='ft-transformer', help='Fusion model in AutoMM.')\n parser.add_argument('--text-backbone', default='google/electra-small-discriminator')\n parser.add_argument('--cat-as-text', default=False)\n parser.add_argument('--data_path', type=str, default='california-house-prices')\n parser.add_argument('--seed', type=int, default=123)\n parser.add_argument('--exp_path', default=None)\n parser.add_argument('--with_tax_values', default=1, type=int)\n return parser\n\n\ndef get_automm_hyperparameters(mode, text_backbone, cat_as_text):\n if mode == \"ft-transformer\":\n hparams = {\"model.names\": [\"ft_transformer\",\n \"hf_text\",\n \"fusion_transformer\"],\n \"model.hf_text.checkpoint_name\": text_backbone,\n \"data.categorical.convert_to_text\": cat_as_text}\n elif mode == \"mlp\":\n hparams = {\"model.names\": [\"categorical_mlp\",\n \"numerical_mlp\",\n \"hf_text\",\n \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": text_backbone,\n \"data.categorical.convert_to_text\": cat_as_text}\n else:\n raise NotImplementedError(f\"mode={mode} is not supported!\")\n return hparams\n\n\ndef preprocess(df, with_tax_values=True, log_scale_lot=True,\n log_scale_listed_price=True, has_label=True):\n new_df = df.copy()\n new_df.drop('Id', axis=1, inplace=True)\n new_df['Elementary School'] = new_df['Elementary School'].apply(lambda ele: str(ele)[:-len(' Elementary School')] if str(ele).endswith('Elementary School') else ele)\n if log_scale_lot:\n new_df['Lot'] = np.log(new_df['Lot'] + 1)\n if log_scale_listed_price:\n log_listed_price = np.log(new_df['Listed Price']).clip(0, None)\n new_df['Listed Price'] = log_listed_price\n if with_tax_values:\n new_df['Tax assessed value'] = np.log(new_df['Tax assessed value'] + 1)\n new_df['Annual tax amount'] = np.log(new_df['Annual tax amount'] + 1)\n else:\n new_df.drop('Tax assessed value', axis=1, inplace=True)\n new_df.drop('Annual tax amount', axis=1, inplace=True)\n if has_label:\n new_df['Sold Price'] = np.log(new_df['Sold Price'])\n return new_df\n\n\ndef set_seed(seed):\n import torch as th\n th.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n\n\ndef train(args):\n set_seed(args.seed)\n train_df = pd.read_csv(os.path.join(args.data_path, 'train.csv'))\n test_df = pd.read_csv(os.path.join(args.data_path, 'test.csv'))\n # For the purpose of generating submission file\n submission_df = pd.read_csv(os.path.join(args.data_path, 'sample_submission.csv'))\n train_df = preprocess(train_df,\n with_tax_values=args.with_tax_values, has_label=True)\n test_df = preprocess(test_df,\n with_tax_values=args.with_tax_values, has_label=False)\n label_column = 'Sold Price'\n eval_metric = 'r2'\n\n automm_hyperparameters = get_automm_hyperparameters(args.automm_mode, args.text_backbone, args.cat_as_text)\n\n tabular_hyperparameters = {\n 'GBM': [\n {},\n {'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}},\n ],\n 'CAT': {},\n 'AG_AUTOMM': automm_hyperparameters,\n }\n if args.mode == 'single':\n predictor = MultiModalPredictor(eval_metric=eval_metric, label=label_column, path=args.exp_path)\n predictor.fit(train_df, hyperparameters=automm_hyperparameters, seed=args.seed)\n elif args.mode == 'weighted' or args.mode == 'stack5' or args.mode == 'single_bag5' or args.mode == 'single_bag4':\n predictor = TabularPredictor(eval_metric=eval_metric, label=label_column, path=args.exp_path)\n\n if args.mode == 'single_bag5':\n tabular_hyperparameters = {\n 'AG_AUTOMM': automm_hyperparameters,\n }\n num_bag_folds, num_stack_levels = 5, 0\n elif args.mode == 'weighted':\n num_bag_folds, num_stack_levels = None, None\n elif args.mode == 'stack5':\n num_bag_folds, num_stack_levels = 5, 1\n else:\n raise NotImplementedError\n predictor.fit(train_df,\n hyperparameters=tabular_hyperparameters,\n num_bag_folds=num_bag_folds,\n num_stack_levels=num_stack_levels)\n leaderboard = predictor.leaderboard()\n leaderboard.to_csv(os.path.join(args.exp_path, 'leaderboard.csv'))\n else:\n raise NotImplementedError\n predictions = np.exp(predictor.predict(test_df))\n submission_df['Sold Price'] = predictions\n submission_df.to_csv(os.path.join(args.exp_path, 'submission.csv'), index=None)\n\n\nif __name__ == '__main__':\n parser = get_parser()\n args = parser.parse_args()\n if args.exp_path is None:\n args.exp_path = f'automm_kaggle_house_{args.mode}_{args.automm_mode}_cat_to_text{args.cat_as_text}_{args.text_backbone}'\n th.manual_seed(args.seed)\n train(args)\n"
},
{
"path": "examples/automm/kaggle_feedback_prize/README.md",
"content": "# Using MultiModalPredictor with Text Normalization\n\nTake [Feedback Prize - Predicting Effective Arguments](https://www.kaggle.com/competitions/feedback-prize-effectiveness) as an example to show how text normalization can be helpful.\n\n## 1. Preprocess data with text normalization\n\nText normalization is the task of mapping non-canonical language, typical of speech transcription and computer-mediated communication, to a standardized writing. \nIt is an up-stream task necessary to enable the subsequent direct employment of standard natural language processing tools and indispensable for languages such as Swiss German, \nwith strong regional variation and no written standard. \nEven though the competition dataset is composed of English only, we found that applying text normalization can reduce `log loss`, the metrics that this competition is evaluating on. \n\n### 1.1 Applying normalization to text columns by enabling hyperparameter configuration\n hyperparameters={\n \"data.text.normalize_text\": True,\n }\n\nUnder the hood, the following process is done when `data.text.normalize_text` is enabled:\n \n #### 1.1.1 Define error handlers for codecs\n def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n def replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n\n #### 1.1.2 Register error handlers for codecs\n codecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\n codecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\n\n #### 1.1.3 Applying a series of decoding and encoding for normalization\n def resolve_encodings_and_normalize(text: str) -> str:\n text = (\n text.encode(\"raw_unicode_escape\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n )\n text = unidecode(text)\n return text\n\n### 1.2 A few examples of normalized texts\n\n # Example-1 pre-normalization\n 'The same technology can make computer-animated faces more expressive\\x97for video games or video surgery. \\x93Most human communication is nonverbal, including emotional communication,\\x94 notes Dr. Huang. \\x93So computers need to understand that, too.\\x94Eckman has classified six basic emotions\\x97happiness, surprise, anger, disgust, fear, and sadness\\x97and then associated each with characteristic movements of the facial muscles. For example, your frontalis pars lateralis muscle (above your eyes) raises your eyebrows when you\\x92re surprised; your orbicularis oris (around your mouth) tightens your lips to show anger. '\n\n # Example-1 post-normalization\n 'The same technology can make computer-animated faces more expressive--for video games or video surgery. \"Most human communication is nonverbal, including emotional communication,\" notes Dr. Huang. \"So computers need to understand that, too.\"Eckman has classified six basic emotions--happiness, surprise, anger, disgust, fear, and sadness--and then associated each with characteristic movements of the facial muscles. For example, your frontalis pars lateralis muscle (above your eyes) raises your eyebrows when you\\'re surprised; your orbicularis oris (around your mouth) tightens your lips to show anger. '\n\n # Example-2 pre-normalization\n '\"Congestion was down\\xa0 60\\xa0 percent\\xa0 in\\xa0 the\\xa0 capital\\xa0 of\\xa0 france\\xa0 after\\xa0 fivedays\\xa0 of\\xa0 intensifying\\xa0 smog.\"smog\\xa0 by\\xa0 meaning\\xa0 pollution\\xa0 went\\xa0 down\\xa0 just\\xa0 60\\xa0 percent\\xa0 in\\xa0 five\\xa0 days. Thats\\xa0 a\\xa0 great\\xa0 adavantage\\xa0 just\\xa0 by\\xa0 limting\\xa0 car\\xa0 usuage. In\\xa0 source\\xa0 number 1\\xa0 explains that \" Passenger\\xa0 cars\\xa0 are\\xa0 responsible\\xa0 for\\xa0 12\\xa0 percent\\xa0 of\\xa0 greenouse\\xa0 gas emissions\\xa0 in\\xa0 Europe.. and up\\xa0 to\\xa0 50\\xa0 perecnt\\xa0 in\\xa0 some\\xa0 car-intensive\\xa0 areas\\xa0 in\\xa0 the\\xa0 Untied States.\" We\\xa0 as\\xa0 a\\xa0 country\\xa0 shoud\\xa0 lower\\xa0 that\\xa0 and\\xa0 the\\xa0 best\\xa0 way\\xa0 is\\xa0 to\\xa0 limting\\xa0 car usuage . Limting car usage is one of te some advantges to lowering pollution ( greenhouse gas, smog). '\n\n # Example-2 post-normalization\n '\"Congestion was down 60 percent in the capital of france after fivedays of intensifying smog.\"smog by meaning pollution went down just 60 percent in five days. Thats a great adavantage just by limting car usuage. In source number 1 explains that \" Passenger cars are responsible for 12 percent of greenouse gas emissions in Europe.. and up to 50 perecnt in some car-intensive areas in the Untied States.\" We as a country shoud lower that and the best way is to limting car usuage . Limting car usage is one of te some advantges to lowering pollution ( greenhouse gas, smog). '\n\nFor details, please refer to \n[`kaggle_feedback_prize_preprocess.py`](./kaggle_feedback_prize_preprocess.py).\n\n## 2. MultiModalPredictor for Training\n\nMultiModalPredictor can automatically build deep learning models with multimodal datasets. \nThe tabular data we have for this Kaggle competition is a perfect example that showcases how easily we could build models with just a few lines of code using MultiModalPredictor. \nFor details, please refer to [`kaggle_feedback_prize_train.py`](./kaggle_feedback_prize_train.py).\n\n### 2.1 Build the MultiModalPredictor\n\nYou can build the predictor as following.\n\n predictor = MultiModalPredictor(\n\t label=\"discourse_effectiveness\", \n\t problem_type=\"multiclass\", \n\t eval_metric=\"log_loss\", \n\t path=save_path, \n\t verbosity=3, \n\t)\n\n - `label` indicates the target value in training data.\n - `problem_type` indicates the type of the problem. It can be \"multiclass\", \"binary\" or \"regression\".\n - `eval_metric` indicates the evaluation metrics of the model which is always the evaluation of the competition.\n - `path` indicates the path to save MultiModalPredictor models.\n - `verbosity` controls how much information is printed.\n\n### 2.2 Train the MultiModalPredictor\n\nThen, you can train the MultiModalPredictor with `.fit()`.\n\n predictor.fit(\n train_data=train_df,\n tuning_data=val_df,\n presets=\"best_quality\",\n hyperparameters={\n \"model.hf_text.checkpoint_name\": \"microsoft/deberta-v3-large\",\n \"data.text.normalize_text\": True,\n \"optim.lr\": 5e-5,\n \"optim.max_epochs\": 7,\n },\n )\n\n - `train_data` is the data used for training.\n - `tuning_data` is the data for validation. If it is empty, the tuning data will be split from training data automatically.\n - `presets` sets a various number of parameters depending on the quality of models one prefers. For details, please refer to [presets section](https://auto.gluon.ai/stable/tutorials/tabular_prediction/tabular-quickstart.html#presets)\n - `hyperparameters` is a Dict which will override the default configs in the training. The configs contain five different types. \n -- `model` contains the parameters which control the models used in the predictor. You can select the model you need and adjust the details. Default is selecting the models determined by the dataset automatically.\n --`optim` contains the configs in the optimization process, including but not limited to max training epochs, learning rate and warm-up.\n\n### 2.3 Save Standalone Model\nModels should be saved for offline deployment for Kaggle competitions, and uploaded to Kaggle as `datasets` after training is done. You can specify the MultiModalPredictor to save a âstandaloneâ model that can be loaded without internet access.\n\n predictor.save(path=save_standalone_path, standalone=True)\n\n## 2. Kaggle Kernel-only Competition with AutoGluon\n\nIn a Kaggle competition, especially a code competition, users cannot obtain AutoGluon resources through the network. \nTo solve the problem, there are two key points:\n\n - Loading AutoGluon and its related libraries through datasets.\n - Using standalone models to avoid model downloading.\n\nThe AutoGluon and its dependencies are currently packaged in a zip file and available for downloading as data in [Kaggle notebook](https://www.kaggle.com/code/linuxdex/get-autogluon-standalone/data). \nYou can download `autogluon_standalone.zip`, unzip it, and upload this folder as a [Kaggle Dataset](https://www.kaggle.com/datasets).\n\nUse the following code to install AutoGluon without network in a kaggle notebook. \n\n import sys\n sys.path.append(\"../input/autogluon-standalone/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8/src/\")\n !pip install --no-deps --no-index --quiet ../input/autogluon-standalone/autogluon_standalone/*.whl\n\nUsing the saved standalone model can avoid downloading models in submission. You can refer to [Section 2.3](#23-save-standalone-model) to save the standalone model.\n\n## 3. Prediction in Kaggle Competitions\n\nNext, let's upload the predictor to Kaggle and use it to generate probabilities on the test set. You can upload the MultiModalPredictor standalone models as datasets to Kaggle directly on a notebook, \nor via [Kaggle API](https://www.kaggle.com/docs/api#interacting-with-datasets). \nMake sure that models are present under the `Input` section in your notebook.\n\nYou can then load the MultiModalPredictor using the following code.\n\n pretrained_model = MultiModalPredictor.load(path=save_standalone_path)\n\nYou can upload the [preprocessing script](./kaggle_feedback_prize_preprocess.py) to Kaggle following the [instructions](https://www.kaggle.com/product-feedback/91185) or paste them directly into a notebook code block.\n\nPreprocess test data with text normalization.\n\n test_df = kaggle_feedback_prize_preprocess.read_and_process_data_with_norm(data_path, \"test.csv\", is_train=False)\n\nWith the `.predict_proba()`, you can get the probabilities of all classes.\n\n test_pred = pretrained_model.predict_proba(test_df)\n \nFor detailed codes, please refer to [`kaggle_feedback_prize_submit.py`](./kaggle_feedback_prize_submit.py).\n\n## 4. Benchmarking model performance with text normalization\n\nWe have benchmarked text normalization effect on different models and hyperparameters. \nFor model evaluation, we fixed 20% of stratified samples from the training data, and we also submitted a couple of large models to Kaggle competition for leadboard scores. \n\n\n| model | lr | lr_decay | cv_k | normalized_text | local_log_loss | kaggle_private | kaggle_public \n| --- | --- | --- | --- |--- |--- |--- |---\n| microsoft/deberta-v3-base | 5e-5 | 0.9 | 3 | Y | 0.5692\n| microsoft/deberta-v3-base | 5e-5 | 0.9 | 3 | N | 0.5835\n| microsoft/deberta-v3-base | 5e-5 | 0.9 | 5 | Y | 0.5694\n| microsoft/deberta-v3-base | 5e-5 | 0.9 | 5 | N | 0.5750\n| microsoft/deberta-v3-large | 5e-5 | 0.9 | 3 | Y | 0.5848\n| microsoft/deberta-v3-large | 5e-5 | 0.9 | 3 | N | 0.5779\n| microsoft/deberta-v3-large | 5e-5 | 0.9 | 5 | Y | 0.5552 | 0.621 | 0.6267\n| microsoft/deberta-v3-large | 5e-5 | 0.9 | 5 | N | 0.5703 | 0.6228 | 0.6296\n| roberta-base | 5e-5 | 0.9 | 3 | Y | 0.5969\n| roberta-base | 5e-5 | 0.9 | 3 | N | 0.5944\n| roberta-base | 5e-5 | 0.9 | 5 | Y | 0.5741\n| roberta-base | 5e-5 | 0.9 | 5 | N | 0.5781\n| roberta-large | 5e-5 | 0.9 | 3 | Y | 0.5739\n| roberta-large | 5e-5 | 0.9 | 3 | N | 0.5850\n| roberta-large | 5e-5 | 0.9 | 5 | Y | 0.5635 | 0.6419 | 0.6399\n| roberta-large | 5e-5 | 0.9 | 5 | N | 0.5657 | 0.6439 | 0.6404\n\nThe results of the benchmark are shown in the table above. It is evident that text normalization is effective in majority of the cases.\n"
},
{
"path": "examples/automm/kaggle_feedback_prize/kaggle_feedback_prize_preprocess.py",
"content": "import os\n\nimport pandas as pd\n\n\ndef get_essay(essay_id: str, input_dir: str, is_train: bool = True) -> str:\n parent_path = input_dir + \"train\" if is_train else input_dir + \"test\"\n essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n essay_text = open(essay_path, \"r\").read()\n return essay_text\n\n\ndef read_and_process_data(path: str, file: str, is_train: bool) -> pd.DataFrame:\n df = pd.read_csv(os.path.join(path, file))\n df[\"essay_text\"] = df[\"essay_id\"].apply(lambda x: get_essay(x, path, is_train=is_train))\n return df\n"
},
{
"path": "examples/automm/kaggle_feedback_prize/kaggle_feedback_prize_submit.py",
"content": "import gc\nimport sys\nimport warnings\n\nimport kaggle_feedback_prize_preprocess\nimport pandas as pd\nimport torch\n\nfrom autogluon.multimodal import MultiModalPredictor\n\nwarnings.filterwarnings(\"ignore\")\nsys.path.append(\"../input/autogluon-standalone/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8/src/\")\n!pip install - -no - deps - -no - index - -quiet .. / input / autogluon - standalone / *.whl\n\n\ndata_path = \"../input/feedback-prize-effectiveness/\"\n\nconfig_1 = {\n \"save_path\": \"../input/feedback_microsoft-deberta-v3-large/microsoft-deberta-v3-large-cv5-lr-5e-05-mepoch-7\",\n \"per_gpu_batch_size_evaluation\": 2,\n \"N_fold\": 5,\n}\nconfig_2 = {\n \"save_path\": \"../input/roberta-large/roberta-large-cv5-lr-5e-05-mepoch-7\",\n \"per_gpu_batch_size_evaluation\": 2,\n \"N_fold\": 5,\n}\n\n\nif __name__ == \"__main__\":\n test_df = kaggle_feedback_prize_preprocess.read_and_process_data(data_path, \"test.csv\", is_train=False)\n\n configs = [config_1, config_2]\n weights = [0.6, 0.4]\n\n all_proba = []\n for config in configs:\n print(config)\n model_proba = []\n for fold in range(config[\"N_fold\"]):\n pretrained_model = MultiModalPredictor.load(path=config[\"save_path\"] + f\"_{fold}\")\n pretrained_model._config.env.per_gpu_batch_size_evaluation = config[\"per_gpu_batch_size_evaluation\"]\n test_proba = pretrained_model.predict_proba(test_df)\n model_proba.append(test_proba)\n\n # free up CPU memory\n del pretrained_model\n torch.cuda.empty_cache()\n gc.collect()\n\n proba_concat = pd.concat(model_proba)\n mean_proba = proba_concat.groupby(level=0).mean()\n all_proba.append(mean_proba)\n\n result = sum([all_proba[i] * weights[i] for i in range(len(configs))])\n result.to_csv(\"submission.csv\", index=False)\n print(result)\n"
},
{
"path": "examples/automm/kaggle_feedback_prize/kaggle_feedback_prize_train.py",
"content": "import argparse\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport torch as th\nfrom kaggle_feedback_prize_preprocess import read_and_process_data\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef get_args() -> argparse.ArgumentParser:\n parser = argparse.ArgumentParser(\n description=\"The example for Kaggle competition Feedback Prize - Predicting Effective Arguments.\"\n )\n parser.add_argument(\"--data_path\", type=str, help=\"The path of the competiton dataset.\", default=\"./data/\")\n parser.add_argument(\"--model_path\", type=str, help=\"The path of the model artifacts.\", default=\"./model/\")\n parser.add_argument(\n \"--label_name\", type=str, help=\"The column name of predictive label.\", default=\"discourse_effectiveness\"\n )\n parser.add_argument(\"--problem_type\", type=str, help=\"The problem type.\", default=\"multiclass\")\n parser.add_argument(\"--eval_metric\", type=str, help=\"The evaluation metric.\", default=\"log_loss\")\n parser.add_argument(\"--lr\", type=float, help=\"The learning rate in the training.\", default=5e-5)\n parser.add_argument(\"--max_epochs\", type=int, help=\"The max training epochs in the training.\", default=7)\n parser.add_argument(\n \"--text_backbone\", type=str, help=\"Pretrained backbone for finetuning.\", default=\"microsoft/deberta-v3-large\"\n )\n parser.add_argument(\"--folds\", type=int, help=\"The folds of the training.\", default=5)\n parser.add_argument(\"--seed\", type=int, help=\"The random seed.\", default=42)\n args = parser.parse_args()\n\n backbone_model = args.text_backbone.replace(\"/\", \"-\")\n args.save_path = args.model_path + \"feedback-{}/{}-cv{}-lr-{}-mepoch-{}\".format(\n backbone_model,\n backbone_model,\n args.folds,\n args.lr,\n args.max_epochs,\n )\n return args\n\n\ndef get_hparams(args: argparse.ArgumentParser) -> dict:\n hparams = {\n \"model.hf_text.checkpoint_name\": args.text_backbone,\n \"data.text.normalize_text\": True,\n \"optim.lr\": args.lr,\n \"optim.max_epochs\": args.max_epochs,\n }\n\n return hparams\n\n\ndef set_seed(seed: int) -> None:\n th.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n\n\ndef train(\n train_df: pd.DataFrame, val_df: pd.DataFrame, args: argparse.ArgumentParser, path: str\n) -> MultiModalPredictor:\n hparams = get_hparams(args)\n\n predictor = MultiModalPredictor(\n label=args.label_name,\n problem_type=args.problem_type,\n eval_metric=args.eval_metric,\n path=path,\n verbosity=3,\n ).fit(\n train_data=train_df,\n tuning_data=val_df,\n presets=\"best_quality\",\n hyperparameters=hparams,\n )\n\n return predictor\n\n\nif __name__ == \"__main__\":\n args = get_args()\n set_seed(args.seed)\n\n train_df = read_and_process_data(args.data_path, \"train.csv\", is_train=True)\n\n y_train = train_df[args.label_name]\n X_train = train_df.drop(args.label_name, axis=1)\n\n # K fold cross validation\n skf = StratifiedKFold(n_splits=args.folds, shuffle=True)\n losses = []\n for i, (train_idx, val_idx) in enumerate(skf.split(X_train, y_train)):\n X_t, X_v = X_train.iloc[train_idx], X_train.iloc[val_idx]\n y_t, y_v = y_train.iloc[train_idx], y_train.iloc[val_idx]\n\n train_df = pd.concat([X_t, y_t], axis=1)\n val_df = pd.concat([X_v, y_v], axis=1)\n path = args.save_path + f\"_{i}\"\n\n predictor = train(train_df, val_df, args, path)\n predictor.save(path, standalone=True)\n"
},
{
"path": "examples/automm/kaggle_pawpularity/README.md",
"content": "īģŋ# How to obtain top 1% in Petfinder Pawpularity with AutoMM\n\nTake [Petfinder Pawpularity competition](https://www.kaggle.com/competitions/petfinder-pawpularity-score/overview) as an example showing how to use MultiModalPredictor to complete a competition.\n\n## 1. MultiModalPredictor for Training\n\nMultiModalPredictor is a tool that can handle diverse data including images, text, numerical data and categorical data. It can automatically identify data types and fuse the pretrained DL backbones. With the tool, you can make the train easily with less code. For details, please refer to [`kaggle_Pawpularity.py`](./kaggle_Pawpularity.py).\n\n### 1.1 Build the MultiModalPredictor\n\nYou can build the predictor as follow.\n\n predictor = MultiModalPredictor(\n\t label=\"Pawpularity\", \n\t problem_type=\"regression\", \n\t eval_metric=\"rmse\", \n\t path=save_path, \n\t verbosity=4, \n\t)\n\n - `label` indicates the target value in training data.\n - `problem_type` indicates the type of the problem. That can be \"Multiclass\", \"Binary\" or \"Regression\".\n - `eval_metric` indicates the evaluation index of the model which is always the evaluation of the competition.\n - `path` indicates the path to save MultiModalPredictor models.\n - `verbosity` controls how much information is printed.\n\n### 1.2 Train the MultiModalPredictor\n\nThen, you can train the MultiModalPredictor with `.fit()`.\n\n predictor.fit( \n\t train_data=training_df,\n\t tuning_data=valid_df, \n\t save_path=save_path, \n\t hyperparameters={\n\t\t \"model.names\": \"['timm_image']\",\n\t\t \"model.timm_image.checkpoint_name\": \"swin_large_patch4_window7_224\",\n\t\t \"model.timm_image.train_transforms\": \"['resize_shorter_side','center_crop','randaug']\",\n\t\t \"data.categorical.convert_to_text\": \"False\",\n\t\t \"env.per_gpu_batch_size\": \"16\",\n\t\t \"env.per_gpu_batch_size_evaluation\": \"32\",\n\t\t \"env.precision\": \"32\",\n\t\t \"optim.lr\": \"2e-5\",\n\t\t \"optim.weight_decay\": \"0\",\n\t\t \"optim.lr_decay\": \"1\",\n\t\t \"optim.max_epochs\": \"5\",\n\t\t \"optim.warmup_steps\": \"0\",\n\t\t \"optim.loss_func\": \"bcewithlogitsloss\",\n\t\t},\n\t\tseed=1,\n\t)\n\n - `train_data` is the data used for training.\n - `tuning_data` is the data for validation. If it is empty, the tuning data will be split from training data automatically.\n - `save_path` indicates the specific path for model saving in a fit process.\n - `hyperparameters` is a Dict which will override the default configs in the training. The configs contain five different types. \n -- `model` contains the parameters which control the models used in the predictor. You can select the model you need and adjust the details. Default is selecting the models determined by the dataset automatically.\n --`data` contains the configs of transforms for different types of data. \n --`env` contains the configs of the training environment.\n --`optim` contains the configs in the optimization process, including but not limited to max training epochs, learning rate and warm-up.\n - `seed` determines the random seed.\n\n### 1.3 Save Standalone Model\nIn MultiModalPredictor, some pre-trained models will be downloaded during training. These models also need to be saved for use in predicting after submission. You can specify the predictor to save a âstandaloneâ model that can be loaded without internet access.\n\n predictor.save(path=save_standalone_path, standalone=True)\n\n## 2. Kaggle Kernel-only Competition with AutoGluon\n\nIn a Kaggle competition, especially a code competition, users cannot obtain AutoGluon resources through the network. \nTo solve the problem, there are two key points:\n\n - Loading AutoGluon and its related libraries through datasets.\n - Using standalone models to avoid model downloading.\n\nYou can download [AutoGluon](https://github.com/autogluon/autogluon) and use the tools to train your model locally.\nFor using AutoGluon in Kaggle submission, it should be uploaded to Kaggle as a dataset. You can create a new dataset called \"auotgluon\" in Kaggle. After that, find AutoGluon at the installation path and upload it into the dataset. \nIn this way, AutoGluon is introduced without network support in submission. \nThrough the code following, you can import AutoGluon into your work.\n\n import sys\n sys.path.append(\"../input/autogluon/\")\n \nIt should be noted that AutoGluon itself needs some dependencies which are not supported in the Kaggle environment. They should be introduced the same way as Auotgluon. \nCurrently, these libraries need to be imported manually:\n\n - typish\n - timm\n - omegaconf\n - antlr4\n - nlpaug\n\nYou can refer to [Kaggle notebook](https://www.kaggle.com/code/linuxdex/get-autogluon-standalone) and run the notebook to get autogluon.multimodal standalone. Missing dependent packages will be downloaded as .whl or .tar.gz in autogluon_standalone. You can download the zip of the folder and create this as a Dataset in Kaggle.\n\nUse the code following to install AutoGluon without network in the kaggle notebook. \n\n import sys\n sys.path.append('../input/autogluon-standalone-install/autogluon_standalone/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8/src/')\n !pip install --no-deps --no-index --quiet ../input/autogluon-standalone-install/autogluon_standalone/*.whl --find-links autogluon_standalone\n\nUsing the saved standalone model can avoid downloading models in submission. You can refer to *1.3* to save the standalone model.\n\n## 3. Prediction in Kaggle Competitions\n\nNext, let's upload the predictor to Kaggle and use it to generate predictions on the test set. You should upload the MultiModalPredictor standalone save files as a dataset to Kaggle and put it in the input data sources of your prediction code. \nYou can load the MultiModalPredictor using the code following.\n\n pretrained_model = predictor.load(path=save_standalone_path)\n\nWith the `.predict()`, you can get the prediction of test datasets.\n\n test_pred = pretrained_model.predict(test_df)\n \nFor detailed codes, please refer to [`Kaggle_Pawpularity_submit.py`](./kaggle_Pawpularity_submit.py).\n\n## 4. Result in Kaggle Competitions\n\nYou can refer to the [kaggle notebook](https://www.kaggle.com/code/linuxdex/use-autogluon-to-predict-pet-adoption) for submission and results.\n\n| ID | Backbone | Fusion | Augment | learning_rate | lr_decay | weight_decay | Max_epochs | Warmup_step | Per_gpu_batch_size | Per_gpu_batch_size_evaluation | Precision | CV | Public_Leaderboard | Private_Leaderboard | Download |\n|----|----------|--------|---------|---------------|----------|--------------|------------|-------------|--------------------|-------------------------------|-----------|----|--------------------|---------------------|----------|\n| 1 | vit_large_patch16_384 | True | randaug | 2e-5 | 1 | 0 | 5 | 0 | 8 | 3 | 32 | 17.3740541397068 | 17.97642 | 17.10867| [result7](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result7_standalone.zip) |\n| 2 | swin_large_patch4_window12_384 | True | randaug | 2e-5 | 1 | 0 | 5 | 0 | 8 | 32 | 32 | 17.4974990118305 | 18.09335 | 17.18875 | [result6](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result6_standalone.zip) |\n| 3 | convnext_large_384_in22ft2k | False | randaug | 5e-5 | 1 | 0 | 10 | 0 | 8 | 4 | 32 | 17.4523797944187 | 18.25999 | 17.20016 | [result26](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result26_standalone.zip) |\n| 4 | swin_large_patch4_window7_224 | False | randaug | 5e-5 | 1 | 0 | 5 | 0 | 16 | 32 | 32 | 17.5192244849318 | 18.03887 | 17.27713 | [result13](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result13_standalone.zip) |\n| 5 | swin_large_patch4_window7_224 | True | randaug | 5e-5 | 1 | 0 | 10 | 0.1 | 16 | 32 | 32 | 17.4848481619876 | 18.15082 | 17.29325 | [result30](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result30_standalone.zip) |\n| 6 | vit_large_patch16_384 | False | randaug | 2e-5 | 1 | 0 | 5 | 0 | 8 | 3 | 32 | 17.5162709909151 | 18.15326 | 17.37978 | [result23](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/result23_standalone.zip) |\n\nThe results of the model are shown in the table above.\n\nIt is obvious that the model using fusion usually has better results.\n\n| BackBone | Fusion_CV | Image_only_CV |\n|----------|-----------|---------------|\n| swin_large_patch4_window7_224 | 17.4848481619876 | 17.5192244849318 |\n| swin_large_patch4_window12_384 | 17.4974990118305 | 17.5871592343891 |\n| convnext_large_384_in22ft1k | 17.6218877694844 | 17.4523797944187 |\n| vit_large_patch16_384 | 17.3740541397068 | 17.5162709909151 | \n| beit_large_patch16_384 | 17.530005178868 | 17.6423355406175 |\n\nWith MultiModalPredictor, You can easily use fusion through modifying hyper parameters instead of changing codes.\n\nThe results of model ensemble are as follows.\n\n| Ensemble IDs | Weights | Public_Leaderboard | Private_Leaderboard | Kaggle screenshot |\n|--------------|---------|--------------------|---------------------|-------------------|\n| [1, 2, 3] | [0.5, 0.25, 0.25] | 17.91944 | 16.97737 | [Kaggle result](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/kaggle-shot/shot_1.png) |\n| [1, 2, 3, 4] | [0.25, 0.25, 0.25, 0.25] | 17.90128 | 16.97970 | [Kaggle result](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/kaggle-shot/shot_2.png) |\n| [1, 2, 4] | [0.4, 0.4, 0.2] | 17.88050 | 16.99416 | [Kaggle result](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/kaggle-shot/shot_3.png) |\n| [1, 2] | [0.5, 0.5] | 17.91753 | 17.01075 | [Kaggle result](http://automl-mm-bench.s3.amazonaws.com/0.5release/petfinder_pawpularity/kaggle-shot/shot_4.png) |\n"
},
{
"path": "examples/automm/kaggle_pawpularity/kaggle_pawpularity_submit.py",
"content": "import warnings\nwarnings.filterwarnings('ignore')\nimport sys\nsys.path.append('../input/autogluon-standalone-install/autogluon_standalone/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8/src/')\n!pip install --no-deps --no-index --quiet ../input/autogluon-standalone-install/autogluon_standalone/*.whl --find-links autogluon_standalone\n\nfrom autogluon.multimodal import MultiModalPredictor\nimport pandas as pd\nimport numpy as np\nimport torch\nimport os\n\n\ndata_path = \"../input/petfinder-pawpularity-score/\"\n\nconfig_6 = {\n \"save_path\": \"../input/pawpularity-automm-result/result6/result\",\n \"per_gpu_batch_size_evaluation\": 32,\n \"N_fold\": 5,\n}\nconfig_7 = {\n \"save_path\": \"../input/pawpularity-automm-result/result7/result\",\n \"per_gpu_batch_size_evaluation\": 3,\n \"N_fold\": 5,\n}\nconfig_13 = {\n \"save_path\": \"../input/pawpularity-automm-result/result13/result\",\n \"per_gpu_batch_size_evaluation\": 32,\n \"N_fold\": 5,\n}\nconfig_26 = {\n \"save_path\": \"../input/pawpularity-automm-result26/result\",\n \"per_gpu_batch_size_evaluation\": 3,\n \"N_fold\": 5,\n}\nconfig_30 = {\n \"save_path\": \"../input/pawpularity-automm-result30/result\",\n \"per_gpu_batch_size_evaluation\": 32,\n \"N_fold\": 5,\n}\n\n\ndef load_data(data_path):\n train_df = pd.read_csv(os.path.join(data_path, \"train.csv\"))\n train_df.rename(columns={\"Id\": \"Image Path\"}, inplace=True)\n train_df[\"Image Path\"] = train_df[\"Image Path\"].apply(lambda s: os.path.join(data_path, \"train\", s + \".jpg\"))\n\n test_df = pd.read_csv(os.path.join(data_path, \"test.csv\"))\n test_df.rename(columns={\"Id\": \"Image Path\"}, inplace=True)\n test_df[\"Image Path\"] = test_df[\"Image Path\"].apply(lambda s: os.path.join(data_path, \"test\", s + \".jpg\"))\n return train_df, test_df\n\n\ntrain_df, test_df = load_data(data_path)\n\nif __name__ == \"__main__\":\n submission = pd.read_csv(\"../input/petfinder-pawpularity-score/sample_submission.csv\")\n \n configs = [config_6, config_7, config_26]\n model_preds = np.empty(shape=[0,submission.shape[0]])\n for perconfig in configs:\n print(perconfig)\n save_standalone_path = perconfig[\"save_path\"] + '_standalone'\n all_preds = []\n for fold in range(perconfig[\"N_fold\"]):\n predictor = MultiModalPredictor(\n label='Pawpularity',\n problem_type='regression',\n eval_metric='rmse',\n path=perconfig[\"save_path\"],\n verbosity=4,\n )\n pretrained_model = predictor.load(path=save_standalone_path + f'_fold{fold}/')\n pretrained_model._config.env.per_gpu_batch_size_evaluation = perconfig[\"per_gpu_batch_size_evaluation\"]\n df_test = pretrained_model.predict(test_df)\n all_preds.append(df_test)\n del predictor\n torch.cuda.empty_cache()\n model_preds = np.append(model_preds, [np.mean(np.stack(all_preds), axis=0)], axis=0)\n \n submission[\"Pawpularity\"] = model_preds[0] * 0.25 + model_preds[1] * 0.5 + model_preds[2] * 0.25 #Model ensemble.\n submission.to_csv(\"submission.csv\", index=False)\n\n print(submission)"
},
{
"path": "examples/automm/kaggle_pawpularity/kaggle_pawpularity_train.py",
"content": "from autogluon.multimodal import MultiModalPredictor\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import root_mean_squared_error\nimport argparse\nimport pandas as pd\nimport numpy as np\nimport torch\nimport os\n\n\ndef get_args():\n parser = argparse.ArgumentParser(description=\"The example for Kaggle competition Petfinder Pawpularity with MultiModalPredictor.\")\n parser.add_argument(\"--data_path\", type=str,\n help=\"The path of the competiton dataset.\",\n default=\"./data/\")\n parser.add_argument(\"--label_column\", type=str,\n help=\"The column name of label.\",\n default=\"Pawpularity\")\n parser.add_argument(\"--problem_type\", type=str,\n help=\"The problem type.\",\n default=\"regression\")\n parser.add_argument(\"--eval_metric\", type=str,\n help=\"The evaluation metric.\",\n default=\"rmse\")\n parser.add_argument(\"--fusion\", type=bool,\n help=\"Whether using model fusion.\",\n default=True)\n parser.add_argument(\"--timm_image_checkpoint_name\", type=str,\n help=\"The name of model for images in timm.\",\n default=\"swin_large_patch4_window7_224\")\n parser.add_argument(\"--image_train_transforms\", type=str,\n help=\"The types for transforming images.\",\n default=\"['resize_shorter_side','center_crop','randaug']\")\n parser.add_argument(\"--categorical_convert_to_text\", type=bool,\n help=\"Whether convert categorical to text.\",\n default=False)\n parser.add_argument(\"--per_gpu_batch_size\", type=int,\n help=\"The batch size in gpu.\",\n default=16)\n parser.add_argument(\"--precision\", type=str,\n help=\"The precision in the training.\",\n default=\"32\")\n parser.add_argument(\"--lr\", type=float,\n help=\"The learning rate in the training.\",\n default=2e-5)\n parser.add_argument(\"--weight_decay\", type=float,\n help=\"The weight decay in the training.\",\n default=0)\n parser.add_argument(\"--lr_decay\", type=float,\n help=\"The learning rate decay in the training.\",\n default=1)\n parser.add_argument(\"--max_epochs\", type=int,\n help=\"The max training epochs in the training.\",\n default=5)\n parser.add_argument(\"--warmup_steps\", type=float,\n help=\"The warmup steps of the training epochs.\",\n default=0)\n parser.add_argument(\"--loss_func\", type=str,\n help=\"The loss function of the training epochs.\",\n default=\"bcewithlogitsloss\")\n parser.add_argument(\"--folds\", type=int,\n help=\"The folds of the training.\",\n default=5)\n parser.add_argument(\"--seed\", type=int,\n help=\"The random seed.\",\n default=1)\n parser.add_argument(\"--trial\", type=int,\n help=\"The id of multiple runs.\",\n default=0)\n args = parser.parse_args()\n\n if args.fusion:\n args.model_names = None\n args.save_path = 'pawpularity_{}_fusion_lr_{}_decay_{}_bsz_{}_mepoch_{}_warmup_{}_trial_{}'.\\\n format(\n args.timm_image_checkpoint_name,\n args.lr,\n args.lr_decay,\n args.per_gpu_batch_size,\n args.max_epochs,\n args.warmup_steps,\n args.trial\n )\n else:\n args.model_names = \"['timm_image']\"\n args.save_path = \"pawpularity_{}_timm_only_lr_{}_decay_{}_bsz_{}_mepoch_{}_warmup_{}_trial_{}\". \\\n format(\n args.timm_image_checkpoint_name,\n args.lr,\n args.lr_decay,\n args.per_gpu_batch_size,\n args.max_epochs,\n args.warmup_steps,\n args.trial\n )\n\n return args\n\n\ndef load_data(data_path: str):\n \"\"\"\n Load training and testing datasets and split folds.\n Parameters\n ----------\n data_path\n The path of training and testing files.\n\n Return\n ----------\n The pure training dataset and the folds split information and the testing dataset.\n \"\"\"\n # Load training dataset.\n train_df = pd.read_csv(os.path.join(data_path, \"train.csv\"))\n train_df.rename(columns={\"Id\": \"Image Path\"}, inplace=True)\n train_df[\"Image Path\"] = train_df[\"Image Path\"].apply(lambda s: os.path.join(data_path, \"train\", s + \".jpg\"))\n\n # Create the split information of folds.\n num_bins = int(np.ceil(2 * ((len(train_df)) ** (1. / 3))))\n train_df_bins = pd.cut(train_df[\"Pawpularity\"], bins=num_bins, labels=False)\n train_df_fold = pd.Series([-1] * len(train_df))\n strat_kfold = StratifiedKFold(n_splits=N_FOLDS, shuffle=True)\n for i, (_, train_index) in enumerate(strat_kfold.split(train_df.index, train_df_bins)):\n train_df_fold[train_index] = i\n train_df_fold = train_df_fold.astype(\"int\")\n\n # Load testing dataset.\n test_df = pd.read_csv(os.path.join(data_path, \"test.csv\"))\n test_df.rename(columns={\"Id\": \"Image Path\"}, inplace=True)\n test_df[\"Image Path\"] = test_df[\"Image Path\"].apply(lambda s: os.path.join(data_path, \"test\", s + \".jpg\"))\n\n return train_df, train_df_fold, test_df\n\n\nif __name__ == \"__main__\":\n args = get_args()\n\n data_path = args.data_path # The path of the training and testing data.\n save_path = args.save_path # The path of saving the model.\n save_standalone_path = save_path + \"_standalone\" # The path of saving the standalone model which includes downloaded model.\n\n N_FOLDS = args.folds # The number of folds.\n\n all_score = [] # The result of folds.\n train_df, train_df_fold, _ = load_data(data_path)\n\n for i in range(N_FOLDS):\n # The predictor in use.\n predictor = MultiModalPredictor(\n label=args.label_column, # label indicates the target value\n problem_type=args.problem_type, # problem_type indicates the type of the problem. It can choose \"multiclass\", # \"binary\" or \"regression\"\n eval_metric=args.eval_metric, # eval_metric indicates the evaluation index of the model\n path=save_path,\n verbosity=4, # verbosity controls how much information is printed.\n )\n\n # Training process.\n training_df = train_df[train_df_fold != i]\n valid_df = train_df[train_df_fold == i]\n predictor.fit(\n train_data=training_df,\n tuning_data=valid_df,\n save_path=save_path + f\"_fold{i}\",\n hyperparameters={\n \"model.names\": args.model_names,\n \"model.timm_image.checkpoint_name\": args.timm_image_checkpoint_name,\n \"model.timm_image.train_transforms\": args.image_train_transforms,\n \"data.categorical.convert_to_text\": args.categorical_convert_to_text,\n \"env.per_gpu_batch_size\": args.per_gpu_batch_size,\n \"env.precision\": args.precision,\n \"optim.lr\": args.lr,\n \"optim.weight_decay\": args.weight_decay,\n \"optim.lr_decay\": args.lr_decay,\n \"optim.max_epochs\": args.max_epochs,\n \"optim.warmup_steps\": args.warmup_steps,\n \"optim.loss_func\": args.loss_func,\n },\n seed=args.seed,\n )\n\n # Manual Validating process.\n valid_pred = predictor.predict(data=valid_df)\n score = root_mean_squared_error(valid_df[\"Pawpularity\"].values, valid_pred)\n print(f\"Fold {i} | Score: {score}\")\n predictor.save(\n path=save_standalone_path + f\"_fold{i}\",\n standalone=True,\n )\n all_score.append(score)\n\n del predictor\n torch.cuda.empty_cache()\n\n print(f\"all-scores: {all_score}\")\n print(f\"mean_rmse: {np.mean(all_score)}\")\n"
},
{
"path": "examples/automm/label_studio_export_reader/LabelStudio_export_file_reader.md",
"content": "# Label-Studio Export file reader\n\nthe Label-Studio export file reader for Autogluon is used for transforming the labeling result in the labeling tool Label-Studio (https://labelstud.io/) into the Dataframe input for Autogluon multimodals.\n\n## Label-Studio Export knowledge\n\n### 1. General\n\nThe **Label-Studio export file** takes in the form like **CSV, JSON, JSON-MIN, etc.** The export form details can be viewed in the Label-Studio official documentation(https://labelstud.io/guide/export.html). \n\n### 2. Getting files from Label-Studio host\n\nFor the annotation of files like image, it usually requires an additional JSON files to indicate their URL. But if user didn't provide the URL itself (just simply dragging the files into the Label-Studio Web UI), Label-Studio will generate a URL with the host of Label-Studio for these files, and user can access them on the condition that the label-studio host is on.These files will be unreachable if the label-studio host is turned off.\n\n### 3. Label-Studio Templates\n\nLabel-Studio annotation templates are a set of default labeling tasks that help users to quickly annotation the data for certain type of tasks. The task templates supported by Label-Studio can be seen at https://labelstud.io/playground/\n\n\n\n## How to Use\n\nWe should create an Label-Studio Reader object first by:\n\n```python\nfrom autogluon.multimodal.utils import LabelStudioReader\n\n# initialize LabelStudioReader with default localhost host\nls = LabelStudioReader() \n\n# set \nls.set_labelstudio_host(\"http://localhost:8080\")\n```\n\nthe `LabelStudioReader` should be initialized with an given label-studio host. If it's not given by the user, the LabelStudioReader's default port is http://localhost:8080, which is Label-Studio's default host & port on a local machine. **If user import the files directly into Label-Studio Web without given the file's original URL, this host param will help user to access these files when users set** `ls_host_on=True`.\n\n\n\n `LabelStudioReader` allows resetting the host by calling `set_labelstudio_host`.\n\n\n\nOn the first PR of this function, we provide 3 types of tasks to handle the Label-Studio export files. **CSV and JSON(JSON-MIN) files are supported so far**. \n\n```python\n# transforming the export files from Label-Studio image classification template (image)\ndf, labels = ls.from_image_classification(\"ic.json\", ls_host_on=False) \n\n# transforming the export files from Label-Studio named entity recognition template (text)\ndf, labels = ls.from_named_entity_recognition('neg.json')\n\n# transforming the export files from user's customized labeling template\ndf, labels=ls.from_customized('custom.csv',\n ls_host_on=True, \n data_columns=['image1','image2','image3'],\n label_columns=['label'])\n```\n\n\n\n### Explanations of parameters\n\n#### `ls_host_on` \n\nIf user follows the Label-Studio's import data instruction(https://labelstud.io/guide/tasks.html#How-to-import-your-data) and provide a list of URLs in a TXT, CSV, or TSV file, or reference the URLs in JSON, their file export will contains the data file's URL. In this way they don't need the Label-Studio host to address the data, so they can set the `ls_host_on` to `False` (default value).\n\nHowever, if the labeling tasks contains data that are imported directly through Label-Studio Web UI, it's recommended to set the `ls_host_on` to `True` and make sure the Label-Studio Web UI host is on.\n\nSome examples are given below to demonstrate the impact of the value of `ls_host_on` .\n\n\n\n#### `data_columns` and `label_columns`\n\nThese two params indicate the data and label contents of an exported file that user want to set. \n\n- exported through CSV: `data_columns` and `label_columns` here refer to a list of the CSV column names of the data or the labels. For example, if a \".csv\" table of a 2-image classificiation export file has two image columns \"image1\" and \"image2\" that contains the image urls of the dataset, and their label column names are \"label\", we should set `data_columns=['image1','image2']` and `label_columns=['label']` to extract the columns\n- exported through JSON-MIN: The JSON-MIN template can be seen in https://labelstud.io/guide/export.html#JSON-MIN, here `data_columns` and `label_columns` here refer to a list of the .json key names of the data or the labels. For the given examples in the link above, we should set `data_columns=['image']` and `label_columns=['tag']` .\n- exported through JSON: The JSON template can be seen in https://labelstud.io/guide/export.html#Label-Studio-JSON-format-of-annotated-tasks, which is rather complex. To handle nested JSON objects, a normalization under the record path `['annotations','result']` is conducted, which provides nested parsing for the `label_columns`. For the given examples of the object detection demo in the link above, we should set `data_columns=['image']` and `label_columns=['value.height','value.rectanglelabels','value.rotation','value.width','value.x','value.y']` .\n\n\n\n\n\n\n\n### Example 1. data with files \n\nwe use the label-studio **Image Classification Template** export file to demonstrate how we deal with data with files. In this use case, user:\n\n- Choose \"Image Classification\" template in Label-Studio\n- Annotate the images in Label-Studio \n- Click \"Export\" and choose the export format\n- Call `from_image_classification` to transform the export file into the Dataframe for Autogluon input (Autogluon's Image Classification support: https://auto.gluon.ai/stable/tutorials/multimodal/image_prediction/beginner_image_cls.html)\n\n\n\nThe params of `from_image_classification` are as follows:\n\n| Param | Type | Optional | Description |\n| :-----------: | :-------: | :------: | :----------------------------------------------------------: |\n| path | Str | | the path of the export file |\n| ls_host_on | Boolean | | Whether user need to access some files through label-studio host. |\n| data_columns | List[Str] | â | the key or column name(s) of the data.By default we use the given name from the image classification template.If users change the name in the template, they should provide the data column names here |\n| label_columns | List[Str] | â | the key or column name(s) of the label, similar to the situation for data_columns |\n\n\n\n\n\n\n\nNormally, data with files like images should be provided with their URL as we mentioned. We preprocess the data in the data columns as follows corresponding to the value of `ls_host_on`:\n\n\n\n| Data | accessible if ls_host_on=True | accessible if ls_host_on=False | Description | Preprocess |\n| :------------------------: | :---------------------------: | :----------------------------: | ------------------------------------------------------------ | ------------------------------------------------------------ |\n| file url, text content,... | â | â | | Not doing anything |\n| \"/data/upload/...\" | â | X | use didn't provide url of the image, just import the image directly in LS | [ls_host_on=True]: join with the label-studio host(e.g: http://localhost:8080/data/upload/...), when the LS host is on, the file is accessible
[ls_host_on=False]: cannot access |\n| \"/data/local-files/...\" | â | â(conditional) | use didn't provide url of the image, instead they set up a local storage in LS. | [ls_host_on=True]: join with the label-studio host(e.g: http://localhost:8080/data/upload/...)
[ls_host_on=False]: reserve the local path of the data files(if the root of the dataset is not moved) |\n| ... | | | | |\n\n\n\nAn demo export files of Label-Studio export file:\n\n\n\nAn demo output with `ls_host_on=True` :\n\n\n\nAn demo output with `ls_host_on=False` :\n\n(the \"uploaded\" data will be warned for not accessible)\n\n\n\n### Example 2. Text annotations (URL not required )\n\nTo we use the label-studio **named entity recognition** export file to demonstrate cases like this. In this use case, user: \n\n- Choose \"named entity recognition\" template in Label-Studio\n\n- Annotate the images in Label-Studio \n\n- Click \"Export\" and choose the export format\n\n- Call `from_named_entity_recognition` to transform the export file into the Dataframe for Autogluon inputīŧAutogluon's NER support: https://auto.gluon.ai/stable/tutorials/multimodal/text_prediction/ner.htmlīŧ\n\n \n\nSince there're no potential URL issues, the `ls_host_on` param is removed in cases like this. The params of `from_named_entity_recognition` are as follows:\n\n| Param | Type | Optional | Description |\n| :-----------: | :-------: | :------: | :----------------------------------------------------------: |\n| path | Str | | the path of the export file |\n| data_columns | List[Str] | â | the key or column name(s) of the data.By default we use the given name from the image classification template.If users change the name in the template, they should provide the data column names here |\n| label_columns | List[Str] | â | the key or column name(s) of the label, similar to the situation for data_columns |\n\n\n\nNOTE:\n\n**Cases like named entity recognition and object detection may have multiple labels on single text or image files, therefore the label columns will be nested. Currently the ` LabelStudioReader` only implements nested data parsing for JSON, CSV and JSON-MIN is not supported yet **. Therefore it's recommended to export JSON files. We will working on nested data parsing on CSV and JSON-MIN in the future.\n\n\n\nThe processed result of a CSV and JSON-MIN file:\n\n\n\n (the Label column are nested data about NER's labeling information)\n\n\n\nThe processed result of a JSON file:\n\n\n\n\n\n### Example 3. Customized template\n\nIf user create their own Label-Studio template and export the annotations, they can call `from_customized` to transform the export file into the Dataframe for Autogluon input. In this case, user should provide the `data_columns` and `label_columns` to identify the data and label content.\n\n\n\nThe params of `from_customized` are as follows:\n\n| Param | Type | Optional | Description |\n| :-----------: | :-------: | :------: | :----------------------------------------------------------: |\n| path | Str | | the path of the export file |\n| ls_host_on | Boolean | | Whether user need to access some files through label-studio host. |\n| data_columns | List[Str] | | the key or column name(s) of the data |\n| label_columns | List[Str] | | the key or column name(s) of the label |\n\n" }, { "path": "examples/automm/memory_bank/README.md", "content": "# Use MultiModal Feature Extraction to Create a Few-shot Memory Bank Model\n\n[memory_bank.py](./memory_bank.py): This example provides a simple and clear way to implement a few-shot memory bank model with AutoGluon MultiModal Feature Extraction according to [Tip-Adapter](https://github.com/gaopengcuhk/Tip-Adapter) [1]. \n\n### 1. Run Example\n\nBefore running the example, the image datasets need to be prepared with additional code. We follow the few-shot dataset design in [CoOP](https://github.com/KaiyangZhou/CoOp). Copy folder `configs` and `datasets` from the repository and rename `datasets` to `imagedatasets` to avoid conflict with `datasets` in Hugging Face. You can download the datasets and the splits under the [direction](https://github.com/KaiyangZhou/CoOp/blob/main/DATASETS.md).\n\nFor text datasets, datasets in Hugging Face are supported. We recommend datasets under [SetFit](https://huggingface.co/datasets?sort=downloads&search=SetFit) [2] for few-shot learning. Remember to change the column names in args if the data column is not named `text` or have multi-text information.\n\nAfter preparing the datasets, run the example:\n\n python memory_bank.py --type clip --dataset food101 --shots 16\n\n- `type` is the type of few-shot learning. You can choose from `clip`, `text`, `image` for different backbone methods.\n- `backbone` is the backbone model of MultiModal Predictor.\n- `data_path` is the path for image dataset.\n- `dataset` is the name of dataset. Support image datasets in COOP and text datasets in Hugging Face.\n- `column_names` is the names of the data columns.\n- `label_column` is the name of the label column.\n- `shots` is the shots for each class in training set.\n- `aug_epochs` is the epochs to create the bank.\n- `model_head_type` is the model head for few-shot classification. You can choose from 'linear' and 'SVM' for different classification heads.\n- `lr` is the learning rate for training the model head.\n- `lr_F` is the learning rate for finetuing the memory bank.\n- `train_epoch` is the training epochs for training the model head.\n- `train_epoch_F` is the training epochs for fine-tuning the memory bank.\n- `init_alpha` is initial values of hyper-parameters in memory bank. `alpha` adjusts the weight of probability between the classifier and memory bank.\n- `init_beta` is initial values of hyper-parameters in memory bank. `beta` modulates the sharpness when converting the similarities into non-negative values.\n- `search_scale` is the search scale of alpha and beta.\n- `search_step` is the steps of searching hyper-parameters alpha and beta.\n\n### 2. Method\n\nThe Memory Bank, inspired by [Tip-Adapter](https://arxiv.org/pdf/2207.09519.pdf), stores image features of the few-shot training set to improve the performance of zero-shot CLIP through feature similarity and serve as an initialization for a trainable classifier. This ProtoNet-like design effectively utilizes few-shot training information, resulting in good performance [3]. It has potential to be widely applied to few-shot classification tasks for images and texts.\n\nThe Memory Bank, which is based on the Tip-Adapter model, uses MultiModal Feature Extraction to acquire diverse multi-modal features. To evaluate its performance, a linear classifier was first trained using these multi-modal features to establish a baseline accuracy. The similarity between the features and the Memory Bank was then incorporated into the baseline prediction probability. Finally, an additional linear adapter was trained, initialized with the Memory Bank, to aid in few-shot classification.\n\nHyper-parameters `alpha` and `beta` which adjust the memory bank are modified through grid search on validation set to attain the superior performance.\n\n### 3. Training Results\n\nThe training results of memory bank can be seen in the followed table. `Acc w/o memory bank` is the few-shot classification accuracy of method. `ACC w. memory bank` and `ACC w. memory bank (+finetune)` show the classification results after introducing and fine-tuning the memory bank, respectively.\n\n| Datasets | Method | BackBone | Head | Shots | lr | lr_F | Acc w/o memory bank| Acc w. memory bank | Acc w. memory bank (+finetune) | \n|----------|--------|----------|------|-------|----|------|--------------------|--------------------|--------------------------------| \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 91.90 | 92.42 | 92.80 | \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 1 | NaN | 1e-3 | 91.90 | 91.99 | 91.97 | \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 64 | NaN | 1e-3 | 91.90 | 92.43 | 93.10 | \n| Food-101 | CLIP | openai/clip-vit-base-patch32 | NaN | 16 | NaN | 1e-3 | 80.42 | 80.88 | 82.01 | \n| Caltech101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 94.48 | 97.32 | 98.80 | \n| DTD | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 54.2 | 69.86 | 72.10 | \n| EuraSAT | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 61.48 | 79.01 | 83.65 | \n| Flower-102 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 79.13 | 96.95 | 96.51 | \n| Oxford Pets | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 93.79 | 94.22 | 95.52 | \n| Stanford Cars | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 78.20 | 84.09 | 87.95 | \n| Food-101 | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 73.66 | 73.64 | 76.18 | \n| Caltech101 | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 96.75 | 96.75 | 97.16 | \n| DTD | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 67.55 | 68.26 | 70.45 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 38.42 | 38.42 | 39.23 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 1 | 1e-2 | 1e-3 | 33.08 | 33.08 | 33.08 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 64 | 1e-2 | 1e-3 | 45.61 | 46.02 | 48.19 | \n| SetFit/sst5 | Text | sentence-transformers/msmarco-MiniLM-L-12-v3 | Linear | 16 | 1e-2 | 1e-3 | 30.18 | 30.86 | 30.59 | \n| SetFit/Emotion | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 43.10 | 43.65 | 43.90 | \n| SetFit/subj | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 90.50 | 90.55 | 90.75 | \n| SetFit/20_newsgroups | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 54.14 | 57.36 | 58.90 | \n| SetFit/enron_spam | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 91.35 | 91.70 | 92.85 | \n| SetFit/SentEval-CR | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 88.31 | 88.58 | 89.24 | \n| Food-101 | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 73.06 | 74.42 | 75.72 | \n| Caltech101 | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 93.10 | 97.16 | 97.44 | \n| DTD | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 69.39 | 70.45 | 70.39 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 30.90 | 39.28 | 39.95 | \n| SetFit/Emotion | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 26.55 | 43.15 | 44.20 | \n| SetFit/20_newsgroups | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 48.43 | 57.90 | 58.72 | \n\n\n---\n\n### Reference\n\n[1] Tip-Adapter: Training-free CLIP-Adapter for Better Vision-Language Modeling.\n\n[2] Efficient Few-Shot Learning Without Prompts. \n\n[3] Prototypical Networks for Few-shot Learning. \n"
},
{
"path": "examples/automm/memory_bank/memory_bank.py",
"content": "\"\"\"\n The derivative application of Tip-Adapter (https://arxiv.org/pdf/2207.09519.pdf).\n Refer to https://github.com/gaopengcuhk/Tip-Adapter\n\"\"\"\n\nfrom autogluon.multimodal import MultiModalPredictor\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport argparse\nfrom tqdm import tqdm\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import SVC\nfrom utils import *\n\n\ndef get_args():\n parser = argparse.ArgumentParser(description=\"MultiModal Fewshot training with memory bank.\")\n parser.add_argument(\n \"--type\",\n type=str,\n default=\"clip\", \n help=\"Choose type of few-shot learning from 'clip', 'text', 'image' for different backbone methods.\",\n )\n parser.add_argument(\"--backbone\", type=str, default=None, help=\"The backbone model of MultiModal Predictor.\")\n parser.add_argument(\"--data_path\", type=str, default=\"./data/\", help=\"The path for image dataset.\")\n parser.add_argument(\"--dataset\", type=str, default=\"food101\", help=\"The name of dataset. Support image datasets in COOP and text datasets in Huggingface.\")\n parser.add_argument(\"--column_names\", nargs=\"+\", default=None, help=\"The name of the data column.\")\n parser.add_argument(\"--label_column\", type=str, default=\"label\", help=\"The name of the label column.\")\n parser.add_argument(\"--shots\", type=int, default=16, help=\"The shots for each class in training set.\")\n parser.add_argument(\"--aug_epochs\", type=int, default=1, help=\"The epochs to create the bank.\")\n parser.add_argument(\"--model_head_type\", type=str, default=\"linear\", help=\"The model head for few-shot classification. Choose from 'linear' and 'SVM'.\")\n parser.add_argument(\"--lr\", type=float, default=1e-2, help=\"The learning rate for training the model head.\")\n parser.add_argument(\"--lr_F\", type=float, default=1e-3, help=\"The learning rate for finetuing the memory bank.\")\n parser.add_argument(\"--train_epoch\", type=int, default=20, help=\"The training epochs for training the model head.\")\n parser.add_argument(\"--train_epoch_F\", type=int, default=20, help=\"The training epochs for finetuning the memory bank.\")\n parser.add_argument(\"--init_alpha\", type=float, default=1.17, help=\"The initial value of hyper-parameter alpha in memory bank. Alpha adjusts the weight of probability between the classifier and memory bank.\")\n parser.add_argument(\"--init_beta\", type=float, default=1., help=\"The initial values of hyper-parameter beta in memory bank. Beta modulates the sharpness when converting the similarities into non-negative values.\")\n parser.add_argument(\"--search_scale\", type=int, nargs=\"+\", default=[10, 10], help=\"The search scale of alpha and beta.\")\n parser.add_argument(\"--search_step\", type=int, nargs=\"+\", default=[200, 20], help=\"The steps of searching hyper-parameters alpha and beta.\")\n args = parser.parse_args()\n\n args.bank_dir = os.path.join('./memory_bank_models', args.dataset)\n os.makedirs(args.bank_dir, exist_ok=True)\n\n if args.column_names is None:\n args.column_names = [\"text\"] if args.type == \"text\" else [\"image\"]\n \n if args.backbone is None:\n if args.type == \"text\":\n args.backbone = \"sentence-transformers/paraphrase-mpnet-base-v2\"\n elif args.type == \"image\":\n args.backbone = \"swin_base_patch4_window7_224\"\n else:\n args.backbone = \"openai/clip-vit-large-patch14-336\"\n \n return args\n\n\nclass AutoMMMemoryBank(nn.Module):\n \"\"\"\n The model to generate few shot predict probability with \n features extracted by AutoGluon MultiModal Predictor.\n \"\"\"\n\n def __init__(\n self, \n bank_keys, \n bank_labels,\n hidden_size, \n num_classes,\n clip_weights=None, \n model_head_type=\"linear\",\n ):\n \"\"\"\n Create the model head and the memory bank.\n\n Parameters\n ----------\n bank_keys\n The content of bank composed of features in the training set.\n bank_labels \n The labels of corresponding bank_keys.\n hidden_size\n The size of features.\n num_classes\n The classes of the dataset.\n clip_weights\n The clip embedding of the semantic text that describes the labels.\n model_head_type \n The type of the few-shot classification head.\n \"\"\"\n super(AutoMMMemoryBank, self).__init__()\n self.bank_keys = bank_keys\n self.bank_values = F.one_hot(bank_labels).float()\n self.adapter = nn.Linear(bank_keys.shape[0], bank_keys.shape[1], bias=False)\n self.adapter.weight = nn.Parameter(bank_keys.t())\n\n self.clip_weights = clip_weights\n \n self.model_head_type = model_head_type\n if clip_weights is None:\n if model_head_type == \"SVM\":\n self.model_head = make_pipeline(StandardScaler(), SVC(gamma=\"auto\", probability=True))\n self.model_head.fit(bank_keys.t().cpu(), bank_labels.cpu())\n else:\n self.model_head = nn.Linear(hidden_size, num_classes, bias=True) if clip_weights is None else None\n else:\n self.model_head = None\n\n def adapt_logits(self, affinity, pure_logits, alpha, beta):\n \"\"\"\n Generate logits with memory bank based on pure_logits and bank output.\n\n Parameters\n ----------\n affinity\n The result of bank similarity. It is based on cosine similarity or a projector initialized with bank_keys.\n pure_logits\n The predict probability of the classifier.\n alpha\n The hyper-parameters of bank model.\n beta\n The hyper-parameters of bank model.\n \n Return\n ------\n The logits with memory bank.\n \"\"\"\n bank_logits = ((-1) * (beta - beta * affinity)).exp() @ self.bank_values\n logits = pure_logits + bank_logits * alpha\n return logits\n \n def change_head_state(self, grad_state):\n \"\"\"\n Change the training state of the model head.\n\n Parameters\n ----------\n grad_state\n The training state of the model head. If \"True\", the model head is trainable. If \"False\", the model head is freezed.\n \"\"\"\n if self.model_head is not None and self.model_head_type == \"linear\":\n for param in self.model_head.parameters():\n param.requires_grad = grad_state\n \n def change_adapter_state(self, grad_state):\n \"\"\"\n Change the training state of the memory bank.\n\n Parameters\n ----------\n grad_state\n The training state of the memory bank. If \"True\", the memory bank is trainable. If \"False\", the memory bank is freezed.\n \"\"\"\n for param in self.adapter.parameters():\n param.requires_grad = grad_state\n\n def forward(self, x, alpha=1, beta=1, pure_logits=None):\n \"\"\"\n Generate three types of logits with features.\n\n Parameters\n ----------\n x\n The image/text features generated by AutoGluon Multimodal Predictor.\n alpha\n The hyper-parameters of memory bank model.\n beta\n The hyper-parameters of memory bank model.\n \n Return\n ------\n The predict probability of the feature.\n - \"pure_logits\"\n The predict probability of classifier.\n - \"adapted_logits\"\n The predict probability composed of classifier and memory bank similarity result.\n - \"adapted_logits_with_finetuning\"\n The predict probability composed of classifier and fine-tuned memory bank result.\n \"\"\"\n if pure_logits is None:\n if self.clip_weights is not None:\n pure_logits = 100. * x @ self.clip_weights\n elif self.model_head_type == \"SVM\":\n pure_logits = torch.tensor(self.model_head.predict_proba(x.cpu())).cuda()\n else:\n pure_logits = self.model_head(x)\n \n affinity = x @ self.bank_keys\n adapted_logits = self.adapt_logits(affinity, pure_logits, alpha, beta)\n\n finetuned_affinity = self.adapter(x)\n adapted_logits_with_finetuning = self.adapt_logits(finetuned_affinity, pure_logits, alpha, beta)\n\n return {\n \"pure_logits\": pure_logits,\n \"adapted_logits\": adapted_logits,\n \"adapted_logits_with_finetuning\": adapted_logits_with_finetuning,\n }\n\n\ndef train_logits(\n args, \n val_features, \n val_labels, \n test_features, \n test_labels, \n predictor, \n memory_bank_model, \n alpha, \n beta, \n loader, \n logits_type=\"pure_logits\"\n):\n \"\"\"\n The training process of AutoMMMemoryBank.\n\n Parameters\n ----------\n args\n The args of the training process.\n val_features, val_labels\n The preprocessed features and labels of validation set.\n test_features, test_labels\n The preprocessed features and labels of test set.\n predictor\n The AutoGluon MultiModal Predictor to extract the features.\n memory_bank_model\n The AutoMMMemoryBank to generate logits and logits with memory bank.\n alpha, beta\n The hyper-parameters of memory bank.\n loader\n The dataset loader for training.\n logits_type\n The target logits of training corresponding to \"pure_logits\", \"adapted_logits\", \"adapted_logits_with_finetuning\".\n \n Return\n ------\n The best model after training.\n \"\"\"\n if logits_type != \"adapted_logits_with_finetuning\":\n memory_bank_model.change_head_state(grad_state=True)\n memory_bank_model.change_adapter_state(grad_state=False)\n lr, train_epoch = args.lr, args.train_epoch\n else:\n memory_bank_model.change_head_state(grad_state=False)\n memory_bank_model.change_adapter_state(grad_state=True)\n lr, train_epoch = args.lr_F, args.train_epoch_F\n \n optimizer = torch.optim.AdamW(memory_bank_model.parameters(), lr=lr, eps=1e-4)\n scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, train_epoch * len(loader))\n memory_bank_model.train()\n\n best_acc, best_epoch = 0.0, 0\n for train_idx in range(train_epoch):\n correct_samples, all_samples = 0, 0\n loss_list = []\n print('Train Epoch: {:} / {:}'.format(train_idx, train_epoch))\n\n for i, (data, target) in enumerate(tqdm(loader)):\n target = target.cuda()\n with torch.no_grad():\n features = extract_embedding(args, data, predictor, args.column_names)\n features /= features.norm(dim=-1, keepdim=True)\n \n logits = memory_bank_model(features, alpha, beta)\n\n loss = F.cross_entropy(logits[logits_type], target)\n\n acc = cls_acc(logits[logits_type], target)\n correct_samples += acc / 100 * len(logits[logits_type])\n all_samples += len(logits[logits_type])\n loss_list.append(loss.item())\n\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n scheduler.step()\n\n current_lr = scheduler.get_last_lr()[0]\n print('LR: {:.6f}, Acc: {:.4f} ({:}/{:}), Loss: {:.4f}'.format(current_lr, correct_samples / all_samples, correct_samples, all_samples, sum(loss_list)/len(loss_list)))\n\n memory_bank_model.eval()\n logits = memory_bank_model(val_features, alpha, beta)[logits_type]\n acc = cls_acc(logits, val_labels)\n print(\"**** val accuracy: {:.2f}. ****\\n\".format(acc))\n if acc > best_acc:\n best_acc = acc\n best_epoch = train_idx\n torch.save(memory_bank_model.state_dict(), args.bank_dir + \"/best_F_\" + str(args.shots) + \"shots.pt\") # nosec B614\n \n memory_bank_model.load_state_dict(torch.load(args.bank_dir + \"/best_F_\" + str(args.shots) + \"shots.pt\")) # nosec B614\n print(f\"**** After fine-tuning {logits_type}, best val accuracy: {best_acc:.2f}, at epoch: {best_epoch}. ****\\n\")\n \n return memory_bank_model\n\n\ndef run_memory_bank(\n args, \n val_features, \n val_labels, \n test_features, \n test_labels, \n predictor, \n memory_bank_model, \n loader,\n):\n \"\"\"\n Test the effectiveness of memory bank. \n Compare the results of pure_logits, adapted_logits and adapted_logits_with_finetuning.\n\n Parameters\n ----------\n args\n The args of the training process.\n val_features, val_labels\n The preprocessed features and labels of validation set.\n test_features, test_labels\n The preprocessed features and labels of test set.\n predictor\n The AutoGluon MultiModal Predictor to extract the features.\n memory_bank_model\n The AutoMMMemoryBank to generate logits and logits with memory bank.\n loader\n The dataset loader for training.\n \"\"\"\n beta, alpha = args.init_beta, args.init_alpha\n\n if args.type != \"clip\" and args.model_head_type ==\"linear\":\n memory_bank_model = train_logits(\n args=args,\n val_features=val_features,\n val_labels=val_labels, \n test_features=test_features, \n test_labels=test_labels, \n predictor=predictor, \n memory_bank_model=memory_bank_model, \n alpha=alpha, \n beta=beta, \n loader=loader, \n logits_type=\"pure_logits\",\n )\n \n with torch.no_grad():\n logits = memory_bank_model(val_features, alpha, beta)\n acc = cls_acc(logits[\"pure_logits\"], val_labels)\n print(\"\\n**** Pure model's val accuracy: {:.2f}. ****\\n\".format(acc))\n \n acc = cls_acc(logits[\"adapted_logits\"], val_labels)\n print(\"**** Model with memory_bank's val accuracy: {:.2f}. ****\\n\".format(acc))\n\n best_beta, best_alpha = search_hp(\n args=args, \n features=val_features, \n labels=val_labels, \n memory_bank_model=memory_bank_model, \n logits_type=\"adapted_logits\",\n )\n\n print(\"\\n-------- Evaluating on the test set. --------\")\n \n with torch.no_grad():\n logits = memory_bank_model(test_features, best_alpha, best_beta)\n acc = cls_acc(logits[\"pure_logits\"], test_labels)\n print(\"\\n**** Pure Model's test accuracy: {:.2f}. ****\\n\".format(acc))\n \n acc = cls_acc(logits[\"adapted_logits\"], test_labels)\n print(\"**** Model with Adapter's test accuracy: {:.2f}. ****\\n\".format(acc))\n\n print(\"\\n-------- Finetune Adapter. --------\")\n memory_bank_model = train_logits(\n args=args, \n val_features=val_features, \n val_labels=val_labels, \n test_features=test_features, \n test_labels=test_labels, \n predictor=predictor, \n memory_bank_model=memory_bank_model, \n alpha=alpha, \n beta=beta, \n loader=loader, \n logits_type=\"adapted_logits_with_finetuning\",\n )\n best_beta, best_alpha = search_hp(\n args=args, \n features=val_features, \n labels=val_labels, \n memory_bank_model=memory_bank_model, \n logits_type=\"adapted_logits_with_finetuning\",\n )\n with torch.no_grad():\n logits = memory_bank_model(test_features, best_alpha, best_beta)\n acc = cls_acc(logits[\"adapted_logits_with_finetuning\"], test_labels)\n print(\"**** Model with Finetuned Adapter's test accuracy: {:.2f}. ****\\n\".format(acc))\n\ndef main():\n\n dataset, train_df, val_df, test_df, num_classes = generate_dataset(args)\n\n if args.type == \"text\":\n predictor = MultiModalPredictor(\n problem_type=\"feature_extraction\",\n hyperparameters={\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.checkpoint_name\": args.backbone,\n },\n num_classes = num_classes,\n )\n elif args.type == \"image\":\n predictor = MultiModalPredictor(\n problem_type=\"feature_extraction\",\n hyperparameters={\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": args.backbone,\n },\n num_classes = num_classes,\n )\n else:\n predictor = MultiModalPredictor(\n problem_type=\"zero_shot_image_classification\",\n hyperparameters={\n \"model.names\": [\"clip\"],\n \"model.clip.checkpoint_name\": args.backbone,\n }\n )\n\n if args.type == \"clip\":\n clip_weights = generate_clip_weights(\n args=args,\n classnames=dataset.classnames, \n template=dataset.template, \n predictor=predictor,\n )\n\n bank_keys, bank_labels = generate_bank_model(\n args=args, \n train_df=train_df, \n predictor=predictor,\n )\n\n val_features, val_labels, test_features, test_labels = extract_val_test(\n args=args, \n predictor=predictor, \n val_df=val_df, \n test_df=test_df,\n )\n\n loader = build_data_loader(\n data_source=train_df, \n batch_size=256, \n shuffle=True, \n column_names=args.column_names,\n label_column=args.label_column,\n )\n \n memory_bank_model = AutoMMMemoryBank(\n bank_keys=bank_keys, \n bank_labels=bank_labels, \n hidden_size=test_features.shape[1], \n num_classes=num_classes, \n clip_weights=clip_weights if args.type == \"clip\" else None, \n model_head_type=args.model_head_type,\n ).cuda()\n\n run_memory_bank(\n args=args, \n val_features=val_features, \n val_labels=val_labels, \n test_features=test_features, \n test_labels=test_labels, \n predictor=predictor, \n memory_bank_model=memory_bank_model, \n loader=loader,\n )\n\n\nif __name__ == \"__main__\":\n args = get_args()\n print(args)\n\n main()\n"
},
{
"path": "examples/automm/memory_bank/utils.py",
"content": "import torch\nimport pandas as pd\nfrom tqdm import tqdm\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import list_datasets, load_dataset\nfrom setfit import sample_dataset\nfrom imagedatasets import build_dataset\nfrom torch.utils.data import Dataset as TorchDataset\n\n\ndef cls_acc(output, target, topk=1):\n pred = output.topk(topk, 1, True, True)[1].t()\n correct = pred.eq(target.view(1, -1).expand_as(pred))\n acc = float(correct[: topk].reshape(-1).float().sum(0, keepdim=True).cpu().numpy())\n acc = 100 * acc / target.shape[0]\n return acc\n\ndef generate_image_df(args, dataset):\n column_names = args.column_names + [args.label_column]\n dataset_df = pd.DataFrame(columns=column_names)\n for data in dataset:\n per_df = pd.DataFrame([[data.impath, data.label]], columns=column_names)\n dataset_df = pd.concat(\n [dataset_df, per_df],\n ignore_index=True,\n )\n return dataset_df\n \ndef generate_dataset(args):\n if args.type == \"text\":\n dataset = load_dataset(args.dataset)\n train_dataset = sample_dataset(dataset[\"train\"], label_column=args.label_column, num_samples=args.shots)\n val_dataset = dataset[\"validation\"] if \"validation\" in dataset else dataset[\"test\"]\n test_dataset = dataset[\"test\"]\n\n train_dataset.set_format(\"pandas\")\n val_dataset.set_format(\"pandas\")\n test_dataset.set_format(\"pandas\")\n\n train_df = train_dataset[:].drop(columns=[\"label_text\"])\n val_df = val_dataset[:].drop(columns=[\"label_text\"])\n test_df = test_dataset[:].drop(columns=[\"label_text\"])\n\n num_classes = train_dataset[args.label_column].max() + 1\n else:\n dataset = build_dataset(args.dataset, args.data_path, args.shots)\n train_df = generate_image_df(args, dataset.train_x)\n val_df = generate_image_df(args, dataset.val)\n test_df = generate_image_df(args, dataset.test)\n num_classes = len(dataset.classnames)\n \n return dataset, train_df, val_df, test_df, num_classes\n\n\ndef extract_embedding(args, data, predictor, column_names):\n features = []\n for per_name in column_names:\n per_features = torch.tensor(predictor.extract_embedding({per_name: data[per_name]})[per_name]).cuda()\n features.append(per_features)\n features = torch.stack(features, dim=1).cuda()\n features = features.mean(dim=1)\n features /= features.norm(dim=-1, keepdim=True)\n return features\n\n\ndef generate_clip_weights(args, classnames, template, predictor):\n with torch.no_grad():\n clip_weights = []\n for classname in classnames:\n classname = classname.replace('_', ' ')\n text = {'text': [t.format(classname) for t in template]}\n class_embeddings = extract_embedding(args, text, predictor, [\"text\"])\n class_embeddings /= class_embeddings.norm(dim=-1, keepdim=True)\n class_embedding = class_embeddings.mean(dim=0)\n class_embedding /= class_embedding.norm()\n clip_weights.append(class_embedding)\n clip_weights = torch.stack(clip_weights, dim=1).cuda()\n \n return clip_weights\n\n\ndef generate_bank_model(args, train_df, predictor):\n bank_keys = []\n bank_labels = []\n with torch.no_grad():\n for augment_idx in range(args.aug_epochs):\n print('Augment Epoch: {:} / {:}'.format(augment_idx, args.aug_epochs))\n train_features = extract_embedding(args, train_df, predictor, args.column_names)\n bank_keys.append(train_features.unsqueeze(0))\n \n for index, per_data in train_df.iterrows():\n bank_labels.append(per_data[args.label_column])\n \n bank_keys = torch.cat(bank_keys, dim=0).mean(dim=0)\n bank_keys /= bank_keys.norm(dim=-1, keepdim=True)\n bank_keys = bank_keys.permute(1, 0)\n bank_labels = torch.tensor(bank_labels).cuda()\n \n return bank_keys, bank_labels\n\n\ndef extract_val_test(args, predictor, val_df, test_df):\n val_features = extract_embedding(args, val_df, predictor, args.column_names)\n test_features = extract_embedding(args, test_df, predictor, args.column_names)\n val_labels = torch.tensor(val_df[args.label_column]).cuda()\n test_labels = torch.tensor(test_df[args.label_column]).cuda()\n return val_features, val_labels, test_features, test_labels\n\n\ndef search_hp(\n args, \n features, \n labels, \n memory_bank_model, \n logits_type=None,\n):\n \"\"\"\n Search the best hyper-parameters of alpha and beta.\n\n Parameters\n ----------\n args\n The args of searching scales and steps.\n features, labels\n The preprocessed features and labels of validation set.\n memory_bank_model\n The AutoMMMemoryBank to generate logits and logits with memory bank.\n logits_type\n The target logits of searching corresponding to \"adapted_logits\" and \"adapted_logits_with_finetuning\".\n \n Return\n ------\n The best hyper-parameters of alpha and beta.\n \"\"\"\n beta_list = [i * (args.search_scale[0] - 0.1) / args.search_step[0] + 0.1 for i in range(args.search_step[0])]\n alpha_list = [i * (args.search_scale[1] - 0.1) / args.search_step[1] + 0.1 for i in range(args.search_step[1])]\n\n best_acc = 0\n best_beta, best_alpha = 0, 0\n\n pure_logits = None\n for beta in beta_list:\n for alpha in alpha_list:\n with torch.no_grad():\n logits = memory_bank_model(features, alpha, beta, pure_logits)\n if pure_logits is None:\n pure_logits = logits[\"pure_logits\"]\n acc = cls_acc(logits[logits_type], labels)\n \n if acc > best_acc:\n print(\"New best setting, beta: {:.2f}, alpha: {:.2f}; accuracy: {:.2f}\".format(beta, alpha, acc))\n best_acc = acc\n best_beta = beta\n best_alpha = alpha\n\n print(\"\\nAfter searching, the best accuracy: {:.2f}.\\n\".format(best_acc))\n\n return best_beta, best_alpha\n\n\nclass Wrapper(TorchDataset):\n\n def __init__(self, data_source, column_names=[\"image\"], label_column=\"label\"):\n self.data_source = data_source\n self.column_names = column_names\n self.label_column = label_column\n \n def __len__(self):\n return len(self.data_source)\n\n def __getitem__(self, idx):\n item = self.data_source.loc[idx]\n data = {}\n for per_name in self.column_names:\n data.update({per_name: item[per_name]})\n return data, item[self.label_column]\n \ndef build_data_loader(\n data_source=None,\n batch_size=64,\n num_workers=8,\n shuffle=False,\n column_names=[\"image\"],\n label_column=\"label\",\n):\n\n data_loader = torch.utils.data.DataLoader(\n Wrapper(data_source, column_names),\n batch_size=batch_size,\n num_workers=num_workers,\n shuffle=shuffle,\n drop_last=False,\n pin_memory=(torch.cuda.is_available())\n )\n assert len(data_loader) > 0\n\n return data_loader\n"
},
{
"path": "examples/automm/object_detection/benchmarking.py",
"content": "import argparse\nimport uuid\nimport os\nimport time\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef main(benchmark_root, dataset_name, presets, seed):\n train_path = os.path.join(benchmark_root, dataset_name, \"annotations\", \"train_train.json\")\n val_path = os.path.join(benchmark_root, dataset_name, \"annotations\", \"train_val.json\")\n if not os.path.exists(val_path):\n val_path = None\n print(f\"Validation path {val_path} does not exist.\")\n test_path = os.path.join(benchmark_root, dataset_name, \"annotations\", \"test.json\")\n\n # Init predictor\n predictor = MultiModalPredictor(\n problem_type=\"object_detection\",\n sample_data_path=train_path,\n path=f\"./AutogluonModels/{dataset_name}_bench_{presets}_seed_{seed}_tune_{uuid.uuid4()}\",\n presets=presets,\n )\n\n # Fit\n start = time.time()\n predictor.fit(\n train_path,\n tuning_data=val_path,\n seed=seed,\n )\n train_end = time.time()\n print(\"The finetuning takes %.2f seconds.\" % (train_end - start))\n\n predictor.evaluate(test_path)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"-r\", \"--benchmark_root\", default=\"./data/AutoMLDetBench\", type=str)\n parser.add_argument(\"-d\", \"--dataset_name\", default=None, type=str)\n parser.add_argument(\"-p\", \"--presets\", default=\"best_quality\", type=str)\n parser.add_argument(\"-s\", \"--seed\", default=0, type=int)\n args = parser.parse_args()\n\n main(\n benchmark_root=args.benchmark_root,\n dataset_name=args.dataset_name,\n presets=args.presets,\n seed=args.seed,\n )\n"
},
{
"path": "examples/automm/object_detection/detection_eval.py",
"content": "\"\"\"\nThe example to evaluate a pretrained object detection model in AutoMM.\n\nAn example to evaluate an MMDetection model on COCO:\n python detection_eval.py \\\n --test_path coco17/annotations/instances_val2017.json \\\n --checkpoint_name yolov3_mobilenetv2_320_300e_coco\n\nAn example to evaluate an MMDetection model on VOC:\n python detection_eval.py \\\n --test_path ./VOCdevkit/VOC2007/Annotations/test_cocoformat.json \\\n --checkpoint_name faster_rcnn_r50_fpn_1x_voc0712\n\"\"\"\n\nimport argparse\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef detection_evaluation(\n checkpoint_name=\"yolov3_mobilenetv2_320_300e_coco\",\n test_path=\"coco17/annotations/instances_val2017.json\",\n num_gpus=1,\n):\n predictor = MultiModalPredictor(\n label=\"label\",\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n import time\n\n start = time.time()\n result = predictor.evaluate(test_path)\n print(\"time usage: %.2f\" % (time.time() - start))\n print(result)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"coco17/annotations/instances_val2017.json\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"yolov3_mobilenetv2_320_300e_coco\", type=str)\n parser.add_argument(\"--num_gpus\", default=1, type=int)\n args = parser.parse_args()\n\n detection_evaluation(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n )\n"
},
{
"path": "examples/automm/object_detection/detection_inference.py",
"content": "import time\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import from_voc\nfrom autogluon.multimodal.utils import from_coco\n\n\n# TODO: update inference API\ndef test_inference(dataset, checkpoint_name):\n assert dataset in [\"coco\", \"voc\"]\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": 1,\n },\n problem_type=\"object_detection\",\n )\n\n if dataset == \"coco\":\n df = from_coco(\"coco17/annotations/instances_val2017.json\")[:10][[\"image\"]]\n elif dataset == \"voc\":\n df = from_voc(\"VOCdevkit/VOC2007\")[:10][[\"image\"]]\n\n start = time.time()\n pred = predictor.predict(df, as_pandas=False) # TODO: disable as_pandas flag for detection\n print(\"time usage: %.2f\" % (time.time() - start))\n\n assert len(pred) == len(df)\n\n assert len(pred[0]) == 80 if dataset == \"coco\" else 20 # COCO has 80 classes, VOC has 20 classes\n assert pred[0][0].ndim == 2 # two dimensions, (# of proposals, 5)\n\n\n\"\"\"\nVOC configs are not supported in mmcv.\nAnd currently for voc inference,\nwe only support checkpoint_name=\"faster_rcnn_r50_fpn_1x_voc0712\"\n\"\"\"\n\n\ndef test_voc_inference(checkpoint_name=\"faster_rcnn_r50_fpn_1x_voc0712\"):\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": 1,\n },\n problem_type=\"object_detection\",\n )\n\n df = from_voc(\"VOCdevkit/VOC2007\")[:10][[\"image\"]]\n\n start = time.time()\n pred = predictor.predict(df, as_pandas=False) # TODO: disable as_pandas flag for detection\n print(\"time usage: %.2f\" % (time.time() - start))\n\n assert len(pred) == len(df)\n assert len(pred[0]) == 20 # VOC has 20 classes\n assert pred[0][0].ndim == 2 # two dimensions, (# of proposals, 5)\n\n\nif __name__ == \"__main__\":\n # test coco inference\n test_inference(\"coco\", \"faster_rcnn_r50_fpn_2x_coco\")\n\n # test voc inference\n # VOC configs are not supported in mmcv.\n # So currently for voc inference,\n # we only support checkpoint_name=\"faster_rcnn_r50_fpn_1x_voc0712\"\n test_inference(\"voc\", \"faster_rcnn_r50_fpn_1x_voc0712\")\n"
},
{
"path": "examples/automm/object_detection/detection_load.py",
"content": "import argparse\nimport os.path\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef load_and_evaluate(\n load_path,\n test_path,\n):\n predictor = MultiModalPredictor.load(load_path)\n\n if os.path.isdir(load_path):\n predictor.set_num_gpus(num_gpus=1)\n\n import time\n\n start = time.time()\n result = predictor.evaluate(test_path)\n print(\"time usage: %.2f\" % (time.time() - start))\n print(result)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--load_path\", type=str)\n parser.add_argument(\"--test_path\", type=str)\n args = parser.parse_args()\n\n load_and_evaluate(\n load_path=args.load_path,\n test_path=args.test_path,\n )\n"
},
{
"path": "examples/automm/object_detection/detection_train.py",
"content": "\"\"\"\nThe example to train an object detection model in AutoMM.\n\nAn example to finetune an MMDetection model on COCO:\n python detection_train.py \\\n --train_path coco17/annotations/instances_train2017.json \\\n --test_path coco17/annotations/instances_val2017.json \\\n --checkpoint_name yolov3_mobilenetv2_320_300e_coco \\\n --num_classes 80 \\\n --lr \\\n --wd \\\n --epochs \n\nAn example to finetune an MMDetection model on VOC:\n First, use this script to convert the VOC dataset to COCO format:\n https://github.com/open-mmlab/mmdetection/blob/9d3e162459590eee4cfc891218dfbb5878378842/tools/dataset_converters/pascal_voc.py\n Then, run:\n python detection_train.py \\\n --train_path ./VOCdevkit/VOC2007/Annotations/cocotrain.json \\\n --test_path ./VOCdevkit/VOC2007/Annotations/test_cocoformat.json \\\n --checkpoint_name yolov3_mobilenetv2_320_300e_coco \\\n --num_classes 20 \\\n --lr \\\n --wd \\\n --epochs \n\"\"\"\n\nimport argparse\nimport os\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import get_detection_classes\n\n\ndef detection_train(\n train_path,\n val_path=None,\n test_path=None,\n checkpoint_name=\"faster_rcnn_r50_fpn_2x_coco\",\n num_classes=80,\n lr=1e-3,\n epochs=50,\n num_gpus=4,\n val_metric=None,\n per_gpu_batch_size=8,\n):\n\n # TODO: add val_path\n # TODO: remove hardcode for num_classes\n\n # TODO: move this code to predictor\n classes = None\n eval_tool = None\n VOC_format = False\n if os.path.isdir(train_path):\n classes = get_detection_classes(train_path)\n eval_tool = \"torchmetrics\"\n VOC_format = True\n\n predictor = MultiModalPredictor(\n label=\"label\",\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n \"optim.val_metric\": val_metric,\n },\n problem_type=\"object_detection\",\n num_classes=num_classes,\n classes=classes,\n )\n\n import time\n\n start = time.time()\n predictor.fit(\n train_path,\n tuning_data=val_path,\n hyperparameters={\n \"optim.lr\": lr / 100, # we use two stage and lr_mult=100 for detection\n \"optim.max_epochs\": epochs,\n \"env.per_gpu_batch_size\": per_gpu_batch_size, # decrease it when model is large\n },\n )\n fit_end = time.time()\n print(\"time usage for fit: %.2f\" % (fit_end - start))\n\n if test_path is not None:\n if (not eval_tool) or eval_tool == \"pycocotools\" or (eval_tool == \"torchmetrics\" and num_gpus == 1):\n print(predictor.evaluate(test_path, eval_tool=eval_tool))\n print(\"time usage for eval: %.2f\" % (time.time() - fit_end))\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"--train_path\", default=\"./VOCdevkit/VOC2007/Annotations/train_cocoformat.json\", type=str\n )\n parser.add_argument(\"--val_path\", default=None, type=str)\n parser.add_argument(\"--test_path\", default=None, type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"yolov3_mobilenetv2_320_300e_coco\", type=str)\n parser.add_argument(\"--num_classes\", default=20, type=int)\n parser.add_argument(\"--lr\", default=1e-3, type=float)\n parser.add_argument(\"--epochs\", default=50, type=int)\n parser.add_argument(\"--num_gpus\", default=4, type=int)\n parser.add_argument(\"--per_gpu_batch_size\", default=8, type=int)\n parser.add_argument(\"--val_metric\", default=None, type=str)\n args = parser.parse_args()\n\n detection_train(\n train_path=args.train_path,\n val_path=args.val_path,\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_classes=args.num_classes,\n lr=args.lr,\n epochs=args.epochs,\n num_gpus=args.num_gpus,\n val_metric=args.val_metric, # \"mAP\" or \"direct_loss\" or None (use default: \"direct_loss\")\n per_gpu_batch_size=args.per_gpu_batch_size,\n )\n"
},
{
"path": "examples/automm/object_detection/download_coco17.sh",
"content": "#!/bin/bash\n# Reference: https://gist.github.com/mkocabas/a6177fc00315403d31572e17700d7fd9\n\nif [ -z \"$1\" ]\n then\n mkdir coco17\n cd coco17\n echo \"extract data in ./coco17\"\n else\n # check if is valid directory\n if [ ! -d $1 ]; then\n echo $1 \"is not a valid directory\"\n exit 0\n fi\n echo \"navigating to\" $1 \"...\"\n cd $1\n mkdir coco17\n cd coco17\n echo \"extract data in coco17 folder in \" $1\nfi\n\nwget http://images.cocodataset.org/zips/train2017.zip\nwget http://images.cocodataset.org/zips/val2017.zip\nwget http://images.cocodataset.org/zips/test2017.zip\nwget http://images.cocodataset.org/zips/unlabeled2017.zip\n\nunzip train2017.zip\nunzip val2017.zip\nunzip test2017.zip\nunzip unlabeled2017.zip\n\nrm train2017.zip\nrm val2017.zip\nrm test2017.zip\nrm unlabeled2017.zip\n\nwget http://images.cocodataset.org/annotations/annotations_trainval2017.zip\nwget http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip\nwget http://images.cocodataset.org/annotations/image_info_test2017.zip\nwget http://images.cocodataset.org/annotations/image_info_unlabeled2017.zip\n\nunzip annotations_trainval2017.zip\nunzip stuff_annotations_trainval2017.zip\nunzip image_info_test2017.zip\nunzip image_info_unlabeled2017.zip\n\nrm annotations_trainval2017.zip\nrm stuff_annotations_trainval2017.zip\nrm image_info_test2017.zip\nrm image_info_unlabeled2017.zip\n"
},
{
"path": "examples/automm/object_detection/download_voc07.sh",
"content": "#!/bin/bash\n# Reference:\n# Ellis Brown\n# https://github.com/amdegroot/ssd.pytorch/blob/master/data/scripts/VOC2007.sh\n\nif [ -z \"$1\" ]\n then\n echo \"extract data in current directory\"\n else\n # check if is valid directory\n if [ ! -d $1 ]; then\n echo $1 \"is not a valid directory\"\n exit 0\n fi\n echo \"navigating to\" $1 \"...\"\n cd $1\nfi\n\necho \"Downloading VOC2007 trainval ...\"\n# Download the data.\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar\necho \"Downloading VOC2007 test data ...\"\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar\necho \"Done downloading.\"\n\n# Extract data\necho \"Extracting trainval ...\"\ntar -xvf VOCtrainval_06-Nov-2007.tar\necho \"Extracting test ...\"\ntar -xvf VOCtest_06-Nov-2007.tar\necho \"removing tars ...\"\nrm VOCtrainval_06-Nov-2007.tar\nrm VOCtest_06-Nov-2007.tar\n"
},
{
"path": "examples/automm/object_detection/download_voc0712.sh",
"content": "#!/bin/bash\n# Reference:\n# Ellis Brown\n# https://github.com/amdegroot/ssd.pytorch/blob/master/data/scripts/VOC2007.sh\n# https://github.com/amdegroot/ssd.pytorch/blob/master/data/scripts/VOC2012.sh\n\nif [ -z \"$1\" ]\n then\n echo \"extract data in current directory\"\n else\n # check if is valid directory\n if [ ! -d $1 ]; then\n echo $1 \"is not a valid directory\"\n exit 0\n fi\n echo \"navigating to\" $1 \"...\"\n cd $1\nfi\n\n# Download the data.\necho \"Downloading VOC2007 trainval ...\"\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar\necho \"Downloading VOC2007 test data ...\"\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar\necho \"Downloading VOC2012 trainval ...\"\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar\necho \"Done downloading.\"\n\n# Extract data\necho \"Extracting trainval ...\"\ntar -xvf VOCtrainval_06-Nov-2007.tar\necho \"Extracting test ...\"\ntar -xvf VOCtest_06-Nov-2007.tar\necho \"Extracting trainval ...\"\ntar -xvf VOCtrainval_11-May-2012.tar\necho \"removing tar ...\"\nrm VOCtrainval_06-Nov-2007.tar\nrm VOCtest_06-Nov-2007.tar\nrm VOCtrainval_11-May-2012.tar\n"
},
{
"path": "examples/automm/object_detection/download_voc12.sh",
"content": "#!/bin/bash\n# Reference:\n# Ellis Brown\n# https://github.com/amdegroot/ssd.pytorch/blob/master/data/scripts/VOC2012.sh\n\nif [ -z \"$1\" ]\n then\n echo \"extract data in current directory\"\n else\n # check if is valid directory\n if [ ! -d $1 ]; then\n echo $1 \"is not a valid directory\"\n exit 0\n fi\n echo \"navigating to\" $1 \"...\"\n cd $1\nfi\n\necho \"Downloading VOC2012 trainval ...\"\n# Download the data.\ncurl -LO http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar\necho \"Done downloading.\"\n\n# Extract data\necho \"Extracting trainval ...\"\ntar -xvf VOCtrainval_11-May-2012.tar\necho \"removing tar ...\"\nrm VOCtrainval_11-May-2012.tar\n"
},
{
"path": "examples/automm/object_detection/download_watercolor.sh",
"content": "#!/bin/bash\n# Reference:\n# https://naoto0804.github.io/cross_domain_detection/\n\nif [ -z \"$1\" ]\n then\n echo \"extract data in current directory\"\n else\n # check if is valid directory\n if [ ! -d $1 ]; then\n echo $1 \"is not a valid directory\"\n exit 0\n fi\n echo \"navigating to\" $1 \"...\"\n cd $1\nfi\n\ncurl -O http://www.hal.t.u-tokyo.ac.jp/~inoue/projects/cross_domain_detection/datasets/watercolor.zip\nunzip watercolor.zip\nrm watercolor.zip\n"
},
{
"path": "examples/automm/object_detection/eval_pretrained_coco_format.py",
"content": "\"\"\"\nThe example to evaluate a pretrained object detection model in COCO format.\n\nAn example to evaluate a pretrained model on COCO dataset:\n python eval_pretrained_coco_format.py \\\n --test_path coco17/annotations/instances_val2017.json \\\n --checkpoint_name yolov3_mobilenetv2_320_300e_coco\n\nAn example to evaluate a pretrained model on VOC dataset (COCO format):\n python eval_pretrained_coco_format.py \\\n --test_path ./VOCdevkit/VOC2007/Annotations/test_cocoformat.json \\\n --checkpoint_name faster_rcnn_r50_fpn_1x_voc0712\n\"\"\"\n\nimport argparse\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef tutorial_script_for_eval_pretrained_coco_format():\n # this code block is used in tutorial\n checkpoint_name = \"yolov3_mobilenetv2_320_300e_coco\"\n num_gpus = -1 # here we use all available GPUs\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n test_path = \"coco17/annotations/instances_val2017.json\"\n\n predictor.evaluate(test_path)\n\n\ndef eval_pretrained_coco_format(\n checkpoint_name=\"yolov3_mobilenetv2_320_300e_coco\",\n test_path=\"coco17/annotations/instances_val2017.json\",\n num_gpus=-1,\n):\n # TODO: replace pipeline with problem type\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n result = predictor.evaluate(test_path)\n\n print(result)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"coco17/annotations/instances_val2017.json\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"yolov3_mobilenetv2_320_300e_coco\", type=str)\n parser.add_argument(\"--num_gpus\", default=-1, type=int)\n args = parser.parse_args()\n\n eval_pretrained_coco_format(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n )\n"
},
{
"path": "examples/automm/object_detection/eval_pretrained_voc_format.py",
"content": "\"\"\"\nThe example to evaluate a pretrained object detection model in VOC format.\n\nAn example to evaluate a pretrained model on VOC dataset (VOC format):\n python eval_pretrained_coco_format.py \\\n --test_path VOCdevkit/VOC2007 \\\n --checkpoint_name faster_rcnn_r50_fpn_1x_voc0712\n\"\"\"\n\nimport argparse\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef tutorial_script_for_eval_pretrained_voc_format():\n # this code block is used in tutorial\n checkpoint_name = \"faster_rcnn_r50_fpn_1x_voc0712\"\n num_gpus = -1 # here we use all available GPUs\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n test_path = \"VOCdevkit/VOC2007\"\n\n result = predictor.evaluate(test_path)\n\n print(result)\n\n\ndef eval_pretrained_voc_format(\n checkpoint_name=\"faster_rcnn_r50_fpn_1x_voc0712\",\n test_path=\"VOCdevkit/VOC2007\",\n num_gpus=-1,\n):\n # TODO: replace pipeline with problem type\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n result = predictor.evaluate(test_path)\n\n print(result)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"VOCdevkit/VOC2007\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"faster_rcnn_r50_fpn_1x_voc0712\", type=str)\n parser.add_argument(\"--num_gpus\", default=-1, type=int)\n args = parser.parse_args()\n\n eval_pretrained_voc_format(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n )\n"
},
{
"path": "examples/automm/object_detection/finetune_coco_format.py",
"content": "\"\"\"\nThe example to finetune an object detection model in a COCO format dataset.\nSee finetune_on_pothole_dataset.py for an example on our provided dataset.\n\"\"\"\n\nimport argparse\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef finetune_coco_format(\n train_path,\n val_path=None,\n test_path=None,\n presets=None,\n checkpoint_name=None,\n num_gpus=None,\n lr=None,\n epochs=None,\n per_gpu_batch_size=None,\n):\n assert train_path is not None, \"train_path must be provided and cannot be None\"\n\n hyperparameters = {}\n if checkpoint_name is not None:\n hyperparameters[\"model.mmdet_image.checkpoint_name\"] = checkpoint_name\n if num_gpus is not None:\n hyperparameters[\"env.num_gpus\"] = num_gpus\n if lr is not None:\n hyperparameters[\"optim.lr\"] = lr\n if epochs is not None:\n hyperparameters[\"optim.max_epochs\"] = epochs\n if per_gpu_batch_size is not None:\n hyperparameters[\"env.per_gpu_batch_size\"] = per_gpu_batch_size\n\n predictor = MultiModalPredictor(\n hyperparameters=hyperparameters,\n problem_type=\"object_detection\",\n sample_data_path=train_path,\n presets=presets,\n )\n predictor.fit(train_path, tuning_data=val_path)\n print(\"time usage for fit: %.2f\" % (predictor._total_train_time))\n\n if test_path is not None:\n predictor.evaluate(test_path)\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--train_path\", default=None, type=str)\n parser.add_argument(\"--val_path\", default=None, type=str)\n parser.add_argument(\"--test_path\", default=None, type=str)\n parser.add_argument(\"-p\", \"--presets\", default=None, type=str)\n parser.add_argument(\"-c\", \"--checkpoint_name\", default=None, type=str)\n parser.add_argument(\"-n\", \"--num_gpus\", default=None, type=int)\n parser.add_argument(\"-l\", \"--lr\", default=None, type=float)\n parser.add_argument(\"-e\", \"--epochs\", default=None, type=int)\n parser.add_argument(\"-b\", \"--per_gpu_batch_size\", default=None, type=int)\n args = parser.parse_args()\n\n finetune_coco_format(\n train_path=args.train_path,\n val_path=args.val_path,\n test_path=args.test_path,\n presets=args.presets,\n checkpoint_name=args.checkpoint_name,\n lr=args.lr,\n epochs=args.epochs,\n num_gpus=args.num_gpus,\n per_gpu_batch_size=args.per_gpu_batch_size,\n )\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "examples/automm/object_detection/finetune_on_pothole_dataset.py",
"content": "\"\"\"\nThe example to finetune an object detection model on pothole dataset.\n\"\"\"\n\nimport argparse\nimport os\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\n\nfrom finetune_coco_format import finetune_coco_format\n\n\ndef download_pothole_dataset():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/object_detection/dataset/pothole.zip\"\n download_dir = \"./pothole\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"pothole\")\n train_path = os.path.join(data_dir, \"Annotations\", \"usersplit_train_cocoformat.json\")\n val_path = os.path.join(data_dir, \"Annotations\", \"usersplit_val_cocoformat.json\")\n test_path = os.path.join(data_dir, \"Annotations\", \"usersplit_test_cocoformat.json\")\n\n return train_path, val_path, test_path\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"-p\", \"--presets\", default=None, type=str)\n parser.add_argument(\"-c\", \"--checkpoint_name\", default=None, type=str)\n parser.add_argument(\"-n\", \"--num_gpus\", default=None, type=int)\n parser.add_argument(\"-l\", \"--lr\", default=None, type=float)\n parser.add_argument(\"-e\", \"--epochs\", default=None, type=int)\n parser.add_argument(\"-b\", \"--per_gpu_batch_size\", default=None, type=int)\n args = parser.parse_args()\n\n train_path, val_path, test_path = download_pothole_dataset()\n\n finetune_coco_format(\n train_path=train_path,\n val_path=val_path,\n test_path=test_path,\n presets=args.presets,\n checkpoint_name=args.checkpoint_name,\n lr=args.lr,\n epochs=args.epochs,\n num_gpus=args.num_gpus,\n per_gpu_batch_size=args.per_gpu_batch_size,\n )\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "examples/automm/object_detection/inference_pretrained_coco_format.py",
"content": "\"\"\"\nThe example to evaluate a pretrained object detection model in COCO format.\nAn example to evaluate a pretrained model on COCO dataset:\n python inference_pretrained_coco_format.py \\\n --test_path coco17/annotations/instances_val2017.json \\\n --checkpoint_name yolov3_mobilenetv2_320_300e_coco \\\nAn example to evaluate a pretrained model on VOC dataset (COCO format):\n python inference_pretrained_coco_format.py \\\n --test_path ./VOCdevkit/VOC2007/Annotations/test_cocoformat.json \\\n --checkpoint_name faster_rcnn_r50_fpn_1x_voc0712 \\\nIf you want to save results, enable the following:\n--save_results\nIf you want to specify a save result path, add the following:\n--result_path VOCdevkit/VOC2007/results.txt\n\"\"\"\n\nimport argparse\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import from_coco, COCODataset\n\n\ndef tutorial_script_for_eval_pretrained_coco_format():\n # this code block is used in tutorial\n checkpoint_name = \"yolov3_mobilenetv2_320_300e_coco\"\n num_gpus = -1 # here we use all available GPUs\n\n # construct the predictor\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n test_path = \"coco17/annotations/instances_val2017.json\"\n\n pred = predictor.predict(test_path, save_results=True)\n\n\ndef eval_pretrained_coco_format(\n checkpoint_name=\"yolov3_mobilenetv2_320_300e_coco\",\n test_path=\"coco17/annotations/instances_val2017.json\",\n num_gpus=-1,\n save_results=True,\n):\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n pipeline=\"object_detection\",\n )\n\n pred = predictor.predict(test_path, save_results=save_results)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"coco17/annotations/instances_val2017.json\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"yolov3_mobilenetv2_320_300e_coco\", type=str)\n parser.add_argument(\"--num_gpus\", default=-1, type=int)\n parser.add_argument(\"--save_results\", action=\"store_true\")\n args = parser.parse_args()\n\n eval_pretrained_coco_format(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n save_results=args.save_results,\n )\n"
},
{
"path": "examples/automm/object_detection/inference_pretrained_voc_format.py",
"content": "\"\"\"\nThe example to evaluate a pretrained object detection model in VOC format.\nAn example to evaluate a pretrained model on VOC dataset (VOC format):\n python inference_pretrained_voc_format.py \\\n --test_path VOCdevkit/VOC2007 \\\n --checkpoint_name faster_rcnn_r50_fpn_1x_voc0712 \\\nIf you want to save results, enable the following:\n--save_results\nIf you want to specify a save result path, add the following:\n--result_path VOCdevkit/VOC2007/results.txt\n\"\"\"\n\nimport argparse\nimport numpy as np\nimport os\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import from_voc\n\n\ndef tutorial_script_for_eval_pretrained_voc_format():\n # this code block is used in tutorial\n checkpoint_name = \"faster_rcnn_r50_fpn_1x_voc0712\"\n num_gpus = -1 # here we use all available GPUs\n\n # construct the predictor\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n test_path = \"VOCdevkit/VOC2007\"\n\n pred = predictor.predict(test_path, save_results=True)\n\n\ndef eval_pretrained_voc_format(\n checkpoint_name=\"faster_rcnn_r50_fpn_1x_voc0712\",\n test_path=\"VOCdevkit/VOC2007\",\n num_gpus=-1,\n save_results=True,\n result_path=None,\n):\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n pipeline=\"object_detection\",\n )\n\n pred = predictor.predict(test_path, save_results=save_results)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"VOCdevkit/VOC2007\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"faster_rcnn_r50_fpn_1x_voc0712\", type=str)\n parser.add_argument(\"--num_gpus\", default=1, type=int)\n parser.add_argument(\"--save_results\", action=\"store_true\")\n args = parser.parse_args()\n\n eval_pretrained_voc_format(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n save_results=args.save_results,\n )\n"
},
{
"path": "examples/automm/object_detection/load_predictor.py",
"content": "\"\"\"\nThe example to load a trained predictor and evaluate.\n\nAn example:\n python load_predictor.py \\\n --test_path \\\n --load_path \\\n --num_gpus \n\n\"\"\"\n\nimport argparse\nimport time\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef tutorial_script_for_save_load_predictor():\n test_path = \"./VOCdevkit/VOC2007/Annotations/test_cocoformat.json\"\n load_path = \"/media/code/autogluon/examples/automm/object_detection/AutogluonModels/ag-20221104_185342\"\n\n print(\"load a predictor from save path...\")\n predictor = MultiModalPredictor.load(load_path)\n predictor.evaluate(test_path)\n\n print(\"load a predictor from save path and change num_gpus to 1...\")\n predictor_single_gpu = MultiModalPredictor.load(load_path)\n predictor_single_gpu.set_num_gpus(num_gpus=1)\n predictor_single_gpu.evaluate(test_path)\n\n\ndef detection_load_predictor_and_eval(test_path, load_path, num_gpus):\n print(f\"loading a predictor from save path and change num_gpus to {num_gpus}...\")\n predictor_single_gpu = MultiModalPredictor.load(load_path)\n predictor_single_gpu.set_num_gpus(num_gpus=num_gpus)\n\n predictor_single_gpu.evaluate(test_path)\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=None, type=str)\n parser.add_argument(\"--load_path\", default=None, type=str)\n parser.add_argument(\"--num_gpus\", default=4, type=int)\n args = parser.parse_args()\n\n detection_load_predictor_and_eval(\n test_path=args.test_path,\n load_path=args.load_path,\n num_gpus=args.num_gpus,\n )\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "examples/automm/object_detection/pretrain_objects365.py",
"content": "import os\nimport time\n\nfrom autogluon.multimodal import MultiModalPredictor\n\ndef main():\n data_dir = \"/media/code/datasets/object365\"\n train_path = os.path.join(data_dir, \"train\", \"annotations\", \"zhiyuan_objv2_train.json\")\n val_path = os.path.join(data_dir, \"val\", \"annotations\", \"zhiyuan_objv2_val.json\")\n\n checkpoint_name = \"yolox_l_8x8_300e_coco\"\n num_gpus = 3\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n \"optim.val_metric\": \"map\",\n },\n problem_type=\"object_detection\",\n sample_data_path=train_path,\n )\n\n start = time.time()\n predictor.fit(\n train_path,\n tuning_data=val_path,\n max_num_tuning_data=5000,\n hyperparameters={\n \"optim.lr\": 1e-3, # we use two stage and detection head has 100x lr\n \"optim.lr_decay\": 0.9,\n \"optim.lr_mult\": 1,\n \"optim.max_epochs\": 12,\n #\"optim.max_steps\": 180000,\n \"optim.warmup_steps\": 0.1,\n \"optim.patience\": 1000,\n \"optim.val_check_interval\": 0.25,\n \"optim.check_val_every_n_epoch\": 1,\n \"optim.top_k\": 20,\n \"env.per_gpu_batch_size\": 6, # decrease it when model is large\n },\n clean_ckpts=False,\n )\n end = time.time()\n\n print(\"This finetuning takes %.2f seconds.\" % (end - start))\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "examples/automm/object_detection/quick_start_on_a_tiny_dataset.py",
"content": "import os\nimport time\nimport json\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef tutorial_script_for_quick_start():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/object_detection_dataset/tiny_motorbike_coco.zip\"\n download_dir = \"./tiny_motorbike_coco\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"tiny_motorbike\")\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n test_path_no_label = os.path.join(data_dir, \"Annotations\", \"test_cocoformat_no_label.json\")\n\n with open(test_path, \"r\") as f:\n test_data = json.load(f)\n \n test_data.pop(\"annotations\")\n test_data.pop(\"categories\")\n print(test_data.keys())\n\n with open(test_path_no_label, \"w+\") as f_nolabel:\n json.dump(test_data, f_nolabel)\n\n checkpoint_name = \"yolov3_mobilenetv2_8xb24-320-300e_coco\"\n num_gpus = -1\n\n # Init predictor\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n sample_data_path=train_path,\n path=\"./quick_start_tutorial_temp_save\",\n )\n\n start = time.time()\n\n # Fit\n predictor.fit(\n train_path,\n hyperparameters={\n \"optim.lr\": 2e-4, # we use two stage and detection head has 100x lr\n \"optim.max_epochs\": 20,\n \"env.per_gpu_batch_size\": 32, # decrease it when model is large\n },\n )\n\n train_end = time.time()\n print(\"The finetuning takes %.2f seconds.\" % (train_end - start))\n\n # Evaluate\n #print(\"Predict...\")\n #predictor.predict(test_path)\n \n #sample_image_path = \"/home/ubuntu/ag/autogluon/examples/automm/object_detection/tiny_motorbike_coco/tiny_motorbike/JPEGImages/000038.jpg\"\n #predictor.predict([sample_image_path]*10)\n #exit()\n \n print(\"Predict no label...\")\n predictor.predict(test_path_no_label)\n\n predictor.evaluate(test_path)\n \n #eval_end = time.time()\n #print(\"The evaluation takes %.2f seconds.\" % (eval_end - train_end))\n\n # Load and reset num_gpus\n #new_predictor = MultiModalPredictor.load(\"./quick_start_tutorial_temp_save\")\n #new_predictor.set_num_gpus(1)\n\n # Evaluate new predictor\n #print(\"Predict with loaded predictor...\")\n #new_predictor.predict(test_path)\n #print(\"Predict no label with loaded predictor...\")\n #new_predictor.predict(test_path_no_label)\n\n from autogluon.multimodal import download\n image_url = \"https://raw.githubusercontent.com/dmlc/web-data/master/gluoncv/detection/street_small.jpg\"\n test_image = download(image_url)\n\n # create a input file for demo\n data = {\"images\": [{\"id\": 0, \"width\": -1, \"height\": -1, \"file_name\": test_image}], \"categories\": []}\n os.mkdir(\"input_data_for_demo\")\n input_file = \"input_data_for_demo/demo_annotation.json\"\n with open(input_file, \"w+\") as f:\n json.dump(data, f)\n\n pred_test_image = predictor.predict(input_file)\n print(pred_test_image)\n\n\nif __name__ == \"__main__\":\n tutorial_script_for_quick_start()\n"
},
{
"path": "examples/automm/object_detection/visualize_results.py",
"content": "\"\"\"\nThe example to visualize detection results in COCO dataset (COCO format):\n python visualize_results.py \\\n --test_path ~/yongxinw-workspace/tools/coco17/annotations/instances_val2017.json \\\n --checkpoint_name vfnet_x101_64x4d_fpn_mdconv_c3-c5_mstrain_2x_coco \\\n --conf_threshold 0.4\nIf you want to specify a folder to save visualizations, add the following:\n--visualization_result_dir VOCdevkit/VOC2007/visualizations\n\"\"\"\n\nimport argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import from_coco_or_voc, visualize_detection\n\n\ndef tutorial_script_for_visualize_detection_results():\n # this code block is used in tutorial\n checkpoint_name = \"vfnet_x101_64x4d_fpn_mdconv_c3-c5_mstrain_2x_coco\"\n num_gpus = -1 # here we use all available GPUs\n\n # construct the predictor\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"object_detection\",\n )\n\n test_path = \"coco17/annotations/instances_val2017.json\"\n visualization_result_dir = \"coco17/visualizations\"\n conf_threshold = 0.4\n\n df = from_coco_or_voc(test_path)[:10][[\"image\"]]\n\n pred = predictor.predict(df)\n\n visualize_detection(\n pred=pred,\n detection_classes=predictor.classes,\n conf_threshold=conf_threshold,\n visualization_result_dir=visualization_result_dir,\n )\n\n\ndef visualize_detection_results(\n checkpoint_name: str = \"faster_rcnn_r50_fpn_1x_voc0712\",\n test_path: str = \"VOCdevkit/VOC2007\",\n num_gpus: int = -1,\n visualization_result_dir: str = \"VOCdevkit/VOC2007/visualizations\",\n conf_threshold: float = 0.3,\n):\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n pipeline=\"object_detection\",\n )\n\n df = from_coco_or_voc(test_path)[:10][[\"image\"]]\n\n pred = predictor.predict(df)\n\n visualize_detection(\n pred=pred,\n detection_classes=predictor.classes,\n conf_threshold=conf_threshold,\n visualization_result_dir=visualization_result_dir,\n )\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--test_path\", default=\"coco17/annotations/instances_val2017.json\", type=str)\n parser.add_argument(\"--checkpoint_name\", default=\"vfnet_x101_64x4d_fpn_mdconv_c3-c5_mstrain_2x_coco\", type=str)\n parser.add_argument(\"--num_gpus\", default=1, type=int)\n parser.add_argument(\"--visualization_result_dir\", default=\"coco17/visualizations/\", type=str)\n parser.add_argument(\"--visualization_conf_threshold\", default=0.3, type=float)\n args = parser.parse_args()\n\n visualize_detection_results(\n test_path=args.test_path,\n checkpoint_name=args.checkpoint_name,\n num_gpus=args.num_gpus,\n visualization_result_dir=args.visualization_result_dir,\n conf_threshold=args.visualization_conf_threshold,\n )\n"
},
{
"path": "examples/automm/pipeline/feature_extraction_example.py",
"content": "# Use STS Benchmark as an example to demonstrate feature extraction pipeline\n\nimport argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import load_dataset\nimport time\n\nfrom sklearn.metrics.pairwise import paired_cosine_distances\nfrom scipy.stats import pearsonr, spearmanr\n\nimport numpy as np\nimport onnx\nimport onnxruntime as ort\nimport torch\nfrom torch import tensor\n\n\ndef evaluate(predictor, df, onnx_session=None):\n labels = df[\"score\"].to_numpy()\n\n # TODO: line below only outputs one embedding since dataloader merge text columns automatically\n # mixEmb = predictor.extract_embedding(valid_df[[\"sentence1\",\"sentence2\"]])[\"sentence1_sentence2\"]\n if not onnx_session:\n QEmb = predictor.extract_embedding(df[[\"sentence1\"]])[\"sentence1\"]\n AEmb = predictor.extract_embedding(df[[\"sentence2\"]])[\"sentence2\"]\n else:\n valid_input = [\n \"hf_text_text_token_ids\",\n \"hf_text_text_valid_length\",\n \"hf_text_text_segment_ids\",\n ]\n QEmb = onnx_session.run(None, predictor.get_processed_batch_for_deployment(data=df[[\"sentence1\"]], valid_input=valid_input))[\n 0\n ]\n AEmb = onnx_session.run(None, predictor.get_processed_batch_for_deployment(data=df[[\"sentence2\"]], valid_input=valid_input))[\n 0\n ]\n\n cosine_scores = 1 - paired_cosine_distances(QEmb, AEmb)\n eval_pearson_cosine, _ = pearsonr(labels, cosine_scores)\n eval_spearman_cosine, _ = spearmanr(labels, cosine_scores)\n\n print(eval_pearson_cosine)\n print(eval_spearman_cosine)\n\n return eval_pearson_cosine, eval_spearman_cosine\n\n\ndef main(args):\n ### Dataset Loading\n train_df = load_dataset(\"wietsedv/stsbenchmark\", split=\"train\").to_pandas()\n val_df = load_dataset(\"wietsedv/stsbenchmark\", split=\"validation\").to_pandas()\n test_df = load_dataset(\"wietsedv/stsbenchmark\", split=\"test\").to_pandas()\n\n start = time.time()\n predictor = MultiModalPredictor(\n pipeline=\"feature_extraction\",\n hyperparameters={\n \"model.hf_text.checkpoint_name\": args.checkpoint_name,\n },\n )\n ag_load = time.time()\n evaluate(predictor, test_df)\n ag_eval = time.time()\n\n ort_sess = ort.InferenceSession(args.onnx_path, providers=[\"CUDAExecutionProvider\"])\n onnx_load = time.time()\n evaluate(predictor, test_df, ort_sess)\n onnx_eval = time.time()\n\n print(\"Autogluon load time: %.4f\" % (ag_load - start))\n print(\"Autogluon eval time: %.4f\" % (ag_eval - ag_load))\n print(\"ONNX load time: %.4f\" % (onnx_load - ag_eval))\n print(\"ONNX eval time: %.4f\" % (onnx_eval - onnx_load))\n\n exit()\n # TODO: support fit after predict for two tower models:\n predictor.fit(\n train_df,\n val_df,\n hyperparameters={\n \"optim.max_epochs\": 1,\n },\n )\n evaluate(predictor, test_df)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--checkpoint_name\", default=\"sentence-transformers/msmarco-MiniLM-L-12-v3\", type=str)\n parser.add_argument(\"--onnx_path\", default=None, type=str)\n args = parser.parse_args()\n\n if not args.onnx_path:\n args.onnx_path = \"../production/\" + args.checkpoint_name.replace(\"/\", \"_\") + \".onnx\"\n\n main(args)\n"
},
{
"path": "examples/automm/production/onnx_text.py",
"content": "# Use STS Benchmark as an example to demonstrate ONNX export and evaluation\n\nimport argparse\nfrom autogluon.multimodal import MultiModalPredictor\nfrom datasets import load_dataset\n\nfrom sklearn.metrics.pairwise import paired_cosine_distances\nfrom scipy.stats import pearsonr, spearmanr\n\nimport onnxruntime as ort\n\n\ndef eval_cosine(predictor, df, onnx_session):\n labels = df[\"score\"].to_numpy()\n valid_input = [\n \"hf_text_text_token_ids\",\n \"hf_text_text_valid_length\",\n \"hf_text_text_segment_ids\", # Remove for mpnet\n ]\n\n QEmb = onnx_session.run(None, predictor._learner.get_processed_batch_for_deployment(data=df[[\"sentence1\"]]))[0]\n AEmb = onnx_session.run(None, predictor._learner.get_processed_batch_for_deployment(data=df[[\"sentence2\"]]))[0]\n\n cosine_scores = 1 - (paired_cosine_distances(QEmb, AEmb))\n eval_pearson_cosine, _ = pearsonr(labels, cosine_scores)\n eval_spearman_cosine, _ = spearmanr(labels, cosine_scores)\n\n print(eval_pearson_cosine)\n print(eval_spearman_cosine)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--checkpoint_name\", default=\"sentence-transformers/msmarco-MiniLM-L-12-v3\", type=str)\n parser.add_argument(\"--model_path\", default=None, type=str)\n parser.add_argument(\"--verbose\", action=\"store_true\")\n args = parser.parse_args()\n if not args.model_path:\n args.model_path = args.checkpoint_name.replace(\"/\", \"_\") + \".onnx\"\n\n # Load Dataset\n val_df = load_dataset(\"wietsedv/stsbenchmark\", split=\"validation\").to_pandas()\n test_df = load_dataset(\"wietsedv/stsbenchmark\", split=\"test\").to_pandas()\n\n # Init Predictor\n predictor = MultiModalPredictor(\n problem_type=\"feature_extraction\",\n hyperparameters={\n \"model.hf_text.checkpoint_name\": args.checkpoint_name,\n },\n )\n\n # Export ONNX model\n onnx_path = predictor.export_onnx(data=val_df, path=args.model_path, verbose=args.verbose)\n\n # Load ONNX model\n ort_sess = ort.InferenceSession(onnx_path, providers=[\"CUDAExecutionProvider\"])\n\n # Evaluate ONNX model\n eval_cosine(predictor, test_df, ort_sess)\n"
},
{
"path": "examples/automm/tabular_dl/README.md",
"content": "# Advanced Tabular DL models in AutoMM\n\n### 1. Run Example\n\n[`example_tabular.py`](./example_tabular.py) : This example provides a use case for the pure _tabular_ data, including pure numerical features and numerical + categorical features, with FT_Transformer [1].\n\nTo run the example:\n\n`python example_tabular.py --dataset_name ad --dataset_dir ./dataset --exp_dir ./result`\n\n- `dataset_name` determines which dataset to run the experiments, refers to [Dataset Section](###2.-Datasets).\n- `dataset_dir` is the path to the dataset(s). If the datasets do not present in this path, it will be automatically downloaded.\n- `exp_dir` is the output path to store the weights and loggings.\n- `gpu_id` specifies the GPU to use (optional).\n- `seed` determines the random seed (optional). Default is 0.\n- `lr` specifies the initial learning rate (optional). Default is `1e-04`.\n- `end_lr` specifies the end learning rate (optional). Default is `1e-04`.\n\n### 2. Datasets\n\nWe borrowed 11 tabular datasets provided by [1], and use identically the same abbreviation as Table 1 in the original paper [1] to name each datasets.\nThe datasets provided by https://github.com/Yura52/tabular-dl-revisiting-models are all in `Numpy.darray` format (can be downloaded from https://www.dropbox.com/s/o53umyg6mn3zhxy/data.tar.gz?dl=1).\nThese Data in `Numpy.ndarray` was first pre-processing into `.csv` format, which can be loaded by `pandas.Dataframe` as the input to `MultiModalPredictor`.\nAll Data can be automatically downloaded from s3 (online connection is necessary) if it does not exist with the given dataset path `dataset_dir`.\n\n### 3. FT-Transformer\n\nFT-Transformer consists of a numerical tokenizer, a categorical tokenizer and the transformer backbone. Some commonly used features are explained below:\n\n- `num_blocks` is the number of transformer blocks in the transformer backbone.\n- `token_dim` is the dimension of the input tokens after categorical/numerical tokenizers.\n- `hidden_size` is the embedding dimension of the ft_transformer backbone.\n- `ffn_hidden_size` is the hidden layer dimension of the FFN (Feed-Forward) layer in ft-transformer blocks.\n- `ffn_dropout` is the dropout rate in feadforward layer.\n- `ffn_activation` determines the activation function in feadforward layer. We support `relu`, `gelu`, `reglu` and `leaky_relu`.\n- `attention_dropout` is the dropout rate in attention layer.\n- `embedding_arch` is a list containing the names of embedding layers as described in [2].\n\nThese features can be tuned using `hyperparameters` in `MultiModalPredictor. For example:\n\n```python\nhyperparameters = {\n \"model.names\": [\"ft_transformer\"],\n \"model.ft_transformer.num_blocks\": 5,\n \"model.ft_transformer.ffn_dropout\": 0.0,\n}\n```\n\n### 4. Results\n\n| Datasets | ca | ad | he | ja | hi | al | ep | ye | co | ya | mi |\n| --------------------- | ---------- | ------ | ---------- | ---------- | ------ | ---------- | ------------ | ---------- | ---------- | ---------- | ---------- |\n| metrics | rmse | acc | acc | acc | acc | acc | acc | rmse | acc | rmse | rmse |\n| problem_type | regression | binary | multiclass | multiclass | binary | multiclass | binary | regression | multiclass | regression | regression |\n| #objects | 20640 | 48842 | 65196 | 83733 | 98050 | 108000 | 500000 | 515345 | 581012 | 709877 | 1200192 |\n| #num. | 8 | 6 | 27 | 54 | 28 | 128 | 2000 | 90 | 54 | 699 | 136 |\n| #cat. | 0 | 8 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |\n| #classes | - | 2 | 100 | 4 | 2 | 1000 | 2 | - | 7 | - | - |\n| Best in [1] | 0.459 | 0.859 | 0.396 | 0.732 | 0.729 | 0.963 | 0.8982 | 8.794 | 0.970 | 0.753 | 0.745 |\n| FT-Transformer in [1] | 0.459 | 0.859 | 0.391 | 0.732 | 0.729 | 0.960 | 0.8982 | 8.855 | 0.970 | 0.756 | 0.746 |\n| AutoMM FT-Transformer | 0.465 | 0.859 | 0.383 | 0.729 | 0.734 | 0.952 | RuntimeError | 8.873 | 0.963 | 0.759 | 0.761 |\n\n`FT-Transformer in [1]` row leverages parameters searching, and `AutoMM FT-Transformer` row use a fixed training configurations.\n\nYou can reproduce the `AutoMM FT-Transformer` row by running:\n\n```bash\nbash run_all.sh\n```\n\nwith overriding the following hyperparameters:\n\n```python\nautomm_hyperparameters = {\n \"data.categorical.convert_to_text\": False,\n \"model.names\": [\"ft_transformer\"],\n \"model.ft_transformer.embedding_arch\": [\"linear\"],\n \"env.batch_size\": 128,\n \"env.per_gpu_batch_size\": 128,\n \"env.inference_batch_size_ratio\": 1,\n \"env.num_workers\": 12,\n \"env.num_workers_inference\": 12,\n \"env.num_gpus\": 1,\n \"optim.max_epochs\": 2000,\n \"optim.weight_decay\": 1.0e-5,\n \"optim.lr_choice\": None,\n \"optim.lr_schedule\": \"polynomial_decay\",\n \"optim.warmup_steps\": 0.0,\n \"optim.patience\": 20,\n \"optim.top_k\": 3,\n}\n```\n\nWe run the experiments on one NVIDIA Tesla T4 GPU with 15360MiB memory.\n\n### Reference\n\n[1] Revisiting Deep Learning Models for Tabular Data, 2021, \n\n[2] On Embeddings for Numerical Features in Tabular Deep Learning, 2022, \n"
},
{
"path": "examples/automm/tabular_dl/dataset.py",
"content": "import abc\nimport os\nimport pandas as pd\nfrom autogluon.multimodal.constants import (\n BINARY,\n MULTICLASS,\n REGRESSION,\n ACC,\n RMSE,\n CATEGORICAL,\n NUMERICAL,\n)\nfrom autogluon.multimodal.utils import download\n\n\n# TODO: release the auto_mm_bench package or reuse the huggingface datasets API\nclass BaseTabularDataset(abc.ABC):\n @property\n @abc.abstractmethod\n def data(self):\n pass\n\n @property\n @abc.abstractmethod\n def label_column(self):\n pass\n\n @property\n @abc.abstractmethod\n def label_type(self):\n pass\n\n @property\n @abc.abstractmethod\n def metric(self):\n pass\n\n @property\n @abc.abstractmethod\n def problem_type(self):\n pass\n\n\nclass AdultTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/adult/train.csv\",\n \"sha1sum\": \"7fca6a419cff0f8504f083f7534119956452a337\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/adult/val.csv\",\n \"sha1sum\": \"80fd57b2848966c8e14f5a96a97f1bfcd41ab42c\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/adult/test.csv\",\n \"sha1sum\": \"2ca8470c2514ba147da593882c76f956681e304a\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"adult\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return BINARY\n\n\nclass AloiTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/aloi/train.csv\",\n \"sha1sum\": \"62f8457a455506d83db0c671fe9d271d376ecb22\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/aloi/val.csv\",\n \"sha1sum\": \"2e0e70dd710a441ac130a7b29a420acac26fbef4\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/aloi/test.csv\",\n \"sha1sum\": \"3f9311df8b1a8cb0c0d9b51ad32e274e7f5eb36b\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"aloi\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return MULTICLASS\n\n\nclass CaliforniaHousingTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/california_housing/train.csv\",\n \"sha1sum\": \"2f8bd84e8665859ea09ec7dfc75540357e0e8b30\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/california_housing/val.csv\",\n \"sha1sum\": \"09b165f96d1d53062dc4b8da1f6257b98a320acc\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/california_housing/test.csv\",\n \"sha1sum\": \"434e092996e94f4067145db7716a7a95849759ec\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"california_housing\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return NUMERICAL\n\n @property\n def metric(self):\n return RMSE\n\n @property\n def problem_type(self):\n return REGRESSION\n\n\nclass CovtypeTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/covtype/train.csv\",\n \"sha1sum\": \"7cd56fed5f667dcebe6fb3eb096bc0166636bc5d\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/covtype/val.csv\",\n \"sha1sum\": \"0e32c887a586963f4e7cb62d16964ec8dd57cf08\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/covtype/test.csv\",\n \"sha1sum\": \"35883acd6db686e148338bb7c6f0c46df0039574\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"covtype\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return MULTICLASS\n\n\nclass EpsilonTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular/epsilon/train.csv\",\n \"sha1sum\": \"8444901bdb20d42359b85ca076eff7f16a34b94c\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular/epsilon/val.csv\",\n \"sha1sum\": \"9d607e0db43979d3d9a6034dc7603ef09934b5c8\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular/epsilon/test.csv\",\n \"sha1sum\": \"0a33633875a87a8c9f316e78d225ddfb41c54718\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"epsilon\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return BINARY\n\n\nclass HelenaTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/helena/train.csv\",\n \"sha1sum\": \"61f290ee851695715662177b5726055cc7f8bccb\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/helena/val.csv\",\n \"sha1sum\": \"67d01556af5f78015de1151e3a3c81f71de6bc11\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/helena/test.csv\",\n \"sha1sum\": \"c55709b71ce79051e5f580e9edb73be81d41bf92\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"helena\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return MULTICLASS\n\n\nclass HiggsSmallTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/higgs_small/train.csv\",\n \"sha1sum\": \"65554dceee2c6647f348153412c099891c3b0516\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/higgs_small/val.csv\",\n \"sha1sum\": \"63ab6b0ac6730ea63c18d106c6eb17f5874042f8\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/higgs_small/test.csv\",\n \"sha1sum\": \"dd372975aae92ffcc3a0335299382f7ff1d889bc\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"higgs_small\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return BINARY\n\n\nclass JannisTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/jannis/train.csv\",\n \"sha1sum\": \"6b2cb95c8858aac965908e6973d8728ce13152c5\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/jannis/val.csv\",\n \"sha1sum\": \"17332c6155d60aeb30ddc201f9da8dc93fa05584\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/jannis/test.csv\",\n \"sha1sum\": \"c27654b99794de445bc4053ccb052800886d0d0a\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"jannis\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return CATEGORICAL\n\n @property\n def metric(self):\n return ACC\n\n @property\n def problem_type(self):\n return MULTICLASS\n\n\nclass MicrosoftTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/microsoft/train.csv\",\n \"sha1sum\": \"40f773e920798759c59d0afc6b83112389953a0a\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/microsoft/val.csv\",\n \"sha1sum\": \"351b43ce2eddb9af3ce38c5404e6c2eb4907392f\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/microsoft/test.csv\",\n \"sha1sum\": \"03a10cb4b34010a41c87e8268f63cfb08ec93711\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"microsoft\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return NUMERICAL\n\n @property\n def metric(self):\n return RMSE\n\n @property\n def problem_type(self):\n return REGRESSION\n\n\nclass YahooTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/yahoo/train.csv\",\n \"sha1sum\": \"9444919528cd4f8429b049e98e67d95ca743b607\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/yahoo/val.csv\",\n \"sha1sum\": \"627ba212d51719449453e24aaa7a14435ca97d8b\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/yahoo/test.csv\",\n \"sha1sum\": \"ad8acb167c1d125c47380e57ac50de120072af23\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"yahoo\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return NUMERICAL\n\n @property\n def metric(self):\n return RMSE\n\n @property\n def problem_type(self):\n return REGRESSION\n\n\nclass YearTabularDataset(BaseTabularDataset):\n _INFO = {\n \"train\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/year/train.csv\",\n \"sha1sum\": \"52a5b79a14a0fd69c94105f3212d9728dfc658ed\",\n },\n \"val\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/year/val.csv\",\n \"sha1sum\": \"f5469142509a75c8ba2915fda74efe1fea221058\",\n },\n \"test\": {\n \"url\": \"https://autogluon.s3.us-west-2.amazonaws.com/datasets/tabular_example/year/test.csv\",\n \"sha1sum\": \"d5cf1b15c255a8a6ac0ab94f83373266944a7675\",\n },\n }\n\n def __init__(self, split=\"train\", path=\"./dataset/\"):\n assert split in [\n \"train\",\n \"val\",\n \"test\",\n ], f\"Unsupported split {split}. Split must be one of train, val, or test.\"\n self._split = split\n self._path = os.path.join(path, \"year\", f\"{split}.csv\")\n download(self._INFO[split][\"url\"], path=self._path, sha1_hash=self._INFO[split][\"sha1sum\"])\n self._data = pd.read_csv(self._path)\n\n @property\n def data(self):\n return self._data\n\n @property\n def label_column(self):\n return \"target\"\n\n @property\n def label_type(self):\n return NUMERICAL\n\n @property\n def metric(self):\n return RMSE\n\n @property\n def problem_type(self):\n return REGRESSION\n"
},
{
"path": "examples/automm/tabular_dl/example_tabular.py",
"content": "import os\nimport argparse\nimport json\nimport pandas as pd\nfrom autogluon.multimodal import MultiModalPredictor\nfrom ray import tune\n\nfrom dataset import (\n AdultTabularDataset,\n AloiTabularDataset,\n CaliforniaHousingTabularDataset,\n CovtypeTabularDataset,\n EpsilonTabularDataset,\n HelenaTabularDataset,\n HiggsSmallTabularDataset,\n JannisTabularDataset,\n MicrosoftTabularDataset,\n YahooTabularDataset,\n YearTabularDataset,\n)\n\nTABULAR_DATASETS = {\n \"ad\": AdultTabularDataset,\n \"al\": AloiTabularDataset,\n \"ca\": CaliforniaHousingTabularDataset,\n \"co\": CovtypeTabularDataset,\n \"ep\": EpsilonTabularDataset,\n \"he\": HelenaTabularDataset,\n \"hi\": HiggsSmallTabularDataset,\n \"ja\": JannisTabularDataset,\n \"mi\": MicrosoftTabularDataset,\n \"ya\": YahooTabularDataset,\n \"ye\": YearTabularDataset,\n}\n\nautomm_hyperparameters = {\n \"data.categorical.convert_to_text\": False,\n \"model.names\": [\"ft_transformer\"],\n \"model.ft_transformer.embedding_arch\": [\"linear\"],\n \"env.batch_size\": 128,\n \"env.per_gpu_batch_size\": 128,\n \"env.inference_batch_size_ratio\": 1,\n \"env.num_workers\": 12,\n \"env.num_workers_inference\": 12,\n \"env.num_gpus\": 1,\n \"optim.max_epochs\": 2000, # Specify a large value to train until convergence\n \"optim.weight_decay\": 1.0e-5,\n \"optim.lr_choice\": None,\n \"optim.lr_schedule\": \"polynomial_decay\",\n \"optim.warmup_steps\": 0.0,\n \"optim.patience\": 20,\n \"optim.top_k\": 3,\n}\n\nhyperparameter_tune_kwargs = {\n \"searcher\": \"random\",\n \"scheduler\": \"FIFO\",\n \"num_trials\": 50,\n}\n\n\ndef main(args):\n\n if args.gpu_id is not None:\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = args.gpu_id\n\n assert args.dataset_name in TABULAR_DATASETS.keys(), \"Unsupported dataset name.\"\n\n ### Dataset loading\n train_data = TABULAR_DATASETS[args.dataset_name](\"train\", args.dataset_dir)\n\n val_data = TABULAR_DATASETS[args.dataset_name](\"val\", args.dataset_dir)\n\n test_data = TABULAR_DATASETS[args.dataset_name](\"test\", args.dataset_dir)\n\n automm_hyperparameters[\"optim.lr\"] = args.lr\n automm_hyperparameters[\"optim.end_lr\"] = args.end_lr\n\n if args.embedding_arch is not None:\n automm_hyperparameters[\"model.ft_transformer.embedding_arch\"] = args.embedding_arch\n\n tabular_hyperparameters = {\n \"GBM\": [\n {},\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n ],\n \"CAT\": {},\n \"XGB\": {},\n \"AG_AUTOMM\": automm_hyperparameters,\n }\n\n if args.mode == \"single\":\n ### model initialization\n predictor = MultiModalPredictor(\n label=train_data.label_column,\n problem_type=train_data.problem_type,\n eval_metric=train_data.metric,\n path=args.exp_dir,\n verbosity=4,\n )\n\n ### model training\n predictor.fit(\n train_data=train_data.data,\n tuning_data=val_data.data,\n seed=args.seed,\n hyperparameters=automm_hyperparameters,\n )\n\n ### model inference\n scores = predictor.evaluate(data=test_data.data, metrics=[test_data.metric])\n with open(os.path.join(args.exp_dir, \"scores.json\"), \"w\") as f:\n json.dump(scores, f)\n print(scores)\n elif args.mode == \"single_hpo\":\n automm_hyperparameters[\"model.ft_transformer.ffn_dropout\"] = tune.uniform(0.0, 0.5)\n automm_hyperparameters[\"model.ft_transformer.attention_dropout\"] = tune.uniform(0.0, 0.5)\n automm_hyperparameters[\"model.ft_transformer.residual_dropout\"] = tune.uniform(0.0, 0.2)\n automm_hyperparameters[\"model.ft_transformer.ffn_hidden_size\"] = tune.randint(150, 300)\n automm_hyperparameters[\"optim.lr\"] = tune.uniform(0.00001, 0.001)\n automm_hyperparameters[\"optim.end_lr\"] = 1e-5\n\n ### model initialization\n predictor = MultiModalPredictor(\n label=train_data.label_column,\n problem_type=train_data.problem_type,\n eval_metric=train_data.metric,\n path=args.exp_dir,\n verbosity=4,\n )\n\n ### model training\n predictor.fit(\n train_data=train_data.data,\n tuning_data=val_data.data,\n seed=args.seed,\n hyperparameters=automm_hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n\n ### model inference\n scores = predictor.evaluate(data=test_data.data, metrics=[test_data.metric])\n with open(os.path.join(args.exp_dir, \"scores.json\"), \"w\") as f:\n json.dump(scores, f)\n print(scores)\n elif args.mode == \"weighted\" or args.mode == \"single_bag5\" or args.mode == \"stack5\":\n if args.mode == \"single_bag5\":\n tabular_hyperparameters = {\n \"AG_AUTOMM\": automm_hyperparameters,\n }\n num_bag_folds, num_stack_levels = 5, 0\n elif args.mode == \"weighted\":\n num_bag_folds, num_stack_levels = None, None\n elif args.mode == \"stack5\":\n num_bag_folds, num_stack_levels = 5, 1\n else:\n raise NotImplementedError\n from autogluon.tabular import TabularPredictor\n\n predictor = TabularPredictor(eval_metric=train_data.metric, label=train_data.label_column, path=args.exp_dir)\n predictor.fit(\n train_data=train_data.data,\n tuning_data=val_data.data if num_bag_folds is None else None,\n hyperparameters=tabular_hyperparameters,\n num_bag_folds=num_bag_folds,\n num_stack_levels=num_stack_levels,\n )\n leaderboard = predictor.leaderboard()\n leaderboard.to_csv(os.path.join(args.exp_dir, \"leaderboard.csv\"))\n else:\n raise NotImplementedError\n scores = predictor.evaluate(data=test_data.data)\n with open(os.path.join(args.exp_dir, \"scores.json\"), \"w\") as f:\n json.dump(scores, f)\n print(scores)\n\n predictions = predictor.predict(data=test_data.data)\n predictions.to_csv(os.path.join(args.exp_dir, \"predictions.csv\"))\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--gpu_id\", default=None, type=str, help=\"Specify the GPU to use.\")\n parser.add_argument(\"--dataset_name\", default=\"ad\", type=str, help=\"Specify the dataset to run the experinments.\")\n parser.add_argument(\"--dataset_dir\", default=\"./dataset\", type=str, help=\"Path to the dataset.\")\n parser.add_argument(\"--exp_dir\", default=None, type=str, help=\"Path to the outputs.\")\n parser.add_argument(\"--lr\", default=1e-04, type=float, help=\"Initial learning rate.\")\n parser.add_argument(\"--end_lr\", default=1e-04, type=float, help=\"End learning rate.\")\n parser.add_argument(\n \"--mode\",\n choices=[\"single\", \"single_hpo\", \"weighted\", \"single_bag5\", \"stack5\"],\n default=\"single\",\n help=\"Method to run with.\",\n )\n parser.add_argument(\"--seed\", default=0, type=int)\n parser.add_argument(\n \"--embedding_arch\",\n type=str,\n nargs=\"+\",\n default=None,\n help=\"Embedding architecture for numerical features in FT_Transformer.\",\n )\n args = parser.parse_args()\n\n if args.exp_dir is None:\n args.exp_dir = f\"./results/{args.dataset_name}\"\n\n main(args)\n"
},
{
"path": "examples/automm/tabular_dl/run_all.sh",
"content": "#!/bin/bash\nset -e\n\nfor dataset_name in ad al ca co ep he hi ja mi ya ye\ndo\n python3 example_tabular.py --dataset_name ${dataset_name} \\\n --dataset_dir ./dataset/ \\\n --exp_dir ./result/${dataset_name} \\\n --lr 1E-4 \\\n --seed 0\ndone\n"
},
{
"path": "examples/automm/text_prediction/README.md",
"content": "# Examples for TextPredictor \n\n## Product Sentiment Classification\n\nHere, we provide the example that shows how to use AutoGluon to achieve top performance in MachineHack Product Sentiment Classification Competition. \nTo join the hackathon, you can first go to [MachineHack Website](https://www.machinehack.com/hackathon) and switch to \"Late Submission\" and \nthen go to \"Product Sentiment Classification\".\nThis will bring you to the link: [link](https://www.machinehack.com/hackathons/product_sentiment_classification_weekend_hackathon_19/leaderboard)\n\n!IMPORTANT, you can not directly access the link and will have to follow the previous steps.\n\n```\nmkdir -p machine_hack_product_sentiment\nwget https://automl-mm-bench.s3.amazonaws.com/machine_hack_product_sentiment/all_train.csv -O machine_hack_product_sentiment/all_train.csv\nwget https://automl-mm-bench.s3.amazonaws.com/machine_hack_product_sentiment/test.csv -O machine_hack_product_sentiment/test.csv\n\nmkdir -p ag_product_sentiment\npython3 run_competition.py --train_file machine_hack_product_sentiment/all_train.csv \\\n --test_file machine_hack_product_sentiment/test.csv \\\n --task product_sentiment \\\n --eval_metric log_loss \\\n --exp_dir ag_product_sentiment \\\n --mode stacking 2>&1 | tee -a ag_product_sentiment/log.txt\n```\nIt will generate a `submission.csv` file and you can try to submit that to the competition leaderboard. \n\n## Predict Price of Book\nHere, we provide the example that shows how to use AutoGluon to achieve top performance in MachineHack Book Price Prediction Hackathon.\nTo join the hackathon, you can first go to [MachineHack Website](https://www.machinehack.com/hackathon) and switch to \"Active\" and \ngo to \"Predict The Price Of Books\".\nThis will bring you to the [link](https://machinehack.com/hackathons/predict_the_price_of_books/overview)\n\n!IMPORTANT, you can not directly access the link and will have to follow the previous steps.\n\nAlso, you will need to install `openpyxl` to read from xlsx file.\n\n```\nbash prepare_price_of_books.sh\npython3 -m pip install openpyxl\nmkdir -p ag_price_of_books\npython3 run_competition.py --train_file price_of_books/Participants_Data/Data_Train.xlsx \\\n --test_file price_of_books/Participants_Data/Data_Test.xlsx \\\n --sample_submission price_of_books/Participants_Data/Sample_Submission.xlsx \\\n --task price_of_books \\\n --eval_metric r2 \\\n --exp_dir ag_price_of_books \\\n --mode stacking 2>&1 | tee -a ag_price_of_books/log.txt\n```\nOnce the script is finished, you will see a `submission.xlsx` file generate in the \n`ag_price_of_books` folder and you can try to submit that to the competition leaderboard.\n\n!IMPORTANT. Try to run the experiment on a p3.2x instance :).\n\n## Predict Salary of Data Scientists\nHere, we provide the example that shows how to use AutoGluon to achieve top performance in MachineHack Data Scientist Salary Prediction Hackathon. \nTo join the hackathon, you can first go to [MachineHack Website](https://www.machinehack.com/hackathon) and switch to \"Active\" and \ngo to \"Predict The Data Scientists Salary In India Hackathon\".\nThis will bring you to the link: [link](https://www.machinehack.com/hackathons/predict_the_data_scientists_salary_in_india_hackathon/overview)\n\n!IMPORTANT, you can not directly access the link and will have to follow the previous steps.\n\nAlso, you will need to install `openpyxl` to read from xlsx file.\n\n```\nbash prepare_data_scientist_salary.sh\npython3 -m pip install openpyxl\nmkdir -p ag_data_scientist_salary\npython3 run_competition.py --train_file data_scientist_salary/Data/Final_Train_Dataset.csv \\\n --test_file data_scientist_salary/Data/Final_Test_Dataset.csv \\\n --sample_submission data_scientist_salary/Data/sample_submission.xlsx \\\n --task data_scientist_salary \\\n --eval_metric acc \\\n --exp_dir ag_data_scientist_salary \\\n --mode stacking 2>&1 | tee -a ag_data_scientist_salary/log.txt\n```\n\nOnce the script is finished, you will see a `submission.xlsx` file generate in the \n`ag_data_scientist_salary` folder and you can try to submit that to the competition leaderboard.\n\n!IMPORTANT. Try to run the experiment on a p3.2x instance :).\n\n## Reach Top-5 Performance in Mercari Price Suggestion\n\nHere, we provide the example that shows how to use AutoGluon to achieve top-5 performance in\n [Mercari Price Suggestion](https://www.kaggle.com/c/mercari-price-suggestion-challenge/data).\nTo run the example, you will need to configure the Kaggle API which will be documented in \nhttps://github.com/Kaggle/kaggle-api and download the dataset.\n\n```\nsudo apt install -y p7zip-full\nbash prepare_mercari_kaggle.sh\n```\n\nAfter you have prepared the dataset, you can use the following command:\n```\nmkdir -p ag_mercari_price_single\npython3 run_competition.py --train_file mercari_price/train.tsv \\\n --test_file mercari_price/test_stg2.tsv \\\n --sample_submission mercari_price/sample_submission_stg2.csv \\\n --task mercari_price \\\n --eval_metric r2 \\\n --exp_dir ag_mercari_price_single \\\n --mode single 2>&1 | tee -a ag_mercari_price_single/log.txt\n```\n\nIn addition, you may run multimodal with weighted ensemble\n```\nmkdir -p ag_mercari_price_weighted\npython3 run_competition.py --train_file mercari_price/train.tsv \\\n --test_file mercari_price/test_stg2.tsv \\\n --sample_submission mercari_price/sample_submission_stg2.csv \\\n --task mercari_price \\\n --eval_metric r2 \\\n --exp_dir ag_mercari_price_weighted \\\n --mode weighted 2>&1 | tee -a ag_mercari_price_weighted/log.txt\n```\nOr stacking\n```\nmkdir -p ag_mercari_price_stacking\npython3 run_competition.py --train_file mercari_price/train.tsv \\\n --test_file mercari_price/test_stg2.tsv \\\n --sample_submission mercari_price/sample_submission_stg2.csv \\\n --task mercari_price \\\n --eval_metric r2 \\\n --exp_dir ag_mercari_price_stacking \\\n --mode stacking 2>&1 | tee -a ag_mercari_price_stacking/log.txt\n```\n\n## Solve GLUE Tasks with AutoGluon Text\n\nHere, we show how you may use AutoGluon Text to solve all tasks in the [GLUE benchmark](https://openreview.net/pdf?id=rJ4km2R5t7).\n \n### Prepare the data\n```bash\npython3 prepare_glue.py --benchmark glue\n```\n\n### Run the benchmark\nRun on all datasets with either a single `TextPredictor` model or the `multimodal` configuration \nin AutoGluon Tabular that will combine the `TextPredictor` model with tabular models from \nAutoGluon-Tabular via a single layer of stack ensembling with 5-fold bagging.\n \n```bash\n# Run single model\nbash run_glue.sh single\n\n# Run 5-fold stacking\nbash run_glue.sh stacking\n```\n\nTo generate the submission file, use `python3 generate_submission.py --prefix autogluon_text --save_dir submission`\n\n### Results\nFor MRPC and STS, we have manually augmented the training and validation data by shuffling the \norder of two sentences.\n\n| | CoLA | SST | MRPC | STS | QQP | MNLI-m | MNLI-mm | QNLI | RTE | WNLI |\n|---------------------------------------|--------|--------|-------------|------------|----------|--------|---------|--------|--------|--------|\n|Metrics | mcc | acc | acc | spearmanr | f1 | acc | acc | acc | acc | acc |\n|Text (Single) - Validation (*) | 0.6782 | 0.9507 | 0.8725 (*) | 0.9047 (*) | 0.8866 | 0.8671 | 0.8696 | 0.9235 | 0.7798 | 0.5634 |\n|Text (Single) - Test (TBA) | - | - | - | - | - | - | - | - | - | |\n"
},
{
"path": "examples/automm/text_prediction/generate_submission.py",
"content": "import pandas as pd\nfrom autogluon.multimodal import MultiModalPredictor\nimport argparse\nimport os\nimport numpy as np\n\n\ndef get_parser():\n parser = argparse.ArgumentParser('Generate GLUE submission folder')\n parser.add_argument('--prefix', type=str, default='autogluon_text')\n parser.add_argument('--save_dir', type=str, default='glue_submission')\n return parser\n\n\ndef get_test_index(path):\n with open(path, 'r', encoding='utf-8') as in_f:\n lines = in_f.readlines()\n index_l = []\n for i in range(1, len(lines)):\n index_l.append(lines[i].split()[0])\n return index_l\n\n\ndef main(args):\n tasks = {\n 'cola': ['CoLA.tsv', 'glue/cola/test.tsv'],\n 'sst': ['SST-2.tsv', 'glue/sst/test.tsv'],\n 'mrpc': ['MRPC.tsv', 'glue/mrpc/test.tsv'],\n 'sts': ['STS-B.tsv', 'glue/sts/test.tsv'],\n 'qqp': ['QQP.tsv', 'glue/qqp/test.tsv'],\n 'mnli_m': ['MNLI-m.tsv', 'glue/mnli/test_matched.tsv'],\n 'mnli_mm': ['MNLI-mm.tsv', 'glue/mnli/test_mismatched.tsv'],\n 'qnli': ['QNLI.tsv', 'glue/qnli/test.tsv'],\n 'rte': ['RTE.tsv', 'glue/rte/test.tsv'],\n 'wnli': ['WNLI.tsv', 'glue/wnli/test.tsv'],\n 'ax': ['AX.tsv', 'glue/rte_diagnostic/diagnostic.tsv']\n }\n\n os.makedirs(args.save_dir, exist_ok=True)\n\n for task, (save_name, test_file_path) in tasks.items():\n if task == 'ax':\n # For AX, we need to load the mnli-m checkpoint and run inference\n test_df = pd.read_csv(test_file_path, sep='\\t', header=0)\n test_index = test_df['index']\n predictor = MultiModalPredictor.load(f'{args.prefix}_mnli_m')\n label_column = predictor.label\n predictions = predictor.predict(test_df)\n else:\n test_index = get_test_index(test_file_path)\n prediction_df = pd.read_csv(f'{args.prefix}_{task}/test_prediction.csv',\n index_col=0)\n label_column = prediction_df.columns[0]\n predictions = prediction_df[label_column]\n if task == 'sts':\n predictions = np.clip(predictions, 0, 5)\n with open(os.path.join(args.save_dir, save_name), 'w') as of:\n of.write('index\\t{}\\n'.format(label_column))\n for i in range(len(predictions)):\n of.write('{}\\t{}\\n'.format(test_index[i],\n predictions[i]))\n\n\nif __name__ == '__main__':\n parser = get_parser()\n args = parser.parse_args()\n main(args)\n"
},
{
"path": "examples/automm/text_prediction/prepare_data_scientist_salary.sh",
"content": "set -ex\n\nmkdir -p data_scientist_salary\nwget https://automl-mm-bench.s3.amazonaws.com/machine_hack_competitions/predict_the_data_scientists_salary_in_india_hackathon/Data.zip -O data_scientist_salary/Data.zip\ncd data_scientist_salary\nunzip Data.zip\n\n"
},
{
"path": "examples/automm/text_prediction/prepare_glue.py",
"content": "# Disclaimer! The script here is partially based on\n# https://github.com/nyu-mll/jiant/blob/master/scripts/download_glue_data.py\n# and\n# https://github.com/nyu-mll/jiant/blob/master/scripts/download_superglue_data.py\nimport os\nimport shutil\nimport argparse\nimport zipfile\nimport json\nimport pandas as pd\nimport pyarrow\nimport pyarrow.json\nfrom autogluon_contrib_nlp.utils.misc import download, load_checksum_stats\nfrom autogluon_contrib_nlp.base import get_data_home_dir\nfrom autogluon_contrib_nlp.data.tokenizers import WhitespaceTokenizer\n\n\n_CITATIONS = \"\"\"\n@inproceedings{wang2019glue,\n title={GLUE: A multi-task benchmark and analysis platform for natural language understanding},\n author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n booktitle={ICLR},\n year={2019}\n}\n\n@inproceedings{wang2019superglue,\n title={Superglue: A stickier benchmark for general-purpose language understanding systems},\n author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and \n Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel},\n booktitle={Advances in Neural Information Processing Systems},\n pages={3261--3275},\n year={2019}\n}\n\"\"\"\n\nGLUE_TASKS = [\"cola\", \"sst\", \"mrpc\", \"qqp\", \"sts\", \"mnli\",\n \"snli\", \"qnli\", \"rte\", \"wnli\", \"diagnostic\"]\nSUPERGLUE_TASKS = [\"cb\", \"copa\", \"multirc\", \"rte\", \"wic\", \"wsc\", \"boolq\", \"record\",\n 'broadcoverage-diagnostic', 'winogender-diagnostic']\n\n_CURR_DIR = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))\n_URL_FILE_STATS = load_checksum_stats(os.path.join(\n _CURR_DIR, 'url_checksums', 'glue.txt'))\n_URL_FILE_STATS.update(load_checksum_stats(os.path.join(\n _CURR_DIR, 'url_checksums', 'superglue.txt')))\n\n\ndef read_tsv_glue(tsv_file, num_skip=1, keep_column_names=False):\n out = []\n nrows = None\n if keep_column_names:\n assert num_skip == 1\n column_names = None\n with open(tsv_file, 'r', encoding=\"utf8\") as f:\n for i, line in enumerate(f):\n line = line.strip()\n if i < num_skip:\n if keep_column_names:\n column_names = line.split()\n continue\n elements = line.split('\\t')\n out.append(elements)\n if nrows is None:\n nrows = len(elements)\n else:\n assert nrows == len(elements)\n df = pd.DataFrame(out, columns=column_names)\n series_l = []\n for col_name in df.columns:\n idx = df[col_name].first_valid_index()\n val = df[col_name][idx]\n if isinstance(val, str):\n try:\n dat = pd.to_numeric(df[col_name])\n series_l.append(dat)\n continue\n except ValueError:\n pass\n finally:\n pass\n series_l.append(df[col_name])\n new_df = pd.DataFrame({name: series for name, series in zip(df.columns, series_l)})\n return new_df\n\n\ndef read_jsonl_superglue(jsonl_file):\n columns = None\n out = []\n with open(jsonl_file, 'r') as f:\n for i, line in enumerate(f):\n line = line.strip()\n sample = json.loads(line)\n if columns is None:\n columns = list(sample.keys())\n else:\n assert sorted(columns) == sorted(list(sample.keys())),\\\n 'Columns={}, sample.keys()={}'.format(columns, sample.keys())\n out.append([sample[col] for col in columns])\n df = pd.DataFrame(out, columns=columns)\n return df\n\n\n# Classification will be stored as pandas dataframe\ndef read_cola(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n if fold == 'test':\n df = pd.read_csv(csv_file, '\\t')\n df = df[['sentence']]\n df_dict[fold] = df\n else:\n df = pd.read_csv(csv_file, '\\t', header=None)\n df = df[[3, 1]]\n df.columns = ['sentence', 'label']\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_sst(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = pd.read_csv(csv_file, '\\t')\n if fold == 'test':\n df = df[['sentence']]\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_mrpc(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n tsv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = read_tsv_glue(tsv_file)\n if fold == 'test':\n df = df[[3, 4]]\n df.columns = ['sentence1', 'sentence2']\n else:\n df = df[[3, 4, 0]]\n df.columns = ['sentence1', 'sentence2', 'label']\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_qqp(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = pd.read_csv(csv_file, '\\t')\n if fold == 'test':\n df = df[['question1', 'question2']]\n df.columns = ['sentence1', 'sentence2']\n else:\n df = df[['question1', 'question2', 'is_duplicate']]\n df.columns = ['sentence1', 'sentence2', 'label']\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_sts(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = read_tsv_glue(csv_file)\n if fold == 'test':\n df = df[[7, 8, 1]]\n df.columns = ['sentence1', 'sentence2', 'genre']\n else:\n df = df[[7, 8, 1, 9]]\n df.columns = ['sentence1', 'sentence2', 'genre', 'score']\n genre_l = []\n for ele in df['genre'].tolist():\n if ele == 'main-forum':\n genre_l.append('main-forums')\n else:\n genre_l.append(ele)\n df['genre'] = pd.Series(genre_l)\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_mnli(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev_matched', 'dev_mismatched', 'test_matched', 'test_mismatched']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = read_tsv_glue(csv_file, 1, True)\n if 'test' in fold:\n df = df[['sentence1', 'sentence2', 'genre']]\n else:\n df = df[['sentence1', 'sentence2', 'genre', 'gold_label']]\n df.columns = ['sentence1', 'sentence2', 'genre', 'label']\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_snli(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n column_names = None\n out = []\n with open(csv_file) as f:\n for i, line in enumerate(f):\n line = line.strip()\n if i == 0:\n column_names = line.split()\n column_names = column_names[:10] + [column_names[-1]]\n continue\n elements = line.split('\\t')\n first_few_elements = elements[:10]\n gold_label = elements[-1]\n out.append(first_few_elements + [gold_label])\n df = pd.DataFrame(out, columns=column_names)\n df = df[['sentence1', 'sentence2', 'gold_label']]\n df.columns = ['sentence1', 'sentence2', 'label']\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_qnli(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = read_tsv_glue(csv_file, 1, True)\n if fold == 'test':\n df_dict[fold] = df[['question', 'sentence']]\n else:\n df_dict[fold] = df[['question', 'sentence', 'label']]\n return df_dict, None\n\n\ndef read_rte(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = read_tsv_glue(csv_file, keep_column_names=True)\n if fold == 'test':\n df_dict[fold] = df[['sentence1', 'sentence2']]\n else:\n df_dict[fold] = df[['sentence1', 'sentence2', 'label']]\n assert df_dict[fold]['label'].isnull().sum().sum() == 0\n return df_dict, None\n\n\ndef read_wnli(dir_path):\n df_dict = dict()\n for fold in ['train', 'dev', 'test']:\n csv_file = os.path.join(dir_path, '{}.tsv'.format(fold))\n df = pd.read_csv(csv_file, '\\t')\n if fold == 'test':\n df = df[['sentence1', 'sentence2']]\n else:\n df = df[['sentence1', 'sentence2', 'label']]\n df_dict[fold] = df\n return df_dict, None\n\n\n# The glue diagnostic will be in MNLI\ndef read_glue_diagnostic(dir_path):\n csv_file = os.path.join(dir_path, 'diagnostic-full.tsv')\n df = pd.read_csv(csv_file, '\\t')\n df.columns = ['semantics', 'predicate', 'logic', 'knowledge', 'domain', 'premise',\n 'hypothesis', 'label']\n return df\n\n\ndef read_cb(dir_path):\n df_dict = dict()\n for fold in ['train', 'val', 'test']:\n columns = ['premise', 'hypothesis']\n if fold != 'test':\n columns.append('label')\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n df = df[columns]\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_copa(dir_path):\n df_dict = dict()\n for fold in ['train', 'val', 'test']:\n columns = ['premise', 'choice1', 'choice2', 'question']\n if fold != 'test':\n columns.append('label')\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n df = df[columns]\n df_dict[fold] = df\n return df_dict, None\n\n\n# passage, question, answer, passage_idx, question_idx, answer_idx\ndef read_multirc(dir_path):\n df_dict = dict()\n for fold in ['train', 'val', 'test']:\n columns = ['passage', 'question', 'answer', 'psg_idx', 'qst_idx', 'ans_idx']\n if fold != 'test':\n columns.append('label')\n out = []\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n with open(jsonl_path, 'r') as f:\n for line in f:\n sample = json.loads(line.strip())\n psg_idx = sample['idx']\n sample = json.loads(line.strip())\n passage = sample['passage']['text']\n for qa in sample['passage']['questions']:\n qst_idx = qa['idx']\n question = qa['question']\n for ans in qa['answers']:\n ans_idx = ans['idx']\n answer = ans['text']\n if fold == 'test':\n out.append((passage, question, answer, psg_idx, qst_idx, ans_idx))\n else:\n label = ans['label']\n out.append((passage, question, answer, psg_idx, qst_idx,\n ans_idx, label))\n df = pd.DataFrame(out, columns=columns)\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_rte_superglue(dir_path):\n df_dict = dict()\n for fold in ['train', 'val', 'test']:\n if fold == 'test':\n columns = ['premise', 'hypothesis']\n else:\n columns = ['premise', 'hypothesis', 'label']\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n df = df[columns]\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_wic(dir_path):\n df_dict = dict()\n meta_data = dict()\n meta_data['entities1'] = {'type': 'entity', 'attrs': {'parent': 'sentence1'}}\n meta_data['entities2'] = {'type': 'entity', 'attrs': {'parent': 'sentence2'}}\n\n for fold in ['train', 'val', 'test']:\n if fold != 'test':\n columns = ['sentence1', 'sentence2', 'entities1', 'entities2', 'label']\n else:\n columns = ['sentence1', 'sentence2', 'entities1', 'entities2']\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n out = []\n for idx, row in df.iterrows():\n sentence1 = row['sentence1']\n sentence2 = row['sentence2']\n start1 = row['start1']\n end1 = row['end1']\n start2 = row['start2']\n end2 = row['end2']\n if fold == 'test':\n out.append([sentence1, sentence2,\n {'start': start1, 'end': end1},\n {'start': start2, 'end': end2}])\n else:\n label = row['label']\n out.append([sentence1, sentence2,\n {'start': start1, 'end': end1},\n {'start': start2, 'end': end2},\n label])\n df = pd.DataFrame(out, columns=columns)\n df_dict[fold] = df\n return df_dict, meta_data\n\n\ndef read_wsc(dir_path):\n df_dict = dict()\n tokenizer = WhitespaceTokenizer()\n meta_data = dict()\n meta_data['noun'] = {'type': 'entity', 'attrs': {'parent': 'text'}}\n meta_data['pronoun'] = {'type': 'entity', 'attrs': {'parent': 'text'}}\n for fold in ['train', 'val', 'test']:\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n samples = []\n for i in range(len(df)):\n text = df.loc[i, 'text']\n if fold != 'test':\n label = df.loc[i, 'label']\n target = df.loc[i, 'target']\n span1_index = target['span1_index']\n span2_index = target['span2_index']\n span1_text = target['span1_text']\n span2_text = target['span2_text']\n # Build entity\n # list of entities\n # 'entities': {'start': 0, 'end': 100}\n tokens, offsets = tokenizer.encode_with_offsets(text, str)\n pos_start1 = offsets[span1_index][0]\n pos_end1 = pos_start1 + len(span1_text)\n pos_start2 = offsets[span2_index][0]\n pos_end2 = pos_start2 + len(span2_text)\n if fold == 'test':\n samples.append({'text': text,\n 'noun': {'start': pos_start1, 'end': pos_end1},\n 'pronoun': {'start': pos_start2, 'end': pos_end2}})\n else:\n samples.append({'text': text,\n 'noun': {'start': pos_start1, 'end': pos_end1},\n 'pronoun': {'start': pos_start2, 'end': pos_end2},\n 'label': label})\n df = pd.DataFrame(samples)\n df_dict[fold] = df\n return df_dict, meta_data\n\n\ndef read_boolq(dir_path):\n df_dict = dict()\n for fold in ['train', 'val', 'test']:\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n df_dict[fold] = df\n return df_dict, None\n\n\ndef read_record(dir_path):\n df_dict = dict()\n meta_data = dict()\n meta_data['entities'] = {'type': 'entity', 'attrs': {'parent': 'text'}}\n meta_data['answers'] = {'type': 'entity', 'attrs': {'parent': 'text'}}\n for fold in ['train', 'val', 'test']:\n if fold != 'test':\n columns = ['source', 'text', 'entities', 'query', 'answers']\n else:\n columns = ['source', 'text', 'entities', 'query']\n jsonl_path = os.path.join(dir_path, '{}.jsonl'.format(fold))\n df = read_jsonl_superglue(jsonl_path)\n df_dict[fold] = df\n out = []\n for i, row in df.iterrows():\n source = row['source']\n passage = row['passage']\n text = passage['text']\n entities = passage['entities']\n entities = [{'start': ele['start'], 'end': ele['end']} for ele in entities]\n for qas in row['qas']:\n query = qas['query']\n if fold != 'test':\n answer_entities = qas['answers']\n out.append((source, text, entities, query, answer_entities))\n else:\n out.append((source, text, entities, query))\n df = pd.DataFrame(out, columns=columns)\n df_dict[fold] = df\n return df_dict, meta_data\n\n\ndef read_winogender_diagnostic(dir_path):\n jsonl_path = os.path.join(dir_path, 'AX-g.jsonl')\n df = read_jsonl_superglue(jsonl_path)\n return df\n\n\ndef read_broadcoverage_diagnostic(dir_path):\n df = pyarrow.json.read_json(os.path.join(dir_path, 'AX-b.jsonl')).to_pandas()\n return df\n\n\nGLUE_TASK2PATH = {\n \"cola\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/cola.zip\", # noqa\n \"sst\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/sst.zip\", # noqa\n \"mrpc\": {\n 'train': \"https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt\",\n 'dev': \"https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc\",\n 'test': \"https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt\"\n },\n \"qqp\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/qqp.zip\", # noqa\n \"sts\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/sts.zip\", # noqa\n \"mnli\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/mnli.zip\", # noqa\n \"snli\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/snli.zip\", # noqa\n \"qnli\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/qnli.zip\", # noqa\n \"rte\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/rte.zip\", # noqa\n \"wnli\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/wnli.zip\", # noqa\n \"diagnostic\": [\n \"https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D\", # noqa\n \"https://www.dropbox.com/s/ju7d95ifb072q9f/diagnostic-full.tsv?dl=1\",\n ],\n}\n\nGLUE_READERS = {\n 'cola': read_cola,\n 'sst': read_sst,\n 'mrpc': read_mrpc,\n 'qqp': read_qqp,\n 'sts': read_sts,\n 'mnli': read_mnli,\n 'snli': read_snli,\n 'qnli': read_qnli,\n 'rte': read_rte,\n 'wnli': read_wnli,\n 'diagnostic': read_glue_diagnostic\n}\n\n\nSUPERGLUE_TASK2PATH = {\n \"cb\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/cb.zip\",\n \"copa\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/copa.zip\",\n \"multirc\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/multirc.zip\",\n \"rte\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/rte.zip\",\n \"wic\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/wic.zip\",\n \"wsc\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/wsc.zip\",\n \"broadcoverage-diagnostic\": \"https://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-b.zip\",\n \"winogender-diagnostic\": \"https://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-g.zip\",\n \"boolq\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/boolq.zip\",\n \"record\": \"https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/record.zip\",\n}\n\nSUPERGLUE_READER = {\n 'cb': read_cb,\n 'copa': read_copa,\n 'multirc': read_multirc,\n 'rte': read_rte_superglue,\n 'wic': read_wic,\n 'wsc': read_wsc,\n 'boolq': read_boolq,\n 'record': read_record,\n 'broadcoverage-diagnostic': read_broadcoverage_diagnostic,\n 'winogender-diagnostic': read_winogender_diagnostic\n}\n\n\ndef format_mrpc(data_dir):\n mrpc_dir = os.path.join(data_dir, \"mrpc\")\n os.makedirs(mrpc_dir, exist_ok=True)\n mrpc_train_file = os.path.join(mrpc_dir, \"msr_paraphrase_train.txt\")\n mrpc_test_file = os.path.join(mrpc_dir, \"msr_paraphrase_test.txt\")\n download(GLUE_TASK2PATH[\"mrpc\"]['train'], mrpc_train_file,\n sha1_hash=_URL_FILE_STATS[GLUE_TASK2PATH[\"mrpc\"]['train']])\n download(GLUE_TASK2PATH[\"mrpc\"]['test'], mrpc_test_file,\n sha1_hash=_URL_FILE_STATS[GLUE_TASK2PATH[\"mrpc\"]['test']])\n assert os.path.isfile(mrpc_train_file), \"Train data not found at %s\" % mrpc_train_file\n assert os.path.isfile(mrpc_test_file), \"Test data not found at %s\" % mrpc_test_file\n download(GLUE_TASK2PATH[\"mrpc\"]['dev'],\n os.path.join(mrpc_dir, \"dev_ids.tsv\"),\n sha1_hash=_URL_FILE_STATS[GLUE_TASK2PATH[\"mrpc\"]['dev']])\n\n dev_ids = []\n with open(os.path.join(mrpc_dir, \"dev_ids.tsv\"), encoding=\"utf8\") as ids_fh:\n for row in ids_fh:\n dev_ids.append(row.strip().split(\"\\t\"))\n\n with open(mrpc_train_file, encoding=\"utf8\") as data_fh, open(\n os.path.join(mrpc_dir, \"train.tsv\"), \"w\", encoding=\"utf8\"\n ) as train_fh, open(os.path.join(mrpc_dir, \"dev.tsv\"), \"w\", encoding=\"utf8\") as dev_fh:\n header = data_fh.readline()\n train_fh.write(header)\n dev_fh.write(header)\n for row in data_fh:\n label, id1, id2, s1, s2 = row.strip().split(\"\\t\")\n if [id1, id2] in dev_ids:\n dev_fh.write(\"%s\\t%s\\t%s\\t%s\\t%s\\n\" % (label, id1, id2, s1, s2))\n else:\n train_fh.write(\"%s\\t%s\\t%s\\t%s\\t%s\\n\" % (label, id1, id2, s1, s2))\n\n with open(mrpc_test_file, encoding=\"utf8\") as data_fh, open(\n os.path.join(mrpc_dir, \"test.tsv\"), \"w\", encoding=\"utf8\"\n ) as test_fh:\n header = data_fh.readline()\n test_fh.write(\"index\\t#1 ID\\t#2 ID\\t#1 String\\t#2 String\\n\")\n for idx, row in enumerate(data_fh):\n label, id1, id2, s1, s2 = row.strip().split(\"\\t\")\n test_fh.write(\"%d\\t%s\\t%s\\t%s\\t%s\\n\" % (idx, id1, id2, s1, s2))\n\n\ndef get_tasks(benchmark, task_names):\n task_names = task_names.split(\",\")\n ALL_TASKS = GLUE_TASKS if benchmark == 'glue' else SUPERGLUE_TASKS\n if \"all\" in task_names:\n tasks = ALL_TASKS\n else:\n tasks = []\n for task_name in task_names:\n if task_name != 'diagnostic':\n assert task_name in ALL_TASKS, \"Task %s not found!\" % task_name\n tasks.append(task_name)\n if \"RTE\" in tasks and \"diagnostic\" not in tasks:\n tasks.append(\"diagnostic\")\n has_diagnostic = any(['diagnostic' in task for task in tasks])\n if has_diagnostic:\n tasks = [ele for ele in tasks if 'diagnostic' not in ele]\n tasks.append('diagnostic')\n return tasks\n\n\ndef get_parser():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--benchmark\", choices=['glue', 'superglue'],\n default='glue', type=str)\n parser.add_argument(\"-d\", \"--data_dir\", help=\"directory to save data to\", type=str,\n default=None)\n parser.add_argument(\n \"-t\",\n \"--tasks\",\n help=\"tasks to download data for as a comma separated string\",\n type=str,\n default=\"all\"\n )\n parser.add_argument('--cache-path', type=str,\n default=os.path.join(get_data_home_dir(), 'glue'),\n help='The temporary path to download the dataset.')\n return parser\n\n\ndef main(args):\n if args.data_dir is None:\n args.data_dir = args.benchmark\n args.cache_path = os.path.join(args.cache_path, args.benchmark)\n print('Downloading {} to \"{}\". Selected tasks = {}'.format(args.benchmark,\n args.data_dir,\n args.tasks))\n os.makedirs(args.cache_path, exist_ok=True)\n os.makedirs(args.data_dir, exist_ok=True)\n tasks = get_tasks(args.benchmark, args.tasks)\n if args.benchmark == 'glue':\n TASK2PATH = GLUE_TASK2PATH\n TASK2READER = GLUE_READERS\n elif args.benchmark == 'superglue':\n TASK2PATH = SUPERGLUE_TASK2PATH\n TASK2READER = SUPERGLUE_READER\n else:\n raise NotImplementedError\n for task in tasks:\n print('Processing {}...'.format(task))\n if task == 'diagnostic' or 'diagnostic' in task:\n if args.benchmark == 'glue':\n reader = TASK2READER[task]\n base_dir = os.path.join(args.data_dir, 'rte_diagnostic')\n os.makedirs(base_dir, exist_ok=True)\n download(TASK2PATH['diagnostic'][0],\n path=os.path.join(base_dir, 'diagnostic.tsv'),\n sha1_hash=_URL_FILE_STATS[TASK2PATH['diagnostic'][0]])\n download(TASK2PATH['diagnostic'][1],\n path=os.path.join(base_dir, 'diagnostic-full.tsv'),\n sha1_hash=_URL_FILE_STATS[TASK2PATH['diagnostic'][1]])\n df = reader(base_dir)\n df.to_parquet(os.path.join(base_dir, 'diagnostic-full.parquet'))\n else:\n for key, name in [('broadcoverage-diagnostic', 'AX-b'),\n ('winogender-diagnostic', 'AX-g')]:\n data_file = os.path.join(args.cache_path, \"{}.zip\".format(key))\n url = TASK2PATH[key]\n reader = TASK2READER[key]\n download(url, data_file, sha1_hash=_URL_FILE_STATS[url])\n with zipfile.ZipFile(data_file) as zipdata:\n zipdata.extractall(args.data_dir)\n df = reader(os.path.join(args.data_dir, name))\n df.to_parquet(os.path.join(args.data_dir, name, '{}.parquet'.format(name)))\n elif task == 'mrpc':\n reader = TASK2READER[task]\n format_mrpc(args.data_dir)\n df_dict, meta_data = reader(os.path.join(args.data_dir, 'mrpc'))\n for key, df in df_dict.items():\n if key == 'val':\n key = 'dev'\n df.to_parquet(os.path.join(args.data_dir, 'mrpc', '{}.parquet'.format(key)))\n with open(os.path.join(args.data_dir, 'mrpc', 'metadata.json'), 'w') as f:\n json.dump(meta_data, f)\n else:\n # Download data\n data_file = os.path.join(args.cache_path, \"{}.zip\".format(task))\n url = TASK2PATH[task]\n reader = TASK2READER[task]\n download(url, data_file, sha1_hash=_URL_FILE_STATS[url])\n base_dir = os.path.join(args.data_dir, task)\n if os.path.exists(base_dir):\n print('Found!')\n continue\n zip_dir_name = None\n with zipfile.ZipFile(data_file) as zipdata:\n if zip_dir_name is None:\n zip_dir_name = os.path.dirname(zipdata.infolist()[0].filename)\n zipdata.extractall(args.data_dir)\n shutil.move(os.path.join(args.data_dir, zip_dir_name),\n base_dir)\n df_dict, meta_data = reader(base_dir)\n for key, df in df_dict.items():\n if key == 'val':\n key = 'dev'\n df.to_parquet(os.path.join(base_dir, '{}.parquet'.format(key)))\n if meta_data is not None:\n with open(os.path.join(base_dir, 'metadata.json'), 'w') as f:\n json.dump(meta_data, f)\n print(\"\\tCompleted!\")\n\n\ndef cli_main():\n parser = get_parser()\n args = parser.parse_args()\n main(args)\n\n\nif __name__ == \"__main__\":\n cli_main()\n"
},
{
"path": "examples/automm/text_prediction/prepare_mercari_kaggle.sh",
"content": "set -ex\n\nkaggle competitions download -c mercari-price-suggestion-challenge\nmkdir -p mercari_price\nmv mercari-price-suggestion-challenge.zip mercari_price/\ncd mercari_price/\nunzip mercari-price-suggestion-challenge.zip\n7za e train.tsv.7z\nunzip test_stg2.tsv.zip\nunzip sample_submission_stg2.csv.zip\ncd ..\n"
},
{
"path": "examples/automm/text_prediction/prepare_price_of_books.sh",
"content": "set -ex\n\nmkdir -p price_of_books\nwget https://automl-mm-bench.s3.amazonaws.com/machine_hack_competitions/predict_the_price_of_books/Data.zip -O price_of_books/Data.zip\ncd price_of_books\nunzip Data.zip\n"
},
{
"path": "examples/automm/text_prediction/run_competition.py",
"content": "import os\nimport json\nimport argparse\nimport pandas as pd\nimport numpy as np\nimport random\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config\n\n\ndef get_parser():\n parser = argparse.ArgumentParser(description='The Basic Example of using AutoGluon Multimodal '\n 'for Text Prediction.')\n parser.add_argument('--train_file', type=str,\n help='The training CSV file.',\n default=None)\n parser.add_argument('--test_file', type=str,\n help='The testing CSV file.',\n default=None)\n parser.add_argument('--sample_submission', type=str,\n help='The sample submission CSV file.',\n default=None)\n parser.add_argument('--seed', type=int,\n help='The seed',\n default=123)\n parser.add_argument('--eval_metric', type=str,\n help='The metric used to evaluate the model.',\n default=None)\n parser.add_argument('--task', type=str,\n choices=['product_sentiment',\n 'mercari_price',\n 'price_of_books',\n 'data_scientist_salary'],\n required=True)\n parser.add_argument('--exp_dir', type=str, default=None,\n help='The experiment directory where the model params will be written.')\n parser.add_argument('--mode',\n choices=['stacking', 'weighted', 'single'],\n default='single',\n help='Whether to use a single model or a stack ensemble. '\n 'If it is \"single\", If it is turned on, we will use 5-fold, 1-layer for stacking.')\n parser.add_argument('--preset', type=str,\n help='Pre-registered configurations',\n choices=['medium_quality_faster_train',\n 'high_quality',\n 'best_quality'],\n default=None)\n return parser\n\n\ndef load_machine_hack_product_sentiment(train_path, test_path):\n train_df = pd.read_csv(train_path)\n test_df = pd.read_csv(test_path)\n feature_columns = ['Product_Description', 'Product_Type']\n label_column = 'Sentiment'\n train_df = train_df[feature_columns + [label_column]]\n test_df = test_df[feature_columns]\n return train_df, test_df, label_column\n\n\ndef load_price_of_books(train_path, test_path):\n train_df = pd.read_excel(train_path, engine='openpyxl')\n test_df = pd.read_excel(test_path, engine='openpyxl')\n # Convert Reviews\n train_df.loc[:, 'Reviews'] = pd.to_numeric(train_df['Reviews'].apply(\n lambda ele: ele[:-len(' out of 5 stars')]))\n test_df.loc[:, 'Reviews'] = pd.to_numeric(\n test_df['Reviews'].apply(lambda ele: ele[:-len(' out of 5 stars')]))\n # Convert Ratings\n train_df.loc[:, 'Ratings'] = pd.to_numeric(train_df['Ratings'].apply(\n lambda ele: ele.replace(',', '')[:-len(' customer reviews')]))\n test_df.loc[:, 'Ratings'] = pd.to_numeric(\n test_df['Ratings'].apply(lambda ele: ele.replace(',', '')[:-len(' customer reviews')]))\n # Convert Price to log scale\n train_df.loc[:, 'Price'] = np.log10(train_df['Price'] + 1)\n return train_df, test_df, 'Price'\n\n\ndef load_data_scientist_salary(train_path, test_path):\n train_df = pd.read_csv(train_path, index_col=0)\n test_df = pd.read_csv(test_path, index_col=None)\n train_df.drop('company_name_encoded', axis=1, inplace=True)\n test_df.drop('company_name_encoded', axis=1, inplace=True)\n return train_df, test_df, 'salary'\n\n\ndef load_mercari_price_prediction(train_path, test_path):\n train_df = pd.read_csv(train_path, sep='\\t')\n test_df = pd.read_csv(test_path, sep='\\t')\n\n train_cat1 = []\n train_cat2 = []\n train_cat3 = []\n\n test_cat1 = []\n test_cat2 = []\n test_cat3 = []\n\n for ele in train_df['category_name']:\n if isinstance(ele, str):\n categories = ele.split('/', 2)\n train_cat1.append(categories[0])\n train_cat2.append(categories[1])\n train_cat3.append(categories[2])\n else:\n train_cat1.append(None)\n train_cat2.append(None)\n train_cat3.append(None)\n\n\n for ele in test_df['category_name']:\n if isinstance(ele, str):\n categories = ele.split('/', 2)\n test_cat1.append(categories[0])\n test_cat2.append(categories[1])\n test_cat3.append(categories[2])\n else:\n test_cat1.append(None)\n test_cat2.append(None)\n test_cat3.append(None)\n\n # Convert to log(1 + x)\n train_df.loc[:, 'price'] = np.log(train_df['price'] + 1)\n # train_df = train_df.drop('category_name', axis=1)\n train_df['cat1'] = train_cat1\n train_df['cat2'] = train_cat2\n train_df['cat3'] = train_cat3\n\n # test_df = test_df.drop('category_name', axis=1)\n test_df['cat1'] = test_cat1\n test_df['cat2'] = test_cat2\n test_df['cat3'] = test_cat3\n\n label_column = 'price'\n ignore_columns = ['train_id']\n feature_columns = []\n for column in sorted(train_df.columns):\n if column != label_column and column not in ignore_columns:\n feature_columns.append(column)\n train_df = train_df[feature_columns + [label_column]]\n test_df = test_df[feature_columns]\n return train_df, test_df, label_column\n\n\ndef run(args):\n if args.task == 'product_sentiment':\n train_df, test_df, label_column = load_machine_hack_product_sentiment(args.train_file,\n args.test_file)\n elif args.task == 'mercari_price':\n train_df, test_df, label_column = load_mercari_price_prediction(args.train_file,\n args.test_file)\n elif args.task == 'price_of_books':\n train_df, test_df, label_column = load_price_of_books(args.train_file, args.test_file)\n elif args.task == 'data_scientist_salary':\n train_df, test_df, label_column = load_data_scientist_salary(args.train_file, args.test_file)\n else:\n raise NotImplementedError\n\n hyperparameters = get_hyperparameter_config('multimodal')\n if args.preset is not None and args.mode in ['stacking', 'weighted']:\n hyperparameters['AG_TEXT_NN']['presets'] = args.preset\n\n if args.mode == 'stacking':\n predictor = TabularPredictor(label=label_column,\n eval_metric=args.eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=train_df,\n hyperparameters=hyperparameters,\n num_bag_folds=5,\n num_stack_levels=1)\n elif args.mode == 'weighted':\n predictor = TabularPredictor(label=label_column,\n eval_metric=args.eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=train_df,\n hyperparameters=hyperparameters)\n elif args.mode == 'single':\n # When no embedding is used,\n # we will just use MultiModalPredictor that will train a single model internally.\n predictor = MultiModalPredictor(label=label_column,\n eval_metric=args.eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=train_df,\n presets=args.preset,\n seed=args.seed)\n else:\n raise NotImplementedError\n if args.task == 'product_sentiment':\n test_probabilities = predictor.predict_proba(test_df, as_pandas=True, as_multiclass=True)\n test_probabilities.to_csv(os.path.join(args.exp_dir, 'submission.csv'), index=False)\n elif args.task == 'data_scientist_salary':\n predictions = predictor.predict(test_df, as_pandas=False)\n submission = pd.read_excel(args.sample_submission, engine='openpyxl')\n submission.loc[:, label_column] = predictions\n submission.to_excel(os.path.join(args.exp_dir, 'submission.xlsx'))\n elif args.task == 'price_of_books':\n predictions = predictor.predict(test_df, as_pandas=False)\n submission = pd.read_excel(args.sample_submission, engine='openpyxl')\n submission.loc[:, label_column] = np.power(10, predictions) - 1\n submission.to_excel(os.path.join(args.exp_dir, 'submission.xlsx'))\n elif args.task == 'mercari_price':\n test_predictions = predictor.predict(test_df, as_pandas=False)\n submission = pd.read_csv(args.sample_submission)\n submission.loc[:, label_column] = np.exp(test_predictions) - 1\n submission.to_csv(os.path.join(args.exp_dir, 'submission.csv'), index=False)\n else:\n raise NotImplementedError\n\n\nif __name__ == '__main__':\n parser = get_parser()\n args = parser.parse_args()\n run(args)\n"
},
{
"path": "examples/automm/text_prediction/run_glue.sh",
"content": "MODE=${1:-single}\n\nfor TASK in cola sst mrpc sts qqp qnli rte wnli\ndo\n TRAIN_FILE=glue/${TASK}/train.parquet\n DEV_FILE=glue/${TASK}/dev.parquet\n TEST_FILE=glue/${TASK}/test.parquet\n python3 run_text_prediction.py \\\n --do_train \\\n --train_file ${TRAIN_FILE} \\\n --dev_file ${DEV_FILE} \\\n --test_file ${TEST_FILE} \\\n --task ${TASK} \\\n --mode ${MODE}\ndone\n\npython3 run_text_prediction.py \\\n --do_train \\\n --train_file glue/mnli/train.parquet \\\n --dev_file glue/mnli/dev_matched.parquet \\\n --test_file glue/mnli/test_matched.parquet \\\n --task mnli_m \\\n --mode ${MODE}\n\npython3 run_text_prediction.py \\\n --do_train \\\n --train_file glue/mnli/train.parquet \\\n --dev_file glue/mnli/dev_mismatched.parquet \\\n --test_file glue/mnli/test_mismatched.parquet \\\n --task mnli_mm \\\n --mode ${MODE}\n"
},
{
"path": "examples/automm/text_prediction/run_text_prediction.py",
"content": "import os\nimport json\nimport argparse\nimport numpy as np\nimport random\nimport pandas as pd\nimport copy\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.core.utils.loaders import load_pd\nfrom autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config\n\n\nTASKS = \\\n {'cola': (['sentence'], 'label', 'mcc', ['mcc']),\n 'sst': (['sentence'], 'label', 'acc', ['acc']),\n 'mrpc': (['sentence1', 'sentence2'], 'label', 'acc', ['acc', 'f1']),\n 'sts': (['sentence1', 'sentence2'], 'score', 'rmse', ['pearsonr', 'spearmanr']),\n 'qqp': (['sentence1', 'sentence2'], 'label', 'acc', ['acc', 'f1']),\n 'mnli_m': (['sentence1', 'sentence2'], 'label', 'acc', ['acc']),\n 'mnli_mm': (['sentence1', 'sentence2'], 'label', 'acc', ['acc']),\n 'qnli': (['question', 'sentence'], 'label', 'acc', ['acc']),\n 'rte': (['sentence1', 'sentence2'], 'label', 'acc', ['acc']),\n 'wnli': (['sentence1', 'sentence2'], 'label', 'acc', ['acc']),\n 'snli': (['sentence1', 'sentence2'], 'label', 'acc', ['acc'])}\n\n\ndef get_parser():\n parser = argparse.ArgumentParser(description='The Basic Example of AutoML for Text Prediction.')\n parser.add_argument('--train_file', type=str,\n help='The training pandas dataframe.',\n default=None)\n parser.add_argument('--dev_file', type=str,\n help='The validation pandas dataframe',\n default=None)\n parser.add_argument('--test_file', type=str,\n help='The test pandas dataframe',\n default=None)\n parser.add_argument('--seed', type=int,\n help='The seed',\n default=123)\n parser.add_argument('--feature_columns', help='Feature columns', default=None)\n parser.add_argument('--label_columns', help='Label columns', default=None)\n parser.add_argument('--eval_metric', type=str,\n help='The metric used to evaluate the model.',\n default=None)\n parser.add_argument('--all_metrics', type=str,\n help='All metrics that we will report. This will usually '\n 'include the eval_metric',\n default=None)\n parser.add_argument('--task', type=str, default=None)\n parser.add_argument('--do_train', action='store_true',\n help='Whether to train the model')\n parser.add_argument('--do_eval', action='store_true',\n help='Whether to evaluate the model')\n parser.add_argument('--exp_dir', type=str, default=None,\n help='The experiment directory where the model params will be written.')\n parser.add_argument('--mode',\n choices=['stacking', 'weighted', 'single'],\n default='single',\n help='Whether to use a single model or a stack ensemble. '\n 'If it is \"single\", If it is turned on, we will use 5-fold, 1-layer for stacking.')\n parser.add_argument('--preset', type=str,\n help='Pre-registered configurations',\n choices=['medium_quality_faster_train',\n 'high_quality',\n 'best_quality'],\n default=None)\n return parser\n\n\ndef train(args):\n if args.task is not None:\n feature_columns, label_column, eval_metric, all_metrics = TASKS[args.task]\n else:\n raise NotImplementedError\n if args.exp_dir is None:\n args.exp_dir = 'autogluon_text_{}'.format(args.task)\n train_df = load_pd.load(args.train_file)\n dev_df = load_pd.load(args.dev_file)\n test_df = load_pd.load(args.test_file)\n train_df = train_df[feature_columns + [label_column]]\n dev_df = dev_df[feature_columns + [label_column]]\n test_df = test_df[feature_columns]\n if args.task == 'mrpc' or args.task == 'sts':\n # Augmenting the un-ordered set manually.\n train_df_other_part = pd.DataFrame({feature_columns[0]: train_df[feature_columns[1]],\n feature_columns[1]: train_df[feature_columns[0]],\n label_column: train_df[label_column]})\n real_train_df = pd.concat([train_df, train_df_other_part])\n real_dev_df = dev_df\n else:\n real_train_df = train_df\n real_dev_df = dev_df\n\n hyperparameters = get_hyperparameter_config('multimodal')\n if args.preset is not None and args.mode in ['stacking', 'weighted']:\n hyperparameters['AG_TEXT_NN']['presets'] = args.preset\n\n if args.mode == 'stacking':\n predictor = TabularPredictor(label=label_column,\n eval_metric=eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=real_train_df,\n tuning_data=real_dev_df,\n hyperparameters=hyperparameters,\n num_bag_folds=5,\n num_stack_levels=1)\n elif args.mode == 'weighted':\n predictor = TabularPredictor(label=label_column,\n eval_metric=eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=real_train_df,\n tuning_data=real_dev_df,\n hyperparameters=hyperparameters)\n elif args.mode == 'single':\n # When no embedding is used,\n # we will just use MultiModalPredictor that will train a single model internally.\n predictor = MultiModalPredictor(label=label_column,\n eval_metric=eval_metric,\n path=args.exp_dir)\n predictor.fit(train_data=real_train_df,\n tuning_data=real_dev_df,\n presets=args.preset,\n seed=args.seed)\n else:\n raise NotImplementedError\n dev_metric_score = predictor.evaluate(dev_df)\n dev_predictions = predictor.predict(dev_df, as_pandas=True)\n test_predictions = predictor.predict(test_df, as_pandas=True)\n dev_predictions.to_csv(os.path.join(args.exp_dir, 'dev_prediction.csv'))\n test_predictions.to_csv(os.path.join(args.exp_dir, 'test_prediction.csv'))\n with open(os.path.join(args.exp_dir, 'final_model_scores.json'), 'w') as of:\n json.dump({f'valid_{eval_metric}': dev_metric_score}, of)\n\n\ndef predict(args):\n if args.use_tabular:\n predictor = TabularPredictor.load(args.model_dir)\n else:\n predictor = MultiModalPredictor.load(args.model_dir)\n test_prediction = predictor.predict(args.test_file, as_pandas=True)\n if args.exp_dir is None:\n args.exp_dir = '.'\n test_prediction.to_csv(os.path.join(args.exp_dir, 'test_prediction.csv'))\n\n\nif __name__ == '__main__':\n parser = get_parser()\n args = parser.parse_args()\n if args.do_train:\n train(args)\n if args.do_eval:\n predict(args)\n"
},
{
"path": "examples/automm/text_prediction/url_checksums/glue.txt",
"content": "https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/cola.zip 19096246cd2a06d8fe2d13880d6cec61149f77c7 376971\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/sst.zip 44f5954391612a8b3d9d65f6d4a824e9ae8d19ce 7439277\nhttps://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt 716e0f67af962f08220b7e97d229b293077ef41f 1047044\nhttps://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc 506c7a1a5e0dd551ceec2f84070fa1a8c2bc4b41 6222\nhttps://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt 4265196c15cf75620b0b592b8b921f543bda7e6c 441275\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/qqp.zip d775bd543ee78e3f64892a43ada949daf93e003d 41696084\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/sts.zip cc66d8533052de6d7475ac56dfce300751e070a4 802872\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/mnli.zip c22c684daa5cc9fad949d09d10ecedf94a2ce053 312783507\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/snli.zip c60db4cc8820749e6af9f713f4d55109dd46e8c1 129820157\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/qnli.zip 6700cb1d2536bf512314b01350f9ac382439218e 10627589\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/rte.zip 2eb8630df898b7d8df14ca9130c1ac1cf79eb376 697150\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/glue/wnli.zip fc9834b5a8af4e1d8412e48bc38b477510a8c2d0 28999\nhttps://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D c137a2020ab489011dc38fde9ee429f4e2c71257 222257\nhttps://www.dropbox.com/s/ju7d95ifb072q9f/diagnostic-full.tsv?dl=1 2f46c4b80fea8d3ea52a28e05467af3332fa65d9 265530\n"
},
{
"path": "examples/automm/text_prediction/url_checksums/superglue.txt",
"content": "https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/cb.zip c16fa0a46f0f888d59767851c44d8db397896fe5 75482\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/copa.zip ef110b215d7ff95a2fd2d0133f0959d324e9eec3 43986\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/multirc.zip 05bfcb1da7ea06742266f24503342fc20b2ab88a 1116225\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/rte.zip 66105efeccc3fc54f9c5539de4c2d393d5bb4d36 750920\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/wic.zip 5b95487a3690abc718bc173ccd35bf084c43b10a 396213\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/wsc.zip 829ec3dd532284281cc19bacf9cded6c11d3452d 32751\nhttps://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-b.zip 8c8874dcace4942dd00cf9f76c2537ea0e2026eb 33950\nhttps://dl.fbaipublicfiles.com/glue/superglue/data/v2/AX-g.zip 949909079262bc4f6fb66bd889707aa71218975f 10413\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/boolq.zip 90bf152c8012869d326260709404ce5111a76b46 4118001\nhttps://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/datasets/text_classification/glue_superglue/superglue/record.zip af2825be511efa8fbc7813756e768efffb8fcc11 51757880\n"
},
{
"path": "examples/image_regression/demo.py",
"content": "from autogluon.multimodal import MultiModalPredictor\n\n\ndef image_regression():\n\n # Prepare data\n from autogluon.multimodal.utils.misc import shopee_dataset\n download_dir = './ag_automm_tutorial_imgcls'\n train_data_path, test_data_path = shopee_dataset(download_dir)\n print(train_data_path)\n\n # Train\n predictor = MultiModalPredictor(problem_type='regression', label=\"label\")\n predictor.fit(train_data=train_data_path,\n hyperparameters={'optim.max_epochs': 3, 'env.batch_size': 8})\n\n # Evaluation on test dataset\n test_result = predictor.predict(test_data_path)\n print('test_result:')\n print(test_result)\n\n test_evaluation = predictor.evaluate(test_data_path)\n print('Evaluation result:', test_evaluation)\n\n result = predictor.predict(test_data_path.iloc[0]['image'])\n print('Prediction result for single image:')\n print(result)\n\n\nif __name__ == \"__main__\":\n image_regression()\n"
},
{
"path": "examples/tabular/distill/example_distill_binary.py",
"content": "\"\"\" Example: distilling AutoGluon's ensemble-predictor into a single model for binary classification. \"\"\"\n\n# NOTE: Distillation can be done in a similar manner for multiclass classification and regression problems.\n# NOTE: To distill CatBoost models in multiclass classification, you need to first run: pip install catboost-dev\n\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\nsubsample_size = 500\ntime_limit = 60\n\nlabel = 'class' # specifies which column do we want to predict\ntrain_file_path = 'https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv'\ntest_file_path = 'https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv'\n\ntrain_data = TabularDataset(train_file_path)\ntrain_data = train_data.head(subsample_size) # subsample for faster demo\n\ntest_data = TabularDataset(test_file_path)\ntest_data = test_data.head(subsample_size) # subsample for faster run\n\n# Fit model ensemble:\npredictor = TabularPredictor(label).fit(train_data, auto_stack=True, time_limit=time_limit)\n\n# Distill ensemble-predictor into single model:\n\ntime_limit = 60 # set = None to fully train distilled models\n\n# aug_data below is optional, but this could be additional unlabeled data you may have. Here we use the training data for demonstration, but you should only use new data here:\naug_data = TabularDataset(train_file_path)\naug_data = aug_data.head(subsample_size)\n\ndistilled_model_names = predictor.distill(time_limit=time_limit, augment_args={'num_augmented_samples': 100}) # default distillation (time_limit & augment_args are also optional, here set to suboptimal values to ensure quick runtime)\n\n# Other distillation variants demonstrating different usage options:\npredictor.distill(time_limit=time_limit, teacher_preds='soft', augment_method='spunge', augment_args={'size_factor': 1}, verbosity=3, models_name_suffix='spunge')\n\npredictor.distill(time_limit=time_limit, hyperparameters={'GBM': {}}, teacher_preds='soft', augment_method='munge', augment_args={'size_factor': 1, 'max_size': 100}, models_name_suffix='munge')\n\npredictor.distill(augmentation_data=aug_data, time_limit=time_limit, teacher_preds='soft', models_name_suffix='extra') # augmentation with \"extra\" unlabeled data.\n\npredictor.distill(time_limit=time_limit, teacher_preds=None, models_name_suffix='noteacher') # standard training without distillation.\n\n# Compare performance of different models on test data after distillation:\nldr = predictor.leaderboard(test_data)\nmodel_to_deploy = distilled_model_names[0]\n\ny_pred = predictor.predict_proba(test_data, model_to_deploy)\nprint(y_pred.head(5))\n"
},
{
"path": "examples/tabular/example_advanced_tabular.py",
"content": "\"\"\" Example script for predicting columns of tables, demonstrating more advanced usage of fit().\n Note that all settings demonstrated here are just chosen for demonstration purposes (to minimize runtime), and do not represent wise choices to use in practice.\n To maximize predictive accuracy, we recommend you do NOT specify `hyperparameters` or `hyperparameter_tune_kwargs`, and instead specify `TabularPredictor(..., eval_metric=YOUR_METRIC).fit(..., presets='best_quality')`\n\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\n\n# Training time:\ntrain_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame\ntrain_data = train_data.head(100) # subsample for faster demo\nprint(train_data.head())\nlabel = 'class' # specifies which column do we want to predict\nsave_path = 'ag_hpo_models/' # where to save trained models\n\nhyperparameters = {\n 'NN_TORCH': {'num_epochs': 10, 'activation': 'relu', 'dropout_prob': space.Real(0.0, 0.5)},\n 'GBM': {'num_boost_round': 1000, 'learning_rate': space.Real(0.01, 0.1, log=True)},\n 'XGB': {'n_estimators': 1000, 'learning_rate': space.Real(0.01, 0.1, log=True)}\n}\n\npredictor = TabularPredictor(label=label, path=save_path).fit(\n train_data, hyperparameters=hyperparameters, hyperparameter_tune_kwargs='auto', time_limit=60\n)\n\nresults = predictor.fit_summary() # display detailed summary of fit() process\nprint(results)\n\n# Inference time:\ntest_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame\nprint(test_data.head())\n\nperf = predictor.evaluate(test_data) # shorthand way to evaluate our predictor if test-labels are available\n\n# Otherwise we make predictions and can evaluate them later:\ny_pred = predictor.predict_proba(test_data)\nperf = predictor.evaluate_predictions(y_true=test_data[label], y_pred=y_pred, auxiliary_metrics=True)\n"
},
{
"path": "examples/tabular/example_custom_feature_generator.py",
"content": "\"\"\"\nExample script for defining and using FeatureGenerators in AutoGluon Tabular.\nFeatureGenerators act to clean and prepare the data to maximize predictive accuracy in downstream models.\nFeatureGenerators are stateful data preprocessors which take input data (pandas DataFrame) and output transformed data (pandas DataFrame).\nFeatureGenerators are first fit on training data through the .fit_transform() function, and then transform new data through the .transform() function.\nThese generators can do anything from filling NaN values (FillNaFeatureGenerator), dropping duplicate features (DropDuplicatesFeatureGenerator), generating ngram features from text (TextNgramFeatureGenerator), and much more.\nIn AutoGluon's TabularPredictor, the input data is transformed via a FeatureGenerator before entering a machine learning model. Some models use this transformed input directly and others perform further transformations before making predictions.\n\nThis example is intended for advanced users that have a strong understanding of feature engineering and data preparation.\nMost users can get strong performance without specifying custom feature generators due to the generic and powerful default feature generator used by AutoGluon.\nAn advanced user may wish to create a custom feature generator to:\n 1. Experiment with different preprocessing pipelines to improve model quality.\n 2. Have full control over what data is being sent to downstream models.\n 3. Migrate existing pipelines into AutoGluon for ease of use and deployment.\n 4. Contribute new feature generators to AutoGluon.\n\"\"\"\n\n################\n# Loading Data #\n################\n\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\ntrain_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification/train_data.csv') # can be local CSV file as well, returns Pandas DataFrame\ntest_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification/test_data.csv') # another Pandas DataFrame\nlabel = 'class' # specifies which column do we want to predict\nsample_train_data = train_data.head(100) # subsample for faster demo\n\n# Separate features and labels\n# Make sure to not include your label/target column when sending input to the feature generators, or else the label will be transformed as well.\nX = sample_train_data.drop(columns=[label])\ny = sample_train_data[label]\n\nX_test = test_data.drop(columns=[label])\ny_test = test_data[label]\n\nprint(X)\n\n##############################\n# Fitting feature generators #\n##############################\n\nfrom autogluon.features.generators import CategoryFeatureGenerator, IdentityFeatureGenerator\n\n# IdentityFeatureGenerator is a 'do-nothing' feature generator if given default arguments. It will simply pass the data along.\nidentity_feature_generator = IdentityFeatureGenerator()\n\n# fit_transform the generator using the input data. This must be done prior to calling transform.\nX_transform = identity_feature_generator.fit_transform(X=X, verbosity=3) # verbosity=3 to log more information during fit.\n# identity_feature_generator.fit(X=X) # This is identical to fit_transform, just without returning X_identity_out\n\n# Because IdentityFeatureGenerator simply passes the data along, nothing changed.\nassert X_transform.equals(X)\n\nidentity_feature_generator = IdentityFeatureGenerator(features_in=['age', 'workclass']) # Limit the valid input to only 'age' and 'workclass' features.\nX_transform = identity_feature_generator.fit_transform(X=X, verbosity=3)\nprint(X_transform.head(5)) # Now the output only contains the two features we declared in the input arguments to the generator, acting as a feature filter.\n\nfrom autogluon.common.features.types import R_INT\nidentity_feature_generator = IdentityFeatureGenerator(infer_features_in_args={'valid_raw_types': [R_INT]}, verbosity=3) # Limit the valid input to only integer features.\nX_transform = identity_feature_generator.fit_transform(X=X)\nprint(X_transform.head(5)) # Now the output only contains the int type features, acting as a type filter.\n\n# Our data contains object features at present, but this is not valid input to models, so lets convert them to category types.\ncategory_feature_generator = CategoryFeatureGenerator(verbosity=3)\nX_transform = category_feature_generator.fit_transform(X=X)\nprint(X_transform.head(5)) # Note that the int features were automatically filtered out of this output. This is due to the defaults of CategoryFeatureGenerator which does not handle features other than objects and categories.\n\n#####################################\n# Create a custom feature generator #\n#####################################\n\nfrom pandas import DataFrame\nfrom autogluon.features.generators import AbstractFeatureGenerator\n\n\n# Feature generator to add k to all values of integer features.\nclass PlusKFeatureGenerator(AbstractFeatureGenerator):\n def __init__(self, k, **kwargs):\n super().__init__(**kwargs)\n self.k = k\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n # Here we can specify any logic we want to make a stateful feature generator based on the data.\n # Just call _transform since this isn't a stateful feature generator.\n X_out = self._transform(X)\n # return the output and the new special types of the data. For this generator, we don't add any new special types, so just return the input special types\n return X_out, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n # Here we can specify the logic taken to convert input data to output data post-fit. Here we can reference any variables created during fit if the generator is stateful.\n # Because this feature generator is not stateful, we simply add k to all features.\n return X + self.k\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_INT]) # This limits input features to only integers. We can assume that the input to _fit_transform and _transform only contain the data post-applying this filter.\n\n\n################################\n# Fit custom feature generator #\n################################\n\nplus_three_feature_generator = PlusKFeatureGenerator(k=3, verbosity=3)\nX_transform = plus_three_feature_generator.fit_transform(X=X)\nprint(X_transform.head(5))\n\n##################################\n# Multi-stage feature generators #\n##################################\n\nfrom autogluon.features.generators import AsTypeFeatureGenerator, BulkFeatureGenerator, DropUniqueFeatureGenerator, FillNaFeatureGenerator, PipelineFeatureGenerator\n\n# BulkFeatureGenerator is an implementation of AbstractFeatureGenerator that allows for advanced multi-stage feature generation.\nbulk_feature_generator = BulkFeatureGenerator(\n generators=[\n [AsTypeFeatureGenerator()], # Stage 1: Convert feature types to be the same as during fit.\n [FillNaFeatureGenerator()], # Stage 2: Fill NaN values of data\n [ # Stage 3: Add 5 to all int features and convert all object features to category features. Concatenate the outputs of each.\n PlusKFeatureGenerator(k=5),\n CategoryFeatureGenerator(),\n ],\n [DropUniqueFeatureGenerator()], # Stage 4: Drop any features which are always the same value (useless).\n ],\n verbosity=3\n)\nX_transform = bulk_feature_generator.fit_transform(X=X) # Fits each stage of the BulkFeatureGenerator sequentially, with inputs to Stage N+1 coming from the output of Stage N.\nprint(X_transform.head(5))\nX_test_transform = bulk_feature_generator.transform(X=X_test) # Can now transform the test data based on the fit data.\nprint(X_test_transform.head(5))\n\n# PipelineFeatureGenerator is an implementation of BulkFeatureGenerator which automatically handles very common necessary stages without requiring them to be defined by the user.\n# It is recommended that users who wish to create custom feature generators as input to predictor.fit() should base their generators off of PipelineFeatureGenerator as a best practice.\n# The following results in the exact same final functionality as bulk_feature_generator, plus many additional edge case handling functionality:\npipeline_feature_generator = PipelineFeatureGenerator(\n generators=[\n # Stage 1: Convert feature types to be the same as during fit. Does not need to be specified.\n # Stage 2: Fill NaN values of data. Does not need to be specified.\n [ # Stage 3: Add 5 to all int features and convert all object features to category features. Concatenate the outputs of each.\n PlusKFeatureGenerator(k=5),\n CategoryFeatureGenerator(),\n ],\n # Stage 4: Drop any features which are always the same value (useless). Does not need to be specified.\n ],\n verbosity=3\n)\nX_transform = pipeline_feature_generator.fit_transform(X=X)\nprint(X_transform.head(5))\nX_test_transform = pipeline_feature_generator.transform(X=X_test)\nprint(X_test_transform.head(5))\n\n###############################\n# Pre-made feature generators #\n###############################\n\nfrom autogluon.features.generators import AutoMLPipelineFeatureGenerator\n\n# This is the default feature generator of AutoGluon, and contains many stages of preprocessing made to handle many types of data.\n# AutoMLPipelineFeatureGenerator is an implementation of PipelineFeatureGenerator\n# For most users, this should be all they need to get high quality features that are ready to fit models.\nauto_ml_pipeline_feature_generator = AutoMLPipelineFeatureGenerator()\nX_transform = auto_ml_pipeline_feature_generator.fit_transform(X=X)\nprint(X_transform.head(5))\nX_test_transform = auto_ml_pipeline_feature_generator.transform(X=X_test)\nprint(X_test_transform.head(5))\n\n###########################################################\n# Specifying custom feature generator to TabularPredictor #\n###########################################################\n\nexample_models = {'GBM': {}, 'CAT': {}}\nexample_models_2 = {'RF': {}, 'KNN': {}}\n\n# Because auto_ml_pipeline_feature_generator is already fit, it doesn't need to be fit again in predictor. Instead, train_data is just transformed by auto_ml_pipeline_feature_generator.transform(train_data).\n# This allows the feature transformation to be completely independent of the training data, we could have used a completely different data source to fit the generator.\npredictor = TabularPredictor(label='class').fit(train_data, hyperparameters=example_models, feature_generator=auto_ml_pipeline_feature_generator)\nX_test_transform_2 = predictor.transform_features(X_test) # This is the same as calling auto_ml_pipeline_feature_generator.transform(X_test)\nassert(X_test_transform.equals(X_test_transform_2))\n# The feature metadata of the feature generator is also preserved. All downstream models will get this feature metadata information to make decisions on how they use the data.\nassert(predictor.feature_metadata.to_dict() == auto_ml_pipeline_feature_generator.feature_metadata.to_dict())\npredictor.leaderboard(test_data)\n\n# We can train multiple predictors with the same pre-fit feature generator. This can save a lot of time during experimentation if the fitting of the generator is expensive.\npredictor_2 = TabularPredictor(label='class').fit(train_data, hyperparameters=example_models_2, feature_generator=auto_ml_pipeline_feature_generator)\npredictor_2.leaderboard(test_data)\n\n# We can even specify our custom generator too (although it needs to do a bit more to actually improve the scores, in most situations just use AutoMLPipelineFeatureGenerator)\npredictor_3 = TabularPredictor(label='class').fit(train_data, hyperparameters=example_models, feature_generator=plus_three_feature_generator)\npredictor_3.leaderboard(test_data)\n"
},
{
"path": "examples/tabular/example_custom_model_tabular.py",
"content": "\"\"\" Example script for defining and using custom models in AutoGluon Tabular \"\"\"\n\nfrom autogluon.core.utils import infer_problem_type\nfrom autogluon.tabular import TabularDataset, TabularPredictor\nfrom autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config\nfrom autogluon.core.data import LabelCleaner\nfrom autogluon.core.models import AbstractModel\n\n\n#########################\n# Create a custom model #\n#########################\n\n# In this example, we create a custom Naive Bayes model for use in AutoGluon\nclass NaiveBayesModel(AbstractModel):\n # The `_preprocess` method takes the input data and transforms it to the internal representation usable by the model.\n # `_preprocess` is called by `preprocess` and is used during model fit and model inference.\n def _preprocess(self, X, **kwargs):\n # Drop category and object column dtypes, since NaiveBayes can't handle these dtypes.\n cat_columns = X.select_dtypes(['category', 'object']).columns\n X = X.drop(cat_columns, axis=1)\n # Add a fillna call to handle missing values.\n return super()._preprocess(X, **kwargs).fillna(0)\n\n # The `_fit` method takes the input training data (and optionally the validation data) and trains the model.\n def _fit(self, X, y, **kwargs):\n from sklearn.naive_bayes import GaussianNB\n # It is important to call `preprocess(X)` in `_fit` to replicate what will occur during inference.\n X = self.preprocess(X)\n self.model = GaussianNB(**self.params)\n self.model.fit(X, y)\n\n\n# Example of a more optimized implementation that drops the invalid features earlier on to avoid having to make repeated checks.\nclass AdvancedNaiveBayesModel(AbstractModel):\n def _preprocess(self, X, **kwargs):\n # Add a fillna call to handle missing values.\n return super()._preprocess(X, **kwargs).fillna(0)\n\n def _fit(self, X, y, **kwargs):\n from sklearn.naive_bayes import GaussianNB\n X = self.preprocess(X)\n self.model = GaussianNB(**self.params)\n self.model.fit(X, y)\n\n # The `_get_default_auxiliary_params` method defines various model-agnostic parameters such as maximum memory usage and valid input column dtypes.\n # For most users who build custom models, they will only need to specify the valid/invalid dtypes to the model here.\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n # Drop category and object column dtypes, since NaiveBayes can't handle these dtypes.\n ignored_type_group_raw=['category', 'object'],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n################\n# Loading Data #\n################\n\ntrain_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame\ntest_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame\nlabel = 'class' # specifies which column do we want to predict\ntrain_data = train_data.head(1000) # subsample for faster demo\n\n#####################################################\n# Training custom model outside of TabularPredictor #\n#####################################################\n\n# Separate features and labels\nX = train_data.drop(columns=[label])\ny = train_data[label]\n\n# Construct a LabelCleaner to neatly convert labels to float/integers during model training/inference, can also use to inverse_transform back to original.\nproblem_type = infer_problem_type(y=y) # Infer problem type (or else specify directly)\nlabel_cleaner = LabelCleaner.construct(problem_type=problem_type, y=y)\ny_clean = label_cleaner.transform(y)\n\nnaive_bayes_model = NaiveBayesModel()\nnaive_bayes_model.fit(X=X, y=y_clean) # Fit custom model\n\n# To save to disk and load the model, do the following:\n# load_path = naive_bayes_model.path\n# naive_bayes_model.save()\n# del naive_bayes_model\n# naive_bayes_model = NaiveBayesModel.load(path=load_path)\n\n# Prepare test data\nX_test = test_data.drop(columns=[label])\ny_test = test_data[label]\ny_test_clean = label_cleaner.transform(y_test)\n\ny_pred = naive_bayes_model.predict(X_test)\nprint(y_pred)\ny_pred_orig = label_cleaner.inverse_transform(y_pred)\nprint(y_pred_orig)\n\nscore = naive_bayes_model.score(X_test, y_test_clean)\nprint(f'test score ({naive_bayes_model.eval_metric.name}) = {score}')\n\n################################################\n# Training custom model using TabularPredictor #\n################################################\n\ncustom_hyperparameters = {NaiveBayesModel: {}}\n# custom_hyperparameters = {NaiveBayesModel: [{}, {'var_smoothing': 0.00001}, {'var_smoothing': 0.000002}]} # Train 3 NaiveBayes models with different hyperparameters\npredictor = TabularPredictor(label=label).fit(train_data, hyperparameters=custom_hyperparameters) # Train a single default NaiveBayesModel\npredictor.leaderboard(test_data)\n\ny_pred = predictor.predict(test_data)\nprint(y_pred)\n\n#######################################################################\n# Training custom model alongside other models using TabularPredictor #\n#######################################################################\n\n# Now we add the custom model to be trained alongside the default models:\ncustom_hyperparameters.update(get_hyperparameter_config('default'))\npredictor = TabularPredictor(label=label).fit(train_data, hyperparameters=custom_hyperparameters) # Train the default models plus a single default NaiveBayesModel\n# predictor = TabularPredictor(label=label).fit(train_data, hyperparameters=custom_hyperparameters, auto_stack=True) # We can even use the custom model in a multi-layer stack ensemble\npredictor.leaderboard(test_data)\n\ny_pred = predictor.predict(test_data)\nprint(y_pred)\n"
},
{
"path": "examples/tabular/example_mitra.py",
"content": "import pytest\nimport openml\nfrom autogluon.core.data.label_cleaner import LabelCleaner\nfrom autogluon.features import AutoMLPipelineFeatureGenerator\nimport numpy as np\nfrom autogluon.core.models import BaggedEnsembleModel\nfrom autogluon.tabular.models.mitra.mitra_model import MitraModel\nfrom autogluon.tabular.testing import FitHelper\nfrom sklearn.metrics import accuracy_score\nimport time\nimport os\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score, log_loss, accuracy_score, root_mean_squared_error, r2_score, mean_squared_error, mean_absolute_error\n\ndef test_mitra():\n model_hyperparameters = {\"n_estimators\": 1}\n\n try:\n model_cls = MitraModel\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n except ImportError as err:\n pytest.skip(\n f\"Import Error, skipping test... \"\n f\"Ensure you have the proper dependencies installed to run this test:\\n\"\n f\"{err}\"\n )\n\ndef run_bagging(task_id, fold, bagging=True, target_dataset=\"tabrepo10fold\", file_name=None, t=\"classification\"):\n\n print('Task id', task_id, 'Fold', fold)\n\n task = openml.tasks.get_task(task_id, download_splits=False)\n X, y, _, _ = task.get_dataset().get_data(task.target_name)\n train_indices, test_indices = task.get_train_test_split_indices(fold=fold)\n x_train, x_test = X.loc[train_indices], X.loc[test_indices]\n y_train, y_test = y[train_indices], y[test_indices]\n\n n_class = len(np.unique(y_train.values))\n if t == \"classification\":\n problem_type = 'multiclass' if n_class > 2 else 'binary'\n elif t == \"regression\":\n problem_type = \"regression\"\n else:\n raise ValueError(f\"Unsupported task type: {t}\")\n\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=y)\n feature_generator = AutoMLPipelineFeatureGenerator()\n x_train = feature_generator.fit_transform(X=x_train, y=y_train)\n y_train = label_cleaner.transform(y_train)\n x_test = feature_generator.transform(X=x_test)\n y_test = label_cleaner.transform(y_test).values\n\n bagged_custom_model = BaggedEnsembleModel(MitraModel(problem_type=problem_type))\n custom_model = MitraModel(problem_type=problem_type)\n bagged_custom_model.params['fold_fitting_strategy'] = 'sequential_local' \n\n time1 = time.time()\n\n try:\n if bagging:\n bagged_custom_model.fit(X=x_train, y=y_train, k_fold=8, save_space=True) # Perform 8-fold bagging\n else:\n custom_model.fit(X=x_train, y=y_train, time_limit=3600)\n except ValueError:\n return\n \n time2 = time.time()\n\n if bagging:\n out = bagged_custom_model.predict_proba(x_test)\n else:\n out = custom_model.predict_proba(x_test)\n \n if n_class == 2 and (out.ndim == 1 or out.shape[1] == 1):\n out = np.vstack([1 - out[:, 0], out[:, 0]]).T if out.ndim > 1 else np.vstack([1 - out, out]).T\n\n time3 = time.time()\n\n train_time = time2 - time1\n infer_time = time3 - time2\n\n if t == \"classification\":\n accuracy = accuracy_score(y_test, out[:,:n_class].argmax(axis=-1))\n ce = log_loss(y_test, out[:,:n_class], labels=list(range(n_class)))\n if n_class == 2:\n roc = roc_auc_score(y_test, out[:,:2][:, 1])\n else:\n roc = roc_auc_score(y_test, out[:,:n_class], multi_class='ovo', labels=list(range(n_class)))\n\n print(f\"accuracy: {accuracy}, ce: {ce}, roc: {roc}\")\n\n file_path = f'/fsx/results/{target_dataset}/{file_name}.csv'\n \n file_exists = os.path.isfile(file_path)\n df = pd.DataFrame({\n \"roc\": roc,\n \"ce\": ce,\n \"accuracy\": accuracy,\n \"time_train_s\": train_time,\n \"time_infer_s\": infer_time,\n }, index=[f'tabrepo_{task_id}' + f'_fold_{fold}'])\n \n elif t == \"regression\":\n\n mse = mean_squared_error(y_test, out)\n mae = mean_absolute_error(y_test, out)\n rmse = root_mean_squared_error(y_test, out)\n r2 = r2_score(y_test, out)\n\n print(f\"mse: {mse}, mae: {mae}, rmse: {rmse}, r2: {r2}\")\n\n file_path = f'/fsx/results/{target_dataset}/{file_name}.csv'\n file_exists = os.path.isfile(file_path)\n df = pd.DataFrame({\n \"mse\": mse,\n \"mae\": mae,\n \"rmse\": rmse,\n \"r2\": r2,\n \"time_train_s\": train_time,\n \"time_infer_s\": infer_time,\n }, index=[f'tabrepo_{task_id}' + f'_fold_{fold}'])\n\n df.to_csv(file_path, mode='a', index=True, header=not file_exists, float_format='%.4f')\n\n bagged_custom_model.delete_from_disk(silent=True)\n\nif __name__ == \"__main__\":\n\n # test_mitra()\n\n # 8 * 6 + 9 * 2 = 66\n # 0-8, 8-16, 16-24, 24-32, 32-40, 40-48, 48-57, 57-66\n tabrepo = [2, 11, 37, 2073, 2077, 3512, 3549, 3560, 3581, 3583, 3606, 3608, 3616, \\\n 3623, 3664, 3667, 3690, 3702, 3704, 3747, 3749, 3766, 3783, 3793, 3799, \\\n 3800, 3812, 3903, 3913, 3918, 9904, 9905, 9906, 9909, 9915, 9924, 9925, \\\n 9926, 9970, 9971, 9979, 14954, 125920, 125921, 146800, 146818, 146819, \\\n 168757, 168784, 190137, 190146, 359954, 359955, 359956, 359958, 359959, \\\n 359960, 359962, 359963, 361333, 361335, 361336, 361339, 361340, 361341, 361345] \n\n # 3 * 3 + 4 * 5 = 29\n # 0-3, 3-6, 6-9, 9-13, 13-17, 17-21, 21-25, 25-29\n amlb = [2073, 146818, 146820, 168350, 168757, 168784, 168911, 190137, 190146, 190392, \\\n 190410, 190411, 359954, 359955, 359956, 359958, 359959, 359960, 359961, 359962, 359963, \\\n 359964, 359965, 359968, 359969, 359970, 359972, 359974, 359975] \n\n # 9 * 5 + 10 * 3 = 75\n # 0-9, 9-18, 18-27, 27-36, 36-45, 45-55, 55-65, 65-75\n tabzilla = [4, 9, 10, 11, 14, 15, 16, 18, 22, 23, 25, 27, 29, 31, 35, 37, 39, 40, 42, \\\n 47, 48, 50, 53, 54, 59, 2079, 2867, 3512, 3540, 3543, 3549, 3560, 3561, 3602, \\\n 3620, 3647, 3731, 3739, 3748, 3779, 3797, 3902, 3903, 3913, 3917, 3918, 9946, \\\n 9957, 9971, 9978, 9979, 9984, 10089, 10093, 10101, 14954, 14967, 125920, 125921, \\\n 145793, 145799, 145847, 145977, 145984, 146024, 146063, 146065, 146192, 146210, \\\n 146800, 146817, 146818, 146819, 146821, 146822] \n\n tabrepo_reg = [167210,359930,359931,359932,359933,359935,359942,359944,359950,359951]\n\n nature_reg = [167210, 359940, 359948, 359939, 359951, 233215, 359944, 359942, 359945, \\\n 360945, 361235, 361236, 361237, 361617, 361243, 361619, 361621, 361251, 361256, \\\n 361258, 361259, 361622, 359934, 359933, 359950, 359932, 359931, 359930]\n\n test_reg = [363612]\n\n dataset_name, target_dataset, start, end = tabrepo, \"tabrepo10fold\", 0, 66\n\n for did in dataset_name[start:end]:\n\n for fold in range(10):\n\n begin_time = time.time()\n\n run_bagging(task_id=did, fold=fold, bagging=True, target_dataset=target_dataset, file_name=f\"mitra_bagging_ft_save_ckpt\", t=\"classification\") \n\n end_time = time.time()\n\n print(end_time - begin_time)"
},
{
"path": "examples/tabular/example_quantile_regression.py",
"content": "\"\"\" Example script for quantile regression with tabular data, demonstrating simple use-case \"\"\"\nimport numpy as np\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\n# Training time:\ntrain_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame\ntrain_data = train_data.head(1000) # subsample for faster demo\nprint(train_data.head())\n\nlabel = 'age' # which column we want to predict\nsave_path = 'ag_models/' # where to save trained models\nquantile_levels = [0.1, 0.5, 0.9] # which quantiles of numeric label-variable we want to predict\n\npredictor = TabularPredictor(label=label, path=save_path, problem_type='quantile', quantile_levels=quantile_levels)\npredictor.fit(train_data, calibrate=True, num_bag_folds=5) # here we fit with 5-fold bagging and calibrate quantile estimates via conformal method\n\n# Inference time:\ntest_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame\ntest_data = test_data.head(1000) # subsample for faster demo\npredictor = TabularPredictor.load(save_path) # unnecessary here, we just to demonstrate how to load previously-trained predictor from file\ny_pred = predictor.predict(test_data)\nprint(y_pred) # each column contains estimates of a particular quantile-level of the label variable\n\n# Check coverage of prediction intervals (ie. how often they contain the observed Y value):\nnum_quantiles = len(quantile_levels)\ny_pred = y_pred.to_numpy()\ny_target = test_data[label].to_numpy()\nfor i in range(num_quantiles // 2):\n low_idx = i\n high_idx = num_quantiles - i - 1\n low_quantile = quantile_levels[low_idx] # which quantile to use for lower end of prediction interval\n high_quantile = quantile_levels[high_idx] # which quantile to use for upper end of prediction interval\n pred_coverage = np.mean((y_pred[:, low_idx] <= y_target) & (y_pred[:, high_idx] >= y_target))\n target_coverage = high_quantile - low_quantile\n print(\"Desired coverage = {:.2f} => Actual coverage of predicted [quantile {}, quantile {}] intervals over test data = {:.2f}\".format(target_coverage, low_quantile, high_quantile, pred_coverage))\n\n# Evaluate performance of every trained model:\nprint(f\"Quantile-regression evaluated using metric = {predictor.eval_metric}\")\nldr = predictor.leaderboard(test_data)\n"
},
{
"path": "examples/tabular/example_simple_tabular.py",
"content": "\"\"\" Example script for predicting columns of tables, demonstrating simple use-case \"\"\"\n\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\n\n# Training time:\ntrain_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame\ntrain_data = train_data.head(500) # subsample for faster demo\nprint(train_data.head())\nlabel = 'class' # specifies which column do we want to predict\nsave_path = 'ag_models/' # where to save trained models\n\npredictor = TabularPredictor(label=label, path=save_path).fit(train_data)\n# NOTE: Default settings above are intended to ensure reasonable runtime at the cost of accuracy. To maximize predictive accuracy, do this instead:\n# predictor = TabularPredictor(label=label, eval_metric=YOUR_METRIC_NAME, path=save_path).fit(train_data, presets='best_quality')\nresults = predictor.fit_summary(show_plot=True)\n\n# Inference time:\ntest_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame\ny_test = test_data[label]\ntest_data = test_data.drop(labels=[label], axis=1) # delete labels from test data since we wouldn't have them in practice\nprint(test_data.head())\n\npredictor = TabularPredictor.load(save_path) # Unnecessary, we reload predictor just to demonstrate how to load previously-trained predictor from file\ny_pred = predictor.predict(test_data)\nperf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)\n"
},
{
"path": "examples/tabular/interpret/SHAP with AutoGluon-Tabular Census income classification.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Explaining AutoGluon-Tabular Predictions with Kernel SHAP for Binary Classification\\n\",\n \"\\n\",\n \"This example uses a processed version of the adult census income dataset from the UCI machine learning data repository, which is a binary classification task. We train an AutoGluon classifier and then explain each of its predictions via [Shapely values](https://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions.pdf) that quantify how much each feature contributed to a particular AutoGluon-prediction deviating from some \\\"baseline\\\" value. We use the [Kernel SHAP variant](https://shap.readthedocs.io/en/latest/generated/shap.KernelExplainer.html) which is appropriate for explaining arbitrary black-box models like the potentially heterogeneous ensemble of many models that AutoGluon-Tabular uses to make its predictions.\\n\",\n \"\\n\",\n \"You must first install the [SHAP package](https://github.com/slundberg/shap/) (`pip install shap`).\\n\",\n \"\\n\",\n \"**References:** This notebook is derived from a [similar notebook](https://shap.readthedocs.io/en/latest/example_notebooks/kernel_explainer/Census%20income%20classification%20with%20scikit-learn.html) that demonstrates how to use Kernel SHAP with sklearn classification models. For more Kernel SHAP examples, you may refer to [this article](https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a). Note that this notebook only demonstrates data that have already been preprocessed to only contain numerical features; handling of data with categorical features is demonstrated in the notebook: \\\"SHAP with AutoGluon-Tabular\\\", which we recommend trying out first.\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\"\n ],\n \"text/plain\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"from autogluon.tabular import TabularPredictor\\n\",\n \"import pandas as pd\\n\",\n \"import sklearn\\n\",\n \"import shap\\n\",\n \"\\n\",\n \"shap.initjs()\\n\",\n \"\\n\",\n \"import warnings\\n\",\n \"warnings.filterwarnings('ignore')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Load the census data\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"X,y = shap.datasets.adult()\\n\",\n \"X_display,y_display = shap.datasets.adult(display=True)\\n\",\n \"X_train, X_valid, y_train, y_valid = sklearn.model_selection.train_test_split(X, y, test_size=0.2, random_state=7)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Train AutoGluon classifier\\n\",\n \"\\n\",\n \"Here we just train directly on the raw data, without any normalizations. We first format the data in a manner suitable for AutoGluon training (pandas DataFrame):\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"\"\n ]\n },\n \"execution_count\": 9,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ROW_INDEX = 0 # index of an example datapoint\\n\",\n \"single_datapoint = X_train.iloc[[ROW_INDEX]]\\n\",\n \"single_prediction = ag_wrapper.predict_binary_prob(single_datapoint)\\n\",\n \"\\n\",\n \"shap_values_single = explainer.shap_values(single_datapoint, nsamples=NSHAP_SAMPLES)\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values_single, X_display.iloc[ROW_INDEX,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can also plot Kernel SHAP explanations aggregated across many predictions, say over the first `N_VAL` datapoints of the validation data. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"e2ba18823ec846498fd75fe1ad11ec16\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \" 0%| | 0/30 [00:00\\n\",\n \"\"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"shap_values = explainer.shap_values(X_valid.iloc[0:N_VAL,:], nsamples=NSHAP_SAMPLES)\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A summary plot is an even better way to see the relative impact of all features over many datapoints. Features are sorted by the sum of their SHAP value magnitudes across all samples.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAjYAAAGKCAYAAAAfVgumAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAByf0lEQVR4nO3dd3gcxfnA8e/cqVuy5d57xTSDh96LCb0ECL2XQCAkkEAILXQChBZKCC2mBPiF3gnVhA4vxnRjbNx7tyxb5e7m98esrLuzZJ9sne50ej/Pc4+2zM6+ezqt3puZ3TXOOZRSSimlckEo0wEopZRSSjUXTWyUUkoplTM0sVFKKaVUztDERimllFI5QxMbpZRSSuUMTWyUUkoplTM0sVFKKaVUo4wx04wxmyUtE2PM7saYq40xR6VQx5XGmL+lL8p6eS2xE6WUUkrlHufcFZmOIZm22CillFJqgxhjxhpjzg2mOxhjnjHGTDTGvG2MeSSplaa3MebVYP0rxpiSdMSkLTaqIXo76gx76aWXADjooIMyHIlSKguY9NX8y8TzvXu2sX09bYypipsf1kCZK4ClzrkRxphOwBfAM3HrLbANsBz4L3AccP8GRt4oTWyUUkoptT5HOOe+rZsxxkgDZfYAfgvgnFtijHk+af1/nXPLgu0/BQanI1DtilJKKaXaLJP0Sqv4Fp8oaWpc0cRGKaWUUs1hHHAigDGmHDgkE0FoYqOUUkq1Wc3aYnM10M0YMxF4DhD8eJoWpWNslFJKqTZr/cmMc25AA8tsMDkubnElcIxzrsoY0x74ALgvKH9l0vYJ881JExullFKqzWrWcTUdgdeMMWGgCHjcOfdWc+4gFZrYKKWUUmqjOecWAKMzHYeOsVFKKaVUztAWG6WUUqrNSvsl3i1OExullFKqzcq9xEa7opRSSimVM7TFRimllGqzcq/FRhMbpZRSqs3SxEYppZRSOSP3EhsdY6OUUkqpnKEtNkoppVSbpS02SqkWkD+phpLXV1I9dUWmQ1FK5TCHSXjlAk1slMoyy1+fQeeLFtDhnmV8P/ppqmdUZDokpZRqNTSxUSrLLH9lOibmp6NLq1n5wdzMBqSUUq2IjrFRKssUbtKxfiYExVt0zlwwSqkclxvdT/G0xUapLFMTP64mBtU/LstYLEqp3JaLY2y0xUapDJi+3HHlh1EWTVrFwMIYy3qUUJsf5tytQvQvCieUHf/8HEZv25PyviUZilYplbtyI5mJ12pbbKy1u1trIxtZx3HW2q+aK6b17GuctfaybIhFZd6Bz0V564NVLJy0mv/MyePRKSGenOjY9+koy8rrExgH/DxhBc+ePyFjsSqlVGuS0RYba+04YAegFogCU4HrROSpNOxrLBARkdPrlonIv4F/N/e+NkQ2xaKaT03UMW6mo1uJYVS3+m9GPy6BATVRACoKw+AcgxYtJ2YMMmcVmwXlDFBlQlTNrWLV6ijvzzP0LDVs0TXxW1ZVjePLn2ro3jHMoF4b/2cd+34ubsZSQrsOwZQUABD932QwEN5lSGLZH+bhpi8htMtgTLvCjd53Y9z8FUSemYDpUEz4iFGYwvy14566CPfDfEI7DMR0DBJE5+C97yA/DDttkvr+aqPE3p2E6VpKaKu+699gyjyYNAd2GA7l7RLrWlhB7PMZmM17EerbsZEKWkb045+hKkJo96EYk3vf1lVT5d5nIBu6oq4RkWuttXnAucDj1tovRWRypgNTamNEY479nonxzgzfc/3PfUKcsYVvJD1hODy7spBVBWFWFfl/0O2rajjxsx/4trSUoXlhCiNRFpeV8PzoTWlXFub55xwfzfF1/WvfECdt5uuqjTh+c/sSvp8WIWTgqlM6MMYWbXjcT42n5pixEI1hbD8K3z+f2j88S/Se9wEI/34PCm473Jd97itqjnzQl92qD4UfXoApLtjgfTcmNm0x1VtcDxXVAERufYfCT/6Iya/vtot+OIWaMXfB6lrMgM4Ufn4hpkspnHEPPPi2L/Snw+CvJ6x3fy4Wo+bAfxB7YyIA+Xf/irzf7Nr4Bu99B7+4GqprYXAP+Pwm6Fjq65q9jKptboK5K6C0kML3zyc0qs8GvhMbp/byl4lc+zoA4RO3peDhEzMSh8oeuTKuJl42JDYAiEjEWns/cBswCphsrT0UuBwYDMwFrg1aNtZird0LuB4YBkSAt4HzRGSBtfYi4Lig3NHBJh2AE4DLRGRIsK4EuAH4JVAMfBDUMSNYPw74AhgA7AMsAC4QkReC9VsBdwKb41ugJgIHiMjSYJ8drbXPNLLtyUmxjAMmAEOA3YHpwB9F5LUmvK0qg6Yuh3dmOMB3KT30TWxNYrN/WQ2fLF3NT93L1pSf0LcbR33xI11Wr+aaY8awyewFLGlfQmVxIQtNmG/msKauf30bW5PYTJkT4ftpvlc25uCVT1ZvVGITGfspRP315k5m4L6eTfSBj9esjz74MQSJTWTsJ/Vlv5xF7MtZhHcctMH7bkzsxW/WJDUAbvxM3KQFmE171sf1uMDqWr9+2mKi70wi74hR8K936yt68O3UEpuZy9YkNQCRhz5ed2Lz2Hs+qQHfcvPed3Dodj6uV7/zSQ3Aymqi/xmfscQm8sBHa6ajj36Ou/9YTEHW/BtQGZF7iU3WjLGx1hYAZwezk6y1Y4AHgd8DnYCTgLustY2dXarxLT5d8YlFL+AOABG5Cd/N87CIlAavaAN13AZsH7z6A4uAl6y18aM5TwJuwSdGdwEPBwkRwN3AG0G83YELgJoUt23IacExlOOTtuestQPWUb5ZVFRU6HQzTHdvB52L1yxiZGezpsyAHnmUVUcoqakfJtaxsoqCaIzlRYUsKy1hSXk7tvh5DjhHqYnRPt+tKTukff12JfmrKSmqPzkN6JG3UfGHRvaoD7pdAZWdCjBxy+qmKyoqEsuWFBDq3ykt72fVgA4kKCnA9OqQUKZmUH0XjwuHCA3rBqEQ0aH1MUaG1ydC69qv6VoKXUvXLAuN7LnuOEfGdVXlhWFofXmzSQ9cXJdP9cDyJh17c07H/77MkC6Ygrys+XvR6canVdMY59z6S6VJ0CqxHT4pKcOPtTlHRB601r4MfCYiV8eVvxMoFpHTrbW7A2+JSINfN6y1BwIPiUi3YH4sSWNs4ltJrLUhYBVwkIi8GawvBZYAu4nIx0G834nIOcH6dsBKYJSIfBWsn4xvWZrWwLGua9s1scSVnykiJ8TV8QHwqohcn9o7vMEy96HIMRMWOG7/IkaPdnD59iHaFdT/g3vj89W89XUN3xUU0anUsNePMyiprOF/s0KYOSv5xfiJ9FhUwee/343tzh5CrH0+d46P0acMLts+RHF+fV3f/FzDc++vpkenMCfv246C/A3/FuZqIkSu+y9u6mLCv96J8E6DcbOWUnvt6xAKkX/5vpieHerL3vAGbsoiwmfsuNb4m+ZU+9BHRG97F8oKKbjjSELb9E+M2zmit79LbPxMwkduRfjgLfyK6QvgumegIA8uPxK6l6e0v9jXs4nc+g6mWxl5V+yHKV3H+KFYDG55Eb6eDsfsDPuPTlgd+c94Yi9+Q2jHgetu+Ukzt7CC2qtfg6oIeZfsQ2hgl4zFopokbc0qEXNawvk+zz3Y6ptwsqEN8rpgjE1HfAvNHsHPgcAe1toL4sqGgfcbqsRaOxrfqrElUIL/IJQ2VLYRXYFC/ABmAERkpbV2AdAXqGuLnxu3vtJaCz4pAzgF33X2gbW2FngMuEpEIils25BpDcxnpg1bbZBR3Qxj9ws3uG6fbYrZZ5u4Jh18UtDrpPeZM3EeJVW+a+PoXYroOMyPW3l4/4br2nxQAZsPap6xLaYgj/yrDkhc1qcjBfce03DZv+zfLPtdn/xTdyT/1B0bXW+MIe/8Pdde0b8b3Hf22svXI7RFbwrGrr/byhcOwYWHNro671dbw6+2bnIMzc10LaPgzl9lOgyVVVp9HrOWbEhsABCRpdba04Ep1tpD8GNKxorIzSlW8STwNHCkiKwIWmxeilsfW8/2C/EtRwPwrS51LTbdgJkpHsNU4NRg283x3VJTgYdSPIZkAxqYf3UD61KtwDcvzObzSTXQqxNzu5eze68Y5QcPyHRYSqkcpYOH00xEllhrb8W3vJwPjLXWfgJ8hG+t2RwwIiINbN4eWA5UWGv7ARcnrZ8HbG+tDYnIWkmOiMSstY8A11hrvweW4cfDTAQ+SyV+a+1JwJsiMifYPoIfRLyhDg0GRY8DfgVY/IBnlaPmfVd/1+FYOETxuZtj8rJmKJxSSmW9bDxj3gH0xA/+PQO4GT+Idy5+cG9j3UtnAqcDFcCzQPK9cB4A2gGLrbXLkgYE1zkfEOBzYEYQx8GNDDRuyJ7AF9baSnzX1ePAoylu25AH8QOQlwNXAIcHrUIqRw3apQt140zzayKsvPADohU1695IKaU2mEl6tX4ZHTysGhcMHn5LRK7NwO71Q5FBE07/H7NenEnHZasorI0w6Im96XT00EyHpZTKnLRlHDXm1wnn+wL3z1af3WRVV5RSCnoMLyOysL5LKr9PU8bAK6VUU7T6PGYt2dgVpVSb1u33W7DykFKqNyug3927ULZzz/VvpJRSCtAWm6wlIrtnOgaVGaH8MBWnlQPQ7aDN1l1YKaU2Qi5eFaUtNkoppZTKGdpio5RSSrVR2mKjlFJKKZXFtMVGKaWUarNyr8VGExullFKqjcrFrihNbJRSSqk2K/cSGx1jo5RSSqmcoS02SimlVBuVi11R2mKjlFJKqZyhLTZKZRkXc0ReriE2PcaM2mn0++WAxss6x9K/fsqit2cxoUdvVv5iOEcfUU5JcQi+mQ7XPgUFeRAOwcoquOgw2FYfqKmU8nKxxUYTG6WyzOSHfiLyQi0An57zCaUDyui0decGy664/2uWXPI+IWBrpnLf4nzGVg/gN6d0hH2ugnnLEjd451uYeR+0K0rvQSilVIZoV5RSWWbl1Ir6GQcrp69stGztT0sT5jtVrGT+gohvnUlOagCWroTFFWsvV0q1USbp1fppYtNCrLX3WmvvynQcKvsNOGogFPvp0kFl9Ni9R6Nly07clFB5IQCLS9vxU7+e7LNnGXRoByfuXl/QBCesQ7eFvl3SFLlSqrVxmIRXLjDOuUzH0CBr7Q7AX4Ad8F1mE4G/i8jDGQ0sBdbaacBlIvJYpmPZQNn5oWhDXnzsBdx8x/6/PoD80vx1lo3Mr6T6hyXM7NCB8l7F9OgelHcOPvsJSosgL+xbarYfBiH9PqNUK5O2jKPCXJBwvi9zt7b67CYrx9hYa/cBXgRuAI4DVgMHAv+01g4Skb9kMj6l1icac5zyeoznJzt26GV4+uAQhbVRXrl4Ajeb7nzVtxs79AvzzKFhOhT688jK+VW8cuGXrJhcwYAZyyieV8Nnf/J5fNGgMjZ5bgwlm3Zaa195H3xF+JjbGVobo5ZConkRlh+wPY8zkuPfeoKS4hDhYQOJfbMQs+dwwk+eiSlqOFl67qcYZ78ZIz8MD+8XYs9+Iaru/pjVl7+J6VyC6VBE9Jt5AIQ26UrZuZsTvvwxnyz961zYZ5Sv6L9fwql3QywG9/8GDrTN/h6vqIhyy50L+XlqNRhDabsQ55zRmUWLozz2f0spLjKcMe1z+rzxFfl7D6H0iaMxhVl5ylNKNaOsbLGx1v4EfCAipyQtPxl4ABgCTAfOAH4L9AeWAzeKyF1B2V8ClwRlq4AHReTSoI7LRGRIXL1jgYiInG6tHQBMDeq+COgOjAPOEJEFQfnfAWcDvYGlwL+DOqPW2peAA4AaIAJ8JCL7xO8jqKM/8HdgJ3zi9gzwZxFZHax3wDnAKcAI4DvgZBGZuBFvbaqy70PRyjw5McYxL8fWzF+7c4h9vp/GvS8u46Gdt1iz/C87GK7cKQzA21d/y/cvzqbX7GX0mLdirTrL9+nNpv/df+2dlR8Py1ettbgyv4h2tVWAb26O4Acgh+48mvC5e65V3jlHhzujVNT4+QHtYcohq1nW6waINfyR6FgwFVMTbNCrE8x+IJg+DeYG4386lcLiRxrcfmM8+uRSXn498X3q0jnM0mVRolE/33/RAi5+5VkASv5xKEVnbdfscSjVArTFpgmyrk3aWjsMn4w01I3zOP4XPAY4C7gSn2CUA1sBnwZ17Ac8HKzvAgwDXmtiKCcCuwJ9gVhSPLOA/YD2wCHAqcDpACJyEDADOF1ESkVknwaOMQ94BZiHT8q2xyc4f0sqejJweHAMM4E7m3gMG6SiokKnN3K6clUV8aIxIOZwJvGcEXX12675ktFIWhmpiTS4r1gjSYdp5EtLdeXqRuJfmZC/RF0Qy7q+/MSvi8bq64yvKJhu7ve5oS9lsVhiSLH49zsay4rPhk7rdFOn0ykXx9hkXWIDdA1+zk5eISI1wCKgG76l5joR+UBEYiKySEQ+D4r+FrhXRF4WkYiIrBCRD5oYx1UiMk9EVgAXAmOstb2COJ4Rkaki4kTkS+BRYK8m1L0tMBS4QEQqRWQ2cBlwqrU2/pN1s4jMEJFqYCzQ/O35DSgrK9PpjZw+cctiDh9qyAvBzr3hvK0NWxzdj3271LL1zPmEnWOHnnD+6NCabbc9YwidB5eysE97VnfNw8VdqFDQp4TBt+3U4L5C95yJC4eIATUU4oxhyZjt+Nfex1JZUEyktITY1iMhL4QZswnFv927wXraty/jgV+EKCuATkXwzzEhQj3KKL55fyjKw/RuT2jTbhAyEDKYoZ2J/e0U6FACHUvh/rPr67z/bL+sQwk88Ju0vM8H79+BQQMLMAbCYShtF+LMUzpz6vGdKCwwdOwQ4lg3HfJC5O87jMJTbVZ8NnRap5s6nV65d1VUNnY4Lwx+9sYPGF7DWluAb71YCAwAJjVSxwDguY2MY1oD032AOdbaY4ALgEH497AA+KQJdfcFFopIZdyyKUARPrFbECybG7e+EmipT7raSPlhw9OHhJOX8qv7t+VXjWzTvlcxx/6fT15efOJFKhfFOPD0AwkXr+fP9PjdMMfvhsF/EAE64fsx4aA1xZKjacjRI0IcPSLx+07xH3ah+A+7NL7RefuuveygbWBJ83c/xSvvEOaGv/RscN2YPev+VPriGz2VUm1FNrbY/AT8DBzbwLqj8Y3jb+KTjcZuobqudRVAu6RlvRooN6CB6VnW2r74bqlrgZ4i0gG4m8RUN8a6zQS6WmtL4pYNwo8FWtjwJqqtWP7DMqovW0XNdVW8td+b1K6oyXRISqkclYtdUVnXYiMizlp7LvC8tXYqcA9+cO0BwO34AcJTrbV3A5dYa7/Ej63pBAwMuqPuBp601r6LT4JKgC2C7qgJQDdr7YHAq/gxMruy9piey6213wb7vhF4S0TmWGs3wSeEC4Faa+32wAnAD3HbzqPxxArgM2AycIu19g/4MULXAP8SER2428b9/O+fffscsGLicua+M5d+h/bPbFBKqZyUK8lMvGxssUFEXsOPWdkV3/qyCLgU+KOIXBoUuwd/OfiDwApgPLBNsP0rwGnA9cAS4EfgF8G6KcDvgPuCdfvir0hK9hjwPr51pQCfvCAiP+Dvr/MCsAy4GHgiadtrgeOttUuttWsNWhaRCP7y9T74gcaf4ZOzP6by/qjc1q5PScJ8Se/kBkallGouuTfGJisv986kuMu9+4rIrAyHkyn6ocigWCTG82c8h5seY6vTtmbQcYMzHZJSKrPSlnEsM39KON+XuxtbfXaTdV1RSrV1obwQ+b/0w4AHHaRJjVIqfXLxW6wmNkoppVQblYtjbDSxSSIi08iVjkallFJqnXLv311WDh5WSimllNoQ2mKjlFJKtVHaFaWUUkqpnJGLiY12RSmllFIqZ2hio5RSSqmcoV1RSimlVBuVi11RmtgolYViqx3RJRCpjpJXmMpzuZVSqulyMbHRriilsszSSStYfF2EpX+L8NLh46ipqM10SEqpnJV7z4rSxEapLDPx/6bigqd7L5m4nJnvzctsQEop1YpoV5RSWcbFEp/ekl+if6ZKqfTQZ0UppdIv6UxTWxnJTBxKqZyXi2NsNLFRKsuYpA7i/FL9M1VKpUcuJjZpHWNjrY1Ya3dP5z7Ws//vrLVHZWr/SgEsW1jD20/O4/M3FhOLrb/h1yUVqVmhg4cb46YuJHb1i8Qe/hCX/MYppdqkJn0VtNaOA3YAks+0O4jIN80VVFNZawcAU4G+IjKrbrmIbJrm/e4OvAu8KyJ7xi0/HrhWRAakc/8q+9VUx7j34p9YttD/ySyeW82+J/Va5zbGJH6DKuyQn7b4WjO3fBWxnW6Aucv9ghlLMJcflNmglGp1cq/FZkPauK8RkWubPZLWKwaMstYeKCIvZzoYlV5LVjtueK8W9+kCNi2MkEeMisUR+g4rYY8juzOlJo9/fBWjb5nhwm0Mi+ZUrUlqAL7/fAXd9unBPz6NsGpWFZt1hq3yqlg1v5pNd+zAkrnVTJjmiJQUUbKqCoCqxdXMumY8NXMq6f6bTWm3eacWO163bBXuhldgdS0cvjX8R6BzKebi/TAlhQ1vVBuBW16EaQvgjDEwenBK+4o9+D58+jPm0K0w+2+x/g0mL6hPagD37kSIS2zcBz/hHv0YRvTA/G5vTCjNF4H++z1473v4xSg4fIf07kupZpKLXVHN1nlvrS0D7gIOAiqAK5LWXwnsLCJ7xy0bB7xVlyhZa7cAbgJGA2FgfF15a+2/gL2BcmAmvkXk8aCqr4KfP1prHXCjiFxjrZ0GXCYijwV17BbUPwKYC9wmIv8M1u0OvAUcB1wPdAH+C5wmIhXrOHQHXAvcZK19TUSiDbw3CccZLHPALiLyQfDe7AIIcCq+i/A64BngX8A2wCTgeBH5YR2xqDT71UsxOr0yg0FLV/Jj3PIZP65i6o+ruGrAIJZUATgWrTbs8v3ihO0XzqrisHsryFsRoWN1hIqVq1i+zP9z/vzNxWu6oUy/nvT7eRYFNbUs+8/PLPrPZAAWP/UzW005hrwOBek/WCB23H3watAY+493IRLz09MXYx4+reGNLn8CbnzOTz/5Afx0N3TtsO79PPoR7vSxALiHPiD0yaUYO2Cd27jBXSEvVB/T0sr6ddMWEdvnVlhd4xdURzAX77/O+jbKi5/B8Xf46fvfhHeugj02T9/+lGomudiB25xfYW4HhgIjgS2AQ/DJSUqstT2B94LXAKAH8Ne4Ih8Ao/CJzdXAWGvtyGDdlsHP4SJSKiLXNFD/QOB14B9AZ+Bk4AZr7ZFxxcLAPkF9w4CtgPNSCP8uoAg4I4WyjdkV+Al/3McDNwMPAucAnYAfgL9vRP0pq6io0OlGpicsdHSrrKIh86etDpIa76sFMH96YlkXg8IVNRQH/4zLautbc+KHiLiQoabQd0FVTlmxZnlkcTU1sytb7tgnzKwPqi6BANxXMxvdNiKT67dZvsq33KxvX/H7icZw385ab2ymJpoQU2xV9ZrpVROm1ic1AF/NTO14N3R6wjQSfD09459Vnc6dadU0G5LYXGqtXZb0CuFbOi4XkXkishz4UxPrPQGYLCI3iEiliNSIyFt1K0XkQRFZLCJREXkS+BrYvQn1H4NvARorIhER+QT4J3B6UrmLRWSliMwHngfs+ioWkRrgz8CV1trSJsQUb5KIPBAc32vAYuC/IvKDiNQCj6cSS3MoKyvT6Uamj9/E8F23chqy1e4dGd29fv64kYatdu+YUKZDl3zKB5WwpMgnLfNLinBBS3BJWZhQ8FUgXBuheFUVqwrz6XbYwDXbl27blaKhHVrseM3x29cH37V+vTl++0a3zTtpD6gbJ7TlANis33r3ZY4YDcF7Qvf2mDGbrj/OrmWwT/0wuvCJO62ZLtljUxjSLVgRwhy97Ua9D+udPnRbKC3yC8rbwQGjM/5Z1encmU4nh0l45YIN6Yq6LnmMjbW2O1AITItbPLWJ9Q7Ad7esJUicrgSOwrdoOKAd0LUJ9fdtIKYp+JalOlERWRg3XwmUBTFcAlwSLJ+ePDBZRP7PWns+PqGL76VI1dyk+VVJy1bVxaIy5/Y9w7w1uBfzf2jPFu2jxGKOisURuvUtYuhWZexXC29Md/QpNWzT08DmXeg1qJgZP1VS2j6foVuV8duiMG9OibF6WRG928GwDl1ZNLuagSPbUbkiyl/vXUDRe1NZmZ/P5J5dGXPCSDbbuyc1cyop/0VfQvktd8Pw0I1H4n6xGayqwe05AvPm99ClFLPT0MY3OmF3GNkXpi+EMVtCcSNjceKYHYYQ+voq+Hom7DQU02PdXVfgB1mHXj4PXvsWyosxuw6vX9ehhNBnl8E7P8DQ7pgt+qZyuBtuiwHw9W0w/mfYdij07ZLe/SnVbHIjmYnXXGNsFgE1+ORkSrBsQFKZCnwyEi/+8pBpwBGN1H8MvmVlH+B7EYlZa4X630iske3izQSSO9kHBcvXS0Sux4+9WZc/AG/gu8riJRy7tXbdl8WorLZ3/xD0b9/gunYFcNjQxBNFvxHt6Dci8aN/2CZh4ntqe/YvBqC0PJ/d8ipYPN+PzRk9dRbtZgygbOduzXgETWP23MT/BDhkq9Q2Gj045UHDa/YztDsM7b7+gvHb5OfBwaMaXtexHRzeIo2c3sDu/qVUK5IrrTTxmiWxEZGotfZx4Cpr7bfAahLHxwB8AVxvrR2NH+x7FjAwbv1j+G6uPwF3AhFg16A7qn0wvxAIWWtPxo+DqbsKaSE+uRkKzKJhTwCXW2tPxHfrbA38Gjh7Q487mYh8aK19HfgjvrWnzhfAUdbaW4Eq/MBgpRrUlVrihxzXLKtutKxSSqlEG9Kmfbm1dmXS60Dgd/iunonAN8BLwJorhERkHHArfgDvXKA78GHc+jn4MTNj8MnJPODCYPXDwKfAZGA2foDy+3HbrgYuB54Ixvxcmhy0iEzFt9icix+/8ih+TNB/NuA9WJc/Acnt6LfhB/9OASYArzTzPlUOSb7zcLgg5TH4SinVJC7plQuM3q1TNUA/FBn04V++ZOK/64eD7fH3bRm0f58MRqSUyrC09RdNN9cnnO/7u0tafd+UPoRGqazT6s8rSqlWIhfH2LTc5RVKqZQUlSc+QiGvWLuilFIqVZrYKJVlNj99GAWbGEwpjDxhEH1375HpkJRSOcskvVo/7YpSKssUlOVTfqb/09zhoFGZDUYpldNycUClJjZKKaVUG6VjbJRSSimlspi22CillFJtVC622Ghio5RSSrVRmtgopZRSKmfk4uBhHWOjlFJKqZyhLTZKKaVUm6VdUUoppZTKEbk4xka7opRKg1gstZ5rF4utZ33D9dQtb2y9UkqlwmESXrlAW2yUakaR2hhPXj+VH2U5fYa148QrB1NctvafWWzBSpbt9yiRCXMpPGJT2j9+JCZc/z3DRRzLHo7x4IXP021UJ35x3w4Uti8AYP6tE5jz54/BGGqqod323Rnx6v7kdSxsseNUSqlspS02SjWjb95fysTPluNiMHNiJZ+8vLDBcqtu+ZDI+DkQc1T/51tqXpyYsL5qgqP6e4eLwfzxS/jusZ8BiCxazew/foirieGqo4SJsvKTBcy789u0H5tSKve4pFcu0BYbpZpROJzYlBsKN9K0mx9e93zSV45QXrAgbCBkIJp4CjL5+h1FKdV0udL9FE/Phko1o8127shWe3WiuCzM8G3as8PBXRssV/LHncgfMxjTqZjis7ah4IBhCeuLRhmKrKGwPJ9+e/Rg0+MHAZDXsYh+9+1OXrdiQh0KcGVFlB/Qjx6/3Sztx6aUUq2BttikmbX2UuBa4GQReTjT8WTa8irHF3NijOgSolf7jf+msLDS8c28GFv0CNGlXWr1TVoUY+5Kx459Q+QntahULKllwcwqeg0qprgsj6qI45M5jr5lhsEdDT8vdUxf7uhfDtNWwJBymLwMRnQydDIxvv16JW7H7owa04vR/fIoKGr4u0OovBj30FGsXlhNx03bY0xiHKtX5cPOhmNO6UO0EpY/+zPhAR0oH5BHlwFRujy/I4t+XEmkX08KMURXRwiX5tdX8ONsmLcMdhwO+XlE51dS891CCkZ1J9ypeK14XDRG9UezCXUqomDThpOxVLnaKLGPfsb07EBoWLeNqksplV652GKjiU0aWWtDwBnAEuBMoE0nNgsrHdveV820ZY72hTDulEK26rnhjYZTFsfY4Z+rWVgJ3drBp2cXM6Djuut78psIxz9bSzQGewwM8cYJBeQFyc28aat54KJJVFVG6dA1n1NvHs7+r4f4bK4jPwSXbGu4/v0otQ5CRSFiGPIMRByU5TtO/H4aBXMr+U+fnqzMz6OsyPDaee3ZrPfaf2aT/reIF6/8gVjU0Wfz9hx1+xaEg+6mH76q5OPnenPeOw+Tv3gWYQyrGMZy2tE+70vCkRqqQwWM63oAnRZ+QzjmyO9axFafHEjxoPbwxPtwwh0QjcGem1Nz+2+Yt9vjxJZWEe5dRs9PTyKvd9maWJxzLDj8WVa/8BMY6PyPfSn79VYb9DtxkSjVv7iH2Ls/QThEwWMnkHf06A2qSymVfrkyriaedkWl1y+A3sCJwI7W2jX9BdbaYdba96y1K6y1X1lrf2etdXHr86y1l1hrJ1lrl1lrP7TW2gwcQ7N59aco05b5Q1xRDf/+OrpR9T31bYSFlX56QSU889366/unRIkGV1i/OzXGxEX1f9Zfj1tCVaWvY/nCWt58ezmfzfXra2Pwz/FRamNAniEWfMuJBJtX1Bo+LyplWkkJK/N9IlNR5Xj6i+oG4/jqpbnEgnEys75ZwcIplWvWffjOMnovmc+gxbMACOHoyAJ6MotwpAaAwlgN3SqWEQ4u965dWMXCp6cFB/kGaw7ynW9YffdHxJZWARCdXcGqF39KiCU6u8InNQAOKu79cr3vY2PcD/N9UgMQjRG576MNrksplX65eLm3JjbpdSbwmoi8AnwN/Bp80gK8BHwFdAcOw7fsxLsKOATYF+gMPAS8bq3tmO6gKyoq0jI9tJMhvsdlWGezUXX2bZeYNAzrbNa7bV0ZgLJC6FlWH0Pn3kUJ9fXoEaMorrGlX4dg24Rbz9QnRp1WVVNeW5tQx+Bu4Qbj6dinvjsovyiEKa7frrwzLC3uQE24vmupmiJWU5Kw16r8goR9lQzr4Osf1qt+YVkx4S17J5SL9Kk/zoqKCkKdiwl1ro8nb1inDf69mF4dcGX1l51HBpZvUD06rdM6XT+tmsY4l4sNUZlnre0FTAeOFJHnrbXn4ZOVXsBo4G2gXERWB+VPAx4QEWOtNcAK4AAR+V9cnd8AN4rIY2kOP20fise/jvDcD1G27R3ijzvlrTW2pKnu+aSWd36OMmZImF9vm7/e8pU1jiveiTBrheO87cPs1C/xaqQPn5vPjB8qGbFdB7baqzNvT4/xzwkxBpcb/ryd4a8fxfhpsaO8LMSyGuhRAvNWge1u+MWqZbz48hI+zGtHVbcSjtiqgHP3KGrwGGuro3z40HSWz69mq0N70m9U+Zp1kYjjjhs+ptu3czl47iTM4iizavoSGdGT4UMWUvjzDKqW1/LFyn7E8svp0KmELgf2pdfZmwQHWQVXPAGzFsPvDoQdR7D89s+o/mAWxQcMoeyULdaKp/rLeay46RNCXUoov2ZXwuVFa5VJVfTDn4n8/T1Mn3Lyr94f007vr6PURkpbU8o35u8J5/vN3XmtvtlGE5s0sdZeDpwL9BaRiLW2EzAHOAtYDfxNRPrGld8beDNIbLoCC/DJTfwvKB+4RkT+mubw9UORYS+99BIABx100FrrVr/wI7Wfz6booGEUbNenpUNTSrW8tCUbXyclNlvkQGKjg4fTIBg0fBpQDsyKGxoTxndHXQh0tdYW17XYAP3iqlgEVAJ7i8jnLRK0ahVWP/MDS454CoCKv31Mt/Fnkj9y465iUkq1XbkyriaejrFJj32BvsCOwKi414HA9sByYAZwg7W2yFo7EPh93cYi4oA7gL9Za4cCWGtLrbW/CLq4VBtV/cGMuJkoNZ/PyVwwSimVhbTFJj1+DTwvIl8kLZ9nrf04WH8wcB+wEPgZeBR/v5s6fwHOA16w1vbBt+B8Avw2zbGrLFa03xAq//4ZxBymfSGFu/Rb/0ZKKdWIXGyx0TE2WcJa+2vgDyIybL2F008/FBm2rjE2NZ/Oomb8PIr2Hkje0M4tHZpSquWlLfv40tyVcL7fyp3b6jMdbbHJEGvtzsBcfGvN5sBFQLqvdlI5oGC7PjpoWCnVLHKxxUYTm8zpCzwOdMF3Rz0F3JDRiJRSSqlWThObDBGRJ4AnMh2HUkqptkxbbJRSSimVI7QrSimllFI5IxevFNH72CillFIqZ2iLjVJKKdVGaVeUUkoppXJGLnZFaWKjVJZy1bl4ylFKZZNYDrbY6BgbpbJM5fzVLL6hloUXR3jjjI+I1cYyHZJSSrUamtgolWW+fegnogv89Mx35zH97bmZDUgplbMcJuGVC7QrSqkss/SnioT5VfNWZSgSpVSuy8UOb01slMoyofzEb00mTxtWlVLpkSutNPH0jKlUlgmZpMQmlHsnHqWUShdtsVEqy0Sqownz0aR5pZRqLtpikwWstcdZa7/ayDqutNa+1VwxqVZu1iJ4cwIsXdnw+qoaePtrmDSnRcIxa51ncu/Eo5TKDi7plQuavcXGWjsO2A04SkT+E7d8O+ATYLqIDNjQ+kXk38C/4+odC0RE5PQNrTOZtbYr8FdgX6ADUAFMAE4VkbnW2t2Bt0Qk5ffPWjsAmAr0FZFZzRWr2kjjp8Bul8PKKujfFT67EbqV16+vrvXrP/sJ8sLwnz/AYdunNaRQQThhPlygiY1SKj20xSZ1PwBnJC07I1i+way1+RuzfRM8BpQBW4lIKbAl8AS5k9CqQOTR//mkBmD6QiKvfgn/+RCKj4Lw4XDoX7mxZCRd/3I/JVeNpWTiFvS9YgEvbHEDdD2Zl2/6kKGXzOa9QZcSyTsS9r8GdrgYepwKNz23Zj9zVsboencE87cIebdEOPvNKM6l9nFyuXfeUUqptEnXGJtngbOstYNE5GdrbRlwOHA9cE5dIWvt0cCfgYFAJfAicIGIVAbrpwEPAXsA2wCnW2uLgMtEZIi19iLguLi6wLewbAb8HdgUCONbis4VkSkpxr8jvsVpAUDw85FgP72A14Cwtbau7+IcEXnYWvsvYG+gHJgJXCsijwdl6rrPfrTWOuBGEbkmmN5FRD4I6t+duNag4Lj+AvQBVgGvi8hJKR6HWo/3S3uxRzAdM4ZX8npyyMlXQlUtAN+PX8DFF/45YZtZ+Z047qjfsuLykxh9zQMcveMv2G1qkLO/9mV9wT89CgdvAyP6cMIrMRat9oujDu79yrH/IMdBg9fOWmI1yWNs9AZ9Sqn00Bab1FXhu4tOC+aPAd4Dku80thw4Fp8I7BK8LksqcwZwAb4F5YX4FSJyU7Cfh0WkNHhF8S0rVwK9gQHASnwrTKr+B9xsrT3TWruVtXZN34CIzAH2A6Jx+3w4WP0BMCo4nquBsdbakcG6LYOfw4NtrllfENbaEuBRfOJUBgwCHmjCcWyQioqKNjM94eC9ueDAE3lyyx055tjfMXlwP4jWJxI1eQ3n/rXhMDFjKIzWUhiNNFgGgOpaKioqaGj877KVqxuMLZJUn2mgjE7rtE63nel00jE2TXM/8Ia19i/AmfhWh47xBUTktbjZydbae4ATk+sRkbqvwauttevdsYh8HTdbba29CvjGWlsiIqnc7ewo4LfAKcAdQR1jgYtFpGod+30wbvZJa+0fgd2B71PYZ2NqgRHW2gkisgR4fyPqSklZWVmbmT59yxAHHHcgt82CPfoazrYhuPkk+P1D4ByjRrTnd1sb/j7ef68xsSh5sRh3P/sAeWHDO2cfz13tRnPQ98KWc2fAlgP8IORZS+C8/WHLgZQBD/wixlaPxKgKEpyDBsMxm5c0GFthSSHxwkXhrHivdFqndToz0+mUbS02xpgxwNFAN+fcQcYYC7R3zr2Tah1pS2xE5Ftr7XTgcqAb8Dq+5WYNa+0Y4ApgBFCI7zZakFTVtKbu21o7GLgZ2A7f0lOXiHYFpqcQ+0rgBuAGa20BfhDxo8CKIN6G9hnCtxIdBfQI9tku2OcGEZFV1tr98S1W11lrfwZuieveUhuprMDwv6PziMQceXX3iznvADh3P4hEoSCf24G/7e7X10ZDhIwhfMFZkHcOR4RCHBpz5F16E4RCkB/8SUWifrBxYETnEKvPD1EViZEXMvX7akAslvi9KRbNle9RSinVOGPMb4Hf4XsmjggWr8YPLdkx1XrSfbn3ffjE5qGgi2iNIGF4HngS6Cci7YE/sfa1resbYNDQ+nvxVzJtEdS7U7C8yampiNSIyIvAW/hupsb2eQxwOn4sUUcRKcePqzHr2AZ8N1m7uPleSfsfJyIHA12Aa4HHgsRNNaO1Eo1QCAry11qfHw4RDhm/LhSqX1dYUJ/UQEJSE68oL7TOpAYgVpP4UXERHWOjlEqPLOuK+j2wt3Pur9T/z5wIDG9KJem+Qd8T+EG0XzSwrgDfSrNURFYHY1HO3YB9zAO2t9aGRKTujWgP/AQss9Z2wY93SZm19tYg9m+AGmBX/ADmG+L2GbbWDhSRqXH7jAALgZC19mT8uJqXg/UL8b+ooUD85d5fACdZa9/FJzUXxMXRHdgZP5h4ubV2WbCqgREbKleYcNKdhzMUh1Iq92VZV1QZPmeA+jwrH/9/OGVpbbERkSoReUtEljawbiVwNnBTcHXR3cCGdLE8gG/xWGytXRYM9D0fPxB5BX5Mysvr2L4hIeBf+G6xpcA9wN+AW4LYJwH/AD4L9nkC8DDwKTAZmA2MJG48jIisxrdePRFsc2mw6lxgCLAE+A8wNimOc4Bp1toK/Ht0kohMa+LxqFYkXJD4Z1k3xkYppZpblj3d+3/AxUnLzgPebUolJtV7aag2RT8UGfTmrz9mxtv1FxBuf8WWbHqi9j4q1YalLeN4yzyccL7f252UsezGGNMTeAk/9KI38DN+WMmBzrl5qdajz4pSKst026pTQmLTaXj7DEajlMpl2TSCzzk31xizDbAt0A/fLfWZc65JYWpio1SW2eyUIXz38Q/UznRsfexm9Nxugy+sU0qpdXLruZihpTnfjfRp8NogmtgolWXChWHKjvDjajY/aGiGo1FK5bJsemSLMWYmjQyFcM71S7UeTWyUUkoplQ2OT5rvib+vzZNNqUQTG6WUUqqNyqauKOfce8nLjDHj8Df4vSPVejSxUUoppdool+7b9G68avyDslOmiY1SSinVRrlw9rTYGGOSb6ZbAuwPvNZA8UZpYqOUUkqpbNA3ab4SuBX/rMaUaWKjlFJKtVGx7Bpjc0pz1KOJjVJZxkVj5D9eSWhKhLnLJtPzhCGZDkkplaMyPcbGGLNnKuWcc++kWqcmNkplmVl3/UDBk6sB+OHE9ykZ2p4O23fLcFRKqVyUBVdFPZhCGQcMSrVCTWyUyjJL/5f4SJQV4xdrYqOUyknOuSZd8ZQKTWyUyjZJT0Ux+nBvpVSaZNOdh5uLJjZKZZlodTRh3lVl02PqlFK5JAu6otYwxrQHrgR2wz/he01wTXmkQvbfmkepNsYkPSnFPxNOKaWaX8wkvjLsHmBr4GqgE/BbYAZwW1Mq0cQmi1lrp1lrk5+doXJcqDCx7ylUpA2rSqk2YR/gcOfcC0A0+HkUcEJTKllvYmOtHWetvSzV5Uqpprn47Qhd/lbNrg/XMH+lY62H22qLjVIqTVzIJLwyLAQsD6ZXGmM6AHOBJt3zImu/Clpr80Wktq3uv7V64ocYH89xHDjYsM+A1t8gOH5qLS+Or6bLiip6RmsZMaqUTW3ZBtW14NOFzHhlFh03LadkeAcm/3cuM3p14sZpnQB4f4Zjr7sqGNFrIHv2r2TQnGXk18aI1TQ8xqby6YlU/W8mxfsOoqSbg0ffI9qvO0uXlRNbsJIyVhAOxSBsCG3em/CZOxF94CNi384lfNTWhHdM+erJdXLVEZbf+jmxhasoO3c0+YPKm6XeBI+8CzIFDtsO9ti8+etXqo3KssHDX+HH17wNvI/vmloJTGpKJc2W2FhrtwBuB7YClgIPATeISNRaOwCYCvQVkVlB+ZOBy0RkSDA/LdhmD2Ab4HRr7Y/AncDmQBSYCBwgIksb2P9YIB9/TckhwELgGhEZG1dmF+AGYGQQ4z3ArSLirLW7A28BpwBXAV2BhP9g1tqtgf8BHUWk1lp7Kv4a/L1E5B1rbXdgDtBLROZba/vhbwe9M/5r+EvAH0SkIqivM3ATvvmtCHgX+K2IzG/g+EqAJ/C/s6NEZGXDv4nMefrHGMe+4v8J3zPB8fGxhm16ZtdfTVPMWBzlmLuW06Gimh0XL2My8MHrSzj3qgEM3rRdk+paMaWCt4//H7HqGNGQYVXnEmIRx09dlsHuPrEh3/BdXgnfDerPuBO68siNzwNrj7kBWPXyZBYe6devuvNjiot/wKyuZi6bsApfXyXV9GAO4P94om9OJPbMBD9/34cUfnsJocFdm/amNGDxuW+y8oGv/D6f/pHek85s3u6zR8fBSXf66XvfALkJthjQfPUr1YY5k1Xn6DOoHzD8O+B6oBw4sSmVNMtXamttB+BN/D/mHsABwKnABU2s6oxgmzLgBeBu4A38IKLuwbqadWz/K+C/QflfA/+w1u4YxDgSeBW4GZ+0HACcS2LfXRj/wK2tgv0l+xKoAnYI5scAk4G9g/m9ge+CpKYIeAf4Hv9k0pFAH4JHr1trDfA8PuHZDOgPVACPJ+/UWtsDeA+fNB2c7qSmoqJig6Zlfv1/4KiDT2au3ug6Mzn99c8VVNVCeU19w51zMGXisibXuWzicmLVPumLhUPEIv69GrJoGfvHlmGcI9/Vt8ws7lDCkvbFGCBUHF67zg+nr5nOZzVmdTUA1ZSuWV5LYUKnlvtyVv1MVS3uu7nN8l7VyNw1y6IzV7Di5wUbXWfCtEypj7s2wupPJzZv/Tqt01k+3YZMd85NAXDOLXDOne6cO8o5931TKkk1sbnUWrss/oVvhahzAD7huFZEqkXkB+BG4PSmBAPcLyJfiogTkdVBnf3wLT21IvKJiFSuY/tPROQxEYmIyFvAM8DJwbrfAE+JyAsiEhWRicBdrJ0J/klElovIquTKRcThk5W9g8RkT+AyfIIDPrF5K5g+EDAicoWIrA5amS4HjrPWhoHRweucuP1dBOxpre0Tt9tNgY+D2M8WkcRrgdOgrKxsg6YPGRKibtxrxyI4cFjxRteZyemdRranV8cQ84oKiQTfIQqLQ2y5XZcm19l12y4UdysCII8YRR3yAQiFDXfvDh93ncGBX9f/Ax8xYxHdl/qPuguSoPg6y4/YFFPsW0Vqyjrh+viYSlm8pkwxlfXXSoYM4aO3hrD/kzd9ygntOKhZ3quSX22yZlnhjr3pMLzHRteZMH3YdpAftAB1bU/xfrZ569dpnc7y6XTKsqui5hlj7jHG7Lz+oo1Ltb34OhG5Nn6BtXZc3GxfYHrwj7/OFNZ+Uuf6TEuaPwWfDHxgra0FHgOuEpFIittPw186Br7VZE9r7S/j1oeAmXHzsaT5hryFT5aeAZYBT+NbhjoDewFnxe2vX5AExnP4Vq2BQCEw31obv74Kn8zVfb0+BViE7zbLajv0Mow/Icz4BY5d+xj6tc/8X8nG6NguxEt/LOeDH2voQgeKVtUwYFgJXXoUNLmu4q5F7PfK3sz7aAEdhnWgqEcxsz5ZRMdBpXTdpAMDduvOHaMWctjspcx4fhaj/vMt4Zj/c4rVrj3GpnB0D3p9eQrVn8+lcKc+mLIQvDGBbkN60G5BmNiKaopdJaYoD6ojhIZ3JzS6H+EjtiL2/TzCew/HdCldq94NUf7nHSjcriexhaspPngIJtzMY6t23wy+uBm+nga7bQp9uqx3E6VUarJgwHC8fYBjgMeNMVHgSeBx59w3TamkuTrCZwL9rbUmLrkZRH2SUNemFj8woVcD9SScwUVkKr5LC2vt5vhuqan4sTgNGdDAfF2CMB14SETOWcdxuKTkrCFv4bvIfgm8GYwheh84G5+wvBe3v0kismlDlVhrp+Mfyd5JRNZ1B7aLgV8Ab1pr929ofFE2GdnFMLJLVv2hbJQuZSEOtUXBXMlG1VXcvZiBh/VfMz/sgN4J6/vu0JUTgAljv2RJbX3ubhrpA88f3pn84Z3rFxy7KwZYV7oS2qovoa2a+n1j/Yr3HNDsdSbYvL9/KaWaVTYNHnbOfYkf8nGRMWY3fJLzjjFmrnNui1Traa6vVq/gWx8usdYWWGuHA38ieLiViCzG/6M/1VobDpKUM9ZXqbX2JGttXQK0DIjgx0E2Zntr7THBPvYEDgceDtbdAxxtrT3IWptvrc2z1o601u7WlAMVkZ/xCdvv8eOKwI/gvhDfFVbXVfYyUGCtvcRaW2atNdba3tbaw+qqwo8A/3vQ2oO1tqu19uikXUaA44BvgXHWWn1oUI4LFSTdx6a49V9dppRSTTQR+AF/g74BTdmwWc6YIrIc34S0NzAfP4D3EfwVQXVOwo87WR4sfzCFqvcEvrDWVuLHmTwOPLqO8v/BD/5dGtR/joh8GMT4bbD/3+Ovi18AjMUPJG6qt/Bf39+Nm29P/fgagjEze+IHDU/EH/fbwKhgfd3VWyY4xgrgE2D35J2JSExEzgi2fz+42krlKJfU9dTY5d5KKbWxnDEJr0wyxpQbY04zxrwN/Iz/f3gj0KQv9CZXbtceXO4dEZGmDlhWa8uND0UrNWHfN1jy39lr5ofcti39ft9gj6ZSqm1IW8bxaP+nEs73J0w/MmPZjTFmFfAR/tYmzzjnlm1IPVl7gz6l2iqTn3he0ad7K6XSJdOtNEkGO+fmrr/YumnnvVJZJq9jYcJ8uF1+hiJRSqmW0xxJDeRQi42InJzpGJRqDl0P7sf8R/09bUIleZTv0tC9IpVSauNl01VRzSVnEhulckW3IwZQdUV7QlMi7PTHvSkZ2iHTISmlclQsu7qimoUmNkploagtIGoLKBvVef2FlVJqA+Vii42OsVFKKaVUxhnvDGPMO8aYr4NluxpjftWUejSxUUoppdqobLqPDXA1cBpwH/7RQuCfHvCnplSiiY1SSinVRmVZYnMycKBz7knq76c2Ff+IppTpGBulslDUQWVUL/NWSqVXlo2xCQMrg+m6xKY0bllKtMVGqSwzY4XjtCl7cPyUvdnh3xGqI3ojaKVUm/AacKsxphD8mBvgGuClplSiiY1SWeacN6Msifonin8yF+7/WhMbpVR6uJBJeGXY+UAP/LMVO+BbavqjY2yUat0+n584P26mPgRTKZUe2TLGxhgTBo4AjsUPHN4e/4iFw5xzFU2pSxMbpbJMbTRxvibacDmllNpY2dJi45yLArc656qccwucc5875+ZtSF2a2CiVbTLeGqyUUhnxkjHmoI2tRK+KUirLZf4KTKVUzsquE0wR8LQx5mNgJvVXRuGcOzHVSjSxUSrbJI0Vdjp2WCmVJlkwYDjet8Fro+R8YmOtvQTYQUQ2uHnLWjsWiIjI6c0WWMP7eQ14V0RuSud+VJZLPs9k1XlHKZVLsuCmfGs4565qjnoynthYay1wGbATUAjMA14FbhSRuRtbv4hcn7S/ccBbInLtxtadVO9o4BJgF6AEWAR8AdwtIu+kGOt+zRmTykJfTYVwCDbrn/o22mKjVOs1cRZUVsPowZmOJOsZY/ZsbJ1zLqX/o5DhwcPW2jHAB8CPwCgRaQ/sBiwOfrYKwXF8CEwBLFAGbA48DhyWwdBUNrnoERj1B9j8fLjq/1LfLnu+UCmlmuKOl2GT88BeCGf+I9PRNMiZUMIrwx5Mer0IvA480JRKMt1icw/wuIisuflO0EpzTd28tfZo4M/AQKASf6AXiEhlsH4a8BCwDzAKmAicLSKfB+uvBHYWkb2ttXfhW1R2sNZeDMwWkeHW2r2A64FhQAR4GzhPRBakeBz/AB4TkYvillUAzwSvVI9lHEFrkrV2AP4ZGScG2/QFPgZOao6WLNXCnIPbX66fv/1l+MtRDRZdqydKW2yUap3i/+bvfxNuOwXaFWUungZk0xgb59zA+Png3jaX4f+fpixj6Zm1dhgwBN+qsS7L8TfsKccnJbvgDzTeWcDvgE7A08Cr1tr2yRWJyLnA+8A1IlIqIsODVdXAuUBXfEtLL+COJhzHYOCJFIqncizJjgJ2BXoD7fBPP02riooKnW7uaWNgYLc1yxjUvdHysVjDN+TLmmPRaZ3W6ZSmI/27rJmO9SiH4oINqiedsuUGfQ0J7m1zHXDR+srGy2SLTdfg5+x1FRKR1+JmJ1tr78G3YsR7UES+ALDW3gj8BjiQ9SdNdfv4IG52nrX2JnwrUCrWOg5r7cHAI/gv34UiUtSEY0l2lYgsCup9HEjrAGaAsrIynU7H9MuXwGWPQzgM1x/XaPnk7xuugTI6rdM6nf3TeU9cABc/BiurCF15FIRCG1RPGzcGaNLt1zOZ2CwMfvYGfmisUDB+5QpgBH5wcRhI7iKaVjchIs5aOwPok2ogwcDf64Et8QN/Df6JoqlYFPzsg+8GQ0ReBMqttTvjW4iacizJ4rudKvHjd1RrNLQX/N8f118uu740KaU2VM9O8PB5mY5i3bLofGOMmUni5RIl+HvbnNOUejLWFSUik4DJwDGNlbHWFgDPA08C/YLBxX9i7V/FgLhtDP45E7MaqbahzO9JYDwwLNhHozE1YBLwM3D0ugo14VhUG5fcGpxlrcNKqRySZV1RxwMnxL32BXo55x5uSiWZHjz8G+Ala+184C4RmWOt7Q6cih84+zK+ZWOpiKy21o7Ej4VJdqq19jngG/zTQUuAVxrZ5zz82J547fHjXyqstf2Ai1M9gKCF6BzgBWvtYuAufFJVDGwXV7QgxWNRbV3yYGEdPKyUSpNsGjwMbOOc+1vyQmPMBc65W1OtJKPXdonIm8DOwEjgG2ttBf7y727AOBFZCZwN3GStXQncTcPjZu4D/g4sxQ+2PUBEljey29vwt89ZZq39Llh2Jn7sSgXwLPBUE4/j9eA4huFbflYC3+HvzbNnUCbVY1EqgT7bWynVRlzRyPL1XWSTwLhWfr/24HLvy0TksUzHkkNa94eilet8V4QlVfXzBw+GFw7LdOOqUiqD0tascvv2byWc73//yd4t3oQTd2O+l/AX/sTHMAi43DmX8l1N9WypVJYpSmpHLdG/UqVUmmTBuBrwN+MDP1A4/opkhx8+8tumVKanTKWyzJiB8PB39fMHDc743UCVUjkqGxKbuhvzGWMeacpTvBvT6hMbERmQ6RiUak537BlGfl7I1Ooyjh5ZwDGbZP7Eo5RS6dYcSQ3kQGKjVK7pUGi4od+nABy070EZjkYplcuyocWmjjGmPXAl/lmRXYgba+Oc65dqPdrGrZRSSrVRWXYfm3uArfGPDuqEH1szA381c8q0xUYppZRqo7IgmYm3D7CJc26xMSbqnHvBGCP4q6VSTm60xUYppZRS2SCEv1kuwEpjTAf8Y4WSb6q7Ttpio5RSSrVRWXbn4a/w42vexj9n8R78DW8nNaUSbbFRKstEYo6HFgzn4unb8c+vopkORymVw7JsjM0Z1D/U+nfAaqAcaNLVUtpio1SWueKDGM8vHQTAWW86hnaMsWc//Q6ilGp+WZDMrOGc+zluegH+UUdNpmdLpbLM4xMTn2jx8Lf6tCilVO4z3hnGmHeMMV8Hy3Y1xvyqKfVoYqNUlllZkzi/tKrhckoptbGyrCvqauA0/IOt6+5bMwv4U1Mq0cRGqSyT/FzajJ9qlFI5K8sSm5OBA51zT1L/MOap+AdhpkwTG6WyTcbPLUoplRFh/FVQUJ/YlMYtS4kmNkplm+QhNZroKKXSJMtabF4FbjXGFIIfcwNcg79BX8raRGJjrb3EWtukN6aBOsZaax9orpiUalTGzy1KqbYiyxKbC4Ce+Jv0dcC31PSniWNssuJyb2utBS4DdgIKgXn4zO1GEZm7sfWLyPVJ+xsHvCUi125s3emsU7VNmT+3KKXaCpcF5xtjTA/n3Dzn3ArgMGNMN3xCM9M5N6+p9WW8xcZaOwb4APgRGCUi7fF3Hlwc/FQqd6yuhl/9DfqdCec9sPZIYRpcpFTb8PlPsOX5MPxceH18pqNRLSf5zsL3Ouc+35CkBrKjxeYe4HERWdPUFLTSXFM3b609GvgzMBCoBF4ELhCRymD9NOAh/AO0RgETgbNF5PNg/ZXAziKyt7X2LmAXYAdr7cXAbBEZbq3dC7geGAZE8Ld0Pk9EFmzsAVpr+wN/x7dIrQaeAf4sIquttQa4FjgFKMMndLeIyJ3W2o74y972xP+uZgFnicj7GxuTypA7XoGnPvLTd74Ke2wGh22fWCbpG5QmOqrNOOlO+GGWnz7qVlj6CIQy/v07p2VB9xOs3QG/+8ZUltFPjLV2GP7hVo+vp+hy4Fj8rZV3CV6XJZU5C38L5k7A08Cr1tr2yRWJyLn4Z1BcIyKlIjI8WFUNnAt0BTYHegF3NP2oEllr84BX8N1r/YHt8QnO34IiY4CTgO1EpAzYFt+CBXAhUBJsVw4chk9u0qqiokKn0zRdvWQFCSqr1yrjYg3fkC8b4tdpnU7rdGXcTZtW11CxfMW6y7eR6XTKkjE2zfr1zbgMfh201u6E/yc+UkR+aMJ25wInisi2wfw04FERuTyYN8B04GIReTy+xSZYP471jIex1h4IPCQi3YL5sUBERBq8xXNjdVprdwTeBTrFtTD9Angen7Tshk/EjgfGiUhV3LZXAvsC5wBfikhL3YJW2wjSZcEy2OtK+HYG7LUFvHIpFOYnFOl8V4Qlcef3gwbBi7/MhsZVpdLsuU/guNuhJgJ3ng5n75vpiLJF2jKOv+z3ecL5/qrXtmnx7MYYswo4gPrjfB44JG4e59w7qdaX6bPlwuBnb6DRxCYYh3MFMAI/uDgMJHcRTaubEBFnrZ0B9Ek1EGvtaHxX1Jb4hMPgr5/fWH2BhXVJTWAKUAR0FZFx1tpL8C1Q/7HWfgJcIiIC3AzkAw8DPa21LwMXicj8ZohLZUK3cvjmdli5GkqLU9okKxqKlWoJh20PK/4N0dhaCb/KaQvww0nqLE6adzThJn0Z7YoSkUnAZOCYxspYawvw2duTQL9gcPGfWPt8PyBuG4O/HXNj3TYNtXw8CYwHhgX7aDSmJpoJdLXWlsQtGwRUESR2InKfiOwM9AAmAM8GyytF5FIR2QzYFJ8A3txMcalMSjGpUarNyQtrUtOCHCbhlZEYnBvgnBu4jleT7jyc6RYbgN8AL1lr5wN3icgca2134FT8rZRfxrfSLA0G247Ej4VJdqq19jngG+B8fKvLK43scx5+bE+89vixPBXW2n7AxRtwLHnW2qKkZZ/hk7dbrLV/wI+VuQb4V9CytC3++D7Dj/OpAKIA1tqDgm0n4a/nr6pbp9oO7RdUSqVLlgweblYZH24uIm8COwMjgW+stRX4cTfd8GNOVgJnAzdZa1cCd9PwYOP78FceLQWOAg4QkeWN7PY2/O1zlllrvwuWnYl/RHoFvsXkqQ04nL/gr3qKf3UBDsR3i83AJzCfAn8MtinFD1JehG9+2yeIH2Aw/o6LK/Bdbatp4o2KVCuUnMnk3nlHKZUlsmTwcLPK6ODh5hIMHr5MRB7LdCw5ovV/KFqx5MHDBw+CF3TwsFJtWdoyjssOGJ9wvr/2la1bfXajZ0ulskwo+enerf40o5TKVrnSShMv411RSqlEwzsnztsemYlDKZX7nEl85YKcaLERkQGZjkGp5nLr7iF2eryWCGG6l8DZo8KZDkkplaNiOdhikxOJjVK5ZNteIe4b9D9m17Tjt4ftQIfC3DvxKKVUumhio1QW6pJfRZf8Kk1qlFJplYtjbDSxUUoppdooTWyUUkoplTNycYyNXhWllFJKqZyhLTZKZSFX5XBLHdGaKOECvSpKKZUeuXKJdzxtsVEqy6z4aQVVl6ym+i9VvLX/W9RW1mY6JKVUjsqGh2A2N01slMoy3978rX/kKbD8u2XMeHZ6ZgNSSuWsmDEJr1ygiY1SWWb5xGUJ88t+aOxZrkoppZLpGBulsoyLJj4syujXD6VUmujl3kqpFuf0WetKqTTRxEYplXYuOZOJaWajlEqPWO7lNTrGpjHW2pOttZM3cNux1toHmjsm1TaYpCsTnCY2SimVslaZ2Fhrz7PWTkla9ltrrbPW7he3rNhaW2WtPbjlo1RqwzigNuSTm5pQCFK4BNM5x7Nf1fCvT6tZVaOJkFIqNc6YhFcuaK1dUW8Dd1hr+4tI3bWwewHfAXsCrwXLdgLCwLimVG6tzW+mOFWWcMtXE/v7O+AgdN4emPKS9Oxo5iK497/QpT2csy8uL0zsn+/jpi8hfMoOfN++C49KDYO7hDh9+0JMAyeSCT07c8Xo7SgMOSKlRfy2dBk2qcySz6bx99eWkt+1jPNOH8wNTy2l4s4viRnDjXtuzokHdeL3o0OU5OfGiUoplR6xHLl3TbxWmdiIyHfW2rn4ZOYha20Y2A04Hbg0ruhewOdAxFp7B/BLoBj4ADhPRGYAWGvHAROAAfjE6HpgXvw+rbX7Av8CzhCRl621pcCVQZ1dgZnAr0Xk/eR4rbXXA0cD3YD5wJ0icnuwrhC4EzgUKArWXyIiT1lrBwD/BLbDf5GfChwjIj82+U1r4yK/vBf3jn/bYm9PJP+9PzT/TmpqYbfLYep8Pz9xFtHuvYld86qP4cEPOfD4C5jmigBYuNJxyZjitaoZv0kfYqvDrO5eCiHDrRTTf3yM87YOGlhnLmL/x6v4tM/msAo+vGseZ97xFpvO8Pu1c+Zxavsj+GI+PHOI3rVYKdW4XGmlidcqu6IC7+ATF4DR+ETkRWCwtbZzsHwv4C3gNmD74NUfWAS8FCREdU4F/g50CH6uYa39NXA/cKCIvBwsfhCfcOwFtAcOBuY2Euv3wM5AGXAGcIO19hfBupOAbYBNRKQ9PrH6Llh3PTAD6A50AU4Glq77bdl4FRUVOTftPpm6Zpn7dGpa9rVyyuz6pAbgk0kJ+w0tWkmneYvWzH8wuarBenpWVXLWp99x02Nvc+inE31Vc9yaMrFvZvBZr0H1u6kpY8TshWvmR81bCCHDx7NjzX6MOq3TOt3y06ppzFpXYLQS1tqTgRtEpKe19s9AHxE5x1r7Gj7peBNYjE8U3gAOEpE3g21LgSXAbiLycdBi87OInJpU/+XAM8ABwP513V7W2rqWl81EpC4JiY9tLBARkdMbif3pYH8XBfu5FJ9YfSwikaR6OgMXicgPG/I+baDW+aFYh8hxDxJ7/HMAQr8aTd7/ndH8O4lGYcdL4LOf/PwVvyLavQfRc54EIDa0G6OPOpfvK3xD6QNHt+O07QvXqubVvf/Lyu+WrZm/5oiduOycXhw1IvgesmgFh14+iRcGjwLglHYLuPqtz1n9rN/v81uN4KJj9+U3owx3760tNkrlgLQ1q5x5zKSE8/19Twxr9U04rbIrKvA20MNaOxKfvNwbLH83mK8FqoAfgUJ8Nw4AIrLSWrsA6At8HCye1sA+ugHnAOfGjeUB32UFMCmVQK215+FbavrgP6DFwOPB6sfwLTK3AUOttW/jE5nJwIX45Oola2074GngzyKyMpX9qnrhR07BHLwlOEfoiK3TtJMwvH0lPPMJdCmDAyxhwAzvgZuxhPyDt2BcYQkvfVvL4C4hdhvSyFCuVZGE2evMPA4e0ad+QZf2PHXpIJ566Qfye5dz+EF9MGccxMqnfqQWQ8/NhvBMYYjDhrb685NSKs1y5TEK8VptYiMiM621k/CtKTsARwWr3gH+DUSA/wELgWp8MjIZ1rTYdMOPi6kTY23z8V1FL1hrIyLyaLB8WvBzKL6bqVHW2p2AG/FdVp+KSDRosTHBcUSC9Tdaa8uBu4CHgF1FZCFwHnCetXYQ8AJwEXDFuvap1mbCIcJHJQ/BTYPSYjhpj4RFob1GrJnuCpzaQCvNuvReexgO+X06cezZneoX5BnKjtkEgOObVLtSqi3LxTE2rTaxCbwNXAD8JCJLgmVf4pOWI4GbRSRmrX0EuMZa+z2wDLgFmAh8tr4diMiH1tp9gNestaUi8g8RWRAkJ/cEXUnTgcFB+eR737QHovgEy1lrDwD2A54CsNbuCSwHvgZWA5VBeay1RwUxTgvK1NStU7lrrX5A7U1SSqmUtebBw+AHBvfAt9IAICJRfEtNj2A9wPmA4K+QmgH0BA4Oyq6XiIwH9gAus9ZeHCw+FX8l1XtABb41pUcDm/8XeASfoCwCjgCei1vfHXgUPyh4Ln5w85nBuq2C+lfiBxSPB25OJWbVmiU/Kyr3vlEppbJDzCS+ckGrHTys0ko/FBn06k6vsvLn+isihpw6hK2vG53BiJRSGZa2lOPk46cknO/HPja41ac3rb0rSqmck/w0b80ylVLpkouDh1t7V5RSOafDyPKE+fJNyhssp5RSam2a2CiVZTa/aHMo9dPlm5XT/5f9MxuQUipnxYxJeOUC7YpSKsuUDS6j6IZi3DLH3ieNIZSv3z+UUumRKwOG42lio1QWMoUG091oUqOUSqtcfAimnjWVUkoplTO0xUYppZRqo/TOw0oppZTKGTrGRimllFI5I1euhIqniY1S2aaqhi3v+IAOUxbBN9VwyRGZjkgppVoNHTysVLa55in6vf0THaYthUsfh7e+ynRESqkcFcMkvHKBJjZKZZuXJXH+1fGZiUMplfOiJvGVC7QrSqlss7omab46M3EopXJeLo6x0RYbpbJMVbuShPnZee0yFIlSSrU+mtgolWWm9ey1ZtoB73fVZ0UppdIjZhJfuUC7opTKMr2WLFozbYAtl87LXDBKqZyWKwOG42lio1SWab90WcL8JrXLMxOIUirnRXNwjI0mNmlmrR0E3AjsApQCSwEBjhKRmnVtq7JDbdTx7SLoXQrd2m3cSWDOSsfCVbB5V1haBTNWwMjOsKIGPpjtKMlzbBPNo1PcNvMjYbpv3CEopVSboYlN+r0KvAEMB1YAvYEDIQfb/3JQdcSx11NRPpwNpfnw+hFhduq9Yb+6FybH+NVLMWqisFsfmLAQllfDiE4wZRnUxny5BzoO5jRmrtnughUD2G58jPO21iFxSqnmlSvjauJpYpNG1trO+ITmlyJS158wC7g3rsyhwOXAYGAucK2I/NtaGwbeBiaLyOlB2eOBW4BRIjK3xQ6kDftgtuPD2X56ZS3c/WWMnXqHN6iu28QnNQDvzapfPnFJYrlYKDGBiWG46TNNbJRSzS+ag9+x9UyZRiKyGPgOeMBae6K1dqS1ds2nyFo7BngQ+D3QCTgJuMtau6uIRIFjgAPrtgXuAY5Nd1JTUVGh08F071JDOO7vvl/7Da+zX/v6ivLW8Zc3sVvvhPkZHbtu1H51Wqd1unVPp1Mu3qDPOOcyHUNOs9Z2AS4A9gU2A5YBdwLXAi8Bn4nI1XHl7wSK41pp9gSeB+YB/xaRq1ogbP1QxPnPxBgPfOMY3glu2jVEcf6G/fUvrXL8cVyMWSvhd1vD29Ph60Vw9HB4dya8NtURjUH5ggXc/fSDdFlVwYsjRzNph6352++H0L9Djpx1lFJNlbY//l3Omptwvn//3p6t/kSjiU0LstaWAL8C7gd+DfwBGADUxhULA++LyP7BNgYYDwwFeopIS6Tx+qHIpOHnwqQ59fPn7gd3npG5eJRSmZa2ZGOns+clnO8//EePVp/Y6BibFiQiq4Cx1trfAqOA6cBYEbl5HZtdChQBnwB34burVC5L/rIRi2UmDqVUztPLvVWTWGs7AhcB/wZ+xLeEHILvkvor8DI+0fkE+AjfWrM5YERErLW7B9vvACwEJlhrTxWRh1r6WFRLcuucVUqp5hLJdABpoIOH06sG6AY8CyzBJyeXAeeJyFMi8gZwBnAzsAh/VdRtQKm1tjvwRFD2OxFZgB9MfLu1dvOWPxTVUlYWFCXMV+YXZigSpZRqfbTFJo1EpBI4bT1lXgFeaWR1z6Sy7wHtmyc6la3eG7wpB3w3bc38R4NHMCZz4Silcph2RSml0m7O8EHcsfN+jJ71M09vvj0ndcjPdEhKqRwVyb28RruilMo2p29TiHFw/3Z7s+fMH9lqx57r30gppTZABJPwygXaYqNUljFH7sh230xgnykfMeKh42Bor0yHpJRSrYYmNkploQXb9GXBNn0ZsdWgTIeilMphtbnRSJNAExullFKqjarVwcNKKaWUyhW16y/S6ujgYaWUUkrlDG2xUUoppdqoVdoVpZRqCf1fm0iHKYshvzfsu3Wmw1FK5ajVuZfXaGKjVNa561W2+MfHfvrNa+HLW2DLgZmNSSmVk2py5N418XSMjVLZ5r436qcd8PC4TEWilFKtjrbYKJVtFlckzs9dmpk4lFK5L/cabDSxUSrr5YczHYFSKlfl4OBh7YpSKttFo5mOQCmlWg1NbJTKdi73vlEppVS6aFeUUlknKZHJ0+8fSqk0ycGuKE1sNoK1dhywA/6u1FFgKnCdiDyVybhUa+cSZyPaFaWUSpPcy2u0K6oZXCMipUBnYCzwuLV2SGZDUq3Bp3Mdox+NsPUjEV6eHGP/Z6IMfSDCc12HJRYM1Z95PpvrsI9G2OqRCO/P8gnQ7ArHmKeijHgowthvYy15CEqpVs8kvVo/bbFpJiISsdbeD9wGjAImW2v/BewNlAMzgWtF5PG6bay1WwA3AaOBMDBeRPYO1vUDbgV2xn+Ffwn4g4gkXQusWqujX4oybYWfPurlGKsifvrEI37DE9U1jJ41lXAsQrdQ/fePY16O8vNyP/2rl6LMPTuP89+N8dZ0n+Sc/t8Ye/Uz9G2fGycopZRqKm2xaSbW2gLg7GB2UvDzA3ySUw5cDYy11o4MyvcE3gteA4AewF+DdUXAO8D3wEBgJNAHuCPtBwJUVFTodAtML6uu73KqjtZPVxYU8v6gTXh/0AgWlXaA2uiabZfXrCnGimr/c3FdRgREHayszZ5j1Gmd1umNn06r3GuwwTjn1l9KNSgYY7MdUA2U4cfanCMiDzZSXoCHROQea+1FwJEisk0D5Y4AbhSRwXHLRgMfASUiku5BF/qhaAH/+ibGmW/GcA4u3tZw39eOhavhj+Ne5OZXHgP8L8KcsCs88nsAHv42xhlvxIg5uGuvEGeNCvHxHMcBz0ZZWgVnbWn4xxi9741SOSZtKYe5aHnC+d7d1KHVpzfaFbXxrhORa621HYEHgT2AB621IeBK4Ch8a4wD2gFdg+0GUN+yk2wg0M9auyxpuQvqmt2M8asMOWXzEL8abnBAaYHhLzs6Kmqg/PZX15QxAHHDZk7aLMQRw+q3Adihl2He2WEqa6FjUas/JymlWlTunTM0sWkmIrLUWns6MMVaewhQCpwO7AN8LyKxoMWm7lM0DTiikeqmA5NEZNM0h60yrF1B/UklP2zoVAyvDN2CMXPfpyAWZVFJKbXlHenZyDZ1CsKGAm2oUUopHWPTnERkCX7A7/X4cTURYCEQstaeCmwZV/wxYLi19k/W2hJrbYG1du9g3ctAgbX2EmttmbXWWGt7W2sPa7mjUZlSUVBCQcz3NnZZtZKfe/XJcERKqZyVg2NsNLFpfncAPfHdRp8Ck/FdRyOB9+sKicgcYHdgDDALmAdcGKxbBewZbDMRWA68jR+IrHLcqHnTEuY3mT41M4EopdqA3MtsdPCwaoh+KDKp92lE5ywlDESMIe+4neHR8zMdlVIqc9I3ePjiisTBw38ta/XZjbbYKJWF6obL5DmH/pkqpVTqdPCwUtmmpDBxvlD/TJVSadLq22fWpl8Flco2Byfd2uiX22cmDqVUG5B7Y2z0q6BS2ebaY5n5xXeUT15E2en7wf6jMx2RUipX5UYuk0ATG6WyTXEhE/6wGwAHHXRQhoNRSqnWRRMbpZRSqq0yuddko2NslFJKKZUzNLFRSimlVM7QxEapLNRuznJ6fDwdFi7PdChKqVyWexdF6RgbpbLO+9+z+2+eIxRz8M/P4Pu/Q/fyTEellMpJOZLNxNEWG6WyzSX/9kkNwJKVcP+bmY1HKZW7crDFRhMbpbLN5LmJ819Py0gYSinVGmlXlFLZJhpNnI/pM0mVUmmSI6008TSxUSrbmKSG1By8z4RSKlvk3vlFExulso1LbqHRFhulVJrkXl6jY2yUyjrJLTTaYqOUUilrcy021loLXAbsBBQC84BXgRtFZO66tm2GfQ8ApgJ9RWRWOvelWrHkFhsdY6OUUilrUy021toxwAfAj8AoEWkP7AYsDn5mnLXWWGvbXMLZlkx5dRZf3PEDS35s5OZ7yS00IcO3Cx1/+TDK/02MrVU8EnPc82WMaz6OMa9SkyClVBPk4OXebe0f6D3A4yLyp7oFQSvNNQDW2hLgBuCXQDE+CTpPRGYE68cBb4nItXXbW2sdsIuIfGCtvRLYBfgUOD0o8g8R+Usw/VXw88dguxtF5Jpg+vfACcCmwJ7W2veAPiKyINiPAX4GrhCRR5vvLVEtaeJ/pvH+ZV8C8M3YyRzxyl6U9ipJKpWYnCyN5bHzk1GWV/t1S6vgrFH130l+906Meyb4bR79Hr47OUx+OEfOUEqp9MrBru4202JjrR0GDAEeX0ex24Dtg1d/YBHwkrU23IRd7QrMAHoBBwOXWGt3CtZtGfwcLiKlInJN3HanAUcBpcB44BPgpLj1Y4By4OkmxLJBKioqdDpN0zM/nrdmurYywuKJy9cqE03qipqc3z5Iarz3Z7uE8u/NqL88/KelMH9VZo9Rp3Vap5t3WjVNm0lsgK7Bz9kNrbTWhvCJxGUiMltEKvGtKJsA2zZhP5NE5F4RiYjIJ8AEwKaw3d9EZIqIREWkGrgPODVu/WnAYyKyugmxbJCysjKdTtP0oL37rJku6lRA1807rlUmnHS594ia5XSPa9TZf6BJKH/QkPq8e+vu0LNdZo9Rp3Vap5t3WjVNW+qKWhj87A380MD6rvjBxFPrFojISmvtAqAv8HGK+0kegFwJpPIJnZY0/zRwh7V25yDeQ4FtUoxBZanBB/ShuEshS3+qoN8ePSjpWrR2oaQWm7JwlM+OD/PyFMcmnWGPfomJzw27hhndPcbiKjhmhCEcyr2mZaVUmuTg6aLNJDYiMslaOxk4BnirgSILgWpgADAZwFpbCnQDZgZlKoB2dRtYa3s1MYy1R342sk5Eqqy1D+Nbar4CJojI103cn8pCvbbrSq/tuq6jxNpXRfVrb/jNVo2fgY4Y3pYaX5VSzSf3Mps2k9gEfoMfMzMfuEtE5lhru+O7fKYCjwDXWGu/B5YBtwATgc+C7b8AjrLW3gpUAdc1cf8L8QnMUCCVy73vAwTYEbi5iftSrZbex0YppTZUm/qaJyJvAjsDI4FvrLUV+CufugHjgPPxicTn+AHAPYGDRaRudOZt+G6hKfixM680cf+rgcuBJ6y1y6y1l66n/ER8MtULeLIp+1I5ZK07ESulVDPJwcu9jdOTZlaz1o4FakTkzBbcrX4oMqn7KbAg7h43R+4A/7kwc/EopTItbSmHuaY64XzvLi9s9elNW+uKalWCS9SPBLbLdCyqJemdh5VSLaTVpzFra1NdUa2JtfZpfDfUDSLybabjUS0olPRnmdeU2ygppVTbpi02WUpEjsh0DCpDRvaBecvq50cPzlgoSinV2miLjVLZ5pZTiBQGrTT9usAZYzIbj1IqdxmT+MoB2mKjVLYZNZC37z+SdvMq2Pk3x0G7Bm7ip5RSzSE3cpkEmtgolYVqyoupKS/WpEYppZpIu6KUUkoplTO0xUYppZRqq7QrSimllFK5I/cyG+2KUkoppVTO0BYbpZRSqq3KvQYbbbFRSimlVO7QFhullFKqrdIWG6WUUkqp7KWJjVJKKaVyhnZFKaWUUm2VdkUppZRSSmUvTWyUUkop1ShjzDRjzGaZjiNV2hWllFJKtVUm9/qitMVGKaWUaqtM0ivVzYw50RjzjTHma2PMc8aYbsHyj40x2wTT9xhjvgum84wxi4wx7Zr9GJJoi41aizHmv0CXhtbl5eV1iUQii1o4pI3WGuNujTFD64y7NcYMrTPu1hgzZDzu151z+6ajYvfHvCY32QTdUn8FRjvn5hpjrgHuBI4C3gb2Aj4HdgZWG2N6AgOAH5xzlc0Ve2M0sVFrWdcfkLVWRMS2ZDzNoTXG3RpjhtYZd2uMGVpn3K0xZmi9cafJHsCrzrm5wfw/ga+C6beBS40x/wYWA+/hE52BwDstEZx2RSmllFKquXwEbA0cgE9y6lpw9gqm004TG6WUUko1xbvA/saYHsH8GcCbAM65amA8cDHwFvAJsBOwRTCddtoVpZrqvkwHsIFaY9ytMWZonXG3xpihdcbdGmOG1ht3c3nLGBOJm/8z8KYxxgE/A7+OW/c2sA3wuXMuaoyZDEx1ztW0RKDGOdcS+1FKKaWUSjvtilJKKaVUztDERimllFI5Q8fYqHWy1pYA/wJGAxHgjyLycgPlDgGuAArxt3l6SERuaclYk+JJNe7ewGP4Ufw/tfTlnNbaYcDDQGf8pZEnishPSWXCwN+BfQEH/FVEHmjJOJOlGPc+wPXA5sCdIvLHFg80SYpxXw4cDUSBWuASEflvS8caF08qMZ8CnA/EgDBwv4j8vaVjTYppvXHHlR0OfAnck8nPSYrv9ZXAb4A5waIPReScloxTrZu22Kj1+SOwQkSGAAcBD1hrSxsoNw84SEQ2A3YEzrbW7tKCcSZLNe6V+ITs2JYMLs69wN0iMgy4G38/iGTHAUOAocAOwJXW2gEtFmHDUon7Z+B04OaWDGw9Uon7M2AbEdkCOBX4P2ttcQvGmCyVmJ8BthSRUfi/vz9Ya7douRAblErcdYn7P4HnWy60RqUUM/CIiIwKXprUZBlNbNT6HEXwxx18cxFgv+RCIvKpiMwJppcDPwD9WzDOZKnGvVxE3gfSfjfMZNbabviWoieCRU8AW1truyYVPQr/DTwmIgvx/wCObLFAk6Qat4hMFpEJ+BazjGtC3P8VkVXB7Nf4FsjOLRZonCbEvEJE6q4EKQHy8a17GdGEzzb4y4JfBia1UHgNamLMKotpYqPWpx8wPW5+BtB3XRtYa0cA29NCd5lsRJPjzoC+wGwRiQIEP+ewdpzZdiypxp1tNiTuE4EpIjKrBeJrSMoxW2sPttZ+h/+s3Cwi37RopIlSittauyXwC+C2Fo9wbU35fBxtrf3aWvuGtXaHlgxSrZ+OsWnjrLXj8f84G9J9A+rrCbwA/KauBScdmjtupZJZa3cDrgHGZDqWVIjIi8CL1tp+wPPW2ldF5MdMx9UYa20+/t4wp4hI1NpW87SCe4HrRKTWWjsGeMFau4mILM50YMrTxKaNE5Gt17XeWjsD36W0MFjUD3/XyYbKdsPfafImEXmqOeNM1pxxZ9BMoLe1Nhyc2MNAr2B5vLpj+TyYT27BaWmpxp1tUo47+Bb+GHBIhpODJr/XIjLDWvsZcCCQqdhTibsnMBh4NUhqygFjrW0vIme2dMCk+F6LyLy46TettTOBzfDPRFJZQLui1Po8RXBHSWvtUPzdJF9PLmSt7Yy/pfZdIvJgi0bYsJTiziQRWQBMAI4JFh0DfBmMo4n3FHCGtTYU9PcfCjzdUnEma0LcWSXVuK212wD/BxwhIuNbNMgkTYh5k7jpLviHFGasKyqVuEVkhoh0EZEBIjIAuB0/liwTSU1T3uvecdOj8E+tztqWsbZIW2zU+twMjLXWTsZf/nqmiFQAWGuvBuaIyL34AYDDgF9ba+turX2HiPwrE0GTYtzBt7Lp+MvUO1hrZwEPiMiVLRTnWcDD1torgKX4MR1Ya18FrhARAR4FtgPqLju9WkSmtlB8jVlv3NbanYEngfb4b+JHA6dl8tJpUnu/7wGKgX/GdY+ckMExK6nEfGZweX0tfrDzXSLyRobirZNK3NkmlZivt9aOxp9XavCfjXmNVahanj5SQSmllFI5Q7uilFJKKZUzNLFRSimlVM7QxEYppZRSOUMTG6WUUkrlDE1slFJKKZUzNLFRKkcYYwYYY5wxpk+a93OWMebRuPnXjDEXpXOfqmHGmMnGmJNTLNsin4+WYIwpDI59RKZjUdlHExvV5hhjBhljnjLGzDPGrDTGzDTGPGeMKQjWn2yMmdzAdo0tPy74h/GXBtaNM8ZUB/tZboz50hhzeHqOLP2MMe2Aq4Er65Y55/Zzzt2UsaDWI/jd7JzpONqCdLzXxpjdjTEJD1J1zlXj71WVTU+OV1lCExvVFr0KzAWGA2XADsB/8Tc22xC/BpYApxljwg2sv8Y5V4p/QvQTwP8ZY4Zt4L4y7XjgG+fclEwHotq8J4A9jTFDMh2Iyi6a2Kg2xRjTGZ/Q3OucW+68Wc65e4NvgU2tbxNgF+Ak/LNv9musrHMugr+rbRjYvIG6zjHGTEhaNtAYEzXGDAjm/xW0MFUYY743xhy7jtiuNMa8lbRsnDHmsrj5zYwx/zXGLDTGzDDG3GCMyV/HIR+Kf3RGg3XGdXecFMRXaYx51RjT0RjzV2PMgqCl7Jy47U8OuhX+ZIyZG5S5JT6O9R23MWYLY8zrwXEsqTtuY8xXQZE3glazBxp5r0qMMXcE+1hkjHneGNMvbv24IKZnghimGGMOaexNijum840xs4Jt/maM6RzUscIYMzG+dcMYk2eMucIY87MxZqkx5m1jzGZx6/ONMbfGvYd/amC/uxhjPgjegynGmD8YY1JO2I0xhxtjvgpaF78yxhyWfExJ5cfWvaeNvdfGmGnBcX0QLBdjzDYN1RG3bJox5nhjTC/gNSAcbLvSGHMSgHNuBf75aQenenyqbdDERrUpzrnFwHfAA8aYE40xI5ty4m/AmcDXzrmX8S1Bv26soPFdXefgb3v/VQNFHgdGGGNGxS07GRjnnJsWzH8AjMI/MPBqYKwxZuSGBG6M6YZ/cN+zQG98y9UY4M/r2Gxr4PsUqj8c2Bn/wM4BwKfAFPxDBU8Bbo9PHPAP+ewHDAriOAi4MG59o8dtjOkZHMd7wb56AH8FcM5tGWy/j3Ou1Dl3eiPx3gZsH7z6A4uAl0xiC9xJwC1AB+Au4GFjTMk63oP+QbyDgvfit/h/0jcDHfHve/wjRy7E38J//+AY3gfeNMa0D9ZfjH+w5Y7AwOBY+9dtHLwfrwb1dwUOAM4FTlhHjGsYY3YE/h3spzNwCfCEMWa7VLZfz3t9FvA7oBP+OWevxh3Xuuqcg/+yEA3qLHXOPRxX5Bv8Z1KpNTSxUW3R7sA44Pf4h97NN8ZcnpTgDDTGLIt/4Vtb1jDGFOH/EdX9c3oQ2M+sPTjz0mD7WcAhwOHOubXG6jjnlgIv4P/xE8RzEvBQXJkHnXOLnXNR59yTwNfB8WyIE4GvnHP/dM7VOOdmAzcEyxvTEViRQt3XOOeWBInky0Ctc+5+51zEOfca/jk8W8WVjwEXOudWB91cN+GTOmC9x30CMNk5d4NzrjI4loSWqnUxxoTw7/NlzrnZzrlK/GdjE2DbuKL/55z7yDkXA+7DJzhD11H1auCqIJ6v8Mns5865T5xzUfzTw4cYYzoE5U8BbnTOTQxaD6/GP4/ogGD9icH6yc651cAfgfhn4vwGeMo590LwPk3EJ2Dr+n3GOxl4xjn3WvB7egV4Djg1xe3X5UHn3BfOuRrgRvx7c2Az1LsCnywptYYmNqrNcc4tcs5d4pzbGv+N+iLgCoKEIjDVOVce/8L/44h3JFCK/wcF/tvyQiC5VeC6oI5uzrkdnXMvrSO8fwHHBt0wewbxPQv+H7Ax5mpjzI9BV8EyYEv8t/MNMRDYKSl5ewjfWtCYpfiHWq7P3LjpVUnzdcvK4uYXOOdWxc1PA/pASsc9AJiUQkyN6Yp/COqaB4s651YCC4C+ceXmxq2vDCbjjyHZgiAJqpP8PtQdb10dfZNiiOHfh7oY+gTz8TEsiKtvIHBM0u/zL/gu0lQk7D8whcT3YENNq5tw/gGFMwh+vxupPX58m1JraGKj2jTn3Crn3Fh8C8CoJm5+Jn68zLfGmHn4FpmOND6IOBVvAtX4rpiTgSeDb+cAx+CTpsOBjkGy9RWND3quANolLesVNz0deCspgesQDHRuzJfABnV9rUe3pG6dAfj3E9Z/3NNYd8vJ+p70uxD/ng+oW2CMKQW6ATNTCb6ZzEyKIRTM18UwO2l9OxKT2unAQ0m/z/bOuU03ZP+BQXH7X9/nCRp/r+PjNvhux7rfb0K9xpg8/HtfJz45TLYZ/jOp1Bqa2Kg2xfhBrDcYP2g2PxiweTj+BPl+E+oZiR83cRg+Iap7bYtv8dh/Q+ILuigeAc4DfklcNxT+22kE/484ZIw5Fd9y0ZgvgK2NMaOD4zwX/62+ziOANcacaowpClpGBhlj9l1Hnc8Dezf5wNYvBNxojCk2xgzCd7PUjaVY33E/Bgw3fvBxiTGmwBgTH+M81pH4BC0jjwDXGGN6BQnWLcBE4LNmOr5UjAUuMsYMC8ZjXQrkAa8E6x8FLjTGDDbGFOO76+LP4fcARxtjDor7bI80xuyW4v4fBg43xvzCGBM2xuyH/wzWdbVOwCegBwaflcOAXZPqaOy9PtUYs3XQEnkhUBJ3XF8Aexk/UL4QuA6IH8A+Dz94OP6zizGmDP/39mKKx6faCE1sVFtTg/82+Cy+CXshcBlwnnPuqSbU82tgvHPuJefcvLjX18BTrGMQcQr+BeyG7w6L/8f6MH4Q7mT8t/eRrCMZc86NA24FXsd3gXQHPoxbPw/YA3+l0zR8N9Nz+G/pjXkU2DJIPprTdPw3+Kn4Y3wd/48b1nPcwQDT3fEDn2fh/xHGDzy+FLja+CuN/tnI/s8HBH+VzQx8983BQaLZUm7GX8L8BjAf3xW5T3D1D/jxT/8FPsG/TzPw7xsAzrlv8eNWfo//fS/AJ0spdVU65z7EjzX6G/6zcBNwvHPuk2D9FPwA4Pvwfzv7As8kVdPYe30f8Peg3qOAA5xzy4N1/8YnJ+PxXV8z8L/nurgmAf8APgu62OoGQx8DvOuc+ymV41Nth/HdnUoplRpjzFnATs65lK62SaG+k/EDd/V+JDnIGDMN//t9bH1lm1BnIfAtPvn8obnqVbkhL9MBKKVaF+fcvcC9mY5DtV3BVWPrGlel2jDtilJKKaVUztCuKKWUUkrlDG2xUUoppVTO0MRGKaWUUjlDExullFJK5QxNbJRSSimVMzSxUUoppVTO+H/T7uJ2k6XvOwAAAABJRU5ErkJggg==\\n\",\n \"text/plain\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" load airtemperature \\\\\\n\",\n \"item_id timestamp \\n\",\n \"Bull_education_Magaret 2016-01-01 00:00:00 0.0000 9.4 \\n\",\n \" 2016-01-01 01:00:00 2.7908 8.9 \\n\",\n \" 2016-01-01 02:00:00 3.7210 8.9 \\n\",\n \" 2016-01-01 03:00:00 2.7908 8.3 \\n\",\n \" 2016-01-01 04:00:00 9.3025 7.8 \\n\",\n \"\\n\",\n \" dewtemperature sealvlpressure \\n\",\n \"item_id timestamp \\n\",\n \"Bull_education_Magaret 2016-01-01 00:00:00 3.3 1028.699951 \\n\",\n \" 2016-01-01 01:00:00 2.2 1028.800049 \\n\",\n \" 2016-01-01 02:00:00 2.2 1029.599976 \\n\",\n \" 2016-01-01 03:00:00 1.7 1029.500000 \\n\",\n \" 2016-01-01 04:00:00 1.7 1029.599976 \"\n ]\n },\n \"execution_count\": 7,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"data = TimeSeriesDataFrame.from_path(\\n\",\n \" \\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/bull/test.parquet\\\", id_column=\\\"id\\\"\\n\",\n \")\\n\",\n \"data.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The goal is to forecast next day's (24 hours) load using historical load and known weather covariates: air temperature, dew temperature and sea level pressure. Since future weather information is not known in advance, weather forecasts are typically used as known covariates.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Sorting the dataframe index before generating the train/test split.\\n\"\n ]\n }\n ],\n \"source\": [\n \"prediction_length = 24\\n\",\n \"train_data, test_data = data.train_test_split(prediction_length=prediction_length)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The following code uses Chronos-2 in the TimeSeriesPredictor to forecast the `load` for the next 24 hours. We use the _univariate_ [Chronos-Bolt (Small)](https://huggingface.co/autogluon/chronos-bolt-small) model as a baseline for comparison.\\n\",\n \"\\n\",\n \"Note that we have specified the target column we are interested in forecasting and the names of known covariates while constructing the TimeSeriesPredictor. Any other columns, if present, will be used as past-only covariates.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {\n \"tags\": [\n \"hide-output\"\n ]\n },\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Beginning AutoGluon training... Time limit = 60s\\n\",\n \"AutoGluon will save models to '/fsx/ansarnd/repos/autogluon/docs/tutorials/timeseries/AutogluonModels/ag-20251214_125334'\\n\",\n \"=================== System Info ===================\\n\",\n \"AutoGluon Version: 1.4.1b20250910\\n\",\n \"Python Version: 3.11.11\\n\",\n \"Operating System: Linux\\n\",\n \"Platform Machine: x86_64\\n\",\n \"Platform Version: #38~22.04.1-Ubuntu SMP Fri Aug 22 15:44:33 UTC 2025\\n\",\n \"CPU Count: 96\\n\",\n \"Pytorch Version: 2.7.1+cu126\\n\",\n \"CUDA Version: 12.6\\n\",\n \"GPU Memory: GPU 0: 39.37/39.38 GB\\n\",\n \"Total GPU Memory: Free: 39.37 GB, Allocated: 0.01 GB, Total: 39.38 GB\\n\",\n \"GPU Count: 1\\n\",\n \"Memory Avail: 1029.12 GB / 1121.80 GB (91.7%)\\n\",\n \"Disk Space Avail: 1758.43 GB / 11459.15 GB (15.3%)\\n\",\n \"===================================================\\n\",\n \"\\n\",\n \"Fitting with arguments:\\n\",\n \"{'enable_ensemble': False,\\n\",\n \" 'eval_metric': MASE,\\n\",\n \" 'hyperparameters': {'Chronos': {}, 'Chronos2': {}},\\n\",\n \" 'known_covariates_names': ['airtemperature',\\n\",\n \" 'dewtemperature',\\n\",\n \" 'sealvlpressure'],\\n\",\n \" 'num_val_windows': 1,\\n\",\n \" 'prediction_length': 24,\\n\",\n \" 'quantile_levels': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\\n\",\n \" 'random_seed': 123,\\n\",\n \" 'refit_every_n_windows': 1,\\n\",\n \" 'refit_full': False,\\n\",\n \" 'skip_model_selection': False,\\n\",\n \" 'target': 'load',\\n\",\n \" 'time_limit': 60,\\n\",\n \" 'verbosity': 2}\\n\",\n \"\\n\",\n \"Inferred time series frequency: 'h'\\n\",\n \"Provided train_data has 718320 rows, 41 time series. Median time series length is 17520 (min=17520, max=17520). \\n\",\n \"\\n\",\n \"Provided data contains following columns:\\n\",\n \"\\ttarget: 'load'\\n\",\n \"\\tknown_covariates:\\n\",\n \"\\t\\tcategorical: []\\n\",\n \"\\t\\tcontinuous (float): ['airtemperature', 'dewtemperature', 'sealvlpressure']\\n\",\n \"\\n\",\n \"To learn how to fix incorrectly inferred types, please see documentation for TimeSeriesPredictor.fit\\n\",\n \"\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'MASE'\\n\",\n \"\\tThis metric's sign has been flipped to adhere to being higher_is_better. The metric score can be multiplied by -1 to get the metric value.\\n\",\n \"===================================================\\n\",\n \"\\n\",\n \"Starting training. Start time is 2025-12-14 12:53:34\\n\",\n \"Models that will be trained: ['Chronos[autogluon__chronos-bolt-small]', 'Chronos2']\\n\",\n \"Training timeseries model Chronos[autogluon__chronos-bolt-small]. Training for up to 29.9s of the 59.9s of remaining time.\\n\",\n \"\\t-1.0865 = Validation score (-MASE)\\n\",\n \"\\t1.23 s = Training runtime\\n\",\n \"\\t0.58 s = Validation (prediction) runtime\\n\",\n \"Training timeseries model Chronos2. Training for up to 58.0s of the 58.0s of remaining time.\\n\",\n \"\\t-0.8172 = Validation score (-MASE)\\n\",\n \"\\t0.78 s = Training runtime\\n\",\n \"\\t1.60 s = Validation (prediction) runtime\\n\",\n \"Training complete. Models trained: ['Chronos[autogluon__chronos-bolt-small]', 'Chronos2']\\n\",\n \"Total runtime: 4.27 s\\n\",\n \"Best model: Chronos2\\n\",\n \"Best model score: -0.8172\\n\"\n ]\n }\n ],\n \"source\": [\n \"predictor = TimeSeriesPredictor(\\n\",\n \" prediction_length=prediction_length,\\n\",\n \" target=\\\"load\\\",\\n\",\n \" known_covariates_names=[\\\"airtemperature\\\", \\\"dewtemperature\\\", \\\"sealvlpressure\\\"],\\n\",\n \" eval_metric=\\\"MASE\\\",\\n\",\n \").fit(\\n\",\n \" train_data,\\n\",\n \" hyperparameters={\\\"Chronos\\\": {}, \\\"Chronos2\\\": {}},\\n\",\n \" enable_ensemble=False,\\n\",\n \" time_limit=60,\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Once the predictor has been fit, we can evaluate it on the test dataset and generate the leaderboard. We see that Chronos-2, which utilizes covariates, produces a significantly more accurate forecast on the test set compared to Chronos-Bolt, which does not utilize covariates.\\n\",\n \"\\n\",\n \"Note that all AutoGluon-TimeSeries models report scores in a \\\"higher is better\\\" format, meaning that most forecasting error metrics like MASE are multiplied by -1 when reported.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Additional data provided, testing on additional data. Resulting leaderboard will be sorted according to test score (`score_test`).\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"| \\n\",\n \" | \\n\",\n \" | load | \\n\",\n \"airtemperature | \\n\",\n \"dewtemperature | \\n\",\n \"sealvlpressure | \\n\",\n \"
|---|---|---|---|---|---|
| item_id | \\n\",\n \"timestamp | \\n\",\n \"\\n\",\n \" | \\n\",\n \" | \\n\",\n \" | \\n\",\n \" |
| Bull_education_Magaret | \\n\",\n \"2016-01-01 00:00:00 | \\n\",\n \"0.0000 | \\n\",\n \"9.4 | \\n\",\n \"3.3 | \\n\",\n \"1028.699951 | \\n\",\n \"
| 2016-01-01 01:00:00 | \\n\",\n \"2.7908 | \\n\",\n \"8.9 | \\n\",\n \"2.2 | \\n\",\n \"1028.800049 | \\n\",\n \"|
| 2016-01-01 02:00:00 | \\n\",\n \"3.7210 | \\n\",\n \"8.9 | \\n\",\n \"2.2 | \\n\",\n \"1029.599976 | \\n\",\n \"|
| 2016-01-01 03:00:00 | \\n\",\n \"2.7908 | \\n\",\n \"8.3 | \\n\",\n \"1.7 | \\n\",\n \"1029.500000 | \\n\",\n \"|
| 2016-01-01 04:00:00 | \\n\",\n \"9.3025 | \\n\",\n \"7.8 | \\n\",\n \"1.7 | \\n\",\n \"1029.599976 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" model score_test score_val \\\\\\n\",\n \"0 Chronos2 -0.696239 -0.817203 \\n\",\n \"1 Chronos[autogluon__chronos-bolt-small] -1.278404 -1.086471 \\n\",\n \"\\n\",\n \" pred_time_test pred_time_val fit_time_marginal fit_order \\n\",\n \"0 2.261695 1.600290 0.782884 2 \\n\",\n \"1 0.618992 0.576958 1.233093 1 \"\n ]\n },\n \"execution_count\": 10,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"predictor.leaderboard(test_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can also use the predictor to compute features importances to understand which exogenous features are affecting the prediction accuracy the most.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Computing feature importance\\n\",\n \"Subsample_size 50 is larger than the number of items in the data and will be ignored\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"| \\n\",\n \" | model | \\n\",\n \"score_test | \\n\",\n \"score_val | \\n\",\n \"pred_time_test | \\n\",\n \"pred_time_val | \\n\",\n \"fit_time_marginal | \\n\",\n \"fit_order | \\n\",\n \"
|---|---|---|---|---|---|---|---|
| 0 | \\n\",\n \"Chronos2 | \\n\",\n \"-0.696239 | \\n\",\n \"-0.817203 | \\n\",\n \"2.261695 | \\n\",\n \"1.600290 | \\n\",\n \"0.782884 | \\n\",\n \"2 | \\n\",\n \"
| 1 | \\n\",\n \"Chronos[autogluon__chronos-bolt-small] | \\n\",\n \"-1.278404 | \\n\",\n \"-1.086471 | \\n\",\n \"0.618992 | \\n\",\n \"0.576958 | \\n\",\n \"1.233093 | \\n\",\n \"1 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" importance stdev n p99_low p99_high\\n\",\n \"airtemperature 0.324309 0.0 5.0 0.324309 0.324309\\n\",\n \"dewtemperature 0.057110 0.0 5.0 0.057110 0.057110\\n\",\n \"sealvlpressure 0.038278 0.0 5.0 0.038278 0.038278\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"predictor.feature_importance(test_data, model=\\\"Chronos2\\\", relative_scores=True)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"With `relative_scores=True`, this method returns relative (percentage) improvements in the `eval_metric` due to each feature. In this example, the `airtemperature` feature is the most important for accurate forecasting, yielding a ~32% error reduction on the test set.\\n\",\n \"\\n\",\n \"Note that covariates may not always be useful and using more covariates does not necessarily imply more accurate forecasts. With Chronos-2, AutoGluon makes it easy for users to quickly validate different configurations and find ones that perform best on held-out data. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Fine-tuning \\n\",\n \"\\n\",\n \"We have seen above how Chronos-2 models can produce forecasts in zero-shot mode, both with and without covariates. AutoGluon also makes it easy to fine-tune Chronos models on a specific dataset to maximize the predictive accuracy.\\n\",\n \"\\n\",\n \"The following snippet specifies two settings for the Chronos-2 model: zero-shot and fine-tuned. `TimeSeriesPredictor` will perform a lightweight fine-tuning of the pretrained model on the provided training data. We add name suffixes to easily identify the zero-shot and fine-tuned versions of the model.\\n\",\n \"\\n\",\n \":::{note}\\n\",\n \"\\n\",\n \"If you are fine-tuning on a machine with multiple GPUs, we strongly recommend setting the `CUDA_VISIBLE_DEVICES` environment variable to ensure that only a single GPU is visible. \\n\",\n \"\\n\",\n \":::\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {\n \"tags\": [\n \"hide-output\"\n ]\n },\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Beginning AutoGluon training... Time limit = 300s\\n\",\n \"AutoGluon will save models to '/fsx/ansarnd/repos/autogluon/docs/tutorials/timeseries/AutogluonModels/ag-20251214_125418'\\n\",\n \"=================== System Info ===================\\n\",\n \"AutoGluon Version: 1.4.1b20250910\\n\",\n \"Python Version: 3.11.11\\n\",\n \"Operating System: Linux\\n\",\n \"Platform Machine: x86_64\\n\",\n \"Platform Version: #38~22.04.1-Ubuntu SMP Fri Aug 22 15:44:33 UTC 2025\\n\",\n \"CPU Count: 96\\n\",\n \"Pytorch Version: 2.7.1+cu126\\n\",\n \"CUDA Version: 12.6\\n\",\n \"GPU Memory: GPU 0: 39.37/39.38 GB\\n\",\n \"Total GPU Memory: Free: 39.37 GB, Allocated: 0.01 GB, Total: 39.38 GB\\n\",\n \"GPU Count: 1\\n\",\n \"Memory Avail: 1028.93 GB / 1121.80 GB (91.7%)\\n\",\n \"Disk Space Avail: 1758.40 GB / 11459.15 GB (15.3%)\\n\",\n \"===================================================\\n\",\n \"\\n\",\n \"Fitting with arguments:\\n\",\n \"{'enable_ensemble': False,\\n\",\n \" 'eval_metric': MASE,\\n\",\n \" 'hyperparameters': {'Chronos2': [{'ag_args': {'name_suffix': 'ZeroShot'}},\\n\",\n \" {'ag_args': {'name_suffix': 'FineTuned'},\\n\",\n \" 'fine_tune': True}]},\\n\",\n \" 'known_covariates_names': ['airtemperature',\\n\",\n \" 'dewtemperature',\\n\",\n \" 'sealvlpressure'],\\n\",\n \" 'num_val_windows': 1,\\n\",\n \" 'prediction_length': 24,\\n\",\n \" 'quantile_levels': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\\n\",\n \" 'random_seed': 123,\\n\",\n \" 'refit_every_n_windows': 1,\\n\",\n \" 'refit_full': False,\\n\",\n \" 'skip_model_selection': False,\\n\",\n \" 'target': 'load',\\n\",\n \" 'time_limit': 300,\\n\",\n \" 'verbosity': 2}\\n\",\n \"\\n\",\n \"Inferred time series frequency: 'h'\\n\",\n \"Provided train_data has 718320 rows, 41 time series. Median time series length is 17520 (min=17520, max=17520). \\n\",\n \"\\n\",\n \"Provided data contains following columns:\\n\",\n \"\\ttarget: 'load'\\n\",\n \"\\tknown_covariates:\\n\",\n \"\\t\\tcategorical: []\\n\",\n \"\\t\\tcontinuous (float): ['airtemperature', 'dewtemperature', 'sealvlpressure']\\n\",\n \"\\n\",\n \"To learn how to fix incorrectly inferred types, please see documentation for TimeSeriesPredictor.fit\\n\",\n \"\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'MASE'\\n\",\n \"\\tThis metric's sign has been flipped to adhere to being higher_is_better. The metric score can be multiplied by -1 to get the metric value.\\n\",\n \"===================================================\\n\",\n \"\\n\",\n \"Starting training. Start time is 2025-12-14 12:54:19\\n\",\n \"Models that will be trained: ['Chronos2ZeroShot', 'Chronos2FineTuned']\\n\",\n \"Training timeseries model Chronos2ZeroShot. Training for up to 149.9s of the 299.9s of remaining time.\\n\",\n \"\\t-0.8172 = Validation score (-MASE)\\n\",\n \"\\t0.89 s = Training runtime\\n\",\n \"\\t1.59 s = Validation (prediction) runtime\\n\",\n \"Training timeseries model Chronos2FineTuned. Training for up to 297.4s of the 297.4s of remaining time.\\n\",\n \"\\t-0.8029 = Validation score (-MASE)\\n\",\n \"\\t123.89 s = Training runtime\\n\",\n \"\\t0.77 s = Validation (prediction) runtime\\n\",\n \"Training complete. Models trained: ['Chronos2ZeroShot', 'Chronos2FineTuned']\\n\",\n \"Total runtime: 127.20 s\\n\",\n \"Best model: Chronos2FineTuned\\n\",\n \"Best model score: -0.8029\\n\"\n ]\n }\n ],\n \"source\": [\n \"predictor = TimeSeriesPredictor(\\n\",\n \" prediction_length=prediction_length,\\n\",\n \" target=\\\"load\\\",\\n\",\n \" known_covariates_names=[\\\"airtemperature\\\", \\\"dewtemperature\\\", \\\"sealvlpressure\\\"],\\n\",\n \" eval_metric=\\\"MASE\\\",\\n\",\n \").fit(\\n\",\n \" train_data=train_data,\\n\",\n \" hyperparameters={\\n\",\n \" \\\"Chronos2\\\": [\\n\",\n \" # Zero-shot model\\n\",\n \" {\\\"ag_args\\\": {\\\"name_suffix\\\": \\\"ZeroShot\\\"}},\\n\",\n \" # Fine-tuned model\\n\",\n \" {\\\"fine_tune\\\": True, \\\"ag_args\\\": {\\\"name_suffix\\\": \\\"FineTuned\\\"}},\\n\",\n \" ]\\n\",\n \" },\\n\",\n \" time_limit=300, # time limit in seconds\\n\",\n \" enable_ensemble=False,\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Here we used the default fine-tuning configuration for Chronos-2 by only specifying `\\\"fine_tune\\\": True`. By default, Chronos-2 is fine-tuned with a low-rank adapter (LoRA) to reduce memory and disk footprint. AutoGluon makes it easy to change other parameters for fine-tuning such as the mode, number of steps or learning rate.\\n\",\n \"```python\\n\",\n \"predictor.fit(\\n\",\n \" ...,\\n\",\n \" hyperparameters={\\\"Chronos2\\\": {\\\"fine_tune\\\": True, \\\"fine_tune_mode\\\": \\\"full\\\", \\\"fine_tune_lr\\\": 1e-4, \\\"fine_tune_steps\\\": 2000, \\\"fine_tune_batch_size\\\": 32}},\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"For the full list of fine-tuning options, see the Chronos-2 documentation in [Forecasting Model Zoo](forecasting-model-zoo.md#autogluon.timeseries.models.Chronos2Model).\\n\",\n \"\\n\",\n \"\\n\",\n \"After fitting, we can evaluate the two model variants on the test data and generate a leaderboard.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 13,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Additional data provided, testing on additional data. Resulting leaderboard will be sorted according to test score (`score_test`).\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"| \\n\",\n \" | importance | \\n\",\n \"stdev | \\n\",\n \"n | \\n\",\n \"p99_low | \\n\",\n \"p99_high | \\n\",\n \"
|---|---|---|---|---|---|
| airtemperature | \\n\",\n \"0.324309 | \\n\",\n \"0.0 | \\n\",\n \"5.0 | \\n\",\n \"0.324309 | \\n\",\n \"0.324309 | \\n\",\n \"
| dewtemperature | \\n\",\n \"0.057110 | \\n\",\n \"0.0 | \\n\",\n \"5.0 | \\n\",\n \"0.057110 | \\n\",\n \"0.057110 | \\n\",\n \"
| sealvlpressure | \\n\",\n \"0.038278 | \\n\",\n \"0.0 | \\n\",\n \"5.0 | \\n\",\n \"0.038278 | \\n\",\n \"0.038278 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" model score_test score_val pred_time_test pred_time_val \\\\\\n\",\n \"0 Chronos2FineTuned -0.677888 -0.802909 2.510460 0.773595 \\n\",\n \"1 Chronos2ZeroShot -0.696239 -0.817203 2.385682 1.586984 \\n\",\n \"\\n\",\n \" fit_time_marginal fit_order \\n\",\n \"0 123.887496 2 \\n\",\n \"1 0.885827 1 \"\n ]\n },\n \"execution_count\": 13,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"predictor.leaderboard(test_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Fine-tuning resulted in a more accurate model, as shown by the better `score_test` on the test set.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## FAQ\\n\",\n \"\\n\",\n \"\\n\",\n \"#### How accurate is Chronos-2?\\n\",\n \"\\n\",\n \"Chronos-2 is the best performing (last updated: Dec 2025) time series foundation model across multiple benchmarks, including [fev-bench](https://huggingface.co/spaces/autogluon/fev-bench), [GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval) and [Chronos Bench II](https://arxiv.org/abs/2403.07815). Details empirical results can be found in the [Chronos-2 technical report](https://arxiv.org/abs/2510.15821). The accuracy of Chronos-2 often exceeds statistical baseline models and task-specific deep learning models such as `DeepAR` and `TemporalFusionTransformer`.\\n\",\n \"\\n\",\n \"#### Does fine-tuning always improve Chronos-2's forecasting accuracy?\\n\",\n \"\\n\",\n \"Fine-tuning a foundation model like Chronos-2 involves many hyperparameter choices. AG-TS provides reasonable defaults that performed well in large-scale benchmarking, but they may not be optimal for every use case. We recommend fine-tuning only when you have a reasonable number of time series and sufficient historical data (e.g., >100 time series with a median history length larger than `3 * prediction_length`), as limited data can lead to overfitting or degraded performance. If you observe degraded accuracy, we recommend increasing the size of the training data and experimenting with different fine-tuning hyperparameters.\\n\",\n \"\\n\",\n \"Alternatively, you can use an ensemble of zero-shot Chronos-2 and fine-tuned Chronos-2 (Small) to construct a robust predictor, available via the `chronos2_ensemble` preset:\\n\",\n \"\\n\",\n \"```py\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length).fit(\\n\",\n \" ...,\\n\",\n \" presets=\\\"chronos2_ensemble\\\",\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"#### What is the recommended hardware for running Chronos models?\\n\",\n \"\\n\",\n \"We recommend using a machine with a GPU for best performance, especially for fine-tuning. For reference, we tested the models on AWS `g5.2xlarge` instances with NVIDIA A10G GPUs (24 GiB GPU memory) and 32 GiB of system memory. However, Chronos-2, Chronos-Bolt, and Chronos (up to small size) can also run on consumer GPUs and CPUs with reasonable inference times.\\n\",\n \"\\n\",\n \"#### Why do my predictions change with the `batch_size`?\\n\",\n \"\\n\",\n \"By default, AutoGluon enables Chronos-2âs cross_learning mode, where the model makes joint predictions across time series within a batch. This often improves accuracy but also makes results sensitive to the `batch_size`. You can disable this mode with:\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.fit(\\n\",\n \" ...,\\n\",\n \" hyperparameters={\\\"Chronos2\\\": {\\\"cross_learning\\\": False}},\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"#### Where can I ask specific questions on Chronos?\\n\",\n \"\\n\",\n \"Members of the AutoGluon team are among the core developers of Chronos. So you can ask Chronos-related questions on [AutoGluon's GitHub](https://github.com/autogluon/autogluon) or on [Chronos' GitHub](https://github.com/amazon-science/chronos-forecasting/discussions).\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"autogluon\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.11\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-ensembles.md",
"content": "(forecasting_ensembles)=\n# Forecasting Time Series - Ensemble Models\n\n:::{note}\nThis documentation is intended for advanced users and may not be comprehensive.\n\nFor a stable public API, refer to the [documentation for `TimeSeriesPredictor`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.html).\n:::\n\nThis page contains the list of time series ensemble models available in AutoGluon.\nThese models combine predictions from multiple base forecasting models to improve accuracy.\n\nThe available hyperparameters for each model are listed under **Parameters**.\n\nThe model names in the `ensemble_hyperparameters` dictionary don't have to include the `\"Ensemble\"` suffix\n(e.g., both `\"SimpleAverage\"` and `\"SimpleAverageEnsemble\"` correspond to {class}`~autogluon.timeseries.models.ensemble.SimpleAverageEnsemble`).\n\n\n## How ensembling works\n\nEnsemble models combine predictions from multiple base forecasting models to produce a final forecast. The ensemble is trained on held-out validation data (backtest windows) to learn how to best combine the base model predictions.\n\nBy default, AutoGluon uses a single {class}`~autogluon.timeseries.models.ensemble.GreedyEnsemble` that learns optimal weights for each base model. You can configure which ensemble models to use via the `ensemble_hyperparameters` argument in [`TimeSeriesPredictor.fit()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.fit.html).\n\n\n## Multi-layer stacking\n\nMulti-layer stacking extends the basic ensembling approach by training ensembles in multiple stages. Each layer of ensembles is trained on different backtest windows, and uses predictions from the previous layer as its inputs.\n\nFor example, with `num_val_windows=(3, 2)` and two ensemble layers:\n\n```\nTime series: [...history...][window 1][window 2][window 3][window 4][window 5]\n ââââââââââ Layer 2 âââââââââââââââââ Layer 3 ââââââ\n```\n\n1. Base models generate predictions for all 5 backtest windows\n2. Layer 2 ensembles are trained on windows 1-3, learning to combine base model predictions\n3. Layer 3 ensembles are trained on windows 4-5, using Layer 1 ensemble predictions as inputs\n4. Final validation scores are computed on windows 4-5\n\nThis approach allows later ensemble layers to correct errors made by earlier layers, often improving forecast accuracy.\n\nTo enable multi-layer stacking, pass a list of dicts to `ensemble_hyperparameters` and a matching tuple to `num_val_windows`:\n\n```{code-cell} ipython3\nfrom autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor\n\nfull_data = TimeSeriesDataFrame('https://autogluon.s3.amazonaws.com/datasets/timeseries/electricity_small/train.csv')\nprediction_length = 48\ntrain_data, test_data = full_data.train_test_split(prediction_length)\n\npredictor = TimeSeriesPredictor(prediction_length=prediction_length, eval_metric=\"MASE\")\npredictor.fit(\n train_data,\n # Base models\n hyperparameters={\"SeasonalNaive\": {}, \"Theta\": {}, \"DirectTabular\": {}, \"ETS\": {}, \"RecursiveTabular\": {}},\n ensemble_hyperparameters=[\n # Layer 2\n {\"WeightedEnsemble\": {}, \"Median\": {}, \"LinearStacker\": {\"weights_per\": \"mt\"}},\n # Layer 3\n {\"WeightedEnsemble\": {}}, # Layer 2\n ],\n num_val_windows=(3, 2), # 3 windows to fit L2 models, 2 windows to fit L3 models\n refit_every_n_windows=\"auto\",\n)\npredictor.leaderboard(test_data)\n```\n\nAfter training the predictor, you can access the validation predictions & targets used to train the ensembles using\n[`backtest_targets()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_targets.html)\nand [`backtest_predictions()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_predictions.html) methods.\n\n\n## Overview\n\n```{eval-rst}\n.. automodule:: autogluon.timeseries.models.ensemble\n```\n\n```{eval-rst}\n.. currentmodule:: autogluon.timeseries.models.ensemble\n```\n\n```{eval-rst}\n.. autosummary::\n :nosignatures:\n\n GreedyEnsemble\n LinearStackerEnsemble\n MedianEnsemble\n PerItemGreedyEnsemble\n PerQuantileTabularEnsemble\n SimpleAverageEnsemble\n TabularEnsemble\n\n```\n\n## Simple averages\n\nSimple ensemble models that combine predictions using mean or median aggregation.\n\n\n```{eval-rst}\n.. autoclass:: SimpleAverageEnsemble\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: MedianEnsemble\n :members: init\n```\n\n\n## Linear ensembles\n\nLinear ensemble models that combine predictions using weighted averages or linear stacking.\n\n\n```{eval-rst}\n.. autoclass:: GreedyEnsemble\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: PerItemGreedyEnsemble\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: LinearStackerEnsemble\n :members: init\n```\n\n\n## Nonlinear ensembles\n\nNonlinear ensemble models that use tabular models to combine predictions from base forecasters.\n\n\n```{eval-rst}\n.. autoclass:: TabularEnsemble\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: PerQuantileTabularEnsemble\n :members: init\n```\n\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-faq.md",
"content": "# AutoGluon Time Series FAQ\n\n## What forecasting tasks can AutoGluon be used for?\nAutoGluon can generate **probabilistic** multi-step-ahead forecasts for one or multiple **univariate** time series.\nFor example, you can use AutoGluon to forecast daily sales of multiple products over the next month.\nThis setting should not be confused with multivariate time series forecasting (see [this discussion](https://stats.stackexchange.com/a/365394/134754) about the difference between multivariate and multiple univariate time series).\n\nAutoGluon also supports additional information, such as time-independent static features (e.g., location of the store)\nand time-dependent covariates (e.g., price of the product each day).\nSee the [In Depth Tutorial](forecasting-indepth.ipynb) for more details.\n\nCurrently, AutoGluon does not support features such as hierarchical forecasting and forecast explainability.\n\n## How can I get the most accurate forecasts?\nTo maximize the forecast accuracy, set `presets=\"best_quality\"` and provide a high `time_limit` when calling {py:meth}`~autogluon.timeseries.TimeSeriesPredictor.fit()`.\n\nYou can typically increase the forecast accuracy even further by increasing `num_val_windows` to 3-5 when calling `predictor.fit()`, but this will require an even longer training time. To speed up training when using multiple validation windows, you can increase the `refit_every_n_windows` argument to `predictor.fit()`.\n\n## How should I choose the evaluation metric?\nSee [Evaluation Metrics](forecasting-metrics.md).\n\n## Are there any restrictions on the data that I can pass to TimeSeriesPredictor?\nSee section \"What data format is expected by `TimeSeriesPredictor`?\" in the [In Depth Tutorial](forecasting-indepth.ipynb).\n\n\n## Can I use GPUs for model training?\nYes! All deep learning models used by `autogluon.timeseries` support GPU training.\nThe models will be automatically trained on a GPU if (1) your machine has a GPU and (2) you installed a PyTorch version with CUDA support.\nMulti-GPU training is not yet supported.\n\n\n## What machine is best for running AutoGluon TimeSeries?\nAutoGluon can be run on any machine including your laptop.\nHaving multiple CPU cores makes training faster for most forecasting models, and deep learning models additionally benefit from a GPU.\nWhen using AWS, we recommend [G4](https://aws.amazon.com/ec2/instance-types/g4/) or [G5](https://aws.amazon.com/ec2/instance-types/g5/) instances with a single GPU and at least 16 CPU cores for fastest training on large datasets.\n\nMachines without a GPU but with a large number of CPU cores are also well suited for training the `TimeSeriesPredictor`.\nAmong CPU-only instances on AWS, we recommend [M6](https://aws.amazon.com/ec2/instance-types/m6i/) instances, where a `m6i.24xlarge` machine should be able to handle most datasets.\n\n\n## Issues not addressed here\nFirst search if your issue is addressed in the [tutorials](index.md),\n[documentation](../../api/autogluon.timeseries.TimeSeriesPredictor.rst), or [Github issues](https://github.com/autogluon/autogluon/issues)\n(search both Closed and Open issues).\nIf it is not there, please open a [new Github Issue](https://github.com/autogluon/autogluon/issues/new) and\nclearly state your issue and clarify how it relates to the module.\n\nIf you have a bug, please include: your code (ideally set `verbosity=4` which will print out more details), the\noutput printed during the code execution, and information about your operating system, Python version, and\ninstalled packages (output of `pip freeze`).\nMany user issues stem from incorrectly formatted data, so please describe your data as clearly as possible.\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-indepth.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"408fb5d1\",\n \"metadata\": {},\n \"source\": [\n \"# Forecasting Time Series - In Depth\\n\",\n \"\\n\",\n \"[](https://colab.research.google.com/github/autogluon/autogluon/blob/master/docs/tutorials/timeseries/forecasting-indepth.ipynb)\\n\",\n \"[](https://studiolab.sagemaker.aws/import/github/autogluon/autogluon/blob/master/docs/tutorials/timeseries/forecasting-indepth.ipynb)\\n\",\n \"\\n\",\n \"\\n\",\n \"This tutorial provides an in-depth overview of the time series forecasting capabilities in AutoGluon.\\n\",\n \"Specifically, we will cover:\\n\",\n \"\\n\",\n \"- What is probabilistic time series forecasting?\\n\",\n \"- Forecasting time series with additional information\\n\",\n \"- What data format is expected by `TimeSeriesPredictor`?\\n\",\n \"- How to evaluate forecast accuracy?\\n\",\n \"- Which forecasting models are available in AutoGluon?\\n\",\n \"- What functionality does `TimeSeriesPredictor` offer?\\n\",\n \" - Basic configuration with `presets` and `time_limit`\\n\",\n \" - Manually selecting what models to train\\n\",\n \" - Hyperparameter tuning\\n\",\n \"\\n\",\n \"This tutorial assumes that you are familiar with the contents of [Forecasting Time Series - Quick Start](forecasting-quick-start.ipynb).\\n\",\n \"\\n\",\n \"## What is probabilistic time series forecasting?\\n\",\n \"A time series is a sequence of measurements made at regular intervals.\\n\",\n \"The main objective of time series forecasting is to predict the future values of a time series given the past observations.\\n\",\n \"A typical example of this task is demand forecasting.\\n\",\n \"For example, we can represent the number of daily purchases of a certain product as a time series.\\n\",\n \"The goal in this case could be predicting the demand for each of the next 14 days (i.e., the forecast horizon) given the historical purchase data.\\n\",\n \"In AutoGluon, the `prediction_length` argument of the `TimeSeriesPredictor` determines the length of the forecast horizon.\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"The objective of forecasting could be to predict future values of a given time series, as well as establishing prediction intervals within which the future values will likely lie.\\n\",\n \"In AutoGluon, the `TimeSeriesPredictor` generates two types of forecasts:\\n\",\n \"\\n\",\n \"- **mean forecast** represents the expected value of the time series at each time step in the forecast horizon.\\n\",\n \"- **quantile forecast** represents the quantiles of the forecast distribution.\\n\",\n \"For example, if the `0.1` quantile (also known as P10, or the 10th percentile) is equal to `x`, it means that the time series value is predicted to be below `x` 10% of the time. As another example, the `0.5` quantile (P50) corresponds to the median forecast.\\n\",\n \"Quantiles can be used to reason about the range of possible outcomes.\\n\",\n \"For instance, by the definition of the quantiles, the time series is predicted to be between the P10 and P90 values with 80% probability.\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"By default, the `TimeSeriesPredictor` outputs the quantiles `[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]`. Custom quantiles can be provided with the `quantile_levels` argument\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(quantile_levels=[0.05, 0.5, 0.95])\\n\",\n \"```\\n\",\n \"\\n\",\n \"## Forecasting time series with additional information\\n\",\n \"In real-world forecasting problems we often have access to additional information, beyond just the raw time series values.\\n\",\n \"AutoGluon supports two types of such additional information: static features and time-varying covariates.\\n\",\n \"\\n\",\n \"```{note}\\n\",\n \"Not all models available in AutoGluon support all types of features & covariates. For an overview, see [Forecasting Model Zoo / Additional features](forecasting-model-zoo.md#additional-features).\\n\",\n \"```\\n\",\n \"\\n\",\n \"### Static features\\n\",\n \"Static features are the time-independent attributes (metadata) of a time series.\\n\",\n \"These may include information such as:\\n\",\n \"\\n\",\n \"- location, where the time series was recorded (country, state, city)\\n\",\n \"- fixed properties of a product (brand name, color, size, weight)\\n\",\n \"- store ID or product ID\\n\",\n \"\\n\",\n \"Providing this information may, for instance, help forecasting models generate similar demand forecasts for stores located in the same city.\\n\",\n \"\\n\",\n \"In AutoGluon, static features are stored as an attribute of a `TimeSeriesDataFrame` object.\\n\",\n \"As an example, let's have a look at the M4 Daily dataset.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"e21f6ccd\",\n \"metadata\": {\n \"tags\": [\n \"remove-cell\",\n \"skip-execution\"\n ]\n },\n \"outputs\": [],\n \"source\": [\n \"# We use uv for faster installation\\n\",\n \"!pip install uv\\n\",\n \"!uv pip install -q autogluon.timeseries --system\\n\",\n \"!uv pip uninstall -q torchaudio torchvision torchtext --system # fix incompatible package versions on Colab\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"c01be409\",\n \"metadata\": {\n \"lines_to_next_cell\": 2,\n \"tags\": [\n \"remove-cell\"\n ]\n },\n \"outputs\": [],\n \"source\": [\n \"import warnings\\n\",\n \"warnings.filterwarnings(action=\\\"ignore\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"dc0c3e40\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"import pandas as pd\\n\",\n \"from autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e70d4f39\",\n \"metadata\": {},\n \"source\": [\n \"We download a subset of 100 time series from the M4 Daily dataset.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"0607284a\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"df = pd.read_csv(\\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_daily_subset/train.csv\\\")\\n\",\n \"df.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"34ee733c\",\n \"metadata\": {},\n \"source\": [\n \"We also load the corresponding static features.\\n\",\n \"In the M4 Daily dataset, there is a single categorical static feature that denotes the domain of origin for each time series.\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"7e37b644\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"static_features_df = pd.read_csv(\\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_daily_subset/metadata.csv\\\")\\n\",\n \"static_features_df.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"542969d0\",\n \"metadata\": {},\n \"source\": [\n \"AutoGluon expects static features as a pandas.DataFrame object. The `item_id` column indicates which item (=individual time series) in `df` each row of `static_features` corresponds to.\\n\",\n \"\\n\",\n \"We can now create a `TimeSeriesDataFrame` that contains both the time series values and the static features.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"96d2c37a\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"train_data = TimeSeriesDataFrame.from_data_frame(\\n\",\n \" df,\\n\",\n \" id_column=\\\"item_id\\\",\\n\",\n \" timestamp_column=\\\"timestamp\\\",\\n\",\n \" static_features_df=static_features_df,\\n\",\n \")\\n\",\n \"train_data.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"da911756\",\n \"metadata\": {},\n \"source\": [\n \"We can validate that `train_data` now also includes the static features using the `.static_features` attribute\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"7996d724\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"train_data.static_features.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"64f9ea6b\",\n \"metadata\": {},\n \"source\": [\n \"Alternatively, we can attach static features to an existing `TimeSeriesDataFrame` by assigning the `.static_features` attribute\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"3a72b3de\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"train_data.static_features = static_features_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"ff8c92ca\",\n \"metadata\": {},\n \"source\": [\n \"\\n\",\n \"If `static_features` doesn't contain some `item_id`s that are present in `train_data`, an exception will be raised.\\n\",\n \"\\n\",\n \"Now, when we fit the predictor, all models that support static features will automatically use the static features included in `train_data`.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=14).fit(train_data)\\n\",\n \"```\\n\",\n \"\\n\",\n \"```\\n\",\n \"...\\n\",\n \"Following types of static features have been inferred:\\n\",\n \"\\tcategorical: ['domain']\\n\",\n \"\\tcontinuous (float): []\\n\",\n \"...\\n\",\n \"```\\n\",\n \"\\n\",\n \"This message confirms that column `'domain'` was interpreted as a categorical feature.\\n\",\n \"In general, AutoGluon-TimeSeries supports two types of static features:\\n\",\n \"\\n\",\n \"- `categorical`: columns of dtype `object`, `string` and `category` are interpreted as discrete categories\\n\",\n \"- `continuous`: columns of dtype `int` and `float` are interpreted as continuous (real-valued) numbers\\n\",\n \"- columns with other dtypes are ignored\\n\",\n \"\\n\",\n \"To override this logic, we need to manually change the columns dtype.\\n\",\n \"For example, suppose the static features dataframe contained an integer-valued column `\\\"store_id\\\"`.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"train_data.static_features[\\\"store_id\\\"] = list(range(len(train_data.item_ids)))\\n\",\n \"```\\n\",\n \"\\n\",\n \"By default, this column will be interpreted as a continuous number.\\n\",\n \"We can force AutoGluon to interpret it a a categorical feature by changing the dtype to `category`.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"train_data.static_features[\\\"store_id\\\"] = train_data.static_features[\\\"store_id\\\"].astype(\\\"category\\\")\\n\",\n \"```\\n\",\n \"\\n\",\n \"**Note:** If training data contained static features, the predictor will expect that data passed to `predictor.predict()`, `predictor.leaderboard()`, and `predictor.evaluate()` also includes static features with the same column names and data types.\\n\",\n \"\\n\",\n \"\\n\",\n \"### Time-varying covariates\\n\",\n \"Covariates are the time-varying features that may influence the target time series.\\n\",\n \"They are sometimes also referred to as dynamic features, exogenous regressors, or related time series.\\n\",\n \"AutoGluon supports two types of covariates:\\n\",\n \"\\n\",\n \"- *known* covariates that are known for the entire forecast horizon, such as\\n\",\n \" - holidays\\n\",\n \" - day of the week, month, year\\n\",\n \" - promotions\\n\",\n \"\\n\",\n \"- *past* covariates that are only known up to the start of the forecast horizon, such as\\n\",\n \" - sales of other products\\n\",\n \" - temperature, precipitation\\n\",\n \" - transformed target time series\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"In AutoGluon, both `known_covariates` and `past_covariates` are stored as additional columns in the `TimeSeriesDataFrame`.\\n\",\n \"\\n\",\n \"We will again use the M4 Daily dataset as an example and generate both types of covariates:\\n\",\n \"\\n\",\n \"- a `past_covariate` equal to the logarithm of the target time series:\\n\",\n \"- a `known_covariate` that equals to 1 if a given day is a weekend, and 0 otherwise.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8ec12a06\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"import numpy as np\\n\",\n \"train_data[\\\"log_target\\\"] = np.log(train_data[\\\"target\\\"])\\n\",\n \"\\n\",\n \"WEEKEND_INDICES = [5, 6]\\n\",\n \"timestamps = train_data.index.get_level_values(\\\"timestamp\\\")\\n\",\n \"train_data[\\\"weekend\\\"] = timestamps.weekday.isin(WEEKEND_INDICES).astype(float)\\n\",\n \"\\n\",\n \"train_data.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"68c62e58\",\n \"metadata\": {},\n \"source\": [\n \"When creating the TimeSeriesPredictor, we specify that the column `\\\"target\\\"` is our prediction target, and the\\n\",\n \"column `\\\"weekend\\\"` contains a covariate that will be known at prediction time.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(\\n\",\n \" prediction_length=14,\\n\",\n \" target=\\\"target\\\",\\n\",\n \" known_covariates_names=[\\\"weekend\\\"],\\n\",\n \").fit(train_data)\\n\",\n \"```\\n\",\n \"\\n\",\n \"Predictor will automatically interpret the remaining columns (except target and known covariates) as past covariates.\\n\",\n \"This information is logged during fitting:\\n\",\n \"\\n\",\n \"```\\n\",\n \"...\\n\",\n \"Provided dataset contains following columns:\\n\",\n \"\\ttarget: 'target'\\n\",\n \"\\tknown covariates: ['weekend']\\n\",\n \"\\tpast covariates: ['log_target']\\n\",\n \"...\\n\",\n \"```\\n\",\n \"\\n\",\n \"Finally, to make predictions, we generate the known covariates for the forecast horizon\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"dfa42122\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"predictor = TimeSeriesPredictor(prediction_length=14, freq=train_data.freq)\\n\",\n \"\\n\",\n \"known_covariates = predictor.make_future_data_frame(train_data)\\n\",\n \"known_covariates[\\\"weekend\\\"] = known_covariates[\\\"timestamp\\\"].dt.weekday.isin(WEEKEND_INDICES).astype(float)\\n\",\n \"\\n\",\n \"known_covariates.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"753cb592\",\n \"metadata\": {},\n \"source\": [\n \"Note that `known_covariates` must satisfy the following conditions:\\n\",\n \"\\n\",\n \"- The columns must include all columns listed in ``predictor.known_covariates_names``\\n\",\n \"- The ``item_id`` index must include all item ids present in ``train_data``\\n\",\n \"- The ``timestamp`` index must include the values for ``prediction_length`` many time steps into the future from the end of each time series in ``train_data``\\n\",\n \"\\n\",\n \"If `known_covariates` contain more information than necessary (e.g., contain additional columns, item_ids, or timestamps),\\n\",\n \"AutoGluon will automatically select the necessary rows and columns.\\n\",\n \"\\n\",\n \"Finally, we pass the `known_covariates` to the `predict` function to generate predictions\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.predict(train_data, known_covariates=known_covariates)\\n\",\n \"```\\n\",\n \"\\n\",\n \"The list of models that support static features and covariates is available in [Forecasting Model Zoo](forecasting-model-zoo.md).\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"18a3c37e\",\n \"metadata\": {},\n \"source\": [\n \"### Holidays\\n\",\n \"Another popular example of `known_covariates` are holiday features. In this section we describe how to add holiday features to a time series dataset and use them in AutoGluon.\\n\",\n \"\\n\",\n \"First, we need to define a dictionary with dates in `datetime.date` format as keys and holiday names as values.\\n\",\n \"We can easily generate such a dictionary using the [`holidays`](https://pypi.org/project/holidays/) Python package.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"547ad391\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!pip install -q holidays\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"84a26868\",\n \"metadata\": {},\n \"source\": [\n \"Here we use German holidays for demonstration purposes only. Make sure to define a holiday calendar that matches your country/region!\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"c1c4bd52\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import holidays\\n\",\n \"\\n\",\n \"timestamps = train_data.index.get_level_values(\\\"timestamp\\\")\\n\",\n \"country_holidays = holidays.country_holidays(\\n\",\n \" country=\\\"DE\\\", # make sure to select the correct country/region!\\n\",\n \" # Add + 1 year to make sure that holidays are initialized for the forecast horizon\\n\",\n \" years=range(timestamps.min().year, timestamps.max().year + 1),\\n\",\n \")\\n\",\n \"# Convert dict to pd.Series for pretty visualization\\n\",\n \"pd.Series(country_holidays).sort_index().head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"4c1eb254\",\n \"metadata\": {},\n \"source\": [\n \"Alternatively, we can manually define a dictionary with custom holidays.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"be8e2a60\",\n \"metadata\": {\n \"lines_to_next_cell\": 1\n },\n \"outputs\": [],\n \"source\": [\n \"import datetime\\n\",\n \"\\n\",\n \"# must cover the full train time range + forecast horizon\\n\",\n \"custom_holidays = {\\n\",\n \" datetime.date(1995, 1, 29): \\\"Superbowl\\\",\\n\",\n \" datetime.date(1995, 11, 29): \\\"Black Friday\\\",\\n\",\n \" datetime.date(1996, 1, 28): \\\"Superbowl\\\",\\n\",\n \" datetime.date(1996, 11, 29): \\\"Black Friday\\\",\\n\",\n \" # ...\\n\",\n \"}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b9b3dd03\",\n \"metadata\": {},\n \"source\": [\n \"Next, we define a method that adds holiday features as columns to a `TimeSeriesDataFrame`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"35777be9\",\n \"metadata\": {\n \"lines_to_next_cell\": 1\n },\n \"outputs\": [],\n \"source\": [\n \"def add_holiday_features(\\n\",\n \" ts_df: TimeSeriesDataFrame,\\n\",\n \" country_holidays: dict,\\n\",\n \" include_individual_holidays: bool = True,\\n\",\n \" include_holiday_indicator: bool = True,\\n\",\n \") -> TimeSeriesDataFrame:\\n\",\n \" \\\"\\\"\\\"Add holiday indicator columns to a TimeSeriesDataFrame.\\\"\\\"\\\"\\n\",\n \" ts_df = ts_df.copy()\\n\",\n \" if not isinstance(ts_df, TimeSeriesDataFrame):\\n\",\n \" ts_df = TimeSeriesDataFrame(ts_df)\\n\",\n \" timestamps = ts_df.index.get_level_values(\\\"timestamp\\\")\\n\",\n \" country_holidays_df = pd.get_dummies(pd.Series(country_holidays)).astype(float)\\n\",\n \" holidays_df = country_holidays_df.reindex(timestamps.date).fillna(0)\\n\",\n \" if include_individual_holidays:\\n\",\n \" ts_df[holidays_df.columns] = holidays_df.values\\n\",\n \" if include_holiday_indicator:\\n\",\n \" ts_df[\\\"Holiday\\\"] = holidays_df.max(axis=1).values\\n\",\n \" return ts_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"5dce18a2\",\n \"metadata\": {},\n \"source\": [\n \"We can create a single indicator feature for all holidays.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"c6efe88c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"add_holiday_features(train_data, country_holidays, include_individual_holidays=False).head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"4cc3a073\",\n \"metadata\": {},\n \"source\": [\n \"Or represent each holiday with a separate feature.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"2fb13e9e\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"train_data_with_holidays = add_holiday_features(train_data, country_holidays)\\n\",\n \"train_data_with_holidays.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"c1186dd7\",\n \"metadata\": {},\n \"source\": [\n \"Remember to add the names of holiday features as `known_covariates_names` when creating `TimeSeriesPredictor`.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"holiday_columns = train_data_with_holidays.columns.difference(train_data.columns)\\n\",\n \"predictor = TimeSeriesPredictor(..., known_covariates_names=holiday_columns).fit(train_data_with_holidays, ...)\\n\",\n \"```\\n\",\n \"\\n\",\n \"At prediction time, we need to provide future holiday values as `known_covariates`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"0e234271\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"known_covariates = predictor.make_future_data_frame(train_data)\\n\",\n \"known_covariates = add_holiday_features(known_covariates, country_holidays)\\n\",\n \"known_covariates.head()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"189f40cc\",\n \"metadata\": {},\n \"source\": [\n \"```python\\n\",\n \"predictions = predictor.predict(train_data_with_holidays, known_covariates=known_covariates)\\n\",\n \"```\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"18b255c5\",\n \"metadata\": {},\n \"source\": [\n \"## What data format is expected by `TimeSeriesPredictor`?\\n\",\n \"\\n\",\n \"AutoGluon expects that at least some time series in the training data are long enough to generate an internal validation set.\\n\",\n \"\\n\",\n \"This means, at least some time series in `train_data` must have length `>= max(prediction_length + 1, 5) + prediction_length` when training with default settings\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length).fit(train_data)\\n\",\n \"```\\n\",\n \"\\n\",\n \"If you use advanced configuration options, such as following,\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length).fit(train_data, num_val_windows=num_val_windows, val_step_size=val_step_size)\\n\",\n \"```\\n\",\n \"then at least some time series in `train_data` must have length `>= max(prediction_length + 1, 5) + prediction_length + (num_val_windows - 1) * val_step_size`.\\n\",\n \"\\n\",\n \"Note that all time series in the dataset can have different lengths.\\n\",\n \"\\n\",\n \"\\n\",\n \"### Handling irregular data and missing values\\n\",\n \"In some applications, like finance, data often comes with irregular measurements (e.g., no stock price is available for weekends or holidays) or missing values.\\n\",\n \"\\n\",\n \"Here is an example of a dataset with an irregular time index:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"41adc03e\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"df_irregular = TimeSeriesDataFrame(\\n\",\n \" pd.DataFrame(\\n\",\n \" {\\n\",\n \" \\\"item_id\\\": [0, 0, 0, 1, 1],\\n\",\n \" \\\"timestamp\\\": [\\\"2022-01-01\\\", \\\"2022-01-02\\\", \\\"2022-01-04\\\", \\\"2022-01-01\\\", \\\"2022-01-04\\\"],\\n\",\n \" \\\"target\\\": [1, 2, 3, 4, 5],\\n\",\n \" }\\n\",\n \" )\\n\",\n \")\\n\",\n \"df_irregular\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"73a5e606\",\n \"metadata\": {},\n \"source\": [\n \"In such case, you can specify the desired frequency when creating the predictor using the `freq` argument.\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(..., freq=\\\"D\\\").fit(df_irregular)\\n\",\n \"```\\n\",\n \"Here we choose `freq=\\\"D\\\"` to indicate that the filled index must have a daily frequency\\n\",\n \"(see [other possible choices in pandas documentation](https://pandas.pydata.org/docs/user_guide/timeseries.html#timeseries-offset-aliases)).\\n\",\n \"\\n\",\n \"AutoGluon will automatically convert the irregular data into daily frequency and deal with missing values.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"d66624c3\",\n \"metadata\": {},\n \"source\": [\n \"--------\\n\",\n \"Alternatively, we can manually fill the gaps in the time index using the method [TimeSeriesDataFrame.convert_frequency()](../../api/autogluon.timeseries.TimeSeriesDataFrame.convert_frequency.rst).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"635cd9e3\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"df_regular = df_irregular.convert_frequency(freq=\\\"D\\\")\\n\",\n \"df_regular\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b61ae82b\",\n \"metadata\": {},\n \"source\": [\n \"We can verify that the index is now regular and has a daily frequency\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8307d936\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"print(f\\\"Data has frequency '{df_regular.freq}'\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"568ce2ce\",\n \"metadata\": {},\n \"source\": [\n \"Now the data contains missing values represented by `NaN`. Most time series models in AutoGluon can natively deal with missing values, so we can just pass data to the `TimeSeriesPredictor`.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"369b6d12\",\n \"metadata\": {},\n \"source\": [\n \"Alternatively, we can manually fill the NaNs with an appropriate strategy using [TimeSeriesDataFrame.fill_missing_values()](../../api/autogluon.timeseries.TimeSeriesDataFrame.fill_missing_values.rst).\\n\",\n \"By default, missing values are filled with a combination of forward + backward filling.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"d334d923\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"df_filled = df_regular.fill_missing_values()\\n\",\n \"df_filled\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"58477368\",\n \"metadata\": {},\n \"source\": [\n \"In some applications such as demand forecasting, missing values may correspond to zero demand. In this case constant fill is more appropriate.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"aafa46d0\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"df_filled = df_regular.fill_missing_values(method=\\\"constant\\\", value=0.0)\\n\",\n \"df_filled\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"160c12e9\",\n \"metadata\": {},\n \"source\": [\n \"## How to evaluate forecast accuracy?\\n\",\n \"\\n\",\n \"To measure how accurately `TimeSeriesPredictor` can forecast unseen time series, we need to reserve some test data that won't be used for training.\\n\",\n \"This can be easily done using the `train_test_split` method of a `TimeSeriesDataFrame`:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"7235f742\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"prediction_length = 48\\n\",\n \"full_data = TimeSeriesDataFrame.from_path(\\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/electricity_small/test.csv\\\")\\n\",\n \"train_data, test_data = full_data.train_test_split(prediction_length)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"beb53f75\",\n \"metadata\": {},\n \"source\": [\n \"We obtained two `TimeSeriesDataFrame`s from our original data:\\n\",\n \"- `test_data` contains exactly the same data as the original `full_data` (i.e., it contains both historical data and the forecast horizon)\\n\",\n \"- In `train_data`, the last `prediction_length` time steps are removed from the end of each time series (i.e., it contains only historical data)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"6525e889\",\n \"metadata\": {\n \"lines_to_next_cell\": 2\n },\n \"outputs\": [],\n \"source\": [\n \"import matplotlib.pyplot as plt\\n\",\n \"import numpy as np\\n\",\n \"\\n\",\n \"item_id = test_data.item_ids[0]\\n\",\n \"max_history_length = 300\\n\",\n \"fig, (ax1, ax2) = plt.subplots(nrows=2, figsize=[10, 4], sharex=True)\\n\",\n \"train_ts = train_data.loc[item_id].iloc[-max_history_length:]\\n\",\n \"test_ts = test_data.loc[item_id].iloc[-(max_history_length + prediction_length):]\\n\",\n \"ax1.set_title(\\\"Train data (past time series values)\\\")\\n\",\n \"ax1.plot(train_ts)\\n\",\n \"ax2.set_title(\\\"Test data (past + future time series values)\\\")\\n\",\n \"ax2.plot(test_ts)\\n\",\n \"for ax in (ax1, ax2):\\n\",\n \" ax.fill_between(np.array([train_ts.index[-1], test_ts.index[-1]]), test_ts.min(), test_ts.max(), color=\\\"C1\\\", alpha=0.3, label=\\\"Forecast horizon\\\")\\n\",\n \"plt.legend()\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"4e94c30d\",\n \"metadata\": {},\n \"source\": [\n \"We can now use `train_data` to train the predictor, and `test_data` to obtain an estimate of its performance on unseen data with the [`evaluate()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.evaluate.html) method.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"ef01f795\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length, eval_metric=\\\"MASE\\\")\\n\",\n \"predictor.fit(train_data, hyperparameters={\\\"SeasonalNaive\\\": {}, \\\"RecursiveTabular\\\": {}, \\\"Chronos\\\": {}}, verbosity=0)\\n\",\n \"predictor.evaluate(test_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"09d19b4b\",\n \"metadata\": {},\n \"source\": [\n \"AutoGluon evaluates the performance of forecasting models by measuring how well their forecasts align with the actually observed time series.\\n\",\n \"When we call [`evaluate()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.evaluate.html), the predictor does the following for each time series in `test_data`:\\n\",\n \"\\n\",\n \"1. Hold out the last `prediction_length` values of the time series.\\n\",\n \"2. Generate a forecast for the held out part of the time series, i.e., the forecast horizon.\\n\",\n \"3. Quantify how well the forecast matches the actually observed (held out) values of the time series using the `eval_metric`.\\n\",\n \"\\n\",\n \"Finally, the scores are averaged over all time series in the dataset.\\n\",\n \"\\n\",\n \"The crucial detail here is that `evaluate` always computes the score on the last `prediction_length` time steps of each time series.\\n\",\n \"The beginning of each time series (except the last `prediction_length` time steps) is only used to initialize the models before forecasting.\\n\",\n \"\\n\",\n \"Note that `evaluate()` returns the score for the best model (based on internal validation performance), which is used by default during [`predict()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.predict.html).\\n\",\n \"To see scores for all models, use the [`leaderboard()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.leaderboard.html) method.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"bd661d2a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"predictor.leaderboard(test_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b9b1c314\",\n \"metadata\": {},\n \"source\": [\n \"For more details about the evaluation metrics, see [Forecasting Evaluation Metrics](forecasting-metrics.md).\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"1c95b5b2\",\n \"metadata\": {},\n \"source\": [\n \"### Backtesting using multiple cutoffs\\n\",\n \"\\n\",\n \"We can more accurately estimate the performance using **backtest** (i.e., evaluate performance on multiple forecast horizons generated from the same time series).\\n\",\n \"First, reserve enough test data for multiple windows:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"0435ecc1\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"num_test_windows = 3\\n\",\n \"train_data, test_data = full_data.train_test_split(num_test_windows * prediction_length)\\n\",\n \"\\n\",\n \"# Fit the predictor\\n\",\n \"predictor = TimeSeriesPredictor(prediction_length=prediction_length, eval_metric=\\\"MASE\\\")\\n\",\n \"predictor.fit(train_data, hyperparameters={\\\"SeasonalNaive\\\": {}, \\\"RecursiveTabular\\\": {}, \\\"Chronos\\\": {}}, verbosity=0)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"db85971e\",\n \"metadata\": {},\n \"source\": [\n \"Now evaluate on each window using the `cutoff` argument to the [`evaluate()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.evaluate.html) method.\\n\",\n \"The `evaluate` method will measure the forecast accuracy using the `prediction_length` time steps after the `cutoff` index as a hold-out set (marked in orange). By default (if no `cutoff` is provided), the cutoff value will be set to `-1 * prediction_length`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8dcf142d\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"for cutoff in range(-num_test_windows * prediction_length, 0, prediction_length):\\n\",\n \" score = predictor.evaluate(test_data, cutoff=cutoff)\\n\",\n \" print(f\\\"Cutoff {cutoff}: score = {score}\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"74e5ab05\",\n \"metadata\": {},\n \"source\": [\n \"The figure below visualizes the backtest splits for `num_test_windows=3` and `prediction_length=3`.\\n\",\n \"By choosing different `cutoff` values we can evaluate the model on different splits. For each split, the forecast accuracy is evaluated on the `prediction_length` time steps (orange) after the `cutoff`.\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"Multi-window backtesting typically results in more accurate estimation of the forecast quality on unseen data.\\n\",\n \"However, this strategy decreases the amount of training data available for fitting models, so we recommend using single-window backtesting if the training time series are short.\\n\",\n \"\\n\",\n \"### Comparing model performance across windows\\n\",\n \"\\n\",\n \"We can compare the performance of different models across the time windows by collecting the [`leaderboard()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.leaderboard.html) scores for each cutoff:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"eed2675a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"leaderboards = []\\n\",\n \"for cutoff in range(-num_test_windows * prediction_length, 0, prediction_length):\\n\",\n \" lb = predictor.leaderboard(test_data, cutoff=cutoff)\\n\",\n \" lb[\\\"cutoff\\\"] = cutoff\\n\",\n \" leaderboards.append(lb)\\n\",\n \"\\n\",\n \"scores_per_window = pd.concat(leaderboards).pivot(index=\\\"model\\\", columns=\\\"cutoff\\\", values=\\\"score_test\\\")\\n\",\n \"scores_per_window\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"ad137e0e\",\n \"metadata\": {},\n \"source\": [\n \"### Visualizing backtest predictions\\n\",\n \"\\n\",\n \"To inspect the predictions and the corresponding targets from each backtest window, use the [`backtest_predictions()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_predictions.html)\\n\",\n \"and [`backtest_targets()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_targets.html) methods:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"bb30b121\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"predictions_per_window = predictor.backtest_predictions(test_data, num_val_windows=num_test_windows)\\n\",\n \"targets_per_window = predictor.backtest_targets(test_data, num_val_windows=num_test_windows)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"9c208b66\",\n \"metadata\": {},\n \"source\": [\n \"Each element in the list corresponds to predictions for one backtest window. Visualize all predictions together using the [`plot()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.plot.html) method:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"2b3cb328\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"item_ids = test_data.item_ids[:4]\\n\",\n \"all_predictions = pd.concat(predictions_per_window)\\n\",\n \"predictor.plot(test_data, all_predictions, max_history_length=300, item_ids=item_ids)\\n\",\n \"\\n\",\n \"# Optional: Plot the cutoff dates with dashed lines\\n\",\n \"for cutoff in range(-num_test_windows * prediction_length, 0, prediction_length):\\n\",\n \" for i, ax in enumerate(plt.gcf().axes):\\n\",\n \" cutoff_timestamp = test_data.loc[item_ids[i]].index[cutoff]\\n\",\n \" ax.axvline(cutoff_timestamp, color='gray', linestyle='--', alpha=0.7)\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"b1859e73\",\n \"metadata\": {},\n \"source\": [\n \"The plot shows the observed time series along with predictions from all backtest windows.\\n\",\n \"The dashed gray lines mark the cutoff points for each backtest window.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"4819b30f\",\n \"metadata\": {},\n \"source\": [\n \"### Internal validation\\n\",\n \"When we fit the predictor with `predictor.fit(train_data=train_data)`, under the hood AutoGluon further splits the original dataset `train_data` into train and validation parts.\\n\",\n \"\\n\",\n \"Performance of different models on the validation set is evaluated using the `evaluate` method, just like described above.\\n\",\n \"The model that achieves the best validation score will be used for prediction in the end.\\n\",\n \"\\n\",\n \"By default, the internal validation set contains a single window containing the last `prediction_length` time steps of each time series. We can increase the number of validation windows using the `num_val_windows` argument.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(...)\\n\",\n \"predictor.fit(train_data, num_val_windows=3)\\n\",\n \"```\\n\",\n \"This will reduce the likelihood of overfitting but will increase the training time approximately by a factor of `num_val_windows`.\\n\",\n \"Note that multiple validation windows can only be used if the time series in `train_data` have length of at least `(num_val_windows + 1) * prediction_length`.\\n\",\n \"\\n\",\n \"Alternatively, a user can provide their own validation set to the `fit` method. In this case it's important to remember that the validation score is computed on the last `prediction_length` time steps of each time series.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.fit(train_data=train_data, tuning_data=my_validation_dataset)\\n\",\n \"```\\n\",\n \"\\n\",\n \"After training is complete, we can check the validation score of each model in the `score_val` column of the [`leaderboard()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.leaderboard.html) (called without passing new data):\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.leaderboard()\\n\",\n \"```\\n\",\n \"\\n\",\n \"Similarly, we can load the validation predictions with the corresponding targets by calling [`backtest_predictions()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_predictions.html) and [`backtest_targets()`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.backtest_targets.html) without passing new data:\\n\",\n \"```python\\n\",\n \"val_predictions = predictor.backtest_predictions()\\n\",\n \"val_targets = predictor.backtest_targets()\\n\",\n \"```\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e28758f3\",\n \"metadata\": {},\n \"source\": [\n \"This is useful for debugging and understanding model behavior on the validation set.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e654b8f4\",\n \"metadata\": {},\n \"source\": [\n \"## Which forecasting models are available in AutoGluon?\\n\",\n \"Forecasting models in AutoGluon can be divided into three broad categories: local, global, and ensemble models.\\n\",\n \"\\n\",\n \"**Local models** are simple statistical models that are specifically designed to capture patterns such as trend or seasonality.\\n\",\n \"Despite their simplicity, these models often produce reasonable forecasts and serve as a strong baseline.\\n\",\n \"Some examples of available local models:\\n\",\n \"\\n\",\n \"- `ETS`\\n\",\n \"- `AutoARIMA`\\n\",\n \"- `Theta`\\n\",\n \"- `SeasonalNaive`\\n\",\n \"\\n\",\n \"If the dataset consists of multiple time series, we fit a separate local model to each time series â hence the name \\\"local\\\".\\n\",\n \"This means, if we want to make a forecast for a new time series that wasn't part of the training set, all local models will be fit from scratch for the new time series.\\n\",\n \"\\n\",\n \"**Global models** are machine learning algorithms that learn a single model from the entire training set consisting of multiple time series.\\n\",\n \"Most global models in AutoGluon are provided by the [GluonTS](https://ts.gluon.ai/stable/) library.\\n\",\n \"These are neural-network algorithms implemented in PyTorch, such as:\\n\",\n \"\\n\",\n \"- `DeepAR`\\n\",\n \"- `PatchTST`\\n\",\n \"- `DLinear`\\n\",\n \"- `TemporalFusionTransformer`\\n\",\n \"\\n\",\n \"This category also includes pre-trained zero-shot forecasting models like [Chronos](forecasting-chronos.ipynb).\\n\",\n \"\\n\",\n \"AutoGluon also offers two tabular global models `RecursiveTabular` and `DirectTabular`.\\n\",\n \"Under the hood, these models convert the forecasting task into a regression problem and fit models like LightGBM from the `autogluon.tabular` module.\\n\",\n \"\\n\",\n \"Finally, an **ensemble** model works by combining predictions of all other models.\\n\",\n \"By default, `TimeSeriesPredictor` always fits a `WeightedEnsemble` on top of other models.\\n\",\n \"This can be disabled by setting `enable_ensemble=False` when calling the `fit` method.\\n\",\n \"\\n\",\n \"For a list of tunable hyperparameters for each model, their default values, and other details see [Forecasting Model Zoo](forecasting-model-zoo.md).\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"e92c5004\",\n \"metadata\": {\n \"lines_to_next_cell\": 3\n },\n \"source\": [\n \"\\n\",\n \"## What functionality does `TimeSeriesPredictor` offer?\\n\",\n \"AutoGluon offers multiple ways to configure the behavior of a `TimeSeriesPredictor` that are suitable for both beginners and expert users.\\n\",\n \"\\n\",\n \"### Basic configuration with `presets` and `time_limit`\\n\",\n \"We can fit `TimeSeriesPredictor` with different pre-defined configurations using the `presets` argument of the `fit` method.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(...)\\n\",\n \"predictor.fit(train_data, presets=\\\"medium_quality\\\")\\n\",\n \"```\\n\",\n \"\\n\",\n \"Higher quality presets usually result in better forecasts but take longer to train.\\n\",\n \"The following presets are available:\\n\",\n \"\\n\",\n \"| Preset | Description | Use Cases | Fit Time (Ideal) |\\n\",\n \"| :------------- | :----------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :--------------- |\\n\",\n \"| `fast_training` | Fit simple statistical and baseline models + fast tree-based models | Fast to train but may not be very accurate | 0.5x |\\n\",\n \"| `medium_quality` | Same models as in `fast_training` + deep learning model `TemporalFusionTransformer` + Chronos-2 (small) | Good forecasts with reasonable training time | 1x |\\n\",\n \"| `high_quality` | More powerful deep learning, machine learning, statistical and pretrained forecasting models | Much more accurate than ``medium_quality``, but takes longer to train | 3x |\\n\",\n \"| `best_quality` | Same models as in `high_quality`, more cross-validation windows | Typically more accurate than `high_quality`, especially for datasets with few (<50) time series | 6x |\\n\",\n \"\\n\",\n \"You can find more information about the [presets](https://github.com/autogluon/autogluon/blob/stable/timeseries/src/autogluon/timeseries/configs/presets_configs.py) and the [models includes in each preset](https://github.com/autogluon/autogluon/blob/stable/timeseries/src/autogluon/timeseries/models/presets.py#L109) in the AutoGluon source code.\\n\",\n \"\\n\",\n \"Another way to control the training time is using the `time_limit` argument.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.fit(\\n\",\n \" train_data,\\n\",\n \" time_limit=60 * 60, # total training time in seconds\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"If no `time_limit` is provided, the predictor will train until all models have been fit.\\n\",\n \"\\n\",\n \"\\n\",\n \"### Manually configuring models\\n\",\n \"Advanced users can override the presets and manually specify what models should be trained by the predictor using the `hyperparameters` argument.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor = TimeSeriesPredictor(...)\\n\",\n \"\\n\",\n \"predictor.fit(\\n\",\n \" ...\\n\",\n \" hyperparameters={\\n\",\n \" \\\"DeepAR\\\": {},\\n\",\n \" \\\"Theta\\\": [\\n\",\n \" {\\\"decomposition_type\\\": \\\"additive\\\"},\\n\",\n \" {\\\"seasonal_period\\\": 1},\\n\",\n \" ],\\n\",\n \" }\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"The above example will train three models:\\n\",\n \"\\n\",\n \"* ``DeepAR`` with default hyperparameters\\n\",\n \"* ``Theta`` with additive seasonal decomposition (all other parameters set to their defaults)\\n\",\n \"* ``Theta`` with seasonality disabled (all other parameters set to their defaults)\\n\",\n \"\\n\",\n \"You can also exclude certain models from the presets using the `excluded_model_type` argument.\\n\",\n \"```python\\n\",\n \"predictor.fit(\\n\",\n \" ...\\n\",\n \" presets=\\\"high_quality\\\",\\n\",\n \" excluded_model_types=[\\\"AutoETS\\\", \\\"AutoARIMA\\\"],\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"For the full list of available models and the respective hyperparameters, see [Forecasting Model Zoo](forecasting-model-zoo.md).\\n\",\n \"\\n\",\n \"### Hyperparameter tuning\\n\",\n \"\\n\",\n \"Advanced users can define search spaces for model hyperparameters and let AutoGluon automatically determine the best configuration for the model.\\n\",\n \"\\n\",\n \"```python\\n\",\n \"from autogluon.common import space\\n\",\n \"\\n\",\n \"predictor = TimeSeriesPredictor()\\n\",\n \"\\n\",\n \"predictor.fit(\\n\",\n \" train_data,\\n\",\n \" hyperparameters={\\n\",\n \" \\\"DeepAR\\\": {\\n\",\n \" \\\"hidden_size\\\": space.Int(20, 100),\\n\",\n \" \\\"dropout_rate\\\": space.Categorical(0.1, 0.3),\\n\",\n \" },\\n\",\n \" },\\n\",\n \" hyperparameter_tune_kwargs=\\\"auto\\\",\\n\",\n \" enable_ensemble=False,\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"This code will train multiple versions of the `DeepAR` model with 10 different hyperparameter configurations.\\n\",\n \"AutGluon will automatically select the best model configuration that achieves the highest validation score and use it for prediction.\\n\",\n \"\\n\",\n \"Currently, HPO is based on Ray Tune for deep learning models from GluonTS, and random search for all other time series models.\\n\",\n \"\\n\",\n \"We can change the number of random search trials per model by passing a dictionary as `hyperparameter_tune_kwargs`\\n\",\n \"\\n\",\n \"```python\\n\",\n \"predictor.fit(\\n\",\n \" ...\\n\",\n \" hyperparameter_tune_kwargs={\\n\",\n \" \\\"num_trials\\\": 20,\\n\",\n \" \\\"scheduler\\\": \\\"local\\\",\\n\",\n \" \\\"searcher\\\": \\\"random\\\",\\n\",\n \" },\\n\",\n \" ...\\n\",\n \")\\n\",\n \"```\\n\",\n \"\\n\",\n \"The `hyperparameter_tune_kwargs` dict must include the following keys:\\n\",\n \"\\n\",\n \"- ``\\\"num_trials\\\"``: int, number of configurations to train for each tuned model\\n\",\n \"- ``\\\"searcher\\\"``: currently, the only supported option is ``\\\"random\\\"`` (random search).\\n\",\n \"- ``\\\"scheduler\\\"``: currently, the only supported option is ``\\\"local\\\"`` (all models trained on the same machine)\\n\",\n \"\\n\",\n \"**Note:** HPO significantly increases the training time for most models, but often provides only modest performance gains.\"\n ]\n }\n ],\n \"metadata\": {\n \"jupytext\": {\n \"cell_metadata_filter\": \"-all\",\n \"main_language\": \"python\",\n \"notebook_metadata_filter\": \"-all\"\n },\n \"kernelspec\": {\n \"display_name\": \"ag\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"name\": \"python\",\n \"version\": \"3.12.9\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-metrics.md",
"content": "---\njupytext:\n text_representation:\n format_name: myst\nkernelspec:\n display_name: Python 3\n name: python3\n---\n(forecasting_metrics)=\n# Forecasting Time Series - Evaluation Metrics\n\n\nPicking the right evaluation metric is one of the most important choices when using an AutoML framework.\nThis page lists the forecast evaluation metrics available in AutoGluon, explains [when different metrics should be used](#which-evaluation-metric-to-choose), and describes how to [define custom evaluation metrics](#custom-forecast-metrics).\n\nWhen using AutoGluon, you can specify the metric using the `eval_metric` argument to `TimeSeriesPredictor`, for example:\n```python\nfrom autogluon.timeseries import TimeSeriesPredictor\n\npredictor = TimeSeriesPredictor(eval_metric=\"MASE\")\n```\nAutoGluon will use the provided metric to tune model hyperparameters, rank models, and construct the final ensemble for prediction.\n\n:::{note}\nAutoGluon always reports all metrics in a **higher-is-better** format.\nFor this purpose, some metrics are multiplied by -1.\nFor example, if we set `eval_metric=\"MASE\"`, the predictor will actually report `-MASE` (i.e., MASE score multiplied by -1). This means the `test_score` will be between 0 (most accurate forecast) and $-\\infty$ (least accurate forecast).\n:::\n\n\nCurrently, AutoGluon supports following evaluation metrics:\n\n```{eval-rst}\n.. automodule:: autogluon.timeseries.metrics\n```\n\n```{eval-rst}\n.. currentmodule:: autogluon.timeseries.metrics\n```\n\n\n```{eval-rst}\n.. autosummary::\n :nosignatures:\n\n SQL\n WQL\n MAE\n MAPE\n MASE\n MSE\n RMSE\n RMSLE\n RMSSE\n SMAPE\n WAPE\n\n```\nAlternatively, you can [define a custom forecast evaluation metric](#custom-forecast-metrics).\n\n## Which evaluation metric to choose?\n\nIf you are not sure which evaluation metric to pick, here are three questions that can help you make the right choice for your use case.\n\n**1. Are you interested in a point forecast or a probabilistic forecast?**\n\nIf your goal is to generate an accurate **probabilistic** forecast, you should use `WQL` or `SQL` metrics.\nThese metrics are based on the [quantile loss](https://en.wikipedia.org/wiki/Quantile_regression) and measure the accuracy of the quantile forecasts.\nBy default, AutoGluon predicts quantile levels `[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]`.\nTo predict a different set of quantiles, you can use `quantile_levels` argument:\n```python\npredictor = TimeSeriesPredictor(eval_metric=\"WQL\", quantile_levels=[0.1, 0.5, 0.75, 0.9])\n```\n\nAll remaining forecast metrics described on this page are **point** forecast metrics.\nNote that if you select the `eval_metric` to a point forecast metric when creating the `TimeSeriesPredictor`, then the forecast minimizing this metric will always be provided in the `\"mean\"` column of the predictions dataframe.\n\n**2. Do you care more about accurately predicting time series with large values?**\n\nIf the answer is \"yes\" (for example, if it's important to more accurately predict sales of popular products), you should use **scale-dependent** metrics like `WQL`, `MAE`, `RMSE`, or `WAPE`.\nThese metrics are also well-suited for dealing with sparse (intermittent) time series that have lots of zeros.\n\nIf the answer is \"no\" (you care equally about all time series in the dataset), consider **scaled** metrics like `SQL`, `MASE` and `RMSSE`. Alternatively, **percentage-based** metrics `MAPE` and `SMAPE` can also be used to equalize the scale across time series. However, these percentage-based metrics have some [well-documented limitations](https://robjhyndman.com/publications/another-look-at-measures-of-forecast-accuracy/), so we don't recommend using them in practice.\nNote that both scaled and percentage-based metrics are poorly suited for sparse (intermittent) data.\n\n**3. (Point forecast only) Do you want to estimate the mean or the median?**\n\nTo estimate the **median**, you need to use metrics such as `MAE`, `MASE` or `WAPE`.\nIf your goal is to predict the **mean** (expected value), you should use `MSE`, `RMSE` or `RMSSE` metrics.\n\n\n\n```{eval-rst}\n.. list-table::\n :header-rows: 1\n :stub-columns: 1\n :align: center\n :widths: 40 20 20 20\n\n * - Metric\n - Probabilistic?\n - Scale-dependent?\n - Predicts median or mean?\n * - :class:`~autogluon.timeseries.metrics.SQL`\n - â
\n -\n -\n * - :class:`~autogluon.timeseries.metrics.WQL`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.metrics.MAE`\n -\n - â
\n - median\n * - :class:`~autogluon.timeseries.metrics.MASE`\n -\n -\n - median\n * - :class:`~autogluon.timeseries.metrics.WAPE`\n -\n - â
\n - median\n * - :class:`~autogluon.timeseries.metrics.MSE`\n -\n - â
\n - mean\n * - :class:`~autogluon.timeseries.metrics.RMSE`\n -\n - â
\n - mean\n * - :class:`~autogluon.timeseries.metrics.RMSLE`\n -\n -\n -\n * - :class:`~autogluon.timeseries.metrics.RMSSE`\n -\n -\n - mean\n * - :class:`~autogluon.timeseries.metrics.MAPE`\n -\n -\n -\n * - :class:`~autogluon.timeseries.metrics.SMAPE`\n -\n -\n -\n```\n\n\n\n## Point forecast metrics\nWe use the following notation in mathematical definitions of point forecast metrics:\n\n- $y_{i,t}$ - observed value of time series $i$ at time $t$\n- $f_{i,t}$ - predicted value of time series $i$ at time $t$\n- $N$ - number of time series (number of items) in the dataset\n- $T$ - length of the observed time series\n- $H$ - length of the forecast horizon (`prediction_length`)\n\n\n```{eval-rst}\n.. autoclass:: MAE\n```\n\n```{eval-rst}\n.. autoclass:: MAPE\n```\n\n```{eval-rst}\n.. autoclass:: MASE\n```\n\n```{eval-rst}\n.. autoclass:: MSE\n```\n\n```{eval-rst}\n.. autoclass:: RMSE\n```\n\n```{eval-rst}\n.. autoclass:: RMSLE\n```\n\n```{eval-rst}\n.. autoclass:: RMSSE\n```\n\n```{eval-rst}\n.. autoclass:: SMAPE\n```\n\n```{eval-rst}\n.. autoclass:: WAPE\n```\n\n\n## Probabilistic forecast metrics\nIn addition to the notation listed above, we use following notation to define probabilistic forecast metrics:\n\n- $f_{i,t}^q$ - predicted quantile $q$ of time series $i$ at time $t$\n- $\\rho_q(y, f) $ - quantile loss at level $q$ defined as\n\n$$\n \\rho_q(y_{i,t}, f_{i,t}^q) = \\begin{cases}\n 2 \\cdot (1 - q) \\cdot (f^q_{i,t} - y_{i,t}), & \\text{ if } y_{i,t} < f_{i,t}^q\\\\\n 2 \\cdot q \\cdot (y_{i,t} - f^q_{i,t} ), & \\text{ if } y_{i,t} \\ge f_{i,t}^q\\\\\n \\end{cases}\n$$\n\n\n\n```{eval-rst}\n.. autoclass:: SQL\n```\n\n```{eval-rst}\n.. autoclass:: WQL\n```\n\n\n## Custom forecast metrics\n\nIf none of the built-in metrics meet your requirements, you can provide a custom evaluation metric to AutoGluon.\nTo define a custom metric, you need to create a class that inherits from `TimeSeriesScorer` and implements the `compute_metric` method according to the following API specification:\n\n```{eval-rst}\n.. automethod:: TimeSeriesScorer.compute_metric\n```\n\n### Custom mean squared error metric\n\nHere is an example of how you can define a custom mean squared error (MSE) metric using `TimeSeriesScorer`.\n\n```{code-cell} ipython3\nimport sklearn.metrics\nfrom autogluon.timeseries.metrics import TimeSeriesScorer\n\nclass MeanSquaredError(TimeSeriesScorer):\n greater_is_better_internal = False\n optimum = 0.0\n\n def compute_metric(self, data_future, predictions, target, **kwargs):\n return sklearn.metrics.mean_squared_error(y_true=data_future[target], y_pred=predictions[\"mean\"])\n```\nThe internal method `compute_metric` returns metric in lower-is-better format, so we need to set `greater_is_better_internal=False`.\nThis will tell AutoGluon that the metric value must be multiplied by `-1` to convert it to greater-is-better format.\n\n:::{note}\nCustom metrics must be defined in a separate Python file and imported so that they can be pickled (Pythonâs serialization protocol). If a custom metric is not picklable, AutoGluon may crash during fit if you enable hyperparameter tuning. In the above example, you would want to create a new python file such as `my_metrics.py` with class `MeanSquaredError` defined in it, and then use it via `from my_metrics import MeanSquaredError`.\n:::\n\nWe can use the custom metric to measure accuracy of a forecast generated by the predictor.\n```{code-cell} ipython3\nimport pandas as pd\nfrom autogluon.timeseries import TimeSeriesPredictor, TimeSeriesDataFrame\n\n# Create dummy dataset\ndata = TimeSeriesDataFrame.from_iterable_dataset(\n [\n {\"start\": pd.Period(\"2023-01-01\", freq=\"D\"), \"target\": list(range(15))},\n {\"start\": pd.Period(\"2023-01-01\", freq=\"D\"), \"target\": list(range(30, 45))},\n ]\n)\nprediction_length = 3\ntrain_data, test_data = data.train_test_split(prediction_length=prediction_length)\npredictor = TimeSeriesPredictor(prediction_length=prediction_length, verbosity=0).fit(train_data, hyperparameters={\"Naive\": {}})\npredictions = predictor.predict(train_data)\n\nmse = MeanSquaredError(prediction_length=predictor.prediction_length)\nmse_score = mse(\n data=test_data,\n predictions=predictions,\n target=predictor.target,\n)\nprint(f\"{mse.name_with_sign} = {mse_score}\")\n```\nNote that the metric value has been multiplied by `-1` because we set `greater_is_better_internal=False` when defining the metric.\n\nWhen we call the metric, `TimeSeriesScorer` takes care of splitting `test_data` into past & future parts, validating that `predictions` have correct timestamps, and ensuring that the score is reported in greater-is-better format.\n\nDuring the metric call, the method `compute_metric` that we implemented receives as input the following arguments:\n\n- Test data corresponding to the forecast horizon\n\n```{code-cell} ipython3\ndata_future = test_data.slice_by_timestep(-prediction_length, None)\ndata_future\n```\n\n- Predictions for the forecast horizon\n\n```{code-cell} ipython3\npredictions.round(2)\n```\n\nNote that both `data_future` and `predictions` cover the same time range.\n\n\n### Custom quantile loss metric\nThe metric can be computed on any columns of the `predictions` DataFrame.\nFor example, here is how we can define the [mean quantile loss](https://otexts.com/fpp3/distaccuracy.html#quantile-scores) metric that measures the accuracy of the quantile forecast.\n\n```{code-cell} ipython3\nclass MeanQuantileLoss(TimeSeriesScorer):\n needs_quantile = True\n greater_is_better_internal = False\n optimum = 0.0\n\n def compute_metric(self, data_future, predictions, target, **kwargs):\n quantile_columns = [col for col in predictions if col != \"mean\"]\n total_quantile_loss = 0.0\n for q in quantile_columns:\n total_quantile_loss += sklearn.metrics.mean_pinball_loss(y_true=data_future[target], y_pred=predictions[q], alpha=float(q))\n return total_quantile_loss / len(quantile_columns)\n```\nHere we set `needs_quantile=True` to tell AutoGluon that this metric is evaluated on the quantile forecasts.\nIn this case, models such as {py:class}`~autogluon.timeseries.models.DirectTabularModel` will train a regression model from `autogluon.tabular` with `problem_type=\"quantile\"` under the hood.\nIf `needs_quantile=False`, these models will use `problem_type=\"regression\"` instead.\n\n### Custom mean absolute scaled error metric\nFinally, here is how we can define the [mean absolute scaled error (MASE) metric](https://otexts.com/fpp3/accuracy.html#scaled-errors).\nUnlike previously discussed metrics, MASE is computed using both **past** and **future** time series values.\nThe past values are used to compute the scale by which we normalize the error during the forecast horizon.\n\n```{code-cell} ipython3\nclass MeanAbsoluteScaledError(TimeSeriesScorer):\n greater_is_better_internal = False\n optimum = 0.0\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"mean_absolute_error\"\n\n def save_past_metrics(\n self, data_past: TimeSeriesDataFrame, target: str = \"target\", seasonal_period: int = 1, **kwargs\n ) -> None:\n seasonal_diffs = data_past[target].groupby(level=\"item_id\").diff(seasonal_period).abs()\n self._abs_seasonal_error_per_item = seasonal_diffs.groupby(level=\"item_id\").mean().fillna(1.0)\n\n def clear_past_metrics(self):\n self._abs_seasonal_error_per_item = None\n\n def compute_metric(\n self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = \"target\", **kwargs\n ) -> float:\n mae_per_item = (data_future[target] - predictions[\"mean\"]).abs().groupby(level=\"item_id\").mean()\n return (mae_per_item / self._abs_seasonal_error_per_item).mean()\n```\nWe compute the metrics on past data using `save_past_metrics` method.\nDoing this in a separate method allows AutoGluon to avoid redundant computations when fitting the weighted ensemble, which requires thousands of metric evaluations.\n\nBecause we set `optimized_by_median=True`, AutoGluon will automatically paste the median forecast into the `\"mean\"` column of predictions.\nThis is done for consistency: if `TimeSeriesPredictor` is trained with a point forecast metric, the optimal point forecast will always be stored in the `\"mean\"` column.\nFinally, the `equivalent_tabular_regression_metric` is used by forecasting models that fit tabular regression models from `autogluon.tabular` under the hood.\n\n\n### Using custom metrics in TimeSeriesPredictor\nNow that we have created several custom metrics, letâs use them for training and evaluating models.\n\n```{code-cell} ipython3\npredictor = TimeSeriesPredictor(eval_metric=MeanQuantileLoss()).fit(train_data, hyperparameters={\"Naive\": {}, \"SeasonalNaive\": {}, \"Theta\": {}})\n```\n\nWe can also evaluate a trained predictor using these custom metrics\n```{code-cell} ipython3\npredictor.evaluate(test_data, metrics=[MeanAbsoluteScaledError(), MeanQuantileLoss(), MeanSquaredError()])\n```\n\nThatâs all it takes to create and use custom forecasting metrics in AutoGluon!\n\nYou can have a look at the AutoGluon source code for example implementations of [point](https://github.com/autogluon/autogluon/blob/master/timeseries/src/autogluon/timeseries/metrics/point.py) and [quantile](https://github.com/autogluon/autogluon/blob/master/timeseries/src/autogluon/timeseries/metrics/quantile.py) forecasting metrics.\n\nIf you create a custom metric, consider [submitting a PR](https://github.com/autogluon/autogluon/pulls) so that we can officially add it to AutoGluon.\n\nFor more tutorials, refer to [Forecasting Time Series - Quick Start](forecasting-quick-start.ipynb) and [Forecasting Time Series - In Depth](forecasting-indepth.ipynb).\n\n\n## Customizing the training loss for individual models\nWhile `eval_metric` is used for model selection and weighted ensemble construction, it usually has no effect on the training loss of the individual forecasting models.\n\nIn some models such as `AutoETS` or `AutoARIMA`, the training loss is fixed and cannot be changed.\nIn contrast, for GluonTS-based deep learning models the training loss can be changed by modifying the `distr_output` [hyperparameter](forecasting-model-zoo.md#deep-learning-models).\nBy default, most GluonTS models set the `distr_output` to the heavyâtailed `StudentTOutput` distribution for increased robustness to outliers.\n\nYou can replace the default `StudentTOutput` with any builtâin `Output` from the [`gluonts.torch.distributions`](https://ts.gluon.ai/stable/api/gluonts/gluonts.torch.distributions.html) module.\nFor example, here we train two versions of PatchTST with different outputs and losses:\n- `NormalOutput` - the model outputs parameters of a Gaussian distribution and trains with the negative log-likelihood loss.\n- `QuantileOutput` - the model outputs a quantile forecast and trains with the quantile loss.\n\n```python\nfrom autogluon.timeseries import TimeSeriesPredictor\nfrom gluonts.torch.distributions import NormalOutput, QuantileOutput\n\npredictor = TimeSeriesPredictor(...)\npredictor.fit(\n train_data,\n hyperparameters={\n \"PatchTST\": [\n {\"distr_output\": NormalOutput()},\n {\"distr_output\": QuantileOutput(quantiles=predictor.quantile_levels)},\n ]\n }\n)\n```\n\nYou can define a custom loss function for the GluonTS models by defining a subclass of [`gluonts.torch.distributions.Output`](https://github.com/awslabs/gluonts/blob/dev/src/gluonts/torch/distributions/output.py)\nand providing it as a `distr_output` to the model.\n\n```python\nfrom gluonts.torch.distributions import Output\n\nclass MyCustomOutput(Output):\n # implement methods of gluonts.torch.distributions.Output\n ...\n\npredictor.fit(train_data, hyperparameters={\"PatchTST\": {\"distr_output\": MyCustomOutput()}})\n```\nYou can find examples of `Output` implementations in the GluonTS code base (e.g., [`QuantileOutput`](https://github.com/awslabs/gluonts/blob/dev/src/gluonts/torch/distributions/quantile_output.py) or [`NormalOutput`](https://github.com/awslabs/gluonts/blob/dev/src/gluonts/torch/distributions/distribution_output.py)).\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-model-zoo.md",
"content": "(forecasting_model_zoo)=\n# Forecasting Time Series - Model Zoo\n\n:::{note}\nThis documentation is intended for advanced users and may not be comprehensive.\n\nFor a stable public API, refer to the [documentation for `TimeSeriesPredictor`](https://auto.gluon.ai/stable/api/autogluon.timeseries.TimeSeriesPredictor.html).\n:::\n\nThis page contains the list of time series forecasting models available in AutoGluon.\nThe available hyperparameters for each model are listed under **Other Parameters**.\n\nThis list is useful if you want to override the default hyperparameters ([Manually configuring models](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-indepth.html#manually-configuring-models))\nor define custom hyperparameter search spaces ([Hyperparameter tuning](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-indepth.html#hyperparameter-tuning)), as described in the In-depth Tutorial.\nFor example, the following code will train a `TimeSeriesPredictor` with `DeepAR` and `ETS` models with default hyperparameters (and a weighted ensemble on top of them):\n\n```\npredictor = TimeSeriesPredictor().fit(\n train_data,\n hyperparameters={\n \"DeepAR\": {},\n \"ETS\": {},\n },\n)\n```\n\nThe model names in the `hyperparameters` dictionary don't have to include the `\"Model\"` suffix\n(e.g., both `\"DeepAR\"` and `\"DeepARModel\"` correspond to {class}`~autogluon.timeseries.models.DeepARModel`).\n\nNote that some of the models' hyperparameters have names and default values that are different from the original libraries.\n\n\n\n## Overview\n\n```{eval-rst}\n.. automodule:: autogluon.timeseries.models\n```\n\n```{eval-rst}\n.. currentmodule:: autogluon.timeseries.models\n```\n\n```{eval-rst}\n.. autosummary::\n :nosignatures:\n\n NaiveModel\n SeasonalNaiveModel\n AverageModel\n SeasonalAverageModel\n ZeroModel\n ETSModel\n AutoARIMAModel\n AutoETSModel\n AutoCESModel\n ThetaModel\n ADIDAModel\n CrostonModel\n IMAPAModel\n NPTSModel\n DeepARModel\n DLinearModel\n PatchTSTModel\n SimpleFeedForwardModel\n TemporalFusionTransformerModel\n TiDEModel\n WaveNetModel\n DirectTabularModel\n PerStepTabularModel\n RecursiveTabularModel\n Chronos2Model\n ChronosModel\n TotoModel\n\n```\n\n## Baseline models\n\nBaseline models are simple approaches that use minimal historical data to make predictions. They serve as benchmarks for evaluating more complex methods.\n\n```{eval-rst}\n.. autoclass:: NaiveModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: SeasonalNaiveModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: AverageModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: SeasonalAverageModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: ZeroModel\n :members: init\n\n```\n\n## Statistical models\n\nStatistical models capture simple patterns in the data like trends and seasonality.\n\n\n```{eval-rst}\n.. autoclass:: ETSModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: AutoARIMAModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: AutoETSModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: AutoCESModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: ThetaModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: NPTSModel\n :members: init\n\n```\n\n\n## Statistical models for sparse data\n\nStatistical models that are built specifically for sparse and nonnegative data, especially for use\nin intermittent demand forecasting.\n\n\n```{eval-rst}\n.. autoclass:: ADIDAModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: CrostonModel\n :members: init\n```\n\n\n```{eval-rst}\n.. autoclass:: IMAPAModel\n :members: init\n```\n\n## Deep learning models\n\nDeep learning models use neural networks to capture complex patterns in the data.\n\n```{eval-rst}\n.. autoclass:: DeepARModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: DLinearModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: PatchTSTModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: SimpleFeedForwardModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: TemporalFusionTransformerModel\n :members: init\n\n\n```\n\n\n```{eval-rst}\n.. autoclass:: TiDEModel\n :members: init\n\n\n```\n\n\n```{eval-rst}\n.. autoclass:: WaveNetModel\n :members: init\n\n\n```\n\n## Tabular models\n\nTabular models convert time series forecasting into a tabular regression problem.\n\n\n```{eval-rst}\n.. autoclass:: DirectTabularModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: PerStepTabularModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: RecursiveTabularModel\n :members: init\n\n```\n\n\n## Pretrained models\n\nDeep learning models pretrained on large time series datasets, able to perform zero-shot forecasting.\n\n\n```{eval-rst}\n.. autoclass:: Chronos2Model\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: ChronosModel\n :members: init\n\n```\n\n\n```{eval-rst}\n.. autoclass:: TotoModel\n :members: init\n\n```\n\n\n## Hyperparameters shared by all models\n\n- **target_scaler** *({\"standard\", \"mean_abs\", \"robust\", \"min_max\", None}, default = None)* - If provided, each time\n series will be scaled as `(y - loc) / scale` before being passed to the model for training / prediction. An inverse\n transformation `y * scale + loc` will be applied to the predictions.\n\n Note that `loc` and `scale` are computed separately for each individual time series.\n\n Available options:\n - `\"standard\"` - standard scaler, `loc = mean(y)`, `scale = std(y)`\n - `\"mean_abs\"` - mean absolute scaler, `loc = 0`, `scale = mean(abs(y))`\n - `\"robust\"` - robust scaler, `loc = median(y)`, `scale = quantile(y, 0.75) - quantile(y, 0.25)`\n - `\"min_max\"` - min-max scaler that converts data into the (0, 1) range, `loc = min(y)`, `scale = max(y) - min(y)`.\n - `None` - no scaling\n\n- **covariate_scaler** *({\"global\", None})* - If provided, the chosen scaling method will be applied to the covariates\n and static features before fitting the model.\n\n Such scaling be helpful for deep learning models that assume that the inputs are normalized.\n\n Available options:\n - `\"global\"` - `QuantileTransform` for skewed features, passthrough for boolean features, and `StandardScaler` for the rest of the features\n - `None` - do not scale the covariates\n\n By default, this parameter is set to `\"global\"` for GluonTS models, and `None` for all other models.\n\n- **covariate_regressor** *({\"LR\", \"GBM\", \"CAT\", \"XGB\", \"RF\", None}, default = None)* - If provided, the chosen tabular\n regression model will be fit on the known covariates & static features to predict the target column at the same time\n step.\n\n The predictions of the regression model will be subtracted from the target column, and the forecasting model will\n be used to forecast the residuals.\n\n At prediction time, the predictions of the regression model will be added to the predictions of the forecasting model.\n\n If you enable the `covariate_regressor`, it is recommended to also enable the `target_scaler`. This will usually\n lead to better accuracy and faster fitting time for the regressor.\n\n If both a `target_scaler` and a `covariate_regressor` are provided, then scaling will be performed before the\n regressor is applied.\n\n\n## MXNet Models\n\nMXNet models from GluonTS have been deprecated because of dependency conflicts caused by MXNet.\n\n\n## Additional features\n\nOverview of the additional features and covariates supported by different models.\nModels not included in this table currently do not support any additional features.\n\n```{eval-rst}\n.. list-table::\n :header-rows: 1\n :stub-columns: 1\n :align: center\n :widths: 55 15 15 15\n\n * - Model\n - Static features (continuous + categorical)\n - Known covariates (continuous + categorical)\n - Past covariates (continuous + categorical)\n * - :class:`~autogluon.timeseries.models.DirectTabularModel`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.models.RecursiveTabularModel`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.models.DeepARModel`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.models.PatchTSTModel`\n -\n - â
\n -\n * - :class:`~autogluon.timeseries.models.TemporalFusionTransformerModel`\n - â
\n - â
\n - â
\n * - :class:`~autogluon.timeseries.models.TiDEModel`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.models.WaveNetModel`\n - â
\n - â
\n -\n * - :class:`~autogluon.timeseries.models.Chronos2Model`\n -\n - â
\n - â
\n```\n\nIn addition to the above table, all models in AutoGluon can handle known covariates & static features if you set the [`covariate_regressor` hyperparameter](#hyperparameters-shared-by-all-models). Note that this may sometime lead to worse forecast accuracy, especially if the features are uninformative.\n"
},
{
"path": "docs/tutorials/timeseries/forecasting-quick-start.ipynb",
"content": "{\n \"cells\": [\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"b0f6df2b\",\n \"metadata\": {},\n \"source\": [\n \"# AutoGluon Time Series - Forecasting Quick Start\\n\",\n \"\\n\",\n \"[](https://colab.research.google.com/github/autogluon/autogluon/blob/master/docs/tutorials/timeseries/forecasting-quick-start.ipynb)\\n\",\n \"[](https://studiolab.sagemaker.aws/import/github/autogluon/autogluon/blob/master/docs/tutorials/timeseries/forecasting-quick-start.ipynb)\\n\",\n \"\\n\",\n \"\\n\",\n \"Via a simple `fit()` call, AutoGluon can train and tune\\n\",\n \"\\n\",\n \"- simple forecasting models (e.g., ARIMA, ETS, Theta),\\n\",\n \"- powerful deep learning models (e.g., DeepAR, Temporal Fusion Transformer),\\n\",\n \"- tree-based models (e.g., LightGBM),\\n\",\n \"- an ensemble that combines predictions of other models\\n\",\n \"\\n\",\n \"to produce multi-step ahead _probabilistic_ forecasts for univariate time series data.\\n\",\n \"\\n\",\n \"This tutorial demonstrates how to quickly start using AutoGluon to generate hourly forecasts for the [M4 forecasting competition](https://www.sciencedirect.com/science/article/pii/S0169207019301128) dataset.\\n\",\n \"\\n\",\n \"## Loading time series data as a `TimeSeriesDataFrame`\\n\",\n \"\\n\",\n \"First, we import some required modules\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"aa00faab-252f-44c9-b8f7-57131aa8251c\",\n \"metadata\": {\n \"tags\": [\n \"remove-cell\",\n \"skip-execution\"\n ]\n },\n \"outputs\": [],\n \"source\": [\n \"# We use uv for faster installation\\n\",\n \"!pip install uv\\n\",\n \"!uv pip install -q autogluon.timeseries --system\\n\",\n \"!uv pip uninstall -q torchaudio torchvision torchtext --system # fix incompatible package versions on Colab\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"843dc3c2\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import pandas as pd\\n\",\n \"from autogluon.timeseries import TimeSeriesDataFrame, TimeSeriesPredictor\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"519d689a\",\n \"metadata\": {},\n \"source\": [\n \"To use `autogluon.timeseries`, we will only need the following two classes:\\n\",\n \"\\n\",\n \"- `TimeSeriesDataFrame` stores a dataset consisting of multiple time series.\\n\",\n \"- `TimeSeriesPredictor` takes care of fitting, tuning and selecting the best forecasting models, as well as generating new forecasts.\\n\",\n \"\\n\",\n \"We load a subset of the M4 hourly dataset as a `pandas.DataFrame`\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"4bcd73a8\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"df = pd.read_csv(\\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_hourly_subset/train.csv\\\")\\n\",\n \"df.head()\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"a64ddd43\",\n \"metadata\": {},\n \"source\": [\n \"AutoGluon expects time series data in [long format](https://doc.dataiku.com/dss/latest/time-series/data-formatting.html#long-format).\\n\",\n \"Each row of the dataframe contains a single observation (timestep) of a single time series represented by\\n\",\n \"\\n\",\n \"- unique ID of the time series (`\\\"item_id\\\"`) as int or str\\n\",\n \"- timestamp of the observation (`\\\"timestamp\\\"`) as a `pandas.Timestamp` or compatible format\\n\",\n \"- numeric value of the time series (`\\\"target\\\"`)\\n\",\n \"\\n\",\n \"The raw dataset should always follow this format with at least three columns for unique ID, timestamp, and target value, but the names of these columns can be arbitrary.\\n\",\n \"It is important, however, that we provide the names of the columns when constructing a `TimeSeriesDataFrame` that is used by AutoGluon.\\n\",\n \"AutoGluon will raise an exception if the data doesn't match the expected format.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"d73ddcc9\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"train_data = TimeSeriesDataFrame.from_data_frame(\\n\",\n \" df,\\n\",\n \" id_column=\\\"item_id\\\",\\n\",\n \" timestamp_column=\\\"timestamp\\\"\\n\",\n \")\\n\",\n \"train_data.head()\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"bfee8b9b\",\n \"metadata\": {},\n \"source\": [\n \"We refer to each individual time series stored in a `TimeSeriesDataFrame` as an _item_.\\n\",\n \"For example, items might correspond to different products in demand forecasting, or to different stocks in financial datasets.\\n\",\n \"This setting is also referred to as a _panel_ of time series.\\n\",\n \"Note that this is *not* the same as multivariate forecasting â AutoGluon generates forecasts for each time series individually, without modeling interactions between different items (time series).\\n\",\n \"\\n\",\n \"`TimeSeriesDataFrame` inherits from [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html), so all attributes and methods of `pandas.DataFrame` are available in a `TimeSeriesDataFrame`.\\n\",\n \"It also provides other utility functions, such as loaders for different data formats (see [TimeSeriesDataFrame](../../api/autogluon.timeseries.TimeSeriesDataFrame) for details).\\n\",\n \"\\n\",\n \"## Training time series models with `TimeSeriesPredictor.fit`\\n\",\n \"To forecast future values of the time series, we need to create a `TimeSeriesPredictor` object.\\n\",\n \"\\n\",\n \"Models in `autogluon.timeseries` forecast time series _multiple steps_ into the future.\\n\",\n \"We choose the number of these steps â the _prediction length_ (also known as the _forecast horizon_) â depending on our task.\\n\",\n \"For example, our dataset contains time series measured at hourly _frequency_, so we set `prediction_length = 48` to train models that forecast up to 48 hours into the future.\\n\",\n \"\\n\",\n \"We instruct AutoGluon to save trained models in the folder `./autogluon-m4-hourly`.\\n\",\n \"We also specify that AutoGluon should rank models according to [mean absolute scaled error (MASE)](https://en.wikipedia.org/wiki/Mean_absolute_scaled_error), and that data that we want to forecast is stored in the column `\\\"target\\\"` of the `TimeSeriesDataFrame`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"f7ef668c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"predictor = TimeSeriesPredictor(\\n\",\n \" prediction_length=48,\\n\",\n \" path=\\\"autogluon-m4-hourly\\\",\\n\",\n \" target=\\\"target\\\",\\n\",\n \" eval_metric=\\\"MASE\\\",\\n\",\n \")\\n\",\n \"\\n\",\n \"predictor.fit(\\n\",\n \" train_data,\\n\",\n \" presets=\\\"medium_quality\\\",\\n\",\n \" time_limit=600,\\n\",\n \")\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"91b3ddd4\",\n \"metadata\": {},\n \"source\": [\n \"Here we used the `\\\"medium_quality\\\"` presets and limited the training time to 10 minutes (600 seconds).\\n\",\n \"The presets define which models AutoGluon will try to fit.\\n\",\n \"For `medium_quality` presets, these are\\n\",\n \"simple baselines (`Naive`, `SeasonalNaive`),\\n\",\n \"statistical models (`ETS`, `Theta`),\\n\",\n \"tree-based models based on LightGBM (`RecursiveTabular`, `DirectTabular`),\\n\",\n \"a deep learning model `TemporalFusionTransformer`,\\n\",\n \"and a weighted ensemble combining these.\\n\",\n \"Other available presets for `TimeSeriesPredictor` are `\\\"fast_training\\\"`, `\\\"high_quality\\\"` and `\\\"best_quality\\\"`.\\n\",\n \"Higher quality presets will usually produce more accurate forecasts but take longer to train.\\n\",\n \"\\n\",\n \"Inside `fit()`, AutoGluon will train as many models as possible within the given time limit.\\n\",\n \"Trained models are then ranked based on their performance on an internal validation set.\\n\",\n \"By default, this validation set is constructed by holding out the last `prediction_length` timesteps of each time series in `train_data`.\\n\",\n \"\\n\",\n \"\\n\",\n \"## Generating forecasts with `TimeSeriesPredictor.predict`\\n\",\n \"\\n\",\n \"We can now use the fitted `TimeSeriesPredictor` to forecast the future time series values.\\n\",\n \"By default, AutoGluon will make forecasts using the model that had the best score on the internal validation set.\\n\",\n \"The forecast always includes predictions for the next `prediction_length` timesteps, starting from the end of each time series in `train_data`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"4a238183\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"predictions = predictor.predict(train_data)\\n\",\n \"predictions.head()\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"bfbca161\",\n \"metadata\": {},\n \"source\": [\n \"AutoGluon produces a _probabilistic_ forecast: in addition to predicting the mean (expected value) of the time series in the future, models also provide the quantiles of the forecast distribution.\\n\",\n \"The quantile forecasts give us an idea about the range of possible outcomes.\\n\",\n \"For example, if the `\\\"0.1\\\"` quantile is equal to `500.0`, it means that the model predicts a 10% chance that the target value will be below `500.0`.\\n\",\n \"\\n\",\n \"We will now visualize the forecast and the actually observed values for one of the time series in the dataset.\\n\",\n \"We plot the mean forecast, as well as the 10% and 90% quantiles to show the range of potential outcomes.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"d2455126\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import matplotlib.pyplot as plt\\n\",\n \"\\n\",\n \"# TimeSeriesDataFrame can also be loaded directly from a file\\n\",\n \"test_data = TimeSeriesDataFrame.from_path(\\\"https://autogluon.s3.amazonaws.com/datasets/timeseries/m4_hourly_subset/test.csv\\\")\\n\",\n \"\\n\",\n \"# Plot 4 randomly chosen time series and the respective forecasts\\n\",\n \"predictor.plot(test_data, predictions, quantile_levels=[0.1, 0.9], max_history_length=200, max_num_item_ids=4);\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"bc2d08f7\",\n \"metadata\": {},\n \"source\": [\n \"## Evaluating the performance of different models\\n\",\n \"\\n\",\n \"We can view the performance of each model AutoGluon has trained via the `leaderboard()` method.\\n\",\n \"We provide the test data set to the leaderboard function to see how well our fitted models are doing on the unseen test data.\\n\",\n \"The leaderboard also includes the validation scores computed on the internal validation dataset.\\n\",\n \"\\n\",\n \"Note the test data includes both the forecast horizon (last `prediction_length` values of each time series) as well as the historical data (all except the last `prediction_last` values).\\n\",\n \"\\n\",\n \"In AutoGluon leaderboards, higher scores always correspond to better predictive performance.\\n\",\n \"Therefore our MASE scores are multiplied by `-1`, such that higher \\\"negative MASE\\\"s correspond to more accurate forecasts.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"2f4f8e9c\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# The test score is computed using the last\\n\",\n \"# prediction_length=48 timesteps of each time series in test_data\\n\",\n \"predictor.leaderboard(test_data)\\n\"\n ]\n },\n {\n \"attachments\": {},\n \"cell_type\": \"markdown\",\n \"id\": \"bd2fdfac\",\n \"metadata\": {},\n \"source\": [\n \"## Summary\\n\",\n \"We used `autogluon.timeseries` to make probabilistic multi-step forecasts on the M4 Hourly dataset.\\n\",\n \"Check out [Forecasting Time Series - In Depth](forecasting-indepth.ipynb) to learn about the advanced capabilities of AutoGluon for time series forecasting.\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"ag\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.14\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"
},
{
"path": "docs/tutorials/timeseries/index.md",
"content": "# Time Series Forecasting\n\nAutoGluon can forecast the future values of multiple time series given the historical data and other related covariates.\nA single call to AutoGluon `TimeSeriesPredictor`'s `fit()` method trains multiple models to generate accurate probabilistic forecasts,\nand does not require you to manually deal with cumbersome issues like model selection and hyperparameter tuning.\n\nUnder the hood, AutoGluon combines various state of the art forecasting algorithms.\nThese include established statical methods like ETS and ARIMA from\n[`StatsForecast`](https://github.com/Nixtla/statsforecast),\nefficient tree-based forecasters like LightGBM based on [AutoGluon-Tabular](https://auto.gluon.ai/stable/tutorials/tabular/index.html), flexible deep learning models like DeepAR and Temporal Fusion Transformer from [GluonTS](https://ts.gluon.ai/), and a pretrained zero-shot forecasting model, [Chronos](https://github.com/amazon-science/chronos-forecasting).\n\nCheck out the [Quick Start Tutorial](forecasting-quick-start.ipynb) to learn how to make accurate forecasts in just 3 lines of code using AutoGluon.\n\n::::{grid} 2\n :gutter: 3\n\n:::{grid-item-card} Quick Start\n :link: forecasting-quick-start.html\n\n Quick start tutorial on fitting models with time series datasets.\n:::\n\n:::{grid-item-card} In-depth Tutorial\n :link: forecasting-indepth.html\n\n Detailed discussion of the time series forecasting capabilities in AutoGluon.\n:::\n\n:::{grid-item-card} Forecasting with Chronos-2\n :link: forecasting-chronos.html\n\n Zero-shot forecasting with pretrained Chronos-2 time series models in AutoGluon.\n:::\n\n:::{grid-item-card} Forecasting Models\n :link: forecasting-model-zoo.html\n\n List of available forecasting models in AutoGluon-TimeSeries.\n:::\n\n:::{grid-item-card} Ensemble Models\n :link: forecasting-ensembles.html\n\n List of available ensemble models in AutoGluon-TimeSeries.\n:::\n\n:::{grid-item-card} Metrics\n :link: forecasting-metrics.html\n\n Evaluation metrics available in AutoGluon-TimeSeries.\n:::\n\n:::{grid-item-card} Custom Models\n :link: advanced/forecasting-custom-model.html\n\n How to add a custom time series forecasting model to AutoGluon.\n:::\n\n::::\n\n\n```{toctree}\n---\nmaxdepth: 2\nhidden: true\n---\n\nQuick Start | \\n\",\n \" | model | \\n\",\n \"score_test | \\n\",\n \"score_val | \\n\",\n \"pred_time_test | \\n\",\n \"pred_time_val | \\n\",\n \"fit_time_marginal | \\n\",\n \"fit_order | \\n\",\n \"
|---|---|---|---|---|---|---|---|
| 0 | \\n\",\n \"Chronos2FineTuned | \\n\",\n \"-0.677888 | \\n\",\n \"-0.802909 | \\n\",\n \"2.510460 | \\n\",\n \"0.773595 | \\n\",\n \"123.887496 | \\n\",\n \"2 | \\n\",\n \"
| 1 | \\n\",\n \"Chronos2ZeroShot | \\n\",\n \"-0.696239 | \\n\",\n \"-0.817203 | \\n\",\n \"2.385682 | \\n\",\n \"1.586984 | \\n\",\n \"0.885827 | \\n\",\n \"1 | \\n\",\n \"
\n\n\n\n \n \n
\n\n### AutoGluon Multimodal's \"Bag of Tricks\" Update\n\nWe are pleased to announce the integration of a comprehensive \"Bag of Tricks\" update for AutoGluon's MultiModal (AutoMM). This significant enhancement substantially improves multimodal AutoML performance when working with combinations of image, text, and tabular data. The update implements various strategies including multimodal model fusion techniques, multimodal data augmentation, cross-modal alignment, tabular data serialization, better handling of missing modalities, and an ensemble learner that integrates these techniques for optimal performance.\n\nUsers can now access these capabilities through a simple parameter when initializing the MultiModalPredictor after following the instruction [here](https://github.com/autogluon/autogluon/blob/2b90eb0f4a848941d70cd387c2fdec67bc67706d/multimodal/src/autogluon/multimodal/learners/ensemble.py#L306-L322) to download the checkpoints:\n```python\nfrom autogluon.multimodal import MultiModalPredictor\npredictor = MultiModalPredictor(label=\"label\", use_ensemble=True)\npredictor.fit(train_data=train_data)\n```\n\nWe express our gratitude to [@zhiqiangdon](https://github.com/zhiqiangdon), for this substantial contribution that enhances AutoGluon's capabilities for handling complex multimodal datasets. Here is the corresponding research paper describing the technical details: [Bag of Tricks for Multimodal AutoML with Image, Text, and Tabular Data](https://arxiv.org/html/2412.16243v1).\n\n\n## Deprecations and Breaking Changes\n\nThe following deprecated TabularPredictor methods have been removed in the 1.3.0 release (deprecated in 1.0.0, raise in 1.2.0, removed in 1.3.0). Please use the new names:\n- `persist_models` -> `persist`, `unpersist_models` -> `unpersist`, `get_model_names` -> `model_names`, `get_model_best` -> `model_best`, `get_pred_from_proba` -> `predict_from_proba`, `get_model_full_dict` -> `model_refit_map`, `get_oof_pred_proba` -> `predict_proba_oof`, `get_oof_pred` -> `predict_oof`, `get_size_disk_per_file` -> `disk_usage_per_file`, `get_size_disk` -> `disk_usage`, `get_model_names_persisted` -> `model_names(persisted=True)`\n\nThe following logic has been deprecated starting in 1.3.0 and will log a `FutureWarning`. Functionality will be changed in a future release:\n\n- (**FutureWarning**) `TabularPredictor.delete_models()` will default to `dry_run=False` in a future release (currently `dry_run=True`). Please ensure you explicitly specify `dry_run=True` for the existing logic to remain in future releases. [@Innixma](https://github.com/Innixma) ([#4905](https://github.com/autogluon/autogluon/pull/4905))\n\n\n## General\n\n\n### Improvements\n- (**Major**) Internal refactor of `AbstractTrainer` class to improve extensibility and reduce code duplication. [@canerturkmen](https://github.com/canerturkmen) ([#4804](https://github.com/autogluon/autogluon/pull/4804), [#4820](https://github.com/autogluon/autogluon/pull/4820), [#4851](https://github.com/autogluon/autogluon/pull/4851))\n\n### Dependencies\n\n- Update numpy to `>=1.25.0,<2.3.0`. [@tonyhoo](https://github.com/tonyhoo), [@Innixma](https://github.com/Innixma), [@suzhoum](https://github.com/suzhoum) ([#5020](https://github.com/autogluon/autogluon/pull/5020), [#5056](https://github.com/autogluon/autogluon/pull/5056), [#5072](https://github.com/autogluon/autogluon/pull/5072))\n- Update spacy to `<3.9`. [@tonyhoo](https://github.com/tonyhoo) ([#5072](https://github.com/autogluon/autogluon/pull/5072))\n- Update scikit-learn to `>=1.4.0,<1.7.0`. [@tonyhoo](https://github.com/tonyhoo), [@Innixma](https://github.com/Innixma) ([#5029](https://github.com/autogluon/autogluon/pull/5029), [#5045](https://github.com/autogluon/autogluon/pull/5045))\n- Update psutil to `>=5.7.3,<7.1.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update s3fs to `>=2024.2,<2026`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update ray to `>=2.10.0,<2.45`. [@suzhoum](https://github.com/suzhoum), [@celestinoxp](https://github.com/celestinoxp), [@tonyhoo](https://github.com/tonyhoo) ([#4714](https://github.com/autogluon/autogluon/pull/4714), [#4887](https://github.com/autogluon/autogluon/pull/4887), [#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update tabpfn to `>=0.1.11,<0.2`. [@Innixma](https://github.com/Innixma) ([#4787](https://github.com/autogluon/autogluon/pull/4787))\n- Update torch to `>=2.2,<2.7`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update lightning to `>=2.2,<2.7`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update torchmetrics to `>=1.2.0,<1.8`. [@zkalson](https://github.com/zkalson), [@tonyhoo](https://github.com/tonyhoo) ([#4720](https://github.com/autogluon/autogluon/pull/4720), [#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update torchvision to `>=0.16.0,<0.22.0`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update accelerate to `>=0.34.0,<2.0`. [@FireballDWF](https://github.com/FireballDWF) ([#5000](https://github.com/autogluon/autogluon/pull/5000))\n- Update lightgbm to `>=4.0,<4.7`. [@tonyhoo](https://github.com/tonyhoo) ([#4960](https://github.com/autogluon/autogluon/pull/4960))\n- Update fastai to `>=2.3.1,<2.9`. [@Innixma](https://github.com/Innixma) ([#4988](https://github.com/autogluon/autogluon/pull/4988))\n- Update jsonschema to `>=4.18,<4.24`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update scikit-image to `>=0.19.1,<0.26.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update omegaconf to `>=2.1.1,<2.4.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update pytorch-metric-learning to `>=1.3.0,<2.9`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update nltk to `>=3.4.5,<4.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update pytesseract to `>=0.3.9,<0.4`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update nvidia-ml-py3 to `>=7.352.0,<8.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update datasets to `>=2.16.0,<3.6.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update onnxruntime to `>=1.17.0,<1.22.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update tensorrt to `>=8.6.0,<10.9.1`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update xgboost to `>=2.0,<3.1`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update imodels to `>=1.3.10,<2.1.0`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n- Update statsforecast to `>=1.7.0,<2.0.2`. [@tonyhoo](https://github.com/tonyhoo) ([#5020](https://github.com/autogluon/autogluon/pull/5020))\n\n\n### Documentation\n- Updating documented python version's in CONTRIBUTING.md. [@celestinoxp](https://github.com/celestinoxp) ([#4796](https://github.com/autogluon/autogluon/pull/4796))\n- Refactored CONTRIBUTING.md to have up-to-date information. [@Innixma](https://github.com/Innixma) ([#4798](https://github.com/autogluon/autogluon/pull/4798))\n- Fix various typos. [@celestinoxp](https://github.com/celestinoxp) ([#4819](https://github.com/autogluon/autogluon/pull/4819))\n- Minor doc improvements. [@tonyhoo](https://github.com/tonyhoo) ([#4894](https://github.com/autogluon/autogluon/pull/4894), [#4929](https://github.com/autogluon/autogluon/pull/4929))\n\n### Fixes and Improvements\n- Fix colab AutoGluon source install with `uv`. [@tonyhoo](https://github.com/tonyhoo) ([#4943](https://github.com/autogluon/autogluon/pull/4943), [#4964](https://github.com/autogluon/autogluon/pull/4964))\n- Make `full_install.sh` use the script directory instead of the working directory. [@Innixma](https://github.com/Innixma) ([#4933](https://github.com/autogluon/autogluon/pull/4933))\n- Add `test_version.py` to ensure proper version format for releases. [@Innixma](https://github.com/Innixma) ([#4799](https://github.com/autogluon/autogluon/pull/4799))\n- Fix `setup_outputdir` to work with s3 paths. [@suzhoum](https://github.com/suzhoum) ([#4734](https://github.com/autogluon/autogluon/pull/4734))\n- Ensure `setup_outputdir` always makes a new directory if `path_suffix != None` and `path=None`. [@Innixma](https://github.com/Innixma) ([#4903](https://github.com/autogluon/autogluon/pull/4903))\n- Check `cuda.is_available()` before calling `cuda.device_count()` to avoid warnings. [@Innixma](https://github.com/Innixma) ([#4902](https://github.com/autogluon/autogluon/pull/4902))\n- Log a warning if mlflow autologging is enabled. [@shchur](https://github.com/shchur) ([#4925](https://github.com/autogluon/autogluon/pull/4925))\n- Fix rare ZeroDivisionError edge-case in `get_approximate_df_mem_usage`. [@shchur](https://github.com/shchur) ([#5083](https://github.com/autogluon/autogluon/pull/5083))\n- Minor fixes & improvements. [@suzhoum](https://github.com/suzhoum) [@Innixma](https://github.com/Innixma) [@canerturkmen](https://github.com/canerturkmen) [@PGijsbers](https://github.com/PGijsbers) [@tonyhoo](https://github.com/tonyhoo) ([#4744](https://github.com/autogluon/autogluon/pull/4744), [#4785](https://github.com/autogluon/autogluon/pull/4785), [#4822](https://github.com/autogluon/autogluon/pull/4822), [#4860](https://github.com/autogluon/autogluon/pull/4860), [#4891](https://github.com/autogluon/autogluon/pull/4891), [#5012](https://github.com/autogluon/autogluon/pull/5012), [#5047](https://github.com/autogluon/autogluon/pull/5047))\n\n--------\n\n## Tabular\n\n### Removed Models\n- Removed vowpalwabbit model (key: `VW`) and optional dependency (`autogluon.tabular[vowpalwabbit]`), as the model implemented in AutoGluon was not widely used and was largely unmaintained. [@Innixma](https://github.com/Innixma) ([#4975](https://github.com/autogluon/autogluon/pull/4975))\n- Removed TabTransformer model (key: `TRANSF`), as the model implemented in AutoGluon was heavily outdated, unmaintained since 2020, and generally outperformed by FT-Transformer (key: `FT_TRANSFORMER`). [@Innixma](https://github.com/Innixma) ([#4976](https://github.com/autogluon/autogluon/pull/4976))\n- Removed tabpfn from `autogluon.tabular[tests]` install in preparation for future `tabpfn>=2.x` support. [@Innixma](https://github.com/Innixma) ([#4974](https://github.com/autogluon/autogluon/pull/4974))\n\n### New Features\n- Add support for regression stratified splits via binning. [@Innixma](https://github.com/Innixma) ([#4586](https://github.com/autogluon/autogluon/pull/4586))\n- Add `TabularPredictor.model_hyperparameters(model)` that returns the hyperparameters of a model. [@Innixma](https://github.com/Innixma) ([#4901](https://github.com/autogluon/autogluon/pull/4901))\n- Add `TabularPredictor.model_info(model)` that returns the metadata of a model. [@Innixma](https://github.com/Innixma) ([#4901](https://github.com/autogluon/autogluon/pull/4901))\n- (Experimental) Add `plot_leaderboard.py` to visualize performance over training time of the predictor. [@Innixma](https://github.com/Innixma) ([#4907](https://github.com/autogluon/autogluon/pull/4907))\n- (**Major**) Add internal `ag_model_registry` to improve the tracking of supported model families and their capabilities. [@Innixma](https://github.com/Innixma) ([#4913](https://github.com/autogluon/autogluon/pull/4913), [#5057](https://github.com/autogluon/autogluon/pull/5057), [#5107](https://github.com/autogluon/autogluon/pull/5107))\n- Add `raise_on_model_failure` `TabularPredictor.fit` argument, default to False. If True, will immediately raise the original exception if a model raises an exception during fit instead of continuing to the next model. Setting to True is very helpful when using a debugger to try to figure out why a model is failing, as otherwise exceptions are handled by AutoGluon which isn't desired while debugging. [@Innixma](https://github.com/Innixma) ([#4937](https://github.com/autogluon/autogluon/pull/4937), [#5055](https://github.com/autogluon/autogluon/pull/5055))\n\n### Documentation\n- Minor tutorial doc improvements/fixes. [@kbulygin](https://github.com/kbulygin) [@Innixma](https://github.com/Innixma) ([#4779](https://github.com/autogluon/autogluon/pull/4779), [#4777](https://github.com/autogluon/autogluon/pull/4777))\n- Add Kaggle competition results. [@Innixma](https://github.com/Innixma) ([#4717](https://github.com/autogluon/autogluon/pull/4717), [#4770](https://github.com/autogluon/autogluon/pull/4770))\n\n### Fixes and Improvements\n- (**Major**) Ensure bagged refits in refit_full works properly (crashed in v1.2.0 due to a bug). [@Innixma](https://github.com/Innixma) ([#4870](https://github.com/autogluon/autogluon/pull/4870))\n- Improve XGBoost and CatBoost memory estimates. [@Innixma](https://github.com/Innixma) ([#5090](https://github.com/autogluon/autogluon/pull/5090))\n- Improve LightGBM memory estimates. [@Innixma](https://github.com/Innixma) ([#5101](https://github.com/autogluon/autogluon/pull/5101))\n- Fixed plot_tabular_models save path. [@everdark](https://github.com/everdark) ([#4711](https://github.com/autogluon/autogluon/pull/4711))\n- Fixed balanced_accuracy metric edge-case exception + added unit tests to ensure future bugs don't occur. [@Innixma](https://github.com/Innixma) ([#4775](https://github.com/autogluon/autogluon/pull/4775))\n- Fix HPO logging verbosity. [@Innixma](https://github.com/Innixma) ([#4781](https://github.com/autogluon/autogluon/pull/4781))\n- Improve logging for use_child_oof=True. [@Innixma](https://github.com/Innixma) ([#4780](https://github.com/autogluon/autogluon/pull/4780))\n- Fix crash when NN_TORCH trains with fewer than 8 samples. [@Innixma](https://github.com/Innixma) ([#4790](https://github.com/autogluon/autogluon/pull/4790))\n- Improve logging and documentation in CatBoost memory_check callback. [@celestinoxp](https://github.com/celestinoxp) ([#4802](https://github.com/autogluon/autogluon/pull/4802))\n- Improve code formatting to satisfy PEP585. [@celestinoxp](https://github.com/celestinoxp) ([#4823](https://github.com/autogluon/autogluon/pull/4823))\n- Remove deprecated TabularPredictor methods: [@Innixma](https://github.com/Innixma) ([#4906](https://github.com/autogluon/autogluon/pull/4906))\n- (**FutureWarning**) `TabularPredictor.delete_models()` will default to `dry_run=False` in a future release (currently `dry_run=True`). Please ensure you explicitly specify `dry_run=True` for the existing logic to remain in future releases. [@Innixma](https://github.com/Innixma) ([#4905](https://github.com/autogluon/autogluon/pull/4905))\n- Sped up tabular unit tests by 4x through various optimizations (3060s -> 743s). [@Innixma](https://github.com/Innixma) ([#4944](https://github.com/autogluon/autogluon/pull/4944))\n- Major tabular unit test refactor to avoid using fixtures. [@Innixma](https://github.com/Innixma) ([#4949](https://github.com/autogluon/autogluon/pull/4949))\n- Fix XGBoost GPU warnings. [@Innixma](https://github.com/Innixma) ([#4866](https://github.com/autogluon/autogluon/pull/4866))\n- Fix `TabularPredictor.refit_full(train_data_extra)` failing when categorical features exist. [@Innixma](https://github.com/Innixma) ([#4948](https://github.com/autogluon/autogluon/pull/4948))\n- Reduced memory usage of artifact created by `convert_simulation_artifacts_to_tabular_predictions_dict` by 4x. [@Innixma](https://github.com/Innixma) ([#5024](https://github.com/autogluon/autogluon/pull/5024))\n- Minor fixes. [@shchur](https://github.com/shchur) ([#5030](https://github.com/autogluon/autogluon/pull/5030))\n- Ensure that max model resources is respected during holdout model fit. [@Innixma](https://github.com/Innixma) ([#5067](https://github.com/autogluon/autogluon/pull/5067))\n- Remove unintended setting of global random seed during LightGBM model fit. [@Innixma](https://github.com/Innixma) ([#5095](https://github.com/autogluon/autogluon/pull/5095))\n\n\n------\n\n## TimeSeries\n\nThe new v1.3 release brings numerous usability improvements and bug fixes to the TimeSeries module.\nInternally, we completed a major refactor of the core classes and introduced static type checking to simplify future contributions, accelerate development, and catch potential bugs earlier.\n\n\n### API Changes and Deprecations\n* As part of the refactor, we made several changes to the internal `AbstractTimeSeriesModel` class. If you maintain a **custom model** implementation, you will likely need to update it. Please refer to the [custom forecasting model tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/advanced/forecasting-custom-model.html) for details.\n\n No action is needed from the users that rely solely on the public API of the `timeseries` module (`TimeSeriesPredictor` and `TimeSeriesDataFrame`).\n\n\n### New Features\n* New tutorial on adding custom forecasting models by [@shchur](https://github.com/shchur) in [#4749](https://github.com/autogluon/autogluon/pull/4749)\n* Add `cutoff` support in `evaluate` and `leaderboard` by [@abdulfatir](https://github.com/abdulfatir) in [#5078](https://github.com/autogluon/autogluon/pull/5078)\n* Add `horizon_weight` support for `TimeSeriesPredictor` by [@shchur](https://github.com/shchur) in [#5084](https://github.com/autogluon/autogluon/pull/5084)\n* Add `make_future_data_frame` method to TimeSeriesPredictor by [@shchur](https://github.com/shchur) in [#5051](https://github.com/autogluon/autogluon/pull/5051)\n* Refactor ensemble base class and add new ensembles by [@canerturkmen](https://github.com/canerturkmen) in [#5062](https://github.com/autogluon/autogluon/pull/5062)\n\n### Code Quality\n* Add static type checking for the `timeseries` module by [@canerturkmen](https://github.com/canerturkmen) in [#4712](https://github.com/autogluon/autogluon/pull/4712) [#4788](https://github.com/autogluon/autogluon/pull/4788) [#4801](https://github.com/autogluon/autogluon/pull/4801) [#4821](https://github.com/autogluon/autogluon/pull/4821) [#4969](https://github.com/autogluon/autogluon/pull/4969) [#5086](https://github.com/autogluon/autogluon/pull/5086) [#5085](https://github.com/autogluon/autogluon/pull/5085)\n* Refactor the `AbstractTimeSeriesModel` class by [@canerturkmen](https://github.com/canerturkmen) in [#4868](https://github.com/autogluon/autogluon/pull/4868) [#4909](https://github.com/autogluon/autogluon/pull/4909) [#4946](https://github.com/autogluon/autogluon/pull/4946) [#4958](https://github.com/autogluon/autogluon/pull/4958) [#5008](https://github.com/autogluon/autogluon/pull/5008) [#5038](https://github.com/autogluon/autogluon/pull/5038)\n* Improvements to the unit tests by [@canerturkmen](https://github.com/canerturkmen) in [#4773](https://github.com/autogluon/autogluon/pull/4773) [#4828](https://github.com/autogluon/autogluon/pull/4828) [#4877](https://github.com/autogluon/autogluon/pull/4877) [#4872](https://github.com/autogluon/autogluon/pull/4872) [#4884](https://github.com/autogluon/autogluon/pull/4884) [#4888](https://github.com/autogluon/autogluon/pull/4888)\n\n### Fixes and Improvements\n* Allow using custom `distr_output` with the TFT model by [@shchur](https://github.com/shchur) in [#4899](https://github.com/autogluon/autogluon/pull/4899)\n* Update version ranges for `statsforecast` & `coreforecast` by [@shchur](https://github.com/shchur) in [#4745](https://github.com/autogluon/autogluon/pull/4745)\n* Fix feature importance calculation for models that use a `covariate_regressor` by [@canerturkmen](https://github.com/canerturkmen) in [#4845](https://github.com/autogluon/autogluon/pull/4845)\n* Fix hyperparameter tuning for Chronos and other models by [@abdulfatir](https://github.com/abdulfatir) [@shchur](https://github.com/shchur) in [#4838](https://github.com/autogluon/autogluon/pull/4838) [#5075](https://github.com/autogluon/autogluon/pull/5075) [#5079](https://github.com/autogluon/autogluon/pull/5079)\n* Fix frequency inference for `TimeSeriesDataFrame` by [@abdulfatir](https://github.com/abdulfatir) [@shchur](https://github.com/shchur) in [#4834](https://github.com/autogluon/autogluon/pull/4834) [#5066](https://github.com/autogluon/autogluon/pull/5066)\n* Fix minor CovariateRegressor bugs by [@shchur](https://github.com/shchur) in [#4849](https://github.com/autogluon/autogluon/pull/4849)\n* Update docs for custom `distr_output` by [@Killer3048](https://github.com/Killer3048) in [#5068](https://github.com/autogluon/autogluon/pull/5068)\n* Minor documentation updates by [@shchur](https://github.com/shchur) in [#4928](https://github.com/autogluon/autogluon/pull/4928) [#5092](https://github.com/autogluon/autogluon/pull/5092)\n* Raise informative error message if invalid model name is provided by [@shchur](https://github.com/shchur) in [#5004](https://github.com/autogluon/autogluon/pull/5004)\n* Gracefully handle corrupted cached predictions by [@shchur](https://github.com/shchur) in [#5005](https://github.com/autogluon/autogluon/pull/5005)\n* Chronos-Bolt: Fix scaling that affects constant series by [@abdulfatir](https://github.com/abdulfatir) in [#5013](https://github.com/autogluon/autogluon/pull/5013)\n* Fix deprecated `evaluation_strategy` kwarg in `transformers` by [@abdulfatir](https://github.com/abdulfatir) in [#5019](https://github.com/autogluon/autogluon/pull/5019)\n* Fix time_limit when val_data is provided [#5046](https://github.com/autogluon/autogluon/pull/5046) by [@shchur](https://github.com/shchur) in [#5059](https://github.com/autogluon/autogluon/pull/5059)\n* Rename covariate metadata by [@canerturkmen](https://github.com/canerturkmen) in [#5064](https://github.com/autogluon/autogluon/pull/5064)\n* Fix NaT timestamp values during resampling by [@shchur](https://github.com/shchur) in [#5080](https://github.com/autogluon/autogluon/pull/5080)\n* Fix typing compatibility for py39 by [@suzhoum](https://github.com/suzhoum) [@shchur](https://github.com/shchur) in [#5094](https://github.com/autogluon/autogluon/pull/5094) [#5097](https://github.com/autogluon/autogluon/pull/5097)\n* Warn if an S3 path is provided to the `TimeSeriesPredictor` by [@shchur](https://github.com/shchur) in [#5091](https://github.com/autogluon/autogluon/pull/5091)\n\n\n\n------\n\n## Multimodal\n\n### New Features\nAutoGluon's MultiModal module has been enhanced with a comprehensive \"Bag of Tricks\" update that significantly improves performance when working with combined image, text, and tabular data through advanced fusion techniques, data augmentation, and an integrated ensemble learner now accessible via a simple `use_ensemble=True` parameter after following the instruction [here](https://github.com/autogluon/autogluon/blob/2b90eb0f4a848941d70cd387c2fdec67bc67706d/multimodal/src/autogluon/multimodal/learners/ensemble.py#L306-L322) to download the checkpoints.\n\n* [AutoMM] Bag of Tricks by [@zhiqiangdon](https://github.com/zhiqiangdon) in [#4737](https://github.com/autogluon/autogluon/pull/4737)\n\n### Documentation\n* [Tutorial] categorical convert_to_text default value by [@cheungdaven](https://github.com/cheungdaven) in [#4699](https://github.com/autogluon/autogluon/pull/4699)\n* [AutoMM] Fix and Update Object Detection Tutorials by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in [#4889](https://github.com/autogluon/autogluon/pull/4889)\n\n### Fixes and Improvements\n* Update s3 path to public URL for AutoMM unit tests by [@suzhoum](https://github.com/suzhoum) in [#4809](https://github.com/autogluon/autogluon/pull/4809)\n* Fix object detection tutorial and default behavior of predict by [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) in [#4865](https://github.com/autogluon/autogluon/pull/4865)\n* Fix NLTK tagger path in download function by [@k-ken-t4g](https://github.com/k-ken-t4g) in [#4982](https://github.com/autogluon/autogluon/pull/4982)\n* Fix AutoMM model saving logic by capping transformer range by [@tonyhoo](https://github.com/tonyhoo) in [#5007](https://github.com/autogluon/autogluon/pull/5007)\n* fix: account for distributed training in learning rate schedule by [@tonyhoo](https://github.com/tonyhoo) in [#5003](https://github.com/autogluon/autogluon/pull/5003)\n\n--------\n\n## Special Thanks\n* [Zhiqiang Tang](https://github.com/zhiqiangdon) for implementing \"Bag of Tricks\" for AutoGluon's MultiModal, which significantly enhances the multimodal performance.\n* [Caner Turkmen](https://github.com/canerturkmen) for leading the efforts on refactoring and improving the internal logic in the `timeseries` module.\n* [Celestino](https://github.com/celestinoxp) for providing numerous bug reports, suggestions, and code cleanup as a new contributor.\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@Innixma](https://github.com/Innixma) [@shchur](https://github.com/shchur) [@canerturkmen](https://github.com/canerturkmen) [@tonyhoo](https://github.com/tonyhoo) [@abdulfatir](https://github.com/abdulfatir) [@celestinoxp](https://github.com/celestinoxp) [@suzhoum](https://github.com/suzhoum) [@FANGAreNotGnu](https://github.com/FANGAreNotGnu) [@prateekdesai04](https://github.com/prateekdesai04) [@zhiqiangdon](https://github.com/zhiqiangdon)\n[@cheungdaven](https://github.com/cheungdaven) [@LennartPurucker](https://github.com/LennartPurucker) [@abhishek-iitmadras](https://github.com/abhishek-iitmadras) [@zkalson](https://github.com/zkalson) [@nathanaelbosch](https://github.com/nathanaelbosch) [@Killer3048](https://github.com/Killer3048) [@FireballDWF](https://github.com/FireballDWF) [@timostrunk](https://github.com/timostrunk) [@everdark](https://github.com/everdark) [@kbulygin](https://github.com/kbulygin) [@PGijsbers](https://github.com/PGijsbers) [@k-ken-t4g](https://github.com/k-ken-t4g)\n\n\n### New Contributors\n* [@everdark](https://github.com/everdark) made their first contribution in [#4711](https://github.com/autogluon/autogluon/pull/4711)\n* [@kbulygin](https://github.com/kbulygin) made their first contribution in [#4777](https://github.com/autogluon/autogluon/pull/4777)\n* [@celestinoxp](https://github.com/celestinoxp) made their first contribution in [#4796](https://github.com/autogluon/autogluon/pull/4796)\n* [@PGijsbers](https://github.com/PGijsbers) made their first contribution in [#4891](https://github.com/autogluon/autogluon/pull/4891)\n* [@k-ken-t4g](https://github.com/k-ken-t4g) made their first contribution in [#4982](https://github.com/autogluon/autogluon/pull/4982)\n* [@FireballDWF](https://github.com/FireballDWF) made their first contribution in [#5000](https://github.com/autogluon/autogluon/pull/5000)\n* [@Killer3048](https://github.com/Killer3048) made their first contribution in [#5068](https://github.com/autogluon/autogluon/pull/5068)\n"
},
{
"path": "docs/whats_new/v1.3.1.md",
"content": "# Version 1.3.1\n\nWe are happy to announce the AutoGluon 1.3.1 release!\n\nAutoGluon 1.3.1 contains several bug fixes and logging improvements for Tabular, TimeSeries, and Multimodal modules.\n\nThis release contains [9 commits from 5 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=5%2F1%2F2025&to=5%2F20%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/1.3.0...1.3.1\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.8, 3.9, 3.10, and 3.11. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.3.1.\n\n--------\n\n## General\n- Version update. [@tonyhoo](https://github.com/tonyhoo) [#5112](https://github.com/autogluon/autogluon/pull/5112)\n\n--------\n\n## Tabular\n\n### Fixes and Improvements\n- Fix TabPFN dependency. [@fplein](https://github.com/fplein) [#5119](https://github.com/autogluon/autogluon/pull/5119)\n- Fix incorrect reference to positive_class in TabularPredictor constructor. [@celestinoxp](https://github.com/celestinoxp) [#5129](https://github.com/autogluon/autogluon/pull/5129)\n\n--------\n\n## TimeSeries\n\n### Fixes and Improvements\n- Fix ensemble weights format for printing. [@shchur](https://github.com/shchur) [#5132](https://github.com/autogluon/autogluon/pull/5132)\n- Avoid masking the `scaler` param with the default `target_scaler` value for `DirectTabular` and `RecursiveTabular` models. [@shchur](https://github.com/shchur) [#5131](https://github.com/autogluon/autogluon/pull/5131)\n- Fix `FutureWarning` in leaderboard and evaluate methods. [@shchur](https://github.com/shchur) [#5126](https://github.com/autogluon/autogluon/pull/5126)\n\n--------\n\n## Multimodal\n\n### Fixes and Improvements\n- Fix multimodal tutorial issue after 1.3 release [@tonyhoo](https://github.com/tonyhoo) [#5121](https://github.com/autogluon/autogluon/pull/5121)\n\n--------\n\n## Documentation and CI\n- Add release instructions for pasting whats_new release notes. [@Innixma](https://github.com/Innixma) [#5111](https://github.com/autogluon/autogluon/pull/5111)\n- Update docker image to use 1.3 release base. [@tonyhoo](https://github.com/tonyhoo) [#5130](https://github.com/autogluon/autogluon/pull/5130)\n\n--------\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur) [@tonyhoo](https://github.com/tonyhoo) [@celestinoxp](https://github.com/celestinoxp)\n\n\n### New Contributors\n- [@fplein](https://github.com/fplein) made their first contribution in [#5119](https://github.com/autogluon/autogluon/pull/5119)\n"
},
{
"path": "docs/whats_new/v1.4.0.md",
"content": "# Version 1.4.0\n\nWe are happy to announce the AutoGluon 1.4.0 release!\n\nAutoGluon 1.4.0 introduces massive new features and improvements to both tabular and time series modules. In particular, we introduce the `extreme` preset to TabularPredictor, which sets a new state of the art for predictive performance **by a massive margin** on datasets with fewer than 30000 samples. We have also added 5 new tabular model families in this release: RealMLP, TabM, TabPFNv2, TabICL, and Mitra. We also release **MLZero 1.0**, aka AutoGluon-Assistant, an end-to-end automated data science agent that brings AutoGluon from 3 lines of code to 0. For more details, refer to the highlights section below.\n\nThis release contains [69 commits from 18 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=5%2F21%2F2025&to=7%2F26%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/1.3.1...1.4.0\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.9, 3.10, 3.11, and 3.12. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.4.0.\n\n--------\n\n## Spotlight\n\n### AutoGluon Tabular Extreme Preset\n\n\n\nAutoGluon 1.4.0 introduces a new tabular preset, `extreme_quality` aka `extreme`.\nAutoGluon's [extreme preset](https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets) is **the largest singular improvement to AutoGluon's predictive performance in the history of the package**, even larger than the improvement seen in AutoGluon 1.0 compared to 0.8.\nThis preset achieves an **88% win-rate** vs Autogluon 1.3 `best_quality` for datasets with fewer than 10000 samples, and a 290 Elo improvement overall on [TabArena](https://tabarena.ai) (shown in the figure above).\n\nTry it out in 3 lines of code:\n\n```python\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(label=\"class\").fit(\"train.csv\", presets=\"extreme\")\npredictions = predictor.predict(\"test.csv\")\n```\n\nThe `extreme` preset leverages a [new model portfolio](https://github.com/autogluon/autogluon/blob/master/tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2025.py), which is an improved version of the `TabArena ensemble` shown in Figure 6a of the [TabArena paper](https://arxiv.org/abs/2506.16791).\nIt consists of many new model families added in this release: TabPFNv2, TabICL, Mitra, TabM, as well as tree methods: CatBoost, LightGBM, XGBoost.\nThis preset is not only more accurate, it also requires much less training time. AutoGluon's `extreme` preset in 5 minutes is able to outperform `best` ran for 4 hours.\n\nIn order to get the most out of the `extreme` preset, a CUDA compatible GPU is required, ideally with 32+ GB vRAM.\nNote that inference time can be longer than `best`, but with a GPU it is very reasonable.\nThe `extreme` portfolio is only leveraged for datasets with at most 30000 samples. For larger datasets, we continue to use the `best_quality` portfolio.\nThe preset requires downloading foundation model weights for TabPFNv2, TabICL, and Mitra during fit. If you don't have an internet connection,\nensure that you pre-download the weights of the models to be able to use them during fit.\nThis preset is considered experimental for this release, and may change without warning in a future release.\n\n### TabArena and new models: TabPFNv2, TabICL, TabM, RealMLP\n\nđ¨What is SOTA on tabular data, really? We are excited to introduce [TabArena](https://tabarena.ai), a living benchmark for machine learning on IID tabular data with:\n\nđ an online leaderboard accepting submissions \nđ carefully curated datasets (real, predictive, tabular, IID, permissive license) \nđ strong tree-based, deep learning, and foundation models \nâī¸ best practices for evaluation (inner CV, outer CV, early stopping) \n\nâšī¸ đđ¯đđĢđ¯đĸđđ° \nLeaderboard: https://tabarena.ai \nPaper: https://arxiv.org/abs/2506.16791 \nCode: https://tabarena.ai/code \n\nđĄ đđđĸđ§ đĸđ§đŦđĸđ đĄđđŦ: \nâĄī¸ Recent deep learning models, RealMLP and TabM, have marginally overtaken boosted trees with weighted ensembling, although they have slower train+inference times. With defaults or regular tuning, CatBoost takes the #1 spot. \nâĄī¸ Foundation models TabPFNv2 and TabICL are only applicable to a subset of datasets, but perform very strongly on these. They have a large inference time and still need tuning/ensembling to get the top spot (for TabPFNv2). \nâĄī¸ The winner does NOT take it all. By using a weighted ensemble of different model types from TabArena, we can significantly outperform the current state of the art on tabular data, AutoGluon 1.3. \nâĄī¸ These insights have been directly incorporated into the AutoGluon 1.4 release with the [extreme preset](https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets), dramatically advancing the state of the art! \nâĄī¸ The models TabPFNv2, TabICL, TabM, and RealMLP have been added to AutoGluon! To use them, run `pip install autogluon[tabarena]` and use the `extreme` preset.\n\nđ¯TabArena is a living benchmark. With the community, we will continually update it! \n\nTabArena Authors: [Nick Erickson](https://github.com/Innixma), [Lennart Purucker](https://github.com/LennartPurucker), [Andrej Tschalzev](https://github.com/atschalz), [David HolzmÃŧller](https://github.com/dholzmueller), [Prateek Mutalik Desai](https://github.com/prateekdesai04), [David Salinas](https://github.com/geoalgo), [Frank Hutter](https://github.com/frank-hutter)\n\n\n### AutoGluon Assistant (MLZero)\n> *Multi-Agent System Powered by LLMs for End-to-end Multimodal ML Automation*\n\nWe are excited to present the [AutoGluon Assistant](https://github.com/autogluon/autogluon-assistant) 1.0 release. Level up from v0.1: v1.0 expands beyond tabular data to robustly support any and many modalities, including **image, text, tabular, audio and mixed-data pipelines**. This aligns precisely with the MLZero vision of comprehensive, modality-agnostic ML automation.\n\nAutoGluon Assistant v1.0 is now synonymous with **\"MLZero: A Multi-Agent System for End-to-end Machine Learning Automation\"** ([arXiv:2505.13941](https://arxiv.org/abs/2505.13941)), the end-to-end, zero-human-intervention AutoML agent framework for multimodal data. Built on a novel **multi-agent architecture** using LLMs, MLZero handles perception, memory (semantic & episodic), code generation, execution, and iterative debugging â seamlessly transforming raw multimodal inputs into high-quality ML/DL pipelines.\n\n- **No-code**: Users define tasks purely through natural language (\"classify images of cats vs dogs with custom labels\"), and MLZero delivers complete solutions with zero manual configuration or technical expertise required.\n- **Built on proven foundations**: MLZero generates code using established, high-performance ML libraries rather than reinventing the wheel, ensuring robust solutions while maintaining the flexibility to easily integrate new libraries as they emerge.\n- **Research-grade performance**: MLZero is extensively validated across 25 challenging tasks spanning diverse data modalities, MLZero outperforms the competing methods by a large margin with a success rate of 0.92 (+263.6\\%) and an average rank of 2.42. \n \n\n\n| Dataset | Ours | Codex CLI | Codex CLI (+reasoning) | AIDE | DS-Agent | AK |\n|-------------|--------------------------|---------------|---------------|----------|--------------|--------|\n| **Avg. Rank â** | **2.42** | 8.04 | 5.76 | 6.16 | 8.26 | 8.28 | \n| **Rel. Time â** | 1.0 | 0.15 | 0.23 | 2.83 | N/A | 4.82 | \n| **Success â** | **92.0%** | 14.7% | 69.3% | 25.3% | 13.3% | 14.7% | \n
\n\n- **Modular and extensible architecture**: We separate the design and implementation of each agent and prompts for different purposes, with a centralized manager coordinating them. This makes adding or editing agents, prompts, and workflows straightforward and intuitive for future development.\n\nWeâre also excited to introduce the newly redesigned **WebUI** in v1.0, now with a streamlined chatbot-style interface that makes interacting with MLZero intuitive and engaging. Furthermore, weâre also bringing **MCP (Model Control Protocol)** integration to MLZero, enabling seamless remote orchestration of AutoML pipelines through a standardized protocolã\n\nAutoGluon Assistant is supported on Python 3.8 - 3.11 and is available on Linux.\n\nInstallation:\n```bash\npip install uv\nuv pip install autogluon.assistant>=1.0\n```\n\nTo use CLI:\n```bash\nmlzero -i \n
\n\n- `Chronos` model: Hyperparameter `optimization_strategy` (deprecated in v1.3.0) has been removed in v1.4.0.\n\n### New Features\n- Add `PerStepTabular` model that fits a separate tabular regression model for each step in the forecast horizon. [@shchur](https://github.com/shchur) ([#5189](https://github.com/autogluon/autogluon/pull/5189), [#5213](https://github.com/autogluon/autogluon/pull/5213))\n- Improve heuristic for long-term forecast unrolling (`prediction_length > 64`) for Chronos-Bolt. [@abdulfatir](https://github.com/abdulfatir) ([#5177](https://github.com/autogluon/autogluon/pull/5177))\n- `RecursiveTabular` model now supports the `lag_transforms` hyperparameter. [@shchur](https://github.com/shchur) ([#5184](https://github.com/autogluon/autogluon/pull/5184))\n\n### Fixes and Improvements\n- Improve the runtime of various `TimeSeriesDataFrame` operations by replacing `groupby` with efficient alternatives based on `indptr`. [@shchur](https://github.com/shchur) ([#5159](https://github.com/autogluon/autogluon/pull/5159))\n- Refactor `DirectTabular` and `RecursiveTabular` models to use a single tabular model under the hood instead of a `TabularPredictor`. ([#5212](https://github.com/autogluon/autogluon/pull/5212))\n- Reorganize `autogluon.timeseries.models.gluonts` namespace. [@canerturkmen](https://github.com/canerturkmen) ([#5104](https://github.com/autogluon/autogluon/pull/5104))\n- Log the full stack trace in case of individual model failures during training. [@shchur](https://github.com/shchur) ([#5178](https://github.com/autogluon/autogluon/pull/5178))\n- Deprecate the `optimization_strategy` hyperparameter for the Chronos (classic) model. [@shchur](https://github.com/shchur) ([#5202](https://github.com/autogluon/autogluon/pull/5202))\n- Fix incompatibility with python 3.9. [@prateekdesai04](https://github.com/prateekdesai04) ([#5220](https://github.com/autogluon/autogluon/pull/5220))\n- Refactor the implementation of `RecursiveTabular` and `DirectTabular` models. [@shchur](https://github.com/shchur) ([#5184](https://github.com/autogluon/autogluon/pull/5184), [#5206](https://github.com/autogluon/autogluon/pull/5206))\n- Fix typos and layout issues in the documentation. [@shchur](https://github.com/shchur) ([#5225](https://github.com/autogluon/autogluon/pull/5225))\n- Fix refit_full failing during ensemble prediction if quantile_levels=[]. [@shchur](https://github.com/shchur) ([#5242](https://github.com/autogluon/autogluon/pull/5242))\n--------\n\n## Multimodal\n\n- Change multilingual preset to use FP32 to avoid DeBERTa BFloat16. [@tonyhoo](https://github.com/tonyhoo) ([#5139](https://github.com/autogluon/autogluon/pull/5139))\n- Update NLTK dependency constraint to <3.10 to address CVE-2024-39705. [@tonyhoo](https://github.com/tonyhoo) ([#5147](https://github.com/autogluon/autogluon/pull/5147))\n\n--------\n\n## Documentation and CI\n- Fix Python syntax in CUDA library path detection. [@tonyhoo](https://github.com/tonyhoo) ([#5166](https://github.com/autogluon/autogluon/pull/5166))\n- Upgrade image to use torch 2.7.1. [@tonyhoo](https://github.com/tonyhoo) ([#5168](https://github.com/autogluon/autogluon/pull/5168))\n- Show tabular model aliases in the documentation. [@shchur](https://github.com/shchur) ([#5183](https://github.com/autogluon/autogluon/pull/5183))\n- Fix lint check. [@prateekdesai04](https://github.com/prateekdesai04) ([#5192](https://github.com/autogluon/autogluon/pull/5192))\n- Add time limit conversion to seconds in benchmark config script. [@tonyhoo](https://github.com/tonyhoo) ([#5224](https://github.com/autogluon/autogluon/pull/5224))\n\n--------\n\n## Special Thanks\n\n- [Lennart Purucker](https://github.com/LennartPurucker) and [David HolzmÃŧller](https://github.com/dholzmueller) for helping to implement TabPFNv2, TabICL, RealMLP and TabM, along with providing improved memory estimate logic for the models.\n- [Steven Shen](https://github.com/HuawenShen) for implementing the front-end web UI and MCP backend for MLZero.\n- [Atharva Rajan Kale](https://github.com/Atharva-Rajan-Kale) for helping to automate and streamline our DLC release process.\n\n## Contributors\n\nFull Contributor List (ordered by # of commits):\n\n[@shchur](https://github.com/shchur) [@Innixma](https://github.com/Innixma) [@tonyhoo](https://github.com/tonyhoo) [@prateekdesai04](https://github.com/prateekdesai04) [@FireballDWF](https://github.com/FireballDWF) [@canerturkmen](https://github.com/canerturkmen) [@abdulfatir](https://github.com/abdulfatir) [@rsj123](https://github.com/rsj123) [@xiyuanzh](https://github.com/xiyuanzh) [@mwhol](https://github.com/mwhol) [@daradib](https://github.com/daradib) [@emmanuel-ferdman](https://github.com/emmanuel-ferdman) [@adibiasio](https://github.com/adibiasio) [@LennartPurucker](https://github.com/LennartPurucker) [@dholzmueller](https://github.com/dholzmueller)\n\n\n### New Contributors\n- [@mwhol](https://github.com/mwhol) made their first contribution in [#5217](https://github.com/autogluon/autogluon/pull/5217)\n- [@daradib](https://github.com/daradib) made their first contribution in [#5231](https://github.com/autogluon/autogluon/pull/5231)\n- [@emmanuel-ferdman](https://github.com/emmanuel-ferdman) made their first contribution in [#5069](https://github.com/autogluon/autogluon/pull/5069)\n"
},
{
"path": "docs/whats_new/v1.5.0.md",
"content": "# Version 1.5.0\n\nWe are happy to announce the AutoGluon 1.5.0 release!\n\nAutoGluon 1.5.0 introduces new features and major improvements to both tabular and time series modules.\n\nThis release contains [131 commits from 17 contributors](https://github.com/autogluon/autogluon/graphs/contributors?from=7%2F28%2F2025&to=12%2F19%2F2025&type=c)! See the full commit change-log here: https://github.com/autogluon/autogluon/compare/1.4.0...1.5.0\n\nJoin the community: [](https://discord.gg/wjUmjqAc2N)\nGet the latest updates: [](https://twitter.com/autogluon)\n\nThis release supports Python versions 3.10, 3.11, 3.12 and 3.13. Support for Python 3.13 is currently experimental, and some features might not be available when running Python 3.13 on Windows. Loading models trained on older versions of AutoGluon is not supported. Please re-train models using AutoGluon 1.5.0.\n\n--------\n\n## Spotlight\n\n### Chronos-2\n\nAutoGluon v1.5 adds support for [Chronos-2](https://huggingface.co/amazon/chronos-2), our latest generation of foundation models for time series forecasting. Chronos-2 natively handles all types of dynamic covariates, and performs cross-learning from items in the batch. It produces multi-step quantile forecasts and is designed for strong out-of-the-box performance on new datasets.\n\nChronos-2 achieves state-of-the-art zero-shot accuracy among public models on major benchmarks such as [fev-bench](https://huggingface.co/spaces/autogluon/fev-bench) and [GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval), making it a strong default choice when little or no task-specific training data is available.\n\nIn AutoGluon, Chronos-2 can be used in **zero-shot mode** or **fine-tuned** on custom data. Both **LoRA fine-tuning** and **full fine-tuning** are supported. Chronos-2 integrates into the standard `TimeSeriesPredictor` workflow, making it easy to backtest, compare against classical and deep learning models, and combine with other models in ensembles.\n```python\nfrom autogluon.timeseries import TimeSeriesPredictor\n\npredictor = TimeSeriesPredictor(...)\npredictor.fit(train_data, presets=\"chronos2\") # zero-shot mode\n```\nMore details on zero-shot usage, fine-tuning and ensembling are available in the [updated tutorial](https://auto.gluon.ai/stable/tutorials/timeseries/forecasting-chronos.html).\n\n### AutoGluon Tabular\n\n**AutoGluon 1.5 Extreme sets a new state-of-the-art on TabArena**, with a 60 Elo improvement over AutoGluon 1.4 Extreme.\nOn average, AutoGluon 1.5 Extreme trains in half the time, has 50% faster inference speed, a 70% win-rate, and 2.8% less relative error compared to AutoGluon 1.4 Extreme. Whereas 1.4 used a mixed portfolio that changed depending on data size, 1.5 uses a single fixed portfolio for all datasets. \n\nNotable Improvements:\n\n1. Added TabDPT model, a tabular foundation model pre-trained exclusively on real data.\n2. Added TabPrep-LightGBM, a LightGBM model with custom preprocessing logic including target mean encoding and feature crossing.\n3. Added early stopping logic for the portfolio which stops training early for small datasets to mitigate overfitting and reduce training time.\n\nAutoGluon 1.5 Extreme uses exclusively open and permissively licensed models, making it suitable for production and commercial use-cases.\n\nTo use AutoGluon 1.5 Extreme, you will need a GPU, ideally with at least 20 GB of VRAM to ensure stability. Performance gains are primarily on datasets with up to 100k training samples.\n\n```python\n# pip install autogluon.tabular[tabarena] # <-- Required for TabDPT, TabICL, TabPFN, and Mitra\nfrom autogluon.tabular import TabularPredictor\npredictor = TabularPredictor(...).fit(train_data, presets=\"extreme\") # GPU required\n```\n\n\nDetails and example usage
\n\n ```python\n # New API: >= v1.4.0\n predictor.fit(\n ...,\n hyperparameters={\n \"RecursiveTabular\": {\"model_name\": \"CAT\", \"model_hyperparameters\": {\"iterations\": 100}}\n }\n )\n # Old API: <= v1.3.1\n predictor.fit(\n ...,\n hyperparameters={\n \"RecursiveTabular\": {\"tabular_hyperparameters\": {\"CAT\": {\"iterations\": 100}}}\n }\n )\n ```\n\n If you provide `tabular_hyperparameters` with a single model in v1.4.0, a warning will be logged and the parameter will be automatically converted to match the new API.\n\n If you provide `tabular_hyperparameters` with >=2 models in v1.4.0, an error will be raised since it cannot automatically be converted to the new API.\n\n| \n TabArena All (51 datasets)\n | \n||||
|---|---|---|---|---|
| Model | \nElo [âŦī¸] | \nImprovability (%) [âŦī¸] | \nTrain Time (s/1K) [âŦī¸] | \nPredict Time (s/1K) [âŦī¸] | \n
| AutoGluon 1.5 (extreme, 4h) | \n1736 | \n3.498 | \n289.07 | \n4.031 | \n
| AutoGluon 1.4 (extreme, 4h) | \n1675 | \n6.381 | \n582.21 | \n6.116 | \n
| AutoGluon 1.4 (best, 4h) | \n1536 | \n9.308 | \n1735.72 | \n2.559 | \n
| Pareto Frontier (Elo) | \nPareto Frontier (Improvability) | \n
|---|---|
| \n | \n \n | \n
[ls_host_on=False]: cannot access |\n| \"/data/local-files/...\" | â | â(conditional) | use didn't provide url of the image, instead they set up a local storage in LS. | [ls_host_on=True]: join with the label-studio host(e.g: http://localhost:8080/data/upload/...)
[ls_host_on=False]: reserve the local path of the data files(if the root of the dataset is not moved) |\n| ... | | | | |\n\n\n\nAn demo export files of Label-Studio export file:\n\n\n\nAn demo output with `ls_host_on=True` :\n\n\n\nAn demo output with `ls_host_on=False` :\n\n(the \"uploaded\" data will be warned for not accessible)\n\n\n\n### Example 2. Text annotations (URL not required )\n\nTo we use the label-studio **named entity recognition** export file to demonstrate cases like this. In this use case, user: \n\n- Choose \"named entity recognition\" template in Label-Studio\n\n- Annotate the images in Label-Studio \n\n- Click \"Export\" and choose the export format\n\n- Call `from_named_entity_recognition` to transform the export file into the Dataframe for Autogluon inputīŧAutogluon's NER support: https://auto.gluon.ai/stable/tutorials/multimodal/text_prediction/ner.htmlīŧ\n\n \n\nSince there're no potential URL issues, the `ls_host_on` param is removed in cases like this. The params of `from_named_entity_recognition` are as follows:\n\n| Param | Type | Optional | Description |\n| :-----------: | :-------: | :------: | :----------------------------------------------------------: |\n| path | Str | | the path of the export file |\n| data_columns | List[Str] | â | the key or column name(s) of the data.By default we use the given name from the image classification template.If users change the name in the template, they should provide the data column names here |\n| label_columns | List[Str] | â | the key or column name(s) of the label, similar to the situation for data_columns |\n\n\n\nNOTE:\n\n**Cases like named entity recognition and object detection may have multiple labels on single text or image files, therefore the label columns will be nested. Currently the ` LabelStudioReader` only implements nested data parsing for JSON, CSV and JSON-MIN is not supported yet **. Therefore it's recommended to export JSON files. We will working on nested data parsing on CSV and JSON-MIN in the future.\n\n\n\nThe processed result of a CSV and JSON-MIN file:\n\n\n\n (the Label column are nested data about NER's labeling information)\n\n\n\nThe processed result of a JSON file:\n\n\n\n\n\n### Example 3. Customized template\n\nIf user create their own Label-Studio template and export the annotations, they can call `from_customized` to transform the export file into the Dataframe for Autogluon input. In this case, user should provide the `data_columns` and `label_columns` to identify the data and label content.\n\n\n\nThe params of `from_customized` are as follows:\n\n| Param | Type | Optional | Description |\n| :-----------: | :-------: | :------: | :----------------------------------------------------------: |\n| path | Str | | the path of the export file |\n| ls_host_on | Boolean | | Whether user need to access some files through label-studio host. |\n| data_columns | List[Str] | | the key or column name(s) of the data |\n| label_columns | List[Str] | | the key or column name(s) of the label |\n\n" }, { "path": "examples/automm/memory_bank/README.md", "content": "# Use MultiModal Feature Extraction to Create a Few-shot Memory Bank Model\n\n[memory_bank.py](./memory_bank.py): This example provides a simple and clear way to implement a few-shot memory bank model with AutoGluon MultiModal Feature Extraction according to [Tip-Adapter](https://github.com/gaopengcuhk/Tip-Adapter) [1]. \n\n### 1. Run Example\n\nBefore running the example, the image datasets need to be prepared with additional code. We follow the few-shot dataset design in [CoOP](https://github.com/KaiyangZhou/CoOp). Copy folder `configs` and `datasets` from the repository and rename `datasets` to `imagedatasets` to avoid conflict with `datasets` in Hugging Face. You can download the datasets and the splits under the [direction](https://github.com/KaiyangZhou/CoOp/blob/main/DATASETS.md).\n\nFor text datasets, datasets in Hugging Face are supported. We recommend datasets under [SetFit](https://huggingface.co/datasets?sort=downloads&search=SetFit) [2] for few-shot learning. Remember to change the column names in args if the data column is not named `text` or have multi-text information.\n\nAfter preparing the datasets, run the example:\n\n python memory_bank.py --type clip --dataset food101 --shots 16\n\n- `type` is the type of few-shot learning. You can choose from `clip`, `text`, `image` for different backbone methods.\n- `backbone` is the backbone model of MultiModal Predictor.\n- `data_path` is the path for image dataset.\n- `dataset` is the name of dataset. Support image datasets in COOP and text datasets in Hugging Face.\n- `column_names` is the names of the data columns.\n- `label_column` is the name of the label column.\n- `shots` is the shots for each class in training set.\n- `aug_epochs` is the epochs to create the bank.\n- `model_head_type` is the model head for few-shot classification. You can choose from 'linear' and 'SVM' for different classification heads.\n- `lr` is the learning rate for training the model head.\n- `lr_F` is the learning rate for finetuing the memory bank.\n- `train_epoch` is the training epochs for training the model head.\n- `train_epoch_F` is the training epochs for fine-tuning the memory bank.\n- `init_alpha` is initial values of hyper-parameters in memory bank. `alpha` adjusts the weight of probability between the classifier and memory bank.\n- `init_beta` is initial values of hyper-parameters in memory bank. `beta` modulates the sharpness when converting the similarities into non-negative values.\n- `search_scale` is the search scale of alpha and beta.\n- `search_step` is the steps of searching hyper-parameters alpha and beta.\n\n### 2. Method\n\nThe Memory Bank, inspired by [Tip-Adapter](https://arxiv.org/pdf/2207.09519.pdf), stores image features of the few-shot training set to improve the performance of zero-shot CLIP through feature similarity and serve as an initialization for a trainable classifier. This ProtoNet-like design effectively utilizes few-shot training information, resulting in good performance [3]. It has potential to be widely applied to few-shot classification tasks for images and texts.\n\nThe Memory Bank, which is based on the Tip-Adapter model, uses MultiModal Feature Extraction to acquire diverse multi-modal features. To evaluate its performance, a linear classifier was first trained using these multi-modal features to establish a baseline accuracy. The similarity between the features and the Memory Bank was then incorporated into the baseline prediction probability. Finally, an additional linear adapter was trained, initialized with the Memory Bank, to aid in few-shot classification.\n\nHyper-parameters `alpha` and `beta` which adjust the memory bank are modified through grid search on validation set to attain the superior performance.\n\n### 3. Training Results\n\nThe training results of memory bank can be seen in the followed table. `Acc w/o memory bank` is the few-shot classification accuracy of method. `ACC w. memory bank` and `ACC w. memory bank (+finetune)` show the classification results after introducing and fine-tuning the memory bank, respectively.\n\n| Datasets | Method | BackBone | Head | Shots | lr | lr_F | Acc w/o memory bank| Acc w. memory bank | Acc w. memory bank (+finetune) | \n|----------|--------|----------|------|-------|----|------|--------------------|--------------------|--------------------------------| \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 91.90 | 92.42 | 92.80 | \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 1 | NaN | 1e-3 | 91.90 | 91.99 | 91.97 | \n| Food-101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 64 | NaN | 1e-3 | 91.90 | 92.43 | 93.10 | \n| Food-101 | CLIP | openai/clip-vit-base-patch32 | NaN | 16 | NaN | 1e-3 | 80.42 | 80.88 | 82.01 | \n| Caltech101 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 94.48 | 97.32 | 98.80 | \n| DTD | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 54.2 | 69.86 | 72.10 | \n| EuraSAT | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 61.48 | 79.01 | 83.65 | \n| Flower-102 | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 79.13 | 96.95 | 96.51 | \n| Oxford Pets | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 93.79 | 94.22 | 95.52 | \n| Stanford Cars | CLIP | openai/clip-vit-large-patch14-336 | NaN | 16 | NaN | 1e-3 | 78.20 | 84.09 | 87.95 | \n| Food-101 | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 73.66 | 73.64 | 76.18 | \n| Caltech101 | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 96.75 | 96.75 | 97.16 | \n| DTD | Image | swin_base_patch4_window7_224 | Linear | 16 | 1e-2 | 1e-3 | 67.55 | 68.26 | 70.45 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 38.42 | 38.42 | 39.23 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 1 | 1e-2 | 1e-3 | 33.08 | 33.08 | 33.08 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 64 | 1e-2 | 1e-3 | 45.61 | 46.02 | 48.19 | \n| SetFit/sst5 | Text | sentence-transformers/msmarco-MiniLM-L-12-v3 | Linear | 16 | 1e-2 | 1e-3 | 30.18 | 30.86 | 30.59 | \n| SetFit/Emotion | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 43.10 | 43.65 | 43.90 | \n| SetFit/subj | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 90.50 | 90.55 | 90.75 | \n| SetFit/20_newsgroups | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 54.14 | 57.36 | 58.90 | \n| SetFit/enron_spam | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 91.35 | 91.70 | 92.85 | \n| SetFit/SentEval-CR | Text | sentence-transformers/paraphrase-mpnet-base-v2 | Linear | 16 | 1e-2 | 1e-3 | 88.31 | 88.58 | 89.24 | \n| Food-101 | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 73.06 | 74.42 | 75.72 | \n| Caltech101 | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 93.10 | 97.16 | 97.44 | \n| DTD | Image | swin_base_patch4_window7_224 | SVM | 16 | NaN | 1e-3 | 69.39 | 70.45 | 70.39 | \n| SetFit/sst5 | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 30.90 | 39.28 | 39.95 | \n| SetFit/Emotion | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 26.55 | 43.15 | 44.20 | \n| SetFit/20_newsgroups | Text | sentence-transformers/paraphrase-mpnet-base-v2 | SVM | 16 | NaN | 1e-3 | 48.43 | 57.90 | 58.72 | \n\n\n---\n\n### Reference\n\n[1] Tip-Adapter: Training-free CLIP-Adapter for Better Vision-Language Modeling.
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" Age Workclass Education-Num Marital Status Occupation \\\\\\n\",\n \"12011 51.0 4 10.0 0 6 \\n\",\n \"23599 51.0 1 14.0 6 12 \\n\",\n \"23603 21.0 4 11.0 4 3 \\n\",\n \"6163 25.0 4 10.0 4 12 \\n\",\n \"14883 48.0 4 13.0 0 1 \\n\",\n \"\\n\",\n \" Relationship Race Sex Capital Gain Capital Loss Hours per week \\\\\\n\",\n \"12011 0 4 0 0.0 0.0 40.0 \\n\",\n \"23599 1 4 1 0.0 0.0 50.0 \\n\",\n \"23603 3 2 1 0.0 0.0 40.0 \\n\",\n \"6163 3 4 1 0.0 0.0 24.0 \\n\",\n \"14883 3 4 1 0.0 0.0 38.0 \\n\",\n \"\\n\",\n \" Country label \\n\",\n \"12011 21 False \\n\",\n \"23599 8 True \\n\",\n \"23603 39 False \\n\",\n \"6163 39 False \\n\",\n \"14883 39 True \"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"label = 'label'\\n\",\n \"feature_names = X_train.columns\\n\",\n \"train_data = X_train.copy()\\n\",\n \"train_data[label] = y_train\\n\",\n \"val_data = X_valid.copy()\\n\",\n \"\\n\",\n \"display(train_data.head())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We then train an AutoGluon classifier:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"No path specified. Models will be saved in: \\\"AutogluonModels/ag-20210428_081203/\\\"\\n\",\n \"Beginning AutoGluon training ... Time limit = 20s\\n\",\n \"AutoGluon will save models to \\\"AutogluonModels/ag-20210428_081203/\\\"\\n\",\n \"AutoGluon Version: 0.2.0b20210427\\n\",\n \"Train Data Rows: 26048\\n\",\n \"Train Data Columns: 12\\n\",\n \"Preprocessing data ...\\n\",\n \"NumExpr defaulting to 4 threads.\\n\",\n \"Selected class <--> label mapping: class 1 = True, class 0 = False\\n\",\n \"Using Feature Generators to preprocess the data ...\\n\",\n \"Fitting AutoMLPipelineFeatureGenerator...\\n\",\n \"\\tAvailable Memory: 4406.31 MB\\n\",\n \"\\tTrain Data (Original) Memory Usage: 0.89 MB (0.0% of available memory)\\n\",\n \"\\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\\n\",\n \"\\tStage 1 Generators:\\n\",\n \"\\t\\tFitting AsTypeFeatureGenerator...\\n\",\n \"\\tStage 2 Generators:\\n\",\n \"\\t\\tFitting FillNaFeatureGenerator...\\n\",\n \"\\tStage 3 Generators:\\n\",\n \"\\t\\tFitting IdentityFeatureGenerator...\\n\",\n \"\\tStage 4 Generators:\\n\",\n \"\\t\\tFitting DropUniqueFeatureGenerator...\\n\",\n \"\\tTypes of features in original data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('float', []) : 5 | ['Age', 'Education-Num', 'Capital Gain', 'Capital Loss', 'Hours per week']\\n\",\n \"\\t\\t('int', []) : 7 | ['Workclass', 'Marital Status', 'Occupation', 'Relationship', 'Race', ...]\\n\",\n \"\\tTypes of features in processed data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('float', []) : 5 | ['Age', 'Education-Num', 'Capital Gain', 'Capital Loss', 'Hours per week']\\n\",\n \"\\t\\t('int', []) : 7 | ['Workclass', 'Marital Status', 'Occupation', 'Relationship', 'Race', ...]\\n\",\n \"\\t0.1s = Fit runtime\\n\",\n \"\\t12 features in original data used to generate 12 features in processed data.\\n\",\n \"\\tTrain Data (Processed) Memory Usage: 0.89 MB (0.0% of available memory)\\n\",\n \"Data preprocessing and feature engineering runtime = 0.2s ...\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\\n\",\n \"\\tTo change this, specify the eval_metric argument of fit()\\n\",\n \"Automatically generating train/validation split with holdout_frac=0.09597665847665848, Train Rows: 23548, Val Rows: 2500\\n\",\n \"Fitting model: KNeighborsUnif ... Training model for up to 19.8s of the 19.8s of remaining time.\\n\",\n \"\\t0.8428\\t = Validation accuracy score\\n\",\n \"\\t0.65s\\t = Training runtime\\n\",\n \"\\t0.22s\\t = Validation runtime\\n\",\n \"Fitting model: KNeighborsDist ... Training model for up to 18.93s of the 18.92s of remaining time.\\n\",\n \"\\t0.8388\\t = Validation accuracy score\\n\",\n \"\\t0.79s\\t = Training runtime\\n\",\n \"\\t0.22s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMXT ... Training model for up to 17.9s of the 17.9s of remaining time.\\n\",\n \"\\t0.864\\t = Validation accuracy score\\n\",\n \"\\t0.81s\\t = Training runtime\\n\",\n \"\\t0.03s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBM ... Training model for up to 17.04s of the 17.04s of remaining time.\\n\",\n \"\\t0.8684\\t = Validation accuracy score\\n\",\n \"\\t0.64s\\t = Training runtime\\n\",\n \"\\t0.03s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestGini ... Training model for up to 16.35s of the 16.34s of remaining time.\\n\",\n \"\\t0.852\\t = Validation accuracy score\\n\",\n \"\\t2.23s\\t = Training runtime\\n\",\n \"\\t0.11s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestEntr ... Training model for up to 13.33s of the 13.33s of remaining time.\\n\",\n \"\\t0.8524\\t = Validation accuracy score\\n\",\n \"\\t2.49s\\t = Training runtime\\n\",\n \"\\t0.11s\\t = Validation runtime\\n\",\n \"Fitting model: CatBoost ... Training model for up to 10.12s of the 10.11s of remaining time.\\n\",\n \"\\t0.8664\\t = Validation accuracy score\\n\",\n \"\\t4.62s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesGini ... Training model for up to 5.49s of the 5.48s of remaining time.\\n\",\n \"\\t0.8464\\t = Validation accuracy score\\n\",\n \"\\t2.62s\\t = Training runtime\\n\",\n \"\\t0.21s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesEntr ... Training model for up to 2.25s of the 2.25s of remaining time.\\n\",\n \"\\t0.8448\\t = Validation accuracy score\\n\",\n \"\\t2.3s\\t = Training runtime\\n\",\n \"\\t0.22s\\t = Validation runtime\\n\",\n \"Fitting model: WeightedEnsemble_L2 ... Training model for up to 19.8s of the -3.6s of remaining time.\\n\",\n \"\\t0.872\\t = Validation accuracy score\\n\",\n \"\\t1.0s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"AutoGluon training complete, total runtime = 24.68s ...\\n\",\n \"TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\\\"AutogluonModels/ag-20210428_081203/\\\")\\n\"\n ]\n }\n ],\n \"source\": [\n \"predictor = TabularPredictor(label=label, problem_type='binary').fit(train_data, time_limit=20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Explain predictions\\n\",\n \"\\n\",\n \"SHAP is intended to explain how much each feature contributes to a particular prediction. In this binary classification context, \\\"how much\\\" is quantified in terms of the deviation between predicted probability of the positive class from a baseline reference value. Let's first print which `y` value AutoGluon considers to be the \\\"positive\\\" class:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {\n \"scrolled\": true\n },\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"positive class: True\\n\"\n ]\n }\n ],\n \"source\": [\n \"print(\\\"positive class:\\\", predictor.positive_class)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next we create a binary classification wrapper class around AutoGluon to allow it to be called for prediction inside of the `shap` package:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"class AutogluonWrapper:\\n\",\n \" def __init__(self, predictor, feature_names):\\n\",\n \" self.ag_model = predictor\\n\",\n \" self.feature_names = feature_names\\n\",\n \" \\n\",\n \" def predict_binary_prob(self, X):\\n\",\n \" if isinstance(X, pd.Series):\\n\",\n \" X = X.values.reshape(1,-1)\\n\",\n \" if not isinstance(X, pd.DataFrame):\\n\",\n \" X = pd.DataFrame(X, columns=self.feature_names)\\n\",\n \" return self.ag_model.predict_proba(X, as_multiclass=False)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next, we define the baseline reference value of the features, which we simply take to be the median value of each column here. AutoGluon predictions will be interpreted in terms of their difference from the prediction for the baseline feature-values.\\n\",\n \"\\n\",\n \"**Note:** We use the median here following the [original notebook from the SHAP package](https://shap.readthedocs.io/en/latest/example_notebooks/kernel_explainer/Census%20income%20classification%20with%20scikit-learn.html). However, this is actually an inappropriate baseline value for ordinally-encoded categorical features, where the median is meaningless (using the mode instead would be better for such categorical features). See also the notebook: \\\"SHAP with AutoGluon-Tabular and Categorical Features\\\".\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Baseline feature-values: \\n\",\n \" Age 37.0\\n\",\n \"Workclass 4.0\\n\",\n \"Education-Num 10.0\\n\",\n \"Marital Status 2.0\\n\",\n \"Occupation 7.0\\n\",\n \"Relationship 3.0\\n\",\n \"Race 4.0\\n\",\n \"Sex 1.0\\n\",\n \"Capital Gain 0.0\\n\",\n \"Capital Loss 0.0\\n\",\n \"Hours per week 40.0\\n\",\n \"Country 39.0\\n\",\n \"dtype: float64\\n\"\n ]\n }\n ],\n \"source\": [\n \"med = X_train.median() # X_train.mode() would be a more appropriate baseline for ordinally-encoded categorical features\\n\",\n \"print(\\\"Baseline feature-values: \\\\n\\\", med)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can now create a `KernelExplainer` which will return Kernel SHAP values to explain particular AutoGluon predictions.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"ag_wrapper = AutogluonWrapper(predictor, feature_names)\\n\",\n \"explainer = shap.KernelExplainer(ag_wrapper.predict_binary_prob, med)\\n\",\n \"\\n\",\n \"NSHAP_SAMPLES = 100 # how many samples to use to approximate each Shapely value, larger values will be slower\\n\",\n \"N_VAL = 30 # how many datapoints from validation data should we interpret predictions for, larger values will be slower\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Let's first explain a prediction for a single datapoint from the training data.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"5866383529ff4b6c893af286a69bc74e\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \" 0%| | 0/1 [00:00\\n\",\n \"| \\n\",\n \" | Age | \\n\",\n \"Workclass | \\n\",\n \"Education-Num | \\n\",\n \"Marital Status | \\n\",\n \"Occupation | \\n\",\n \"Relationship | \\n\",\n \"Race | \\n\",\n \"Sex | \\n\",\n \"Capital Gain | \\n\",\n \"Capital Loss | \\n\",\n \"Hours per week | \\n\",\n \"Country | \\n\",\n \"label | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 12011 | \\n\",\n \"51.0 | \\n\",\n \"4 | \\n\",\n \"10.0 | \\n\",\n \"0 | \\n\",\n \"6 | \\n\",\n \"0 | \\n\",\n \"4 | \\n\",\n \"0 | \\n\",\n \"0.0 | \\n\",\n \"0.0 | \\n\",\n \"40.0 | \\n\",\n \"21 | \\n\",\n \"False | \\n\",\n \"
| 23599 | \\n\",\n \"51.0 | \\n\",\n \"1 | \\n\",\n \"14.0 | \\n\",\n \"6 | \\n\",\n \"12 | \\n\",\n \"1 | \\n\",\n \"4 | \\n\",\n \"1 | \\n\",\n \"0.0 | \\n\",\n \"0.0 | \\n\",\n \"50.0 | \\n\",\n \"8 | \\n\",\n \"True | \\n\",\n \"
| 23603 | \\n\",\n \"21.0 | \\n\",\n \"4 | \\n\",\n \"11.0 | \\n\",\n \"4 | \\n\",\n \"3 | \\n\",\n \"3 | \\n\",\n \"2 | \\n\",\n \"1 | \\n\",\n \"0.0 | \\n\",\n \"0.0 | \\n\",\n \"40.0 | \\n\",\n \"39 | \\n\",\n \"False | \\n\",\n \"
| 6163 | \\n\",\n \"25.0 | \\n\",\n \"4 | \\n\",\n \"10.0 | \\n\",\n \"4 | \\n\",\n \"12 | \\n\",\n \"3 | \\n\",\n \"4 | \\n\",\n \"1 | \\n\",\n \"0.0 | \\n\",\n \"0.0 | \\n\",\n \"24.0 | \\n\",\n \"39 | \\n\",\n \"False | \\n\",\n \"
| 14883 | \\n\",\n \"48.0 | \\n\",\n \"4 | \\n\",\n \"13.0 | \\n\",\n \"0 | \\n\",\n \"1 | \\n\",\n \"3 | \\n\",\n \"4 | \\n\",\n \"1 | \\n\",\n \"0.0 | \\n\",\n \"0.0 | \\n\",\n \"38.0 | \\n\",\n \"39 | \\n\",\n \"True | \\n\",\n \"
\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\"\n ]\n },\n \"metadata\": {\n \"needs_background\": \"light\"\n },\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"shap.summary_plot(shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A dependence plot can be used to visualize how the number of years of education increases the chance of making over 50K annually.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAeUAAAFACAYAAACC4WQ/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0F0lEQVR4nO3deZxcZZX/8c/pTkIIaxICMUBYhMgiW3IYQRFwAVFkREEhA0FFCDDqjPpjBgVZBASEUUdZhkQUQ9iEAdFhQITfDxRE1AOERQSCJBAISwIEEiBb9/P7494OlerqdFV1ddfS3/frVa+q+9ztVJY+/Tz3WSylhIiIiNRfW70DEBERkYySsoiISINQUhYREWkQSsoiIiINQklZRESkQSgpi4iINIgh9Q6gVtx9AjADGA28AhwVEbOLjjkVOBzoAFYAJ0fEbfm+nwMfBRbmh18fEd/t7b5f/epX04UXXlirryEiMlhYba/2me7je9ONtb3HAGilmvKlwMURMQG4GJhW4pg/A7tHxM7A0cAv3H3tgv3nRcSu+avXhAywaNGiPoYtIiKSaYmk7O4bAxOBa/Kia4CJ7j6m8LiIuC0i3so3Hyb7TW30gAUqIiL9xEq8mk9LJGVgc+D5iOgAyN/n5+U9OQr4e0Q8V1D2DXd/xN1vcvft+y9cERGprdZIyi3zTLkS7r4PcBawX0HxKcALEdHp7kcBv3H3rbsSfdH5U4GpAGPHjh2IkEVEZI2aMwkXa5Wa8jxgU3dvB8jfx+Xlq3H3PYErgYMj4omu8oh4PiI6889XAOsCm5W6WURMjwiPCB85cmTNv4yIiFSqNWrKLZGUI+JlYBYwOS+aDDwYEQsKj3P33YFfAIdGxANF+zYt+Pwxsh7az/dj2CIiIqtppebr44EZ7n4a8BrZM2Pc/RbgtIgI4BJgbWCau3edNyUiHsnP3QToBN4A/jEiVg7wdxARkao0Z824mGnpxr6ZMmVKmjlzZr3DEBFpNjUep3x4iXHK1zZdpm6lmrKIiAxaTZd/S1JSFhGRsnV0JO795UssnL+MSfuNZvz269Y7pJySsoiIDDJ3XfsCd179IgCP/O41vjZtB9bfaFido4JWScot0ftaREQGxktz3l71efnSTl55cVkdoymkIVEiIjLI7LzvKCzPHBuPH86m24yob0C5hHV7NSM1X4uISNneu9dIRo9bi9deWs7Wu6zHsOHt9Q6ppSgpi4hIRd619QjetXVj1JDf0Zw142JKyiIi0vRKNVc3Y5pWUhYRkRZQXQo2s7nA0vwFcFJK6TYzS8AjZLM8AkxJKT3S1yh7o6QsIiKD3aEppUdLlL8/pbRkIANRUhYRkRbQjI3V3WlIlIiINL0+Dom6ysweNrNLzGzDgvK7zGyWmZ1rZmvVNuLSlJRFRKQFdJ88xMymmlkUvKaWOPGDKaVdgN3zEy/Ky8enlBzYG9gBOLX/v4Oar0VEpAWUWu8wpTQdmL7G81Kal78vM7NLgF8Xlb9hZpcB36htxKWppiwiIi2g8mk2zWwdM9sg/2zA4cAsMxtpZmvn5UOAQ4FZ/RP36lRTFhGRplfltJqbADeYWTvQDjwG/DOwHTAtHxY1FLgXNV+LiIiUq/KknFJ6GtitxK4XgJ37GlE1lJRFRKTpNesCFMWUlEVEpAUoKYuIiDSEVqkpq/e1iIhIg1BNWUREml6r1JSVlEVEpAUoKYuIiDQE1ZRFREQahpKyiIhIQ2iVmrJ6X4uIiDQIJWUREZEGoeZrERFpeq3SfK2kLCIiLUBJWUREpCGopiwiItIgWiUpq6OXiIhIg2iZmrK7TwBmAKOBV4CjImJ20TGnAocDHcAK4OSIuC3fNwK4HJgErAROjIibB+4biIhItVRTbjyXAhdHxATgYmBaiWP+DOweETsDRwO/cPe1830nAm9ExDbAQcBl7r7uAMQtIiJ9ZiVezaclkrK7bwxMBK7Ji64BJrr7mMLjIuK2iHgr33yY7G9tdL59GHkiz2vYAXy8n0MXEZEaSFi3VzNqiaQMbA48HxEdAPn7/Ly8J0cBf4+I5/Lt8cAzBfuf7eV8ERFpEK2SlFvmmXIl3H0f4CxgvyrPnwpMBRg7dmwNIxMRkeo0ZxIu1io15XnApu7eDpC/j8vLV+PuewJXAgdHxBMFu54FtijYHl/qfICImB4RHhE+cuTIGn0FERGpVirxakYtkZQj4mVgFjA5L5oMPBgRCwqPc/fdgV8Ah0bEA0WXuR44Lj9uW2B34Df9GLaIiNSImq8bz/HADHc/DXiN7Jkx7n4LcFpEBHAJsDYwzd27zpsSEY8AFwA/d/enyIZMTY2IxQP8HUREpCrNmYSLWUrNWslvDFOmTEkzZ86sdxgiIs2mpln0FTu5WzIbnc5pukzdSjVlEREZpJq1ubpYSzxTFhERqYaZzTWzx81sVv76WF6+h5k9ZGZPmtlvzWzjgYhHSVlERAa7Q1NKu+av28ysjWyUzpdTShOA3wPnDUQgSsoiItL0atz7ehKwNKV0T759KfC5PgdZBiVlERFpeqWSsplNNbMoeE3t4fSrzOxhM7vEzDakaIbHlNJCoM3MRvX391BHLxERaQHda8YppenA9F5O/GBKaZ6ZrQX8J3AR8Muah1cm1ZRFRKTpVTujV0ppXv6+jGwuiw9QNMOjmW0EdKaUXq1lzKUoKYuISNOr5pmyma1jZhvknw04nGx2yPuBtc1sr/zQ48lmfex3ar4WEZGmV2XHrk2AG8ysHWgHHgP+OaXUaWZTgGlmNhyYCxxZq1jXRElZRERaQOVJOaX0NLBbD/vuBXbqY1AVU/O1iEizu+dv8H8uh6t/X+9I6kYLUoiISP09Ng8+cjosX5ltd3TClH3rGlI9tMoqDqopi4g0s1lz3knIAH+eXb9Y6qhVaspKyiIizWzvHWCj9bPP7W3wj7vXN566sRKv5qPmaxGRZrbZRnD/BfD/HoFdtoTdtq53RHXRrDXjYkrKIiLNbvwY+MKH6x1FXbXKM2UlZRERaXqqKYuIiDSIVknK6uglIiLSIFRTFhGRFtAaNWUlZRERaXrq6CUiItIg9ExZREQaRlreUe8Q6qpVZvRSTVlEpImllZ3M3+WnrHhsIbbhcMY98EWGbrVhvcMacM2ahIuppiwi0sQWnfMHVjy2EIC0aCkLj/qfOkdUH6nEqxmppiwi0sTS0tWbrQdvM7ZqyiIiUmcbnvoB2setC4ANH8JG0z9e54jqQ8+URUSk7trWHspm875Cx/zFtI8Zga01OH+sN2sSLjY4//ZERFqItRlDNlu/3mHUVbM+Qy6mpCwiIk1PNWUREZEGoaQsIiLSINR8LSIi0iBUU24w7j4BmAGMBl4BjoqI2UXH7A+cA+wEXBgRJxbsOwP4Z2B+XvSHiPjyAIQuIiJ9pKTceC4FLo6IK939SGAa8OGiY54GjgEOBYaXuMYVhYlaRESaQ6s0X7fE5CHuvjEwEbgmL7oGmOjuYwqPi4inImIWsHJgIxQRkf7Ul8lDzOx0M0tm9t58O5nZw2Y2K3/t1G+BF2mVmvLmwPMR0QEQER3uPj8vX1DBdQ7Pm7hfBE6PiD+WOsjdpwJTAcaOHdunwEVEpH7MbCKwB/BM0a73p5SWDHQ8LVFTrpFLga0iYmfgAuBX7j661IERMT0iPCJ85MiRAxqkiIh0V01N2czWAi4GTuj3AMvUa1J29yHu/oi7l3oG2yjmAZu6eztA/j4uLy9LRLwYESvyz7fn5763H2IVEZEaK7VKlJlNNbMoeE0tOu1M4MqU0twSl7wrb7o+N0/eA6LXpBwRK4ENaeDn6BHxMjALmJwXTQYejIiym67dfdOCz7sCWwJP1CxIERHpN6Vqyiml6SklL3hN7zrezPYEHLikxOXGp5Qc2BvYATh1YL5F+c+UfwR8192/mSfpirj7e4Afk/0BrFe4LyKGVXq9HhwPzHD304DXgKPye98CnBYR4e57AdcC6wPm7ocDX4qI24Bz3H0S0AEsB6ZExIs1ik1ERPpVxUOi9gG2B+aYGcBmwG1m9sWU0m8BUkpvmNllwDdqGemaWEq9V4DdfTZZzXE58ALQ2bUvIiaUcf5fgMeBq4G3CvdFxO8qirjBTJkyJc2cObPeYYiINJuaDiy+3/6rWzKblE4o+x5mNhf4JPA8sDSl9LaZDQF+CryaUvp6rWJdk3Jrymf38T7vAfbo6h0tIiJSSzV8vrodMM3MEjAUuJdGa76OiBl9vM9fgHcDT/bxOiIiIt30dUavlNKWBZs79+lifVD2OGV33x04mmzs7zzgZxHxlzJP/yJwmbvfRtb8vUpEXF1uDCIiIqU0bE/kCpWVlN39YLJZsn4JPAhsDfzO3Y+IiF+WcYlDyKa83IXVnyknsufMIiIiVescZHNfnw4cEhG3dBW4+8eB88gSdW9OBj4ZEb+pPEQREZE1a5UFKcqd0WtLoDih3gZsUeb5KT9eRERqrHPR2yy7bTYdzyyqdyh1U2rykGZUblJ+BvhoUdlHgGfLPP9nwBfKPFZERMrUufBNXt3tEl4/4Ape2eHHrLiv7IkMW0pfFqRoJOU2X59FNhf0fwNzyGrOhwCfL/N8B77u7t+ge0ev/cu8hoiIFFl++9/pnLso23hrBUuveoihe2xe15jqoVmTcLFyh0Td4O4vkCVhJ+t9vV9E3Fvmfe7OXyIiUkPtE0ZDm0Fn1mDbvv2YXs5oTc3aXF2sx6Ts7jdExCH55y9GxOVkg6grFhHfqTI+ERFZg6GTNmX96w9j2X//laETx7H2Cf9Q75DqYjDUlD9S8PlHwOXV3sTd39/Tvgpq2yIiUsLwz+zI8M/sWO8w6mowJOW/uvs1wCPAMHc/udRBEXFOGfe5p0RZV2tDexnni4iItLw1JeUjgW8CHyJLnPuVOCYBvSbliFitl7e7jyObT/vmsiMVERHpQcs/U46IOcBxAO4+KyI+VKubRsR8d/9X4AHgxlpdV0REBqfB0Hy9SkTs2g/3XgvYuB+uKyIig0zL15RrqcTz6HWAg4HbB+L+IiLS2gZVTbkGip9HLwauA344QPcXEZEWpqRcgVo+jxYRESnWWe8AamSgasoiIiL9JrUNopqyu7cD3yKbZnPjiNjA3T8GbBURl67hvDms+fl7ioh3VxKwiIhIsdQaObmiBSk+CpxEtuITwJNk6yn3mJSBb/dQvilwIrB+mfcXERHp0aCqKQP/BOwZES+4+2V52Vyy1aJ6FBFXFW67+wiyCUm+QbY+80mVBCsiIlJKKnch4gZXblIeAbxcVDYMWFrOye5uwLHAGWTJfH/NeS0iIrWS2gdXTfkB4IvAZQVl/wT8ubcT3f0TwAXAcOBrEXFdpUGKiIisSecga74+EbjL3Q8HRrj7/5Ctq7zGoU7ufgewK9k81xdFxMo+xCoiIlLSoGq+johH3X174CjgceAZ4JiIeKmXUz9MNnzsfOB8dy917WEVRSwiIlJksHX0IiIWAN+v8PqaNERERPrdoBoS1dNayrDm9ZQj4nfVBCUiIlKJvtSUzex0so7IO6WUHjWzPYBpwNpknZOPTCkVd3buF+W2wu9X9Po82Rf4aKU3dPdHKj1HRESkP5jZRGAPsseymFkbcCXw5ZTSBOD3ZHNyDIhynyl3a4Z2968AY6q45xZVnCMiIg0ireigY/EKhowaXu9QVumsoqJsZmsBFwOTgbvy4knA0pTSPfn2pWS15aP7GmM5+tJf7b+A46s4r0Va/kVEBp+lDy/gyc1/yhOjL+W5I24lpcZYyTi1WbdXGc4ErkwpzS0oG09eawZIKS0E2sxsVE0D7kFfkvIuVJdge3wGLSIijW3Bd//MypfeAuD1q5/g7fteqHNEmWTdX2Y21cyi4DW163gz25NsaO8l9Yu6u3I7et3O6gtLrANMBH5Q6Q0j4txKzxERkcbQvsFaq223rb9WD0cOrGTd64gppenA9B5O2QfYHphj2bmbAbcBP6bgMauZbQR0ppRerXHIJZU7JOqeou3FwMnl9q5293WAfyH7rWS9wn0RsX+ZMfR2jwnADGA08ApwVETMLjpmf7Ka+k7AhRFxYsG+drK/jAPIfgE5LyIKZzATERn0Nv7u+1kx/02Wz17EqC/vzPAdR9c7JKDyZ8oppfMo6MBlZnOBTwKPAVPNbK/8ufLxwPXlXNPMNgAOBDZLKZ1vZmOBtpTS/HLjKrej13fKvWAPfgbsBtwEvNnHa/XkUuDiiLjS3Y8k687+4aJjngaOAQ4lm/az0BHANsC2ZIn9QXe/IyLm9lO8IiJNZ8iYEWxx86fqHUY3tZo8JKXUaWZTgGlmNpx8SFRv55nZJLKFll4AtiKbNGtn4DjgkHLv32NSdvdx5VwgIsr5DWB/YEI+AUnNufvGZM3p++VF1wAXufuYwntGxFP58QeXuMxhwE8iohNY4O43AZ8lm7dbREQaWF8nD0kpbVnw+V6yFtVK/Cfw7ymly83stbzsXuDySi6yppryc6z+HLmY5fvby7jPK8CSCuKq1ObA8xHRARARHe4+Py8v9xeB1XrcAc/m53fj7lOBqQBjx46tNmYREamRUs+UB9iOwM/zzwkgpbTEzNap5CJrSspbVRdXSScDP3b3kyJiQB6W96eIWNV5YMqUKY0xHkBEZBCrZpxyjS2gqHJnZtsAz1dykR6TckQ809O+KlxFVqM+2t07iu5TiwUp5gGbunt7XktuB8bl5eV6lqzH3V/y7eKas4iINKgGqCnPAK41s38DLH/G/H3gJ5VcpOwFKdx9O2Bfslm8Vn37iDizjNMrno6zEhHxsrvPIpuV5cr8/cEKn2FfDxzr7jeSdfQ6GPhgjUMVEZF+0AALUnyPbLjwLcC6wJ3Aj8hG9ZSt3HHKk8nayh8m6032MNnkIb8v5/wBWpjieGCGu58GvEa2zCTufgtwWkSEu+8FXAusD1i+PvSXIuI2YCbwPqBrGNWZETFnAOIWEZE+6qxzTTml1AGcApxiZhvlM4FVrNya8inAlIi4zt1fi4jd3f1oYLtyb+Tuh5ANR9qcrFn5soi4oeKIexARj5Ml1eLyTxR8vodsgHip8zuAE2oVj4iIDJwGqCmvUm1ChvKn2RxP98HTVwBTyjk57608HXgQ+GH+Ps3djyvz/iIiIj1KZt1eA8nMOs2so8TrbTN73MxOM7Ne+1CVW1NeBGyQv7/k7tuTDXMqt6v314BPRMSfugryccAzyCb5EBERqVoDdPT6GnAsWcXzGbKOw/9KVoFdAvwbWc48aU0XKTcp3wF8mmwQ9HX59grg1jLPH8c7vZq73A9okK+IiPRZAzRffxE4qHDFKTO7E7gxpbSbmf0R+BW1SMoRUbiO5OnA42SdpWaUGezjZNNYziwomww8Web5IiIijWxroHiGy/nAuwFSSg+b2ZjeLlJu7+vxEfEsQEQk4OrKYuUk4Nb82fIcYEuyhaQ/saaTREREylGrua/74EHge2b2zZTSMjNbCzg3L8fMtiZ77LtG5Xb0etrdb3f3w9294nW68iFRO5KN31pC1uy94wANlRIRkRZX745eZM+TPw4sMrNnyPpgfSIvB3gXvTRdQ/nPlLcFvkCW9S9x92uBn0VElBttPuZXaymLiEjN1bumnFKabWY7AnuS9aN6Hngd+ArwLymlP5RznXKfKc8he5Z8urt/BPg8cKe7Px0Ru5Q6x90/GxHX55//aQ3XrrQpXEREunR2wtEXw7X3wKSt4dcnw+j1ej+v1dS/9zUppQ4z+wvZqoPnA3sAf6zkGmVPs1ngLrJOXpsDe6/huNN5Z2zzd3s4pprn0yIi0uXmgBl3Zp/vfQIuuAnOK2sKiZZS75qyme1AtnbykcAIssfDH08p/baS61Qy9/XOZF2+/wlYRjb26tiejo+I9xZ8ruWKUyIi0qW4htgANcZ6qNc4ZTObQraU7wfIpqA+g2wRpr8Csyq9Xlkdvdz9QeA+sgfVRwFbRsS3I+KpMs8/pYfyb5UbqIiIlHDgJPjCh2DtYfCB7eDET9U7orpI1tbtNUBmkE05fWBKadeU0oUppaqXKC63pvwT4OqIWFTlfU6idBP2v6HOXyIi1Wtrg8u/mr0GsTo2X59Ktq7DTWZ2C/Az4H+rvVi5Hb0uqebi7j4u/9jm7u+iYMlHsh7dy6q5roiIvOONU+/krWv/yrBJ72LDnx5E2zq1WKa+udSr+Tql9F0zOwc4gOyR7g1k45E3JOuF/XIl16umo1clniPrzNX1uYsBHWS/YYiISJWW3v53Fp99NwBvP/UqQ7bfiPVP36fOUdVBHR+lp5QS2fwbt5rZu8hqzkcDfzGzX6aUPlfutfo7KW9F9kc1i2z95S6dwIKIWNrP9xcRaWnp9dUbHDtfH5w/VhtgQQoAUkovAGeZ2dlkk4lMreT8fk3KEfFM/nHD/ryPiMhgNfwf38NaH92KZXfMoX3rkaz7tT3qHVJd1HtIVLG89nxL/ipbr0nZ3bcBdgIeioinqwsP3H07YF9gDAUNDRFxZrXXFBEZ7GxYOxvdPoXOV9/GNhyONVhyGiiNUlPuqzUmZXf/DPALoB1Y7u6fiYiKsn5+ncnAz8nGcO2cv+8C/L7Sa4mISHdto9audwh11SpJubeBXN8GTgbWI5uh6+Qq73MKMCUidgfeyt+PBx6o8noiIiKrNMCCFDXRW1LeCvh+RLwJ/ADYpsr7jOedKTe7XAEMvrngRESk5gZLUm6PiE6AiFgBVDv4bRGwQf75JXffHhgFrFPl9URERPrMzG4ys4fM7EEzu9vMds3L55rZ42Y2K399bCDi6a2j1zB3L2yyHl60TUScU8Z97gA+DVwOXJdvryAb1yUiItInfagZfz6l9DqAmX2KbEauifm+Q1NKj9YgvLL1lpTvA/Yr2P5T0XYCek3KEXF0webpwONkK03NKC9MERGRnlWblLsScm4Dsnk06maNSTki9q31DSNCyzWKiEhNlRqnbGZTWX3yjukppekljrsM2J9suO4BBbuuMjMD7gFOTiktqmXMpVS1jIa7m7sf6O6/LvP437r7h4vKPuLuar4WEZE+K9XRK6U0PaXkBa9uCRkgpXRMSmk82QijC/LiD6aUdgF2J0vWFw3E96hoRq98gYljgC+RLeN4XZmnTqT7mOS7ycZAi4iI9EktelunlGaa2XQzG51SmpeXLTOzS4CyKqF9Vc6MXkY2f+dx+ftCsmkzJ0XEI2XepxMYCqwsKGunrlOIi4hIq6gmKZvZusDIrgRsZgcBrwJLzWyDlNLrefP14WRrOPS7NTZfu/upwBzgJrJOXYeQjTl+HXipgvvcDxQv9vkVNHmIiIjUQJXjlNcBrjezR8xsFvB14CBgE+AuM3sYeBSYAPxzP4W+mt5qyt8hWxfy4MLpNd290vucBNzl7ocAT5KtpfwesrmwRURE+qSamnJK6SWgpxU8dutTQFXqraPXFOAx4H/c/UF3/6q7j+KdNZLLEhEPAzsA/w28QbYI9A4R8VAVMYuIiKymVWb06m1I1FXAVfkMXFPJxhifT/Y82KlgSaqIeJF3erWJiIjUTGrOHNxNWb2vI+JvwNfd/ZvA54BjgZvdPSLiH3o7v3gWsKJrlzMjmIiISI+atWZcrKIhURGxDJgJzHT3HVh9UPaa7Fe0PY5ssYt7KGNGMBERkTUZlEm5UEQ8BnytzGM/VFzm7l8BxlR7/xLXm0A2bedoss5pR0XE7KJj2oEfk83YkoDzIuKyfN8ZZL3r5ueH/yEivlyr+EREpP90Doak7O6z6aVTV0RMqPLe/0WWAE+v8vxilwIXR8SV7n4kMA34cNExR5AtP7ktWfJ+0N3viIi5+f4rIuLEGsUjIiIDJLXItBe91ZTPLvhswMXUbqzWLtRo8hB335hs1rCuZvJrgIvcfUxELCg49DDgJ/lylAvc/Sbgs6gDmohIUxsUzdcRsdoqTu7+g+Kycrj77axe416HLIn+oNJr9WBz4PmI6ACIiA53n5+XFybl8cAzBdvP5sd0Odzd9wdeBE6PiD+Wupm7r5rkfOzYsTX6CiIiUq1WScpVLUhRhXuAPxS8bgAOiIhTBuj+5bgU2CoidiarOf/K3UeXOjAipkeER4SPHDlyQIMUEZHuBsU45VqJiO/08y3mAZu6e3teS24n6+E9r+i4Z4EtgL/k26tqzvk46q54b3f3ecB7gd/1c+wiIiJAPyZld9+7nOMionj1qIpFxMvuPguYDFyZvz9Y9DwZ4HrgWHe/kayj18HAB/N4N42I5/PPuwJbAk/0NTYREel/g2LykBK9r9d39ycLj1lD7+v/W7TdxuoduxLQAQwrL9ReHQ/McPfTgNeAowDc/RbgtIgIsjHW7wO6hkqdGRFz8s/nuPukPKblwJTC2rOIiDSuQTEkitV7X1ckIoZ2fXb3o4BPkS1MMYds4pBzqeH6lBHxOFnCLS7/RMHnDuCEHs7/fK1iERGRgdWsz5CLVdT7ug++A+wcEYvz7afc/UvAQ2S1VxERkaoNiqTs7kMAi4gVBWVfAHYFfh8RN5Z5n/WB4cDigrLhwAaVBCsiIlJKqzRf9zYk6hfAF7s23P3bwHRgL7LVo44p8z43A790933dfSt3/xDZMo43VxGziIjIapJ1fzWj3pKys3ri/CpwTEQ4cCQ9PJ8t4ctkPZlvBf6evz9F7WYHExGRQSxh3V7NqLekPDIi5gPkaypvAFyX77uJbNhQj9x9R4CIWBIRXwJGAGOBERFxNFDWsCkREZE16TTr9mpGvSXlN9193fyzA49GxNJ82+i993bxNJWvRMTL+dzTANeWH6qIiEhprTKjV29J+W7gLHffDjgO+E3BvvcAL/RyfvGfSm/bIiIiFRssSfkksrWHHyPrQV24gMQRZHNar0nxso+9bYuIiFSs07q/mlFv45TnANu7+6iIeLVo9/lkM1+JiIjUVbPWjIuVNfd1iYRMRCwq49Rh7n5ywfbwou2hxSeIiIhUqrNFnob29ypR9wH7FWz/qWj7vn6+v4iIDAKDqqZcrYjYtz+vLyIiAs37DLlYbx29REREGl6145TN7CYze8jMHjSzu81s17x8gpn90cyezN+37c/4uygpi4jIYPb5lNIuKaXdgP8AfpaXXwpcnFKaAFwMTBuIYJSURUSk6VU7Tjml9HrB5gZAp5ltDEwErsnLrwEmmtmYmgZdQn939BIREel3pZ4pm9lUYGpB0fSU0vQSx10G7E82odUBwObA8ymlDoCUUoeZzc/LF9Q8+AJKyiIi0vRKLUCRJ+BuSbjEcccAmNkU4ALg1FrHVy41X4uISNOrxYIUKaWZwIeA54BNzawdIH8fB8yrZcylKCmLiEjTqyYpm9m6ZrZ5wfZBwKvAy8AsYHK+azLwYEqpX5uuQc3XIiLSAqocp7wOcL2ZrQN0kCXkg1JKycyOB2aY2WnAa8BRtYp1TZSURUSk6VUzzWZK6SVgjx72PQ68r49hVUxJWUREmp6m2RQREWkQrTLNppKyiIg0vWp6WzciJWUREWl6WrpRRESkQXS0Rk7WOGURGRyWv93BGy8tJaVU71CkH9Ri8pBGoJqyiLS8F/72Btf/26MsXbySbT84moPP3AFra84f2lJaq3T0Uk1ZRFren65+jqWLVwIw++5XeOFvi+sckdRaJ9bt1YxUUxaRlrf2hkNXfbY2GL6+fvS1mo4mba4upn+ZItLy9pm6JUsXr2TR828z8dPjGLX5iHqHJFKSkrKItLzh6w3lU2dsX+8wpB+1yjPllknK7j4BmAGMBl4BjoqI2UXHtAM/JlvEOgHnRcRlve0TEZHG1tGkz5CLtVJHr0uBiyNiAnAxMK3EMUcA2wDbAnsCZ7j7lmXsE5Eae2PJcs6d9gJ/ferNeociLaDDur+aUUskZXffGJgIXJMXXQNMdPcxRYceBvwkIjojYgFwE/DZMvaJDLiFbyY+NXMpO/34LX7ylxX1Dqem7n90CRd85G42O+cebv/UnZz6o+frHZI0uVYZp9wSSRnYHHg+IjoA8vf5eXmh8cAzBdvPFhyzpn0iA+6bty3n14938OhLieN+tZwnF3bWO6SaOfu8Z5nw4qsAjF7yNs/eNr/OEUmz6zDr9mpGLfNMeSC5+1RgKsDYsWPrHI20qlfffmfmqZRg0dutMxPVkBGr/+hZOkw/iqRvVtY7gBpplZryPGDTvLNWV6etcXl5oWeBLQq2xxccs6Z9q4mI6RHhEeEjR46sQfgi3Z28z1BG5yN3Ju/czu6b9e9/17R8JSuvvZ+Vv3q436eivH76BP531214ZvT63D1hM0753rb9ej9pfaopN5CIeNndZwGTgSvz9wfzZ8OFrgeOdfcbyXppHwx8sIx9IgPON2vnuX8fwevLYJN1+/8HzLJPX0bnLY8BMOQrezPswkP79X7X/mq3fr2+DC4rmzMHd9MqNWWA44GvuvuTwFfzbdz9Fnf3/JiZwNPAbOA+4MyImFPGPpG6GD7UBiQhp6UrViVkgI4bH+r3e4rU0kqs26sZtURNGSAiHgfeV6L8EwWfO4ATeji/x30irc6GD8V2Gkd6JOtw1fYPW/RyhkhjWdGcObiblknKItI3w+/4Miv+8y5sxDCGfH3feocjUpEVTfoMuZiSsogAYBuvx7BzDqp3GCJVaZWR/ErKIgLA3Dvm88ezH6Z9rXb2+d4kNtl1VL1DEinbWy1SU26ljl4iUqXOjsSdJwZL5r/N63OW8PuTH6h3SCIVedu6v5qRasoiAp2JzhXvzBjWsax1Zg+TwWF5k/a2LqaasojQNrSND5y2C21D2xi6zhDef+rO9Q5JpDJW4tXbKWajzewWM3vCzB4xsxvNbEy+L5nZw2Y2K3/t1J/hd1FNWaRBLV/awW9++jyvvbicfzhwI7bfY8N+vd92h23FhEO3xNrAWuT5nEgvEnB+SukuADO7ADgP+FK+//0ppSUDGZBqyiIN6rc/n8+fb1nI7Afe4Npz5/Day8v6/Z5t7aaELM3JrPurFymlV7sScu4+Vp9uecApKYtU4PVlif/3bCfzl/T/4hCLXl6+6nPHysSSV1tlyn2RxmNmbWQTSP26oPiuvOn6XDNbayDiUFIWKdPCtxK7XdHBR67rZLufdfDAS/2bmPc4aAxD18p+2996l/UYt82Ifr2fSFMrUVM2s6lmFgWvqWu4woXAEuCifHt8SsmBvYEdgFP7+RsAeqYsUrZb5yTmvJ59Xrwcrnysk4mbtPfb/bbZbX2+cdl7WfzqCjbZam3a29WsLNKjEv89UkrTgem9nmr2H8C2wEEppc783Hn5+xtmdhnwjVqG2xMlZZEybTvSMLKeIQATRvZ/klxv1FDWGzW03+8j0vyq+/9oZucAk4ADU0rL8rKRwNKU0ttmNgQ4FJhVo0DXSElZpEx7jDOuOrCNG55M7D7WOG4X1VxFGkYV/x3NbEfgW8CTwL15J8c5wPnANDNLwFDgXtR8LdJ4Jm/fxuTt6x2FiHRTRVJOKf11DWfWZbC+krKIiLSA1mi5Uu9rEQFg/hV/5/+udxV3jrmWV+6YX+9wRCpTxYxejUhJWUToXNnJj344j3887kA+fcR+zDz9yXqHJFKh1sjKSsoiwsqOxPcPmMRbI4bx+vprc+773lvvkEQq0xo5Wc+URQRoNzqGtq+amnDR+pqoRJpMkybhYkrKIoKZ0dZmdOaDsEeNaJGfcDKItMa/WTVfiwhD243vf2wobQZrDYELDxxW75BEKqPmaxFpJV/bcwjHTGxnSBsMH9qkP9Fk8GqR1c2UlEVklXXXao0fbCLNSs3XIiIiDUI1ZRERaX4t0sijmnI9xFNwwJlwyPnwzMv1jkZEpAW0Rk8v1ZQHWmcnfOJsWPBGtr3gDfj92fWNSUSk2TVnDu5GSXmgLV3xTkIGmLewfrGIiLSKFknKar4eaCPWgq99Mvvc1gbf/HR94xERaQlqvpZq/fBoOP5jsPYwGD+m3tGIiDS/5szB3Sgp18t7Nq13BCIi0mCUlEVEpPm1SE1Zz5RFJHPfk7DPGXDAOTD7hXpHI1IZs+6vJtT0NWV3HwFcDkwCVgInRsTNPRx7LHAS2e9UtwL/EhGd7r4vcAvQtbL7soh4X3/HLtIwUoKDzoeFi7PtIy6EP59T35hEBqGmT8rAicAbEbGNu28L3O3u20TEksKD3H0r4HRgN+AVsqR8JHBFfshjEeEDGLdI41jRQXplyaoWwM4XFqkZTZpLc1aMu2mF/3eHAdMAImI2EMDHSxx3KHBTRCyIiE7gJ/m5IjJsCH/abS8AOsy4b7e96xyQSKU0JKpRjAeeKdh+Fti8iuMmuPsDwArgkoiYUetARRrViuWd3Dz2/dz9ofeysq2dtg035P31DkqkEs2Zg7tp+KScJ8rxPezepEa3eQDYPCJez5u573D35yPijh5imgpMBRg7dmyNQhCpn6HD2thhjw147L5se699R9Y3IJFBquGTckRMXNN+d38W2AJYkBeNB+4scWjXcRQcNy+/x6p5LyNijrvfBHwAKJmUI2I6MB1gypQpqZzvIdLodrnncdaZtZj2jk4mjH8Tjt6s3iGJlK9Fasqt8Ez5euA4gLyj1+7Ab0ocdwNwsLuPcfc24Fjguvy8d7m75Z9HAfsDs/o/dOmrFa8s5ZmzZjHv+4/QuXRlvcNpWp1LV7L4t8+xycuvs9Eri3ntV3PrHZLIoNQKSfkCYEN3fwq4GZgaEYsB3P1Mdz8eICKeBs4C7gNmA08DV+bXOAR41N1nAb8HroiIXw3ot5CqPPzR3zD3tAd4+sS/8Pjn7653OE2rbfgQRuw6etX2eu+v1ZMhkQFSxThlMxttZreY2RNm9oiZ3WhmY/J9e5jZQ2b2pJn91sw27vfvAFhKan3tiylTpqSZM2fWO4xBqeOtldyzzhWrttfabB32mKcO9dVasfBtXrzor7SPGMLYr76XtrUb/umWNLeaNjjbd5d3S2bplGFrvIeZjQJ2TindlW9fAIwia0l9EvhCSukeM/s2sHVK6ehaxlxKK9SUZZBqHzGEDT/8rlXboz5ZqtO9lGvoRmuz+RnOuH/fVQlZmk8VI6JSSq92JeTcfWR9jyYBS1NK9+TllwKfq2G0PdL/PGlqO/3vfrx8zdO0jRjCmM9uVe9wRKSBmNmqkTK56Sml6T0c2wacAPyaoiG0KaWFZtZmZqNSSq/2Z8xKytLU2oYPYewXJ9Q7DBGptxI14zwBl0zCJVwILAEuAuq20L2SsoiItIDqH1Gb2X8A2wIHpZQ6zWy1IbRmthHQ2d+1ZNAzZRERaQVVzrJpZueQPUM+OKW0LC++H1jbzPbKt48nG37b71RTFhGRQcnMdgS+RdbT+l7LhlHNSSl92symANPMbDgwl2wBo36npCwiIs2vitbrlNJfezozpXQvsFPfgqqcmq9FREQahGrKIiLS/DT3tYiIiNSSasoiItL8ypjruhkoKYuISPNrjZysBSn6yt0XUDAdW4PbCFhY7yD6ib5b82rl76fv1rOFEXFArYJpFUrKg4i7R0R4vePoD/puzauVv5++m1RKHb1EREQahJKyiIhIg1BSHlzKXS2lGem7Na9W/n76blIRPVMWERFpEKopi4iINAiNUx5E3P104Axgp4h4tM7h1IS7Dwd+CHwUWAr8MSKm1jeq2nH3TwJn8c5idN+JiBvrG1V13P0/gEOALSn4N+juE4AZwGjgFeCoiJhdrzirUeq7uftoYCbwbmA5MBs4LiIW1C3QKvX0d1ewv+V+ttSLasqDhLtPBPagecZUl+t8smQ8ISJ2Ak6tczw14+5G9kN9SkTsCkwBZrh7s/6/vQnYm+7/Bi8FLo6ICcDFwLQBjqsWbqL7d0vA+RHxnvzf5t+B8+oQWy3cROm/u1b+2VIXzfqfWyrg7muR/bA7od6x1JK7rwscBZwaEQkgIl6qb1Q11wlskH/eEHghIjrrF071IuKeiJhXWObuGwMTgWvyomuAie4+ZqDj64tS3y0iXo2IuwqK7gO2GNDAaqTU94PW/dlST0rKg8OZwJURMbfegdTYu8maO09393D3u9x9r3oHVSv5LxqfA37l7s+Q1VaOqmtQtbc58HxEdADk7/Pz8paRt26cAPy63rHUWKv+bKkbJeUW5+57Ag5cUu9Y+kE7sDXwYD6z0EnAje6+fn3Dqg13HwJ8C/hURGwBHARcl7cQSHO5EFgCXFTvQGqlxX+21I2ScuvbB9gemOPuc4HNgNvcff+6RlUbzwIryZs+I+JPZHPxTqhnUDW0KzAuIv4AkL+/Sfb32SrmAZu6eztA/j4uL28JeSepbYHDmvXRQw9a+WdL3Sgpt7iIOC8ixkXElhGxJfAc8LGI+G2dQ+uziFgI3AnsB6t68W4MPFXPuGroOWAzd38PgLtvD2xC1mGoJUTEy8AsYHJeNJms5aPpeiiX4u7nAJOAgyNiWb3jqaVW/tlST5o8ZJDJf6P9ZKsMW3D3rYGfkQ2nWQGcEhG31jeq2nH3I4BvknX4Ajg9Im6qX0TVc/cfA58BxpK1aLwSETu6+3ZkQ6JGAq+RDYl6on6RVq7UdyPrD/Ao8CTwdn7onIj4dF2C7IOe/u6KjplLC/1sqRclZRERkQah5msREZEGoaQsIiLSIJSURUREGoSSsoiISINQUhYREWkQSsoiNeDuK9193zre/6/ufli97i8itaGlG0VKcPe7gD3Jxj4X2jMiHhn4iDLuviUwB9g8Ip7rKi8eM9oP992XbKKWOyPiwwXlRwJn55NHiEgfKSmL9OysiDi73kE0kE5gV3f/ZETcXO9gRFqRkrJIhdx9PbKFBQ4CFgOnFe0/A9grIj5aUHYXcEdXknf3ncnWgp5EtrDGA13Hu/vlwEfJlmqcR1YTvTq/1EP5+xPunoDvRcRZ+WxK346IK/Nr7JNffzvgBeCHETEt37cvcAdwBHAOsBFwG/CliFi8hq+egLOB89391q6VnYq++2rfMy9LwAcj4p78z+aDQABHkz1C+y5wA3A5sDvZDFhHRsTf1hCLSEvSM2WRyv0n2QIDOwA7A58iS6xlcfd3Ab/LX1uSTV14XsEh95AtRrEh2dJ4P3f3HfJ9u+Tv74mIdSPirBLX3wr4DfBfZNOPfgE4190/W3BYO7B/fr0JwG7Av5QR/kXAcODYMo7tyd7AbLLvfSRwAfBT4MvAKOBvwI/7cH2RpqWaskjPTnH3E4vKRpHVMA+MiBcB3P0koJL5jKcAT0XEuQVld3R9iIifFpRfm8ewL/BYmdefTFbz/nm+fZ+7TwOOAa4vOO6bEbEEWOLuN5Etw7dGEbHc3b8F/MjdrywznmJPRsRl+edb3f0V4LaumrG7Xw1cVeW1RZqakrJIz75b/EzZ3TcB1gLmFhTPqfC6W5I10Xbj7m3AGcBhZDXJBKwDjKng+puXiOnvZDX6Lh1FKzG9CayXx3AycHJe/kxxJ7KI+IW7f51s/epqFo54oWj7raKyt7piERlslJRFKrMQWE6WWLuWUNyy6JjFZIm00LiCz3OBQ3u4/mSyGu3+wGMR0enuAVi+v5z1eOcBnygq25oy1yiOiHPInjWvyf8BfkvWvF5ote/u7uMQkbIpKYtUICI68ubV77j7o2RL8p1XdNj9wDnuPomsY9bxwFYF+68kaxo/CbgQWAnsHRF3AOvn2wuANnf/Atlz367ezgvIEvO2ZOvXlnINcKq7HwVcDUwEjgNOqPZ7F4uIP7j7b4ATyWrZXe4HDnP3HwBLyTpxiUiZ1NFLpGenuvuSotcngX8lax5+HHgE+B9gVU/kiLgL+AFZZ6sXgE2APxTsn0/2jHg/ssT6IvBv+e4ZwJ+Ap4DnyTqT3V1w7tvAqcA17r7I3U8pDjoi5pDVlL9Ctq7vTODUiLiuj38exU4CNigq+yFZR62/A7OA/63xPUVamtZTFhERaRCqKYuIiDQIJWUREZEGoaQsIiLSIJSURUREGoSSsoiISINQUhYREWkQSsoiIiINQklZRESkQSgpi4iINIj/D1LRg4h/QN3lAAAAAElFTkSuQmCC\\n\",\n \"text/plain\": [\n \"
\"\n ]\n },\n \"metadata\": {\n \"needs_background\": \"light\"\n },\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"shap.dependence_plot(\\\"Education-Num\\\", shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Scoring Overall Feature Importance via Permutation Shuffling\\n\",\n \"\\n\",\n \"Note that if you'd like to understand how much each feature contributes to AutoGluon's general predictive accuracy (rather than explaining individual predictions), AutoGluon offers a built-in method for this based on [permutation-shuffling](https://explained.ai/rf-importance/):\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 13,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Computing feature importance via permutation shuffling for 12 features using 1000 rows with 3 shuffle sets...\\n\",\n \"\\t42.76s\\t= Expected runtime (14.25s per shuffle set)\\n\",\n \"\\t13.0s\\t= Actual runtime (Completed 3 of 3 shuffle sets)\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"\"\n ],\n \"text/plain\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"from autogluon.tabular import TabularPredictor\\n\",\n \"import pandas as pd\\n\",\n \"import numpy as np\\n\",\n \"import sklearn\\n\",\n \"import shap\\n\",\n \"import time\\n\",\n \"\\n\",\n \"shap.initjs()\\n\",\n \"\\n\",\n \"import warnings\\n\",\n \"warnings.filterwarnings('ignore')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Load the diabetes data\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"X,y = shap.datasets.diabetes()\\n\",\n \"X_train,X_valid,y_train,y_valid = sklearn.model_selection.train_test_split(X, y, test_size=0.2, random_state=0)\\n\",\n \"\\n\",\n \"def print_accuracy(f):\\n\",\n \" print(\\\"Root mean squared test error = {0}\\\".format(np.sqrt(np.mean((f(X_valid) - y_valid)**2))))\\n\",\n \" time.sleep(0.5) # to let the print get out before any progress bars\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Train AutoGluon regressor\\n\",\n \"\\n\",\n \"Here we just train directly on the raw data, without any normalizations. We first format the data in a manner suitable for AutoGluon training (pandas DataFrame):\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"\\n\"\n ]\n }\n ],\n \"source\": [\n \"X_train_summary = shap.kmeans(X_train, 10)\\n\",\n \"print(\\\"Baseline feature-values: \\\\n\\\", X_train_summary)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can now create a `KernelExplainer` which will return Kernel SHAP values to explain particular AutoGluon predictions.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 7,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Root mean squared test error = 58.16574872551215\\n\"\n ]\n }\n ],\n \"source\": [\n \"ag_wrapper = AutogluonWrapper(predictor, feature_names)\\n\",\n \"print_accuracy(ag_wrapper.predict)\\n\",\n \"\\n\",\n \"explainer = shap.KernelExplainer(ag_wrapper.predict, X_train_summary)\\n\",\n \"\\n\",\n \"NSHAP_SAMPLES = 100 # how many samples to use to approximate each Shapely value, larger values will be slower\\n\",\n \"N_VAL = 30 # how many datapoints from validation data should we interpret predictions for, larger values will be slower\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Let's first explain a prediction for a single datapoint from the training data.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 8,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"925d80dd82b44faa97b4e1ba254ffba9\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [-2.16673763 -7.30257651 -6.01534141 17.9040135 -0.69770945 2.19206997\\n\",\n \" -4.45803757 -8.42249673 5.32206295 12.13680085]\\n\"\n ]\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"\\n\",\n \"\"\n ]\n },\n \"execution_count\": 8,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"ROW_INDEX = 0 # index of an example datapoint\\n\",\n \"single_datapoint = X_train.iloc[[ROW_INDEX]]\\n\",\n \"single_prediction = ag_wrapper.predict(single_datapoint)\\n\",\n \"\\n\",\n \"shap_values_single = explainer.shap_values(single_datapoint, nsamples=NSHAP_SAMPLES)\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values_single, X_train.iloc[ROW_INDEX,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can also plot Kernel SHAP explanations aggregated across many predictions, say over the first `N_VAL` datapoints of the validation data. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"34cc0b9bfde1438697ab689efb00eb98\",\n \"version_major\": 2,\n \"version_minor\": 0\n },\n \"text/plain\": [\n \"HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 0. 0. 63.69359417 27.21267897 5.20834155 0.\\n\",\n \" 0.81633909 0. 10.26999198 6.50870779]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 0. 0. 29.25021484 15.93078397 0. 3.27738944\\n\",\n \" 0. 1.91848041 38.04089951 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999996\\n\",\n \"phi = [ 14.31946816 -7.39413279 4.36798833 9.06797456 6.19110434\\n\",\n \" -6.47101103 7.10822616 -8.03039118 -21.11975486 -7.23973024]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999998\\n\",\n \"phi = [ -8.789158 16.17749534 -11.73169955 -8.39858956 2.89905291\\n\",\n \" 0. 4.43960454 -10.28228143 -20.66888232 5.25473967]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [ -5.64068976 7.21626128 38.76103664 31.74885774 0.\\n\",\n \" 0. -11.4491603 -5.51146652 -22.49884925 3.81547525]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999996\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999996\\n\",\n \"phi = [-6.06175334 0. 37.89447948 -5.00720861 0. -5.77670438\\n\",\n \" 0. -6.42181305 39.9725613 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999998\\n\",\n \"phi = [ -2.13544767 14.5719235 -14.68747995 -3.57124352 3.85603983\\n\",\n \" 0. -8.21865402 -8.29701616 -16.13847713 3.15103834]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999996\\n\",\n \"phi = [ 8.66347225 0. 22.11892852 23.12263857 0. 0.\\n\",\n \" 0. 0. 43.27191463 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ -1.85292786 8.43368626 27.50072162 -7.14809398 6.88768633\\n\",\n \" 0. 0. -6.57793464 -35.94251127 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [ 0. 0. 32.49094562 0. 0. 3.14016984\\n\",\n \" 0. 0. 39.56861988 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [ 0. 13.31560366 -24.49352188 -5.19353812 6.52024096\\n\",\n \" 15.04173621 6.3924407 -8.90559676 41.73577965 2.12396274]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999999\\n\",\n \"phi = [ 5.881003 -6.98106869 30.7259501 -9.44596626 2.04919406 -7.79317557\\n\",\n \" 0. 22.37259822 7.03326867 -7.23217134]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999999\\n\",\n \"phi = [ 0. -10.2075933 -14.80481836 -16.87522515 8.60859206\\n\",\n \" 13.81357314 9.49536535 -6.88215379 -20.0630101 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ -5.69405509 8.46227189 -14.50971968 -21.38935346 5.68266072\\n\",\n \" 10.26003665 5.75481235 -6.71902121 -34.47367198 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 9.24290337 0. 53.14152246 28.05650298 0. 5.4803388\\n\",\n \" 0. 20.01019078 40.48546858 16.92750748]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999998\\n\",\n \"phi = [ 0. -15.28472701 -10.86835208 -10.3318947 0.\\n\",\n \" 0. 0. -5.94133372 -28.92029046 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999996\\n\",\n \"phi = [ 8.61725094 13.42064196 -20.98804736 0. 8.03119977\\n\",\n \" 23.0467636 0. -5.60104253 4.76458275 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0000000000000002\\n\",\n \"phi = [ -8.54535662 11.64960096 -15.67058107 -3.45940766 0.\\n\",\n \" -4.86732447 -12.72642341 -7.12263442 -32.26967056 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0000000000000002\\n\",\n \"phi = [ 13.98822924 13.25414658 -16.04534911 -11.6820341 5.79664317\\n\",\n \" 9.56973722 0. -4.74629304 -35.95275212 -3.35903446]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 5.03014177 3.76477239 33.59227563 24.85266859 3.37472313 5.54220647\\n\",\n \" 3.45679307 9.31849101 6.81283955 18.40771986]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [ -4.0384511 -11.83896209 18.05952073 21.75582647 9.17814365\\n\",\n \" 14.85817747 2.65287958 -4.62941164 -24.99065342 -5.88777864]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ -8.76483426 8.66939548 7.89887873 -27.46385583 3.4606591\\n\",\n \" 4.71626958 0. -10.32863648 12.3987181 1.47140203]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 12.35058303 -10.85969224 -22.48586052 23.6638099 0.\\n\",\n \" -2.83632124 3.40961245 -8.23314868 29.51155125 5.38550021]\\n\",\n \"num_full_subsets = 1\\n\"\n ]\n },\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 0. 0. 28.40184855 -18.76787593 4.58736196\\n\",\n \" 10.40780163 0. -6.89448539 -39.76472483 -2.61805289]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999997\\n\",\n \"phi = [ 0. -5.97731577 30.36177332 -17.87193754 -10.1351185\\n\",\n \" 0. 5.51059166 18.40536102 33.5459797 5.10274154]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0000000000000002\\n\",\n \"phi = [ 0. 9.83915705 12.30750499 -8.15644688 -19.68456459\\n\",\n \" 3.88742185 0. -7.41810269 35.59772399 0. ]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ 9.86495019 6.46338744 -18.67973685 22.83503627 -1.82620075\\n\",\n \" 2.46297072 -12.10575575 -3.93432058 -18.80142066 -5.43351429]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 10.0\\n\",\n \"np.sum(self.kernelWeights) = 1.0\\n\",\n \"phi = [ -2.67919902 7.73234112 -14.55077222 -22.84749361 0.\\n\",\n \" 16.16904114 -13.86821077 -8.14549584 -29.40078509 8.88033649]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999998\\n\",\n \"phi = [ 0. 0. 0. 11.12557672 2.99050478 13.52778722\\n\",\n \" 0. 0. 43.14263999 11.82022432]\\n\",\n \"num_full_subsets = 1\\n\",\n \"remaining_weight_vector = [0.32588454 0.24829299 0.21725636 0.20856611]\\n\",\n \"num_paired_subset_sizes = 4\\n\",\n \"weight_left = 0.6072380418010941\\n\",\n \"np.sum(w_aug) = 9.999999999999998\\n\",\n \"np.sum(self.kernelWeights) = 0.9999999999999998\\n\",\n \"phi = [ -4.67329699 -10.86118878 20.79620634 -4.8511668 -1.24456468\\n\",\n \" 4.42538701 2.35501036 15.42071128 6.61113335 -9.26519008]\\n\"\n ]\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"\\n\",\n \"\"\n ]\n },\n \"execution_count\": 9,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"shap_values = explainer.shap_values(X_valid.iloc[0:N_VAL,:], nsamples=NSHAP_SAMPLES)\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A summary plot is an even better way to see the relative impact of all features over many datapoints. Features are sorted by the sum of their SHAP value magnitudes across all samples.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 10,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAegAAAFfCAYAAACFn1/BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABiGElEQVR4nO3dd3gcxf348fdcUW+W5N57BYO9NNN7dehJ6ISEBL4klBQgQAjll1BDCin03kIHh947mHG3weAuV/Vykk5X5/fHnqU7WZJP9kl3On9ez3OPtszOzq7u9rMzO7urjDEIIYQQIrU4kl0AIYQQQmxLArQQQgiRgiRACyGEEClIArQQQgiRgiRACyGEEClIArQQQgiRgiRACyGE2CUopdYqpaa1m6aVUocopW5SSv0ojjxuUErd2XOlbOPqjZUIIYQQqcwYc32yy9Ce1KCFEELs8pRSjyilfhkZLlRKvaCUWq6Uek8p9Vi7WvNQpdTrkfmvKaVyeqJMUoMWQgixK3leKdUSNT6hgzTXA7XGmElKqWJgHvBC1HwL2AuoB94CzgLuT3RB0z1Ay3NMRdzmzJkDwOzZs5NcEiHSmuqZXE+JPd6bFztbz2nGmKWtiymlO0hzKPArAGNMjVLq5Xbz3zLG1EWW/woYu2OF7po0cQshhEgDqt2nR0XXwEP0UGVXArQQQggR60PgXAClVBFwYjIKIQFaCCFEGkhoDfomYIBSajnwEqCxrzf3qnS/Bi2EEGKXsP2gbIwZ1cE0KzL4YdTkJuAMY0yLUqoA+BS4L5L+hnbLx4wnkgRoIYQQaSCh1537AW8opZxAFvCUMebdRK4gHhKghRBCiCjGmApgZrLLIQFaCCFEGujxntu9TgK0EEKINJB+AVp6cQshhBApSGrQQggh0kD61aAlQAshhEgDEqCFEEKIFCQBWgghhEhBEqCFEEKIlGPaBeh0CNfSi1sIIYRIQRKghRBCiBQkTdxCCCHSQDo0aseSAC2EEKLPS8dr0BKghRBCpIF0CMmx5Bp0gtz8SZAJ//Zz2vMBPD6T7OKIFLH2tQ28cvhbvPnDD/Gsa0x2cUS0G/8L4y+B0++ARm+XScNbPHiOeZD6iXfgu+fLXiqg6B7V7tP3SYBOgE/Kwlz/UYgVNYYXlof582ehZBdJpABfvZ9Pr5hL/SoPFXOr+PK6+ckuktjqw6Vww39h5WZ4/gu49aUuk3uvep3gW98T/r6K5v97mdCKql4qqIiXQcV80oEE6ASob1djbj8udk0hX4iwP9w6HmgIJLE0IkZ9c7vxpi6Tm/qWqBGD8fh6oFBCxOqVa9CWZa0FBgHBqMn7aa2X9Mb6e9oxYx0cN87B6yvDjCyE3+4rl/YF5AzIZtrFE1n6n+9w57nY88ppyS6S2Oq4GXDMnvDmAhg1AH5zYpfJs64/guBn6zBVTWT8xMI1Y2gvFVTELz1qzdGUMT1f24sE6Ou01k/0+Mpi9WpVtsZrKMwEpyP9vii7gjlz5gAwe/bshObrbwjgzHTgzHQmNF+RADUeKMwB5/b/NyYQwjT6cPTL6YWCpbUeOUAG1U9jjvcu82CfPxAntKpnWdalwBVAKdAAPKq1viaR6+gtl73awv1z/Uwe4ODV83IYWtj11YBL3wvxwBLDlBJ49WQnfr/hxMeaWVEV5pezMrj9uKzWtMsqDSe/FGRTI/xhloOr9t324LByc5Ar7q9nfXWIiqJs+o/J5tWTnIws7Po7t2xJMw/8p5xQyHDOBQPYa5+8mPnvvO/hyWdryct1cPkl/clXIZ678TtqK/ysKyrE07+AgnovU6dmc+HP+/PGDcvYuLCeUfsWc8JNU3G6Y/fD22/U8fLz1RQWuvi/ywYxfGTm9nZt2vHX+/n8/E+oXlDDsOOGsffd+6CcDhq/qWPxyR/g29jMqGt3Z/Tvd+syn8XXzmftYyvJG1fAfk8eRPYQOxAEKr2sPPENmhdUUXLWeEbefwhK9fljD80fllF+1muY5gD9/3kE+WdN2f5C1z4Ff3sNxgyEV66y/wLBhZtoOPVpTEUjOTcfQfbl+8cu94/X4NonoX8hvPA72HOMPf2VrzFXPEzYZ6gMj0EdMY3Sh47Fd8bjhL5Yi+vEaWQ9cRaeexZSe81HOEuz6f/cyWTOHJTgvSF2Xt//TbSXsGvQlmVNAG4FTtBa5wNTgVejktxlWVaNZVkLLcv6RaLW2xM+XRPkH5/58QZg/sYwN73b9fWmj9cb7l5g8AZhXjnc/EWY69/xsXhzGG8A7vjIz7wNbR3HfvtBiBW10BSAqz8Ks65+24r+nS81srYiRCgEJdVevt0U4vrPwtuka+/RBypoqA/R1Bjm4XvLiW4haWwK8eDjNXhbDJXVIR55soZ3711H7SYfBA0jqupo8gSpwcWShc08+68NrJtbS9AfZuXHVXz7dnnMuupqgzz7ZBW+FkNFeYBnntw1O86suO87Kr+oJNwSouzFdWx4bYM9/Xea5u8bCDUFWXXNfLxrPJ3mUT23klX3fkfIG6J+SS3f3r60dd6W2xbQ9EU5piVE1YPLaXizrMe3qTdUXvQOoU2NhOt8VPz0TUxgO50rF62FP78IzT5YWgbXPd06q+mK1wmvrsE0+mn69RuEt0Tt64o6uPwhaGyBNeX2MIDXBz+/B1XfjLPFS7F/Nd7XV9H00xcIfbASWoIE/7sQ/0NzqbnsXYzHT3BNPTWXv5vwfSF2nnQS61oQ+xRmqmVZeVrrOq311vsRzgPGAAOB3wF/7o0g7fF4dng4WiAQ6DBNPOk7K0+0xqbG7abpLJ94ytPZ9FCoe73NW1raOsrEs66+OJyRkbHDy0br7v5pbm7uKHna7ued2a7O0kdrbGyMGm6CDlocPJ6Ob3sLBoMdTu9uOWW46+HES7/brBJ6DdqyrFOAi4F9gMXATVrrtztIdy1wjNb6wIStvGM7vHE70sT9YKSJ+5VIE/dJUU3ct7Vr4j7l5SAbPfE3cQ8Yk80rcTZxP3hPOcFg/E3cz9/0PTXlvi6buEfvV8LxN05J6ybuHb0GvbWJu2ZhDUOPbdfEfcoH+DY0M/q63Rl1dXxN3PnjC9j3iXZN3Ce9iXdBFcVnjpMm7r+9BmMHwsvtmrhPexpT3kkT992vwTVPwoBCeD6qifvh9ztu4j7zCUKfr2lr4r53IbW/jzRxP38ymTOkiXsn9MiX169+EXO8zzD39vkfSY90ErMsKwO4CLgNKNFaN7eb/3vgeK31AQlfeSy530nErac6iRljCNT4cffLQEkHQiEkQMcpYZ3ELMuaCIwGPga8QD12gOxvWdYY4AsgAByA3ZHs5kStW4hUFfQE+Gr2e9R9XUX+lCL2feMIMkqztr+gEKKb+nw83kYir0FnANcDm4E64FLgVCAXuAuoBGqBf2E3fd+dwHULkZI2/ncNdV/bnec839RR9vDKJJdIiPSUjp3EElaDjjx0ZFYns/dM1HqE6EtcBe4ux4UQojPyyCshetCQ00ZRN7eKirc3Ubz/AEZcMD7ZRRIiLaVLrTmaBGghepByKKbeuRdTk10QIdKeBGghhBAi5aRjDVreZiWEEEKkIKlBCyGESAPpV4OWAC2EEKLPkyZuIYQQQvQKqUELIYTo89KxBi0BWgghRBqQAC2EEEKknHSsQcs1aCGEECIFSQ1aCCFEnyc1aCHi0BIw/O+7EPM2hZNdFCGE6LOkBi0SKhgyHPGYn8/K7OB832w3F1ryNRNC9CypQQuxHStrTGtwBnh0USiJpRFC7DpUu0/fJwFa7LRg2NASNAAMzlcUZ7fN221AevxQhBCpzaBiPulAArTYKW+tCdPv7hC5fw9x+9wwTd4wo/0t5PoC7JEf4o6jpHlbCNHzdskAbVnWWsuyzk7UCi3LesOyrCsTlZ9Irt9+FKYxAGEDV38c5u53vVRWhShp9lNT1sLn3wWSXUQhhOiTer16o7U+trfXKXpOrrttONMFeZmxZ645melxJiuESHXpd6yR9sc09Zevwzz7XRhrkOKvhzrIcNpf3oUr/Nz3qoecLAdX/DCfof1dnaYFeHtFkJs+CFCaq/jX7AyGFsQ2utx/lJNzXg+xug4GZRlacjMIFodo9BnO283JARMzOizfyx828daXXkYNduEwhnVbghyxTw4nHZrbY/ukKyZsKP8oi6YNLj6tWscBPxkZ13JNVT7ev2M5zTV+9v7JaEbPKu00bUtDgI9u/YaGTV6mnzGSCUcPjmsdDesa+eLGRQS8Qfb67TQGziyJa7n2vBuaWPLrr/HX+pl4zW70P3Tb9Zc9u4bVD64gb0w+02+bSc3iWpbcuZSMggxm/nkGucO6//9pfPZbGv76Na6RhWROKsT3xkrc0wdiGloIr6sn97J9CBkHNTd/gbcqSOaeAxly3+FkjCzABMNs+c0nNH+5mYKTx5K5zxA2XP81rqJMRv7zADJH5m+/AL4AXP4QzF+N78h92LTARdjjZ9Cf9iP3wKHd3p5eV9UAlzwEZdXw2xPg1H16vwzNPvjVA7CsDM4/DC46uvfLsB0m2QXoAcqYrjfLsqy1wEPAUcAewHLgYq3115ZlPQI4gQBwCtAE/Bb4FrgfmARo4Cyt9aZIfh8C72qt/1/Ct2Zb6fg/264PysIc9mxbT+rbDnJw5d4OgiHDCb+rwOO1d8v0cW5OP71fh2kBPD7DoFubaY60Uh8/0cn/zsnaZn2Xvh3kbr01DwMeu817TLFi1VV526RfvtbPRbdUt44rTOu1lruvKmXKmI6Dek9a+L/NvHXnytbxk26azMSDOg+2W7165SJWf1wJgCvTwc/mHEhWgbvDtO/dvJRvXt4IgHIqzn35QAqGZHeYNtqc0z+kYkENABmFbs6aewIOZ/drC1+e9D6V720GwJnn4qiVp+LKbTtHb1zl4Z39XrOvVwCjzhvLqjnrCTYGARh44EAO/e8h3VpncH0D68fcA8EwijBugq3zFGG7zuNUtITdNJkcttaCcg8bxuj3TqXqbwvYcsXHrcv4s7MIRr6/BYcPZdK7s7dfiD89D9c9BcBK9qCZInsfFGUypfJClCvFu+Kc80944lN72O2ENXfD0OLeLcM1T8AtL7aNz7sDZozd0dx6pKpbo66OOd4Xm1v7fJU63m/mRcBlQDHwPPC6ZVkFkXmnAS9E5t2MHZhvAk4GBmIHyRsTWOa4eTyeXXK4vJkYZbV+AHwB0xqcAarqgtuk3dJkWvPx+ExrcAbYVN92cI1e78a66OvMqvXnt8VjOk5f3kRnahtCnW5XTw4318ReK2+q8ce3bKW3dTjoC+NvCnaa3lvjb51mQobqjXVxlc1b5Wsd99cHCPtDO7SNvoqW1vFQY5BQczAmTe36mtbgDODd4m0NzgAtlS3dX29ZFQTtkzfV7ny5tSNPyKCMIfq4HdzSjMfjIdjuC2q8beUJRNJsrwz+9ZVt+dJ28heq8xFu6fz/lSrDwY01rcMEQlDT2Otl8G+oIkZ5/U7nmXi77m1WD2qt52mt/cBtgBc4ITLvfa31a1rrMPAYkAs8rrXeoLVuxg7oVqILHo/8/Pxdcnj2GMVeg+xpQ/Lgsr0zAcjNcnDmkXYTpdMBPz2hYJu0F+/haM1nSIGDi/a2a1iZLrj+sKxt1gVw5f6ZFEVmTe6HfZB3Ozh0UgbegNkm/YEzith9nH2gzM9RFOTYP6ZpYzPYa2pWUvbbbscOxF1onxyUjMph8mH941p2vwvG4sy0f0ZTZw+hYHB2p+n3PHc07lwnAGMOGcComYPiKtuev5qEctn7aPpFE3Blu3ZoG8dfOQ1Hhl3WUb+YQGb/rJg0ww4czsAjhwDgLnAz6YqpTPjpeAAcbgdTL5/S7fUWzRpNzqkTATC5WTgn2M3zjv45OCK198wTJ5H1gwm4iZwkuRT9/7A3+fn5FP98Gq5hditM1p79Kbl4NwCU28GQP8yMqwwZl8+GQUUADBzrg0iNuf+VM3HmZXRrHyZj2HXtyZBr/4b54b4wbXivlyHjNydCaaROdsg0OHy3nc4z0dKxF3e8Tdw3a60fjJr2KfAqMAUIaq1/FjXPAAdqrT+NjJ8PXKe1HhcZ/xBp4u5xwbBhbb0ddHPcsV/WLTUhMt2KfvmO7aYFWFsbJj9TUZLT+Ze+wWeobIbRRbDHPX6WVNi7/rQpDp47fdsm61DYsLkqRHGBA6Wgpj7MoFInTkfyflivvDSHgMfByWcci9Mdf7Ont96PvylEYRzN1T5PgJa6AAXDslEq/m31VrcQ8oXJG5IT9zIdrr+qhVBTkJyR2156APtafNO6RjJLs3Dn2031TRuacGW7yCzJ3KF1GmMIrqnHUZyFI9dNaF09jiH5EAgRrmrGOaYfAMHVdYSag7gG5+EqbduX4eYAgY2NZIwqQLmd+NZ5cOS6cJduf3+3amqBzbUwagDBWh9hb4iMET0XLBKursmuOY8eAN343iRUoxe21NllcDp3Jqce2YAqdW3M8b7U/KnPR+l4O4mN2jpgWZYCRgAbsAO0SEEuh2Jcv47nDSp2xp0WYFS/7QergkxFQSa0BE1rcAb4fH3Hz+N2OhTDBrR9/YYOSP51QIcLMvuFuxWcAbILM8gujC9tZr6bzPyOr1F3uY6Sba/974jM0izo4tK6cijyRscGrh3pGBaTp1K4xxS1jrvGbb1+6sZR2LZd7rH96GjPOHLcZI5v+4LG1TGsvdwsGGd3inP137mTnKQoyrU/yZSXDeO6cVIkdlq8R6ILLMuaYVmWG/gdkAO81nPFEn1Vlktx5Ji2r9XsCTt1pi2EELuseGvQ9wH/wO7F/R1wvNa63rKScmlZpLhXz3DzzNIQOW7FaVOSXzMWQqS/dLnuHG2716D7uLTeOJFYc+bMAWD27Dhu3RFC7KgeiaQV6g8xx/sB5uY+H7HlQSVCCCHSQJ+Px9uQAC2EEKLPS8fmUgnQQggh+rx0vAYtPXiEEEKIFCQ1aCGEEH1eOtagJUALIYRIAxKghRBCiJQjNWghhBAiBUkvbiGEECIFSQ1aCCGESEnpF6DlNishhBAiBUkNWgghRJ8nTdxCCCFECpJOYkKkMGMM8z6qo6YiwMyDiygZmNGt5f3fhQmuNpQPqWbgzJIeKmUfFQzB/e9AXRNceCSUFiS7RELEkBq0ECnsvRcqef2pCgA+f6uGq/4xnuxcZ1zLbvqsgob/BMHAG+9+xAnPH0rp7v16srh9yyX3wX3v2MNPfgyL/woO6cIiUkc6Bmj5hYm0seqb5tbhhtogVZv9cS9b/nVVaxuZCRoqFlQnunh920fftA0vWw/VnuSVRYgOqXafvi8hAdqyrLWWZZ2diLyE2FGT9sxrHS4e4GbA0PibuIccMKD11+DMdDBon/6JLl7fduyebcPWWCjJT15ZhOiAafdJB9LELdLGwbNLKR2UQU1lgOn7FZCZHV/zNsBAq5TCy10E1xiO+NmhFE8q7MGS9kF3/QT2Hm9fgz7rIGneFqIXSIDuY0Jhw13asKLO8JNpDvYbEn9TzoJywz2LwowoUPxuL4XbAXcvMCytMpwxSXHoiG0Puq8uDzHnuxD7DXfg8xvmbwpx2jQX9QHFO6vDHDrKwZm7xR8It6pe6WHxf8vI7Z/JjPNGE2gJ88VTGwgFw4w8ZCBfaC8FBU4OOqqQP73tY8O3XvYb4+SMo/N45kMvwZDhvKNy6V/Utu4NK5pZ8VUd/QZkkJvvIhQy3PtgJXpdiP32zObC04q6LFPuah+uZQHC39TA5J4P0IF1DVTfOQ9HjouS3++FsyirdV79fxbgm19B3ukTyDlqdLfybX5sEf5Pysg8dhzZp0ze4fKF564l9OBnqDGlOH9zBOEjZ1J/y+eYC9/A5Q7jnj6I7CtmwZOfw6ffw3HTUSdbsXl8X07ob++jinNg/ADMp6twHDYR5xl7db3yhmb48wvQ2AK/PRFGDdjh7RC7hnS8Bq2M2fnGAMuy1gIPAUcBewDLgYu11l9blvUI4AbCwIlAJXCz1vqRnV7x9qVLS0erGz4LceMX9mbluuG7C5wMzd/+F7Paaxj3QIg6nz3+G0sxLF9xxQdhADKcsOQ8JxOK2/L6cn2Y/R/wEd66F0NhMAaHUxF2twXG185wc9z4+IO0vynIY7M/oqUuAMC004ezuTLMugX1hJVizZABBCLtzf5h2WyshexIIXIKnJR57TJOGObiqWvs3tZN9QH+cuE3+Jrt7TnotAFs8Dn5z0JA2el/e0ouPz4ij45UvbSW5ae8a48o2O2D4yk8eHDc27QjVo5/mMDKOgDyThjN8DknAVB/70IqL4p0yHI5GD7/XDJ3i6/J3fvCt9Se9pw9oqDkw/PIPGhkt8tmttTjH/9HaLS/MM7rjqXqo3r8n6wji5bWQ2H+6aPJfO7DyPoUfHQt6sCJdh6+AP6x18PGum3yd71+Cc5jp3VegJNuhVfm2sNjBsLKf7f+H0Wf1yP/yFXq9pjj/VhzZZ//wiSyneoi4DKgGHgeeN2yrK33YvwQeCsy7xfAfyzLmpXAdXfI4/Gk3fDiqtZJNAVgyabmuJYta6A1OAMsqoR5m9o6UflDsLzGxCy7tCLcFpyh9WcVJtbcspZubUvF2urW4AxQ9b2HitVNAAQdjtbgDNBSHWwNzgCNjaHW4ZUbg4Qj8zauqWsNzgCb13j5viwQc1Bf9F3n+6p5cU3bBhloWlzTo//HcHOgNTgDeBdWtG2j3tRWlmAY/zfVceff9HVZzHYEl1TsUDmblpa1BmcAs2gD/sUVqPb1lMXro9ZnYMn6tnyqmzoMzgAtc1d3XYbF69oSry7Hs6myW+WX4dQfTjQT+Xaabb+lfVYiA/SDWut5Wms/cBvgBU6IzPtSa/2E1jqotX4XeAE4P4Hr7lB+fn7aDZ85SeGIfPcmF8MBo3PiWnZKCcwYaI8r4KzJirOnZeCOfANGFcABQ1XMskePc9A/156f6QSXsoPhgBwoyIzknQE/3D27W9syZGJ/Bu1e1Do+8bghTD3SbsJ0h0KUZLcF5FG75VDlbrsSM35MW8evo/fKwhHZGaOnlDBotF0OpWCPQ4o5+qA8MkJ2QHdgOPnQtnt325et5OSRmCw7L1dxJv2OG96j/0dHjpv8k8e1Ti86Z0rrcPHZu9lNGoBzaB7Zhw6PO//CH09H5brt/VCcTeZx43aonLmzJqKmDbFHHArHmXuRd/Y0wjgIbz34ORSOCw6E3MiXoSQPjt29LZ/BhajDJ7bmSVbk/1iQRfbpbU3cHZbh7IPalpttkT90QNfpZbjPDSdaOnYSS2QT981a6wejpn0KvApMAZxa63Oi5v0/YIbW+ridXnnX0uX/FGNRhWF1veGwEYrCzPjPFJv8hnfWGYblK6xB9nLfVBmW1xgOHq4oyd42r80ew+dlYaYPUoTCsKw8zAGjHPhCirmbwliDHYws6v7ZarAlRNkXVeSUZjJotyIA1s6rIxQMM3x6IUuXtVBQ6GTcuCw+WBlkyfIWDh7vYvrELL5a7iMYgv0mZ7QGaACfN8TKBR6KBmQwdJx94rJ0SRNff+Nn/31zmDAys8syvXb/yzhXBzj0kuPIHJbb7W3qLhMK0/jmWhy5bnIPGR4zz/9dDf5lVWQfOAxn/5xOcuhYcHUtgQVbyNh3KM6hO/5AEdPgJfzed6jRJTj2sMvnfWcNpqEFhwrjGlOMa4/BmNUVsGAd7DsWNbQ4Ng9/kPCby1Aleajh/Qh/vRaHNRI1Mo4HwXy0DDxeOGZPcHW/n4NIWT1SvV2h7ow53o83v+3z1ehEBujHtdZ/iIwrYB1wNfZ16bFa6wOj0j8BNGutf77TK+9aWgZosS1PU5gHnq+nzhPmh8fkMXVc18G4I3PmzAFg9uzZiS6eEKJNjwTO79VfYo73E8xv+nyATmQv7gssy3oJWAJcAeQAr2EH6H0tyzoDeBY4GDgVOCKB6xa7uLufqONj7QVgyfc+nrh9EDlZciuQEKLvSuQR7D7gH0At8CPgeK11fWTes8BxkXkPApdorT9L4LrFLm5LVbB1uNlraGhs35VNCJHO0rGTWEJq0FrrUZHBGztJ4tVa/ywR6xKiIycdnsedD9cSDsMBM7IYWCLXLIXYlaTj9Ux5UIlIC4ftm8OUsRk0NIUZN8KNkntmhdilpEutOZoEaJE2BvV3MUgeoS3ELkkC9A7QWp/f0+sQQgixa5MmbiGEECIFpWMNWu5DEUIIIVKQ1KCFEEL0eelYg5YALYQQos+Ta9BCCCFECpIatBBCCJGCJEALIYQQKUiauIUQQogUlI41aLnNSgghhEhBEqBFn9PQGOLq2ys4/ZKN3P1YLYl4p3nQG6T2/hAVvw/y3qVzCQfkbVhC9C2q3afvkwAt+pwX32xk6fd+vD7DWx83oZe07HSe3zy1Bv+3BuODNW9sZOWr6xNQUiFEb5HXTQqRAsLtaszhRFR2w7F5mnA6djkRIn2l4y9WArRICVW1IR5/qZ5A0HDmDwoYNsgNgDGG51+tY9NTKyglwMG/HMeoPUpQX/kwDQEOsLLZa/es1nxWlAf5yxvNZLkVV5+Qw6DC+N4LPemM0Xzy5Arcm/3k7N6fcT8Y3iPbKYToGelSa44mAVqkhNvvq+abFX4Avlvt58FbBwPwzqfNzPvbt4zbUI4XeOsXVdx89mGsG1QCg6BgiMLhaPthnvnvBjbW2lXqsqoQL15WFNf6P18W4I1x02GcPT55RYCZ0+IL7kKI5JMa9E6yLGswsAyo0VqP6811i9S2qTzYOlxRHSIYNLhcik3lQXK9vtZ5JhDGWeOFQdkArKxr+1n6AqY1OAOsrYq/7Tt6/QCbKoLM7PZWCCGSJSw16J12LzAfGNXL6xUp7geH5/HO/evI8gWYfPIwXC5F9fJ6hldW89nIAXw3oJBsf4BDQw0cX1fFO2EHRS0+jqlp4YVn8lmwSdG/n5Mj9szklRUGlz/E9N2yuXdRmDMnK/Izuv7xHrxPNu/+dz15NS0Ehxex357ZrfN887fg+2w9WQeNIGP6wIRud3BDA80vf497QjHZR43ZqbzC9S00Pf0NjpJsck6bhFKdbPOrc2F9NfxwFvQvtKe9Pg/WVMAp+8Dg4tj06ypgjoYpw+Gw3XaqjEKI+CU0QFuWdSlwBVAKNACPaq2vicw7J7K+J4DrErle0fdNKNtMw7KVABS+UE3F3jN47axPCfnDvH/SAawe2A+AlcvWcMbHy6iw3DTmZPN1RSaLX2vEl5kBwLqibBoGFaKAR8rgkbIwDy6BL89y4ugsYAHODQ3M0t9DGJzrN+CoKIV+hfi+2siWAx+HQBgynQz+8nwy9khMkA7VeNmyzyOENjUCUPLg8eRdMH2H8jKhMOWHPol/QTkABVftR79bD9024Z0vw+8es4f/9j9YdBc8/D788n572h0vw+K/QkGOPV5eB3tdCZUN9vhTV8AZB+5QGYXoSel4DTpht1lZljUBuBU4QWudD0wFXo3MGwT8P+CiRK1PpJf1H21pHa5f3ciatzcR8odpznC1BmeAJcMH0pyZQWNOVA03EpwBSpvs69jR16O+3gJVzV2vf+NnFRBpEQ/5wmz6sgqAlnfX2sEZwBei5b213d62zvgXlrcGZwDv6yt3OK/Q5sbW4NxlXq/PbxteuRm+32TXnrdaVwnLom4x0yvbgnP75YVIIabdJx0k8j7oIPbd4VMty8rTWtdprb+MzLsHuENrXZbA9W2Xx+OR4T4yPMgqbZ2WNySbEqsA5VRk+4MMq65vnTdhczXZPj+ZPn/rNHcg0Dpck+OmvSklUJLddRkGzSxpfbaBcikG7lmMx+Mhc/9hsLUTmlOROWtowrbdPzIbR3HbiUbmQSN2OE/nwFxcE9qaprMOGtFx+gOntO2YIcV4BuTGThtQCBOHtKWfPgpT0FbGlr3bmuGT/Z2R4b49nGjpeB+0SsRTmLayLOsU4GJgH2AxcBN2c/clwIFa67BlWecD1/VSJ7F0OZFKeyZs+O75dTSXtzDhtBHkDc5h89wqNn1Rib8khycqM+k/MIOTTD2PfhZihT8LhWG3Aj9jxmSyqNpFcZGTfY4q4qsaB9ZAKPMo6nzw890V/XO6/sE2bm7mv8e9hWmE3CHZnPzSoWQVZwLgfX8tvo/KyDp8FFmRIJoogeVVNP33W9wTisk9Y+pO5RXa0ojngYU4S7LJu3BPlKuD829j4JH37WvQ5x4CowbY0x/7wL4GfdZBMG5w7DJL18ELX9rXoE+ftVNlFIIeeszXp+qBmOP9AeZnSY3SSqkjgR8DA4wxs5VSFlBgjHk/7jwSGaC3siwrA7s5+zZgDnAcsPVxT5lADlALHK61XpTwArSRAJ1m6hpCnP3rza3jo4a6+eeNO39NeME/lzPv79+2js/643SmnL1znbaEEB3qkcD5SbsAfWASA7RS6lfAZcADwO+NMYVKqanA/caYuM9yE9ZJzLKsicBo4GPAC9RjB8jLgF9HJT0duBQ4EChHiG7IylRkZym8LfZvsbgoMVdpcgZmxY4PyOokpRAiFaVYs/blwOHGmLVKqasi05YDE7uTSSJ7cWcA12N3DgNYCZyqtd4cnciyrFogpLXekMB1i11EVqaD639VylOvNpCf6+DnZxQlJN8Jp45k3nsLCaw2TD95CqOOGpKQfIUQu6R8YGtvy601ezfg7zh5x3qkiTuFpPXGicSaM2cOALNnz05ySYRIaz1S1f1IPRRzvD/YXJDMJu7ngQXGmD8ppWqMMcVKqSuBPYwxZ8abjzzqUwghRJ+XYrWxXwFzlFIXAvlKqe8AD3BCdzKRAC2EEKLPS6Vr0MaYzUqpvYC9gRHYzd1zjTHdeveeBGghhBB9XorVoDH29eOvIp8dIgFaCCFEn5dKNWil1Ho6OWcwxsT9MAUJ0EIIIfq8VArQwNntxgdj33L8THcykQAthBBCJJAx5qP205RSHwJvAn+PNx8J0EIIIfq8bvW+Sg4f9sO84iYBWgghRJ9nHKnTxK2UuqndpBzsR16/0Z18JEALIYTo80zqxGeA4e3Gm4C7gMe7k4kEaCGEEH1eKtWgjTE/SUQ+EqCF2CpscDf67FcyqtT5sQshts8k5r05O0wpdVg86brzukkJ0EIAVDVw8GUvU7CuDu7W8M4NkJ+d7FIJIeJknEk/qX4wjjQGiPs9thKghQC47207OAN8tQKe/BguOjqpRRJC9B3GmG710I5HkhsFhEgNDVk5MeNN2VJ7FqIvCTtUzCcdSA1aCOBfex/BgL1WcdCab3l90p5k7DaLi5NdKCFE3JJ9DTqaUqoAuAE4GCgl6hWb8qhPIbrL5eRnP2wLyQ+6U+jXLoTYrlTqxQ38GxgG3AQ8gf3oz98BL3QnEzkKieSr9oBeCV5f0orgb/cYouZAz74b57saw7fVO7aOTY2G+eWGYDjV3t8jRPIYFftJsqOAU40xrwChyN8fAed0JxMJ0CK5Fq+F8ZfAXlfCPldDQ3NSipHpjB3PdffcL/zPX4aZ9FCIKQ+HuO7TULeWfWtNmLEPhJj5eIjjXghLkBYiwjhUzCfJHEB9ZLhRKVUIbAbGdTcTIRKj2gMfLbP/bk9ZJXy8DP7zJtQ22tOWrIO7XoX6ps6XW1MOr8+DdxZBRR20+O181lXsVNFbgrHjzcGO00VbW2/4eL2hJdhxkPSHDJ9sMKyuMyyrMnxUFubj9WFundtWXb9jrr3sBo+dV5M/Nq8F5Qa9pW3aP+ab1rK+s86wsAK+rzF8uiG2Rr2qzl53ICQBXOwawir2k2SLsK8/A3yC3eT9H+D77mTSrWvQlmVdClyBfdG7AXhUa32NZVkjsB9jdgD2fV5zgN9orT2WZf0UuBnYQ2tdYVnWAGAh8AetdTz3jYm+YE057Pd7KK+DgUXwxS0wemDHad9fAsf/yQ6upQWx8258Fh75AL66zc4n2v80nHwbBCO1zqIcGFoCy9ZDphvm/B6O3GOHip/d7peQ5+46sL2xOsxJr4Txh2DvQfDRj51kudqOCv6Q4bBnQ3y2ERwKOqvo+sPwz/lhrvo4THMQppXC52c6yc9Q/OHTEP/vS3vBy2cq/nqokypvW0YOBV9sCnP5B4awgWNGKV471cHLKww/+l+YYBgOGa5453QHruTXKITYlVxIW8ewy4A/A0XAud3JJO4atGVZE4BbgRO01vnAVOBVy7KygPeBb7Df1DEF++L43wEiQfgd4EnLstzAU8A7EpzTzHOf28EZ7L/Pf9F52gfetYMzQFXDtvPXVcKcr7edfu/bbcEZoK7ZDs4AvgDc/+6OlBwAbxCG1Ndw7LfzGeCpw+PvOqDdt9jgjxRl7hbQW2LnL6iAzzbaw9trhf7bvHBrjX1pFXy03l7g7gVtC/5rgcEYw7qo3RU28PS3pjX/N9caVtbCfxYZgpFK+ofrDUurul6/EOkgxZq41xljVgEYYyqMMT8zxvzIGPNNdzLpThN3EPuMYKplWXla6zqt9ZfACYDSWl+vtfZqrWuBPwBnWZa19crexcAQYC4wKDLe4zwejwz31vD4wcQYP7jT9L4Rxa3Dnf2QmocUbrts+3UAuKK+wpH5O1L+vao3sOzOX/P6Q7ey+K7fMa25ssv04/u1rTbTCcPzY9MUqUay4myfGl3Ytg+cCga47Ovw44va0owrgsbGRka27RYUUJLdtmxBBmSHG2OWy3XDkNwU+p7IsAz3kBTrJLZFKfVvpdQBO5OJMib+a1SWZZ2CHVz3ARZjdyGfjl19b3/hMAsYq7XeGFn2Cuxm8Au01g/vTKG7QS7A9aZ/vQHvLoIjpsMlx3aezh+wm7K/3QDnHWpfe56/GrIy7J7cJ+4N53fwWFuvD65/xm4iz86Ak/aGKcPhwfdgwhC44Ud2U/eOuPG/cMN/28bv/hn88rhOk7cEDX/8LMzKOvj57oqjR297rvveujD/XmgYmGNoCSlW1RncDhhZAMtrYGMjnDxe8ecDFLd8ZVhaDedMUZw83s5rfYPhuk/DhAzcuL+DsUWKu3SI33zY9rW+bAaEjWJTI/xmLwf7DVE0+Q1/+CxMWQP8ck/FISOkq4lIKT0SPp8d/N+Y4/0PN/8oaWFaKbUncAbwYyAEPAM8ZYxZ0q18uhOgt7IsKwO4CLgN+AVwldZ6ahfpJwFfYd8Pdir29egtnaVPIAnQIi5rHviM0Rf+pXV840t/ZOhJ05NYoo59scmw/1Oh1i/2I8c4OG+aBGDRp/RI4HxmSGyA/vGm5AXoaEqpg7GD9anAZmPM7vEuG3cnMcuyJmJfY/4Y8GJ3ITfA88AfLMu6BrgbaMRuzt5ba/2SZVk5wHPA37TWf7QsKxN42rKsI7TW3bvHRIge8ujkfTFHns7R3y3k+d33ZfSI3fhVsgvVgf2GKJ6d7eDllYZ9BysJzkJEpMB1584sB74FyoDx3VmwO7/uDOB67Hu56oBLgVO11s3AYdidw5ZjB+73gD0iy/0LqABujIz/CijBfgyaECnhkC2r+P0HLzOrbAU3vPM8+9ZvTHaROnXaRAdPHO/klzMkOAuxVSpdg1ZKFSmlfqqUeg9YDRyC3eI8oFv57EgTdx+S1hsnEuiGZ+zr4lv946fwq+OTVx4h0lePhM+nhj8bc7w/c/0Pk3kNuhn4HHgaeMEYU7cj+cizuIUA2Duq5cnhgJljk1cWIUS3GZVSTdxjjTGbdzYTCdBCABw3k6+vOYziZeWMvfgUmDUp2SUSQnRDCjw9rFUigjNIgBai1ZZ9R7Jl35GMPXrPZBdFCNFNKVaDTggJ0EIIIfq8ZHcM6wkSoIUQQvR54TSsQct9GkIIIfq8FLvNSimlLlRKva+UWhyZdpBS6ofdyUcCtBBCCJFYNwE/Be4DRkSmbQCu6k4mEqCFEEL0eUapmE+SnQ+cYIx5hrbncawBxnQnE7kGLYQQos9LgaAczYn92GtoC9B5UdPiIjVoIYQQfV4qXYMG3gDuUkplgn1NGrgZmNOdTCRACyGE6POMQ8V8kuwKYBD2uykKsWvOI+nmNWhp4hYCCPuCDLljI9nLvJSf/CYD7j0K5ZTzVyH6ilRp4lZKOYHTgDOBAuzAvN4Y0+1XLMsRSAig4d5FFHziwV0TpOHBJXie/jbZRRJCdEOq1KCNMSHgLmNMizGmwhjz9Y4EZ5AALQQAgXp/zHjIE0hSSYQQaWCOUmr2zmYiTdxCACtGDUUV5lNa72FLcRGe4YPpl+xCCSHilyJN3BFZwPNKqS+A9US9+tgYc268mUiAFgLwu1x8tfsUMnx+fFkZHJZaP3YhxHakQMewaEsjn50iAVoIILPFjzsYxDgdZASCZAZ2vInbBILgcqJ2IMj7g4YMV0odaIToE1KlkxiAMebGROTT4wHasqwBwJ3AwUAJsAV4ELhVa226WlaI3uIw4XbjO/bVDN/6Oua6l6AwG8fLv0QdOCG+5cKGc55s4ql5fib0d/D2xfmMLHbuUBmE2BUZlTpdqpRSh3U2zxjzfrz59MYW5QHfAIcA+cBJwC+w7xMTIn5eHzT7WkdDgTD+piDhkMEX6dQVDoTxN3Zc+w3U+jCdBN6CWi/ZjS2oUJjcBi+5td5t0oSCYXxNQcJhQ+PmJkwwBHVNmECQYHUT3rI6zDUvYkJhTE0T4d8+G9dmGWN4eVmAp+bZHdW+rwxz67st29kVhqrGMIGQvT2+oGFDQ5hQ2B4PhgxN3nBXWeyQZm+YYFDOq0XqSZVe3BEPtvu8CrwJPNCdTBJag7Ys61LswFsKNACPaq2vAW6NSrbUsqxnsAP2XYlcv0hjj34AF/4HjIF7fsHG3fbgrUu/xt8YxFmYQYsPhuxRRMOiKgKeALudP5b9rt4NgJA3yIJj3qXu43JyJxcy44OjyRyYHZN9BkEGb6hrHXc5Y4PbxkV1vPK7RdS1KNYP60+9O4u9yr7h0g+fYEPxSP554I8IKcVtxrD10NDZyUC0uhbD4c+FmF8Wu75F5Z0H1yffaeLulxoJAVsG5HLRUVlc/06AsD9MUbbi8eOc/PuJWjzNhtMPy+HSHxZutxzxuP/JWl5/r5G8XAd/uLyUCWMzE5KvEImQYk3co6PHI/dGXwd4upNPwmrQlmVNwA7EJ2it84Gp2GcN7dM5sIPzokStW+wCfv0wBIIQDMHlDzH378vxNwYBCNX7wRg2LazD1xwCYMkjq2jY0ARA+bNrqfu4HICmb+vZ8O/vtsl+VPMKhvjXkxluYYRvDcOb18TM/+L+1fg8QWoLcqh3ZwHw9YgpfDNoPG9MmoUnK5cRteVEHyLUyJLtbtbDSw3zy4FMF+Rn2D1RXQ6+qHBS6902wLf4Df982X6crxMorvZy0ychwn47oNd5DVf9rwVPs73sc+83s6EiuN1ybM/GzQFef89eb2NTmKdfbtjpPIXYVUTujf4TcGV3lktkE3cQUMBUy7LytNZ1WusvO0h3F9AP+7p0j/J4PDKcLsP5UTXe/GzcOW3XZ2PCWGREuRSuTCcejwdXvjs6Ba5817b5O13s3fQVx9b/jxnN8zDujJg0KsPOWIVjg2ZWwEdW0G6a9rkyYuYxomS725UfvUhhFhRnQ24mWRmKDOe26V1OyHS3nQaElcLliC1TTlS7mNMBWRlqp/d/ZqYiutUwO2vn85ThXXs44VS7T+o5EujWdScVTzNcvCzLOgW4GNgHWAzcpLV+O2r+XcDxwOFa6w0JW3Hn5GJZuvjiO/i/+yAchn9eSP3IkXxw3UKaq1pwFWXR0hxizMEDqJ1fSXNFC3v+30QmnGi/htUYw4rfaCpfKaNw1gCmPDALR2ZsB6zNN36F64bHyFN1NJgSHP/8Gf0v2b11fsNmL2/d/A21VX7K83KpbjQcXP4tJ679hHryeWzykVQNGMh56ltGfTYfNW0Ijkd+iirK6XKzgmHD/70b5r11BmsglJWHqGsx3HKEm5Mmd9xJ7PNlPm5+ykNlC2SPy+Hao7O48FU/5bUhZgxx8NhsN/98pp7KuhDnHpvHsft1XYZ4vftJIy++7qGkn5PLflZMabHcBCJ2SI+Ez3/s9XbM8f7Sr49KWphWSsXc+wzkYN8bfYkx5tG480lkgN7KsqwM4CLgNuye2y3AvcB+wBFa6x167NkOkAAt4rL2L4t4/cnNtGRlkNPs48QrRjP4vEnJLpYQ6ahHAuff93kn5nh/2VdHJjNAH9xuUhPwvTGmW9eGEnYKbFnWRGA08DHgxX6Lh8FuRn8SmAQcorWuStQ6hUiUzUX5eLOqCTscNOVkUp6Xx+BkF0oIEbdU6iQG7GWM2eYyrlLq18aYuDtHJ7KNKgO4HrtzGMBK4FRgJvBjwAestSxra/pPtNbHJnD9QuwwlenCk5+LcThwhMI4suQeZCH6khQL0NfTcT+r6+jG3UsJC9Ba6yXArE5mp9SeE6I9L06Mw+4zGXY68IZT56EHQojtS4UAHfWAEqdS6lBiY98YunmblfTyEAIoGhh7T29Rf7nHVwjRbQ9G/mYBD0VNN9hP0fxVdzKTAC0EsMcRJcz9ZDGezRnMOnYcE/ctSnaRhBDdkAo16K0PKFFKPdadt1Z1RgK0EIBSiiF7NQFNzJo9KNnFEUJ0UyoE6K0SEZxBArQQQog0kEoBWilVANyA/ZKoUqKuRRtjRsSbj/SEEUII0eel2Msy/g3MAG4CirGvPZcBf+1OJlKDFkII0eelUg0aOAqYbIypVkqFjDGvKKU0MIduBGmpQQshhBCJ5cB+WBdAo1KqENgMjOtOJlKDFkII0eelWA16Efb15/eAT7CbvBuB77uTidSghRBC9HlGqZhPkl0IrI0MX4b9+OsioFu9u6UGLYQQos9LgaDcyhizOmq4AvjZjuQjNWghoihft17XKoRIEalUg1a2C5VS7yulFkemHaSU+mF38pEALQQQrvUy4dcrmH76UioOfpxwcyDZRRJCdEMqBWjs26t+CtwHbL3veQNwVXcykQAtBNB47wJyVnoB8H1cRvMTS5JcIiFEdxgV+0my84ETjDHPYD+HG2AN9gsz4iYBWghgXSC2O8amkDtJJRFCpAEndq9taAvQeVHT4iIBWgjgpVm7s2zAQLxOF18PHcZb1uRkF0kI0Q0p1sT9OnCXUioT7GvSwM3YDyqJmwRoIYD9v15DbgWUh0rovzGAtXh9soskhOiGFAvQvwYGYz+spBC75jySbl6DltushABGVzewMWp8WE1D0soihOi+cPKDMkqpQcaYLcaYBuBkpdQA7MC83hizpbv5SYAWKau5qoXlz6wlI9/F5DPG4Mzo2Qafyv4F1BbnUVrZwKgeXZMQItEMyQ/Q2E8KK4gav8cYc8qOZiYBWqSkcMjw2jmfUrfKA0D18noOvmVmj62vql8eeu9xoBSrxw5iTEEOw3tsbUKIREuBZm1gm7OEQ3Yms7gCtGVZlwEXA0OBWuBJ4DqtdciyrAnA/cCe2N3IHwL+prVWkWVdwJXY3c4HAMuAy7TWemcKLtKbr95Pw4p6svwhHGGoemI1gat2w12U0Zom6Avx1v/7hlUfVRIKGjIKMzjikH5U/mc5m0aXsGZEKYUDM/nRb0aSta6WzT97h+CWJh6bNoX3xo9m8qQc/vqLfhTmOKgfWgiqErBfW9cwqLBb5Q1trMdz5rOE19bhPmocgQ/XoHLc5D98Cq4ZQ2PSBkOGn9/wHR96cnji9fvZz7MJsvMhYOBkC/5+Nir6YLOuEs76J6yvhqt/ABcfRV2L4acveDnnzqfYZ+N6is/Zg8y/nLzjO1yIPi5FArTZfpL4xdtmuAE4FrvqfiJwAfCzSPCdg/1g8IHAydjPII12Y2SZY4AS7AD+pmVZ/Xa69CJtZfXLICMUBqUIOxUtxrDizqUxaRa9sIHv360gFDBgIFzhpez386gr97Mwt5D6miBl3zbx2gMb2XT+W/iWVLHInc8zUydTmZHFx6vD/OftZgAGr6skx9vC4Koasr0tDNxQ2a3yNl/5FsGP1xIuq8P3gCa8sprQ4i14Lnhxm7SPPrueh9VIjl8+n1krlsGWRlhTAxtq4e534JX5sQv85nH47Dsoq4JLHoZ1ldzyVZiJj37AkQuXUFBZR/CuDwm+vqx7O1kIkWgupdShSqnDlFKHtR+PTItbXAFaa/2C1nqN1tporRcAjwOHA/sCo4CrtNZerfVqot51aVmWAi4Ffqe1Xq21DmmtH8R+7dbx3SnojvB4PDLcV4cbPDicbWfERkGoORiTprHeSzQVNigDYYeCqLNpf0sY0xwEwOdyxizT7DN4PB7619fzo/c/5bgv5/Gj9z+lX2NTt8psmv10xDQFtknf3BICICewdZnYM/+WqrrY/KPzNga8fuqaA2QH2j3trMmfGv87GZbhOIYTLUV6cVdgV0IfjHyq240/0J3MlDHbr5FblnUGdrfxMdjN4hnAl8A9wJ1a6+FRaY8A3tFaK8uy+kcK3EBs1d8N3Ky1vrU7hd0BCW1uEL3r+ydX8dW18yFs6Jfr5rA3jyRnZF7rfG+dn+d+OZ/qVU32BKfiwNEZ1D2xitUzhrF2SDE5BS7O++MYCpdtZtNZbxD0hbj9qAP4fNxIhucbHv9NCcNKnFRf/RF1t33VmnfJXw6l6Nd7xV3W4KLN1B/zCGZLI679RxD8cgNkOsl/5kdkzo69p9rTHOSo68tYrgr44pEbmFi1GXKKoDkAR0yF//0alRn1oJR5q+HYW6GyAS47Fv52HuvqDWfcX8Pdd9zLpIpyHMdMJuvVC1Hu2BMQIVJQj0TPa2YviDne/3nOninR5r0zthugLcsajv3arFOAN7TWfsuy7gQs4DrgXaCf1tobSX8B8GAkQCvAAxyqtf665zajUxKg+7iQP0zIG8Rd4I69Lhsl4A0R8IXILnCjHIqQN4gj00kwYHC6FQ6HvZwJhjGhMIShRTnIzlSteVZe8R4Nf5vXmmfxbQfR78p9u1VWYwy0BFHZbowvCE6FcnUeMJvqWsgtzIRmH+RmYZp8qNzMjhOHw+ALQnZGbB5+Q07A3/lyQqSeHgmcV/9gYczx/tZX9+jzATqeJu68SLpKIGBZ1r7AOZF5XwJlwC2WZWVZljUauHzrglprA/wduNOyrPEAlmXlWZZ1tGVZQxK3GSJdOTMcZBRmdBqcAdzZTnKKMlCRQOzMdqEcCnemozU4AyiXA0emC0e2i5wsR2yeOe0e7bkDNVGlFCrbzkdluroMzgC5RVl2U3xulr1MV0HW4dgmOAPkZigJzkKQMk3cCbXdAK21/hb4I/AKUAdcDTwdmRcEfgDMwA7gL2Nfn46+ILd12Vcsy2oAVgAXxbNuIXrLlowcvh9Qwkt7TGJVaT/KM7KTXSQhRDekY4CO6xp0d1iW9QvgN1rrCQnNeMdIE7eIyzP3ruPc8gEEXE6y/QFemlTL0T8eluxiCZGOeiR6/vbkJTHH+ztf2q3PR+mdflCJZVkHYPfKXg3shn3P8xM7m68Qvemb8YMJVNvD3gw3340exNHJLZIQYheXiGbm4cAHQBP2PdEvAbckIF8hes3ew9vOVR0KZg6RKzBC9CUp9j7ohNjpGrTW+mki16SF6KtOGO/g6hELWdbUj4uPGMv+wyVAC9GXpMizuBNKnsUtRMSswnJmFZZz7NjxyS6KEKKbUuFtVokmAVoIIUSfly49t6NJgBZCCNHnSYAWQgghUlA4/eKzPCxECCGESEVSgxZCCNHnSRO3EEIIkYLCcpuVEEIIkXqkBi1EmgqEDH9fM4VlniJOc/j5+7HumDdhCSFSm3QSEyJN3atDfLplIKrWwX2fB/jv0lCyiySE6IawUjGfdCA1aCGAmi0+zl25gZxQiAa3i6qqkcjPQ4i+Ix2buKUGLQQwrbGJnJBday4IBJnc2JTkEgkhdnVSRRACcDX7Y8cDwSSVRAixI9LxGrQEaCGAQgIc+/E8+tc0sKV/ETk/PjTZRRJCdEM6vs1KmrhFnxJoCLDp5TLq5lcnNN/hq8sZXFWHKxxmWHkNgzdUdTuPUNjw1SIv85a1JLRsQojtk05iO8GyrF8BvwKGAjXAH7XWD/XW+kXfF2wO8unRb+H5ph4U7HnPfgz/8ZiE5J3ra6Exajzb5+t2Hrc/UMtHX3sBOOnwXC76cVFCyiaE2L50CcrReiVAW5Z1HXAOcCYwH+gHlPbGukXfEw6FWXbXN9QuqWH47OGMPn00AA3L6uzgDGBgw3/XUmqVsOZajfebOtz9sxhw9jgGXTCxNa/1G/w891IdLhcUmyDe+iAHHFPM5D3zYtaZkxVgS78cShpaqC7MYlxmgKaKFr7667dU+uDVfSbTnOnm6r0UWjezYmOAic0NDN1YzZgDBzD5xKGtwRng7c+a+enEZppv/hBVmEnerUfjGNi2TuMP0nT9e4SWVZB13p5knjatw31h7n0f5iyA/cfD1bNR3T0IPfQevPgl7DMerj0Nmn1w9RNQVgmXnQCH7969/IRIUXINejssy7oUuAI7+DYAjwK3A9cAp2itdSRpdeQjxDZWPLySZX9dBsCm9zaTPyaf0pmlOHOdMel8VS0s+8E7eL+ta51W/8FmsscXUnjgIIwx/PmOcmrr7N7Z7lCIEm8L3y9u5Np/jqeoxN263POTJ/PT3x9CntePJzuDl8dWoa6ez6a51fz7sBksWe8CDIsXNtOvohmAz0wmxy9rZO2nVaxvAEcYwpGHm3i9IWqOfBxHtZ02vKWRojfOa11f860f473tEwD8b6yg36L+uKYOjNk+8+5SuOhhe+S1hTCwEC44OP4d+fEy+Om/IsvPg9ICWLQW7n3bnvbuYlj9HxjUL/48hUhR6fioz4Rdg7YsawJwK3CC1jofmAq8CuwLZAPTLctaY1nWZsuynrYsa2AX2SWEx+OR4T443LQuqrHZQNX3kXO5oCGao9BJy2oP7dUtqwAgEDCtwRkgFKl9BvyGzRvqY9a7qbSEv937Fk/85WVuf/hdNhQXU19ml6MqP6c1bdDblp9RisbMDDvN2mbygkEyQyEyQyFy/IHW4AwQXl0bs42hyLg9Eqbx283b7pPVFTHb5f92w7Zpuhj2flMWu2NWl9uf1gR+mlZt7FaeMizDiRoW25fITmJBQAFTLcvK01rXaa2/pK0p+2hgH2AydsB+IoHr7lB+fr4M98Hh0T8ajbvArt0WjMtnzHF2E3f+lCJKDxkEgHI7GH/xFIZeEds0nDWugCGnTwAgI8PBEYe25Z8Xsm+dGjc1hwlTS2LWe/i8VYyqtIP2xI01HLB4HdPPGwfAYd+sRRn75MDaLYucTDvQF/taGNzQSMGQbA4+aygFuQ6yQ2EyQ2FKRuTgPHsPewVKkX35fjHbmH2hBbl2cHftOZjCY6dsu09Omgmj+9vDpflk/Oywbu3P7NMPgHGD7ZF+eXDeIXDJseCONJwdsTu5+0zuVp4yLMOJGk40o1TMJx0oY8z2U8XJsqxTgIuxA/Fi4CbsYPwycKTW+t1Iuj2wr0Xna6178okQids40ataqlpoXNtI0ZQiXDltV2LCwTD1C2vIHJBFzgj7mm7T4moCtX4cGQ5ydyvGmeeOyWvNWh8ZGYocN9TXBhk+JhunK/YH/O6fv6H5tgWt46V37Musn4+ldrWHYEuIusGF1LWANQiqG8Jsqg4xIt/g3eylZGweGbkuGhtDLF/pw+FWTBmXSVamg4DeiCrIxDVh2y4Xoc0ewmtrce05GJXl3mY+gPF4YekGmDAIVbIDB7dGLywpg/GD7SZugHUVsKUOZo4Fl7PLxYXoAT0SPU//ybqY4/1zD4/s81E6odegtdYvAi9alpUBXAS8AkyKzJZgKeKWVZpFVmnWNtMdLgf9rNhgl7t7yTbpoo0eldk63K9/RodpPp82nJrxFUzeUsuCYf2ZNn4ws4B+Y+yg2D8qbWmhk9JCO7AV9W/LOy/PibVHDtHc1tBOy+UcnI9zcNdBV+Vnw37ju0zTpbxs2G9i7LSRA+yPEGlEenF3wbKsicBo4GPAC9RjB+Vy4HXg95ZlLQACwB+At3q49ixE3HZfuYUf7z8NoxSuUIj/ra0ARiW7WEKIOKVjJ7FE1qAzgOuxO4cBrARO1Vq3WJZ1DvBPYC128H4bu4YtREqYXlHD319cxOIh/dmrrJzxoyZufyEhRMoIpV98TlyA1lovAWZ1Mq8G+x5oIVLS5sH9mFy9lJkbKvHkZlE+sIhhyS6UECJu0sQtRJoKhMLMnTmODH8Af6abQ0LhZBdJCLGLk2dxCwHkqxDZnmbCQE59E7mEtruMECJ1hFXsJx1IgBYCGH7YYHKbmymqrKcgHGDIAdLLWYi+JIyK+aQDaeIWAug/vZii37oIrjMcfeHhFIzK2/5CQoiUEZJr0EKkL9dgB67BSHAWog9Kl2btaBKghRBC9HmhNGnWjiYBWgghRJ+XjvdBSycxIYQQIgVJDVoIIUSfJw8qEUIIIVKQ9OIWIo05X2nBuSzAxi3fM/TCCckujhCiG4LJLkAPkAAtBLDlqdVkPNgMwLdffkHWiFxKju78VZFCiNSSjjVo6SQmBOBZXBsz3ri0tpOUQohUFFSxn3QgNWghAAIh7NeXK/tvUF6WIURfEpT7oIVITyEFDgx2kE7PHqFCiL5FArQQQLA4A1+WE7c/jD/TQaAoI9lFEkJ0QyANz6nlGrQQQP/6TTQWuKntn0VzvovSxvIu04c2eQiuq+twXp3X8F1FiFDY9EBJhRAdCSgV80kHvVKDtizr/wHHA1OBj7XWR/TGeoWIV2m4jgPrPqfKXcpAfzlFoeM6Tdt0j6b+kjcgbMi7/iAKbjykdd5XZUGOfqCZ+hY4cryT1y/IweVMj4OFEKkskOwC9IDeauJeBVwPHA1M6qV1ih21ZB088ylMGgrnHNJ7612wGp7/AqaNgDMO7LHVhJdvIfzEV4SCDkLKjWv/kfgowul3kOMPoHDRbArZ+k6rcHkjzf+Zi6Mgk+oxw1h2wzICpcPJCfsYcttcyqaM4rnCgWSHQrw130t9SyYA76wIcdHNmxla6GK34S6OPrqQuuoA87/0UJnhZlW/PIIoMt1w9mQHk0okkAuxo5rTpNYcLaEB2rKsS4ErgFKgAXhUa32N1vrhyPyZiVyf6AGbauDAa6HevieY2ia49PieX++6CjjwOmhqsccbW+DCIxO+GlPVSOCAOwlVe2khDyI9P/0/mMl37LE1FVNMNnmACYepPexhQt9U0uDKYkHJGHwZOTTnuAFY2a8/N7/tYmORD4BBLSHItXNRxuBZ0cwah4M180F/3UTLFi++Frvp+6PhQb4dVATAPYtCLP+Jk9Kc9DvICNEbvGn400nYNWjLsiYAtwInaK3zsZuzX01U/jvC4/HIcHeHl5W1BWcg8NHS3lnvorVtwRkIfLSkR9ZlviuH6ibCuCDqtoxQVRNtFC1ZOXb6uhZC31QCUJthR96gq+1nU5edxcaitvdHh8OGmZU1jPQ0ccLGCnKiLkNv2hSgpaVtwqAmX+twtRe+r0389sqwDKfqcKL5UTGfdKCMSUxHFsuyxgDLgPOA17XWjR2kuQE4oBevQUsvne6q9sD0K2BjjT3++GVw9sE9v97yOtj9CqioB6Xg2d/AabMSvhrT4MU//U+E19bi3VqDdjnYcP4RVD+4FoeBkENR8u8Dmf6LcQDUHvAAgc/KaHJmoEvH4Xc7aMq1e3mrYJi79t2D5QOLATh0cwVHbyqnPD+XnDDU5GTTmGU3eU+YkIlnvZeG+hAGeGfUAFb3zwdgVAEsOs9JQWZ6HFiE6EKPfMnV5TUxx3vzt+I+/2NKWBO31nq1ZVlnARcDD1iWtRi4SWv9dqLWIXpBST7MvR3+p2HiUDh4au+sd2AR6Dvg9XkwdQQcMLlHVqMKssn48krCryzClZFByAeuvYYRfLeaYY6vCIdcOJwhWlxtDyopevtcWp5ZSkFhJoePH8T3962AokyyB2SR7wtw/IEFvKGyyVeGCRX5eLNLWFhQSP6KWlpawgSy3UwZ4mK//fJobAixbEETTblu9sjKwu1UhIATxykJzkLsjDT8+SSsBh3NsqwM4CLgNqBEa90cmX4DUoMWKaj+mndZ/MgqthT1Y1h1FdOu3pP8K/ZLdrGESEc9U4P+dW1sDfqufn0+ZCesBm1Z1kRgNPAx4AXqsQNk2LIsN+CMrM9hWVYWYLTWvs7yE6I3rcsr5tNJ2QCsHDSU0txC8pNcJiHEri2RDyrJwL6VajNQB1wKnKq1bgHuxw7a1wKHRoa/S+C6hdgptf2LY8eLCpNUEiHEDlEq9pMGEnkNegnQYa8erfX5wPmJWpcQiZZV5UGFwhinA0cwRFbNNn0chRCpLD1icgx5FrcQQFFJBsPWlOPPdJPZ4ie/ZHiyiySE6Jb0i9DyLG4hgLHnjCVzKuSYFsacOJyRJ49MdpGEELs4qUELAbiyXWT9n32/8n6z901yaYQQ3ZZ+FWgJ0EIIIdKABGghhBAiFaVfhJYALYQQou9Lv/gsAVoIIUQ6SL8ILQFaCCFE35d+8VlusxJCCCFSkdSghRBC9H1SgxYiPYVaQvD3ZrjYw8KLvsCEwttfSAiRQlS7T98nAVoIYN1DK8j9pIl+W5qoemQ5G55dm+wiCSG6I/3iswRoIQBaPt9Mgd9HVihIka+Flq8rkl0kIUR3pOHbrCRACwHUFGXHjNcXZSWpJEIIYZMALQTwxsxx1OVkEgbKC3N4b/fRyS6SEKI70rCJW3pxCwFsKcrnf4fsxrD6JlYXF5Cfl739hYQQogdJgBYC2Gv1ZhrrmwAYU9PAkLJKYGhyCyWE6IY0qTZH6ZUAbVnWW8DuQC5QDzwH/F5r7euN9QuxPaO31LEkanx4eT2dBWhjDM981sKq8hA/sDLZfaS7V8oohOhC+sXnXqtBXwV8q7X2WZY1EHgW+CNwTS+tXyRDKAT/mweZLjhmBqyrgC++h/4FUNkAI0qhrAZmjobxgzvPpqoZ33trcE0sIWOPQT1SVH+Gm8asLLL9fpoyM/jO58axuIX9pmXicCj8AcPH87wsanZS6Qnz+lsNOI3huc+aeeRn+fjqQ0yanIUjy8mL83y46vxMnJTNqoCTycXwXXmYEYWKfUc4t1n312VB1tSEOXqim8LsNDzKCNEb0vCnk9AAbVnWpcAVQCnQADyqtb5Ga72wXdIwMDGR6xYp6Ed/gRe+tIfPPQRemQv1zVEJMuw/OZnwyQ0wY9uOWeFaL5V7PUBobR04FcUv/pDsHyT+q1MzoJDykrafw6K1GTz671qO3CuL635SxHV3VXKfP5/6bCfgZLdsF8Prmgn6DX+/vZFwCPILHMwf1o+Ry2vwup28MT6LgDOMA0O4IQAhwwMnZ/BTq63G/YT2ce7TzRgDkwc6+PryAnIz0/BII0SPS7/fTcJ6cVuWNQG4FThBa50PTAVejZr/b8uymoAtwHTgL4lat0hBgWBbcAZ46at2wTnqq9fsg1d1h9n4v9hgB2eAkMH73DcJLypA2NH24w4BjW67pvv+vBaqa4PMXReiPrstsG4qsG/Dyg+GCYfsaZ6GMM6yJjLCho352QSc9jaGUeC2h59ZHIxZ7zMLAxhjD39bHmbx5lBPbJ4Q6S8Ne3En8jarIPZumWpZVp7Wuk5r3XqE1lr/H5AH7AbcA2xI4Lo75PF4ZDhZw24XoSnDWqcHJw5u9/AAE/lEzBjdYT6uyaWQ3VazDU8p7pkyB8P4HA58Tgf1GW78DvunMX64m6ICF0NywmQE24JnYUsAgIyoJmmHAxoKMjBAsddPa+QFCNrD00rb8vB4PMwY2tbkXZilGFviSP7/ToZluBeGxfYpE30Q2UmWZZ0CXAzsAywGbtJav91Buh8CV2it90vYyjuWuI0T3be5Bu58BbIy4MqT4KNl8NZCcDvBH4R++VDrhQMnwRn7d5qN77MyvE8vwzWllNyLLVQPPCXo3r9v4jPd0jo+cEwWo3bP44wjcykucLK5Ish9bzTxeSCD/SdlMLamkabGMCccnsfK71pYs9rHzJk5BEuzuGeOh+zqFgbulkdZZhajC6CsIsiIIsUVs9y4nFG19bDh7x/7WF0d5oJ9MpgxTG6sEGmvR+q36o/emOO9ubHvd+hIaIDeyrKsDOAi4DagRGvd3G7+mcA/tNalCV95LAnQIi7/uXsLX8xt+5oec2Q+Z57bP4klEiJt9UyAvqFdgL6h7wfohJ2uW5Y1ERgNfAx4sW+nMsAIy7ImAe8CzdjXn68H3kjUuoXYWQ5lyPG2UOjxUluQg9ORn+wiCSG6I02evx0tke1pGdiBd2pkfCVwKvbZ0pXAI4ATKAdeBG5M4LqF2CnZLT7O+t/nZPsCNOZkUn3sMckukhBiF5ewAK21XgLM6mR2Z9OFSAm7r1hP0Gd3/Mpr9tF/zWZgSHILJYSIX/pVoOVlGUIADK3fEjM+pGFLJymFEKkp/e6zkgAtBFCyTyFD2EABdQxjHYUz+yW7SEKI7ki/+CwBWggAddEhhPd1MCx/Pf3PGY86c59kF0kIsYuTmy6FAFR2Bguu2RuA2bNnJ7k0QgghAVoIIUQ6SJNm7WjSxC2EEEKkIKlBCyGE6PvkQSVCCCFECkq/+CxN3EIIIUQqkgAthBBCpCBp4hZCCNH3pWETtwRoIYQQaSD9IrQEaCGEEH1f+sVnuQYthBBCpCKpQQshhOj7pAYthBBCiN4gAVoIIYRIQRKghRBC9H1xvA9aKbVWKTWtF0u1UyRACyGEEClIArQQQoi+T6nYT9yLqXOVUkuUUouVUi8ppQZEpn+hlNorMvxvpdSyyLBLKVWllMrtke2IIgFaCCFE3xdHE/c2i9jN3bcCRxljdgeWAndHZr8HHB4ZPgDwKqUGA3sB3xpjmhJW9k6k9W1WSqm3gNIdXd7lcpUGg8GqBBapz9hVt31X3W6QbZdt7zVvGmOOSXSm5reuHbnR6lDgdWPM5sj4vcCiyPB7wLVKqSeBauAj7IA9Gnh/J4sbl7QO0Dv7JbAsS2utrUSVpy/ZVbd9V91ukG2XbRftfA7MAI7HDtYfARdgB+jre6MA0sQthBBiV/UBcJxSalBk/ELgHQBjjA+YD1wNvAt8CewP7B4Z7nFpXYMWQggh2nlXKRWMGv898I5SygCrgV9EzXsP+5rz18aYkFJqJbDGGOPvjYJKgO7afckuQBLtqtu+q243yLbvqnaZbTfGjOpk1qOdpL8FuCVq/LgeKFanlDGmN9cnhBBCiDjINWghhBAiBUkTdxcsyzoE+xrEZVrrf0amDQQeB0YBXuDnWuuvklXGRLIs61/YtxH4gEbs7daReWm73VtZljUBu6mrBPu2inO11iuSW6rEsyyrBPt/ORbwAyuAX2itKy3L2hf7VpNsYC1wtta6Illl7UmWZf0RuAHYTWu9NN233bKsLOCvwBFAC/CF1vrnu8r3vi+SGnQnLMvKB24D3mg36xbgY631BOAS4AnLstLlRWdvYB+spmNv53+j5qXzdm91D/CvyDb+C/tgnY4McLvWeqLWejdgFXCrZVkO4Angksg++Bj7IQ5px7KsGcC+wLrI+K6w7bdjB+YJkf/7HyLTd5XvfZ8jAbpzdwF3AO1v4P8h9hcarfWn2LXNtLiHUGv9P611IDL6BTAscuCCNN5uAMuyBmDf8/h0ZNLTwAzLsvonr1Q9Q2tdo7X+MGrSl8BIYCbQEvn/gv3//mEvF6/HWZaViR2ILo6anNbbbllWHnAu8AettQHQWpfvSt/7vkgCdAcsyzoWKNRaP99uegmgtNbRQbsMGN6b5eslvwRe01qHd5HtHg5s1FqHACJ/N5Fe27iNyAnYxcCrwAgiNUqAyP/bYVlWcZKK11NuAp7QWq+Nmpbu2z4Wu/n6j5ZlacuyPrQs6wB20e99X7FLXoO2LGs+9g+yIxOxm7aO7L0S9Y7tbPfArT9Sy7J+DJwJHNRbZRNJczd2f4N/AicnuSw9zrKs/bBbfq5Odll6mRMYAyzQWv/Osqx9gDnA6cktlujKLhmgtdYzOpsXOascDMy1LAvsZ3nPtiyrWGt9k2VZWJZVGlWbHAGs7/FCJ0BX272VZVknA38CDtdal0eWq+7L2x2n9cBQy7KcWuuQZVlOYAjptY0xLMu6ExgPzI60lJRhN3VvnV8KhLXWNckqYw84GJgMrIn8vocBbwH/IL23vQwIEmnK1lp/ZVlWFXaHz13qe9+XSBN3O1rrT7XWA7TWo7TWo4DngT9qrW+KJHkOuAhag3k2MC8phU0wy7JOwL72fnS75j9I4+0GiPTWXQicEZl0BnZtozJphepBlmX9Gfu660laa19k8jwgO/L/Bfv//VwyytdTtNa3aq2HRP2+NwBHY/c3Sdttj5xYf0CkZTDSc3sA8D270Pe+r9kla9A76WrsHsznYZ99nqO1Die5TInyMPZtN89Hahdg16SrSe/t3uoi4FHLsq4HarE71aQdy7KmYj/e8Hvg88j/eo3W+mTLss4B7o3ckrMWODtpBe1FkRaEdN/2i4CHLMv6CxDA/g3XWZa1S3zv+yJ5kpgQQgiRgqSJWwghhEhBEqCFEEKIFCQBWgghhEhBEqCFEEKIFCQBWgghhEhBEqBFSlJKjVJKGaXUsB5ez0VKqcejxt9QSl3Zk+sUHVNKrVRKnR9n2l75fvQGpVRmZNsnJbssIrVIgO7jlFJjlFLPKaW2KKUalVLrlVIvKaUyIvPPV0qt7GC5zqafFTnw/bGDeR8qpXyR9dQrpRYopU7tmS3reUqpXOznMt+wdZox5lhjzO1JK9R2RP43B2w/pdhZPbGvlVKHKKWC0dOMMT7sB6Xckch1ib5PAnTf9zqwGfsZ4vnAftiPLtzRV0H+AqgBfqqUcnYw/2ZjTB72u2OfBv6rlJqwg+tKtrOBJcaYVckuiNjlPQ0cppQal+yCiNQhAboPU0qVYAfme4wx9ca2wRhzT+SsvLv5TQYOBM7Dfh75sZ2lNcYEgX9jP4R/tw7yukQptbDdtNFKqZBSalRk/OFIjd+jlPpGKXVmF2W7QSn1brtpHyqlrosan6aUekspVamUKlNK3aKUcnexyScB73SWZ1Qz6nmR8jUppV5XSvVTSt2qlKqItFxcErX8+ZHmyquUUpsjaf4SXY7tbbdSanel1JuR7ajZut1KqUWRJG9HWjEe6GRf5Sil/h5ZR5VS6mWl1Iio+R9GyvRCpAyrlFIndraTorbpCqXUhsgydyqlSiJ5NCillkfXNpVSLqXU9Uqp1UqpWqXUe0qpaVHz3Uqpu6L24VUdrPdApdSnkX2wSin1G6VU3CeeSqlTlVKLIq09i5RSJ0fN26YFSSn1yNZ92tm+VkqtjWzXp5HpWim1V0d5RE1bq5Q6Wyk1BPud687Iso1KqfMAjDENwNfAD+LdPpH+JED3YcaYamAZ8IBS6lyl1JTuHMA68HNgsTHmf9g18190llDZTeiXYD8ycFEHSZ4CJiml9oiadj7woTFmbWT8U2APoAi7qfkRpdSUHSm4UmoA8BHwIjAUuyXhSOxHWnZmBvBNHNmfChyA/YKQUcBXwCrslwr8BPhbdADEfunCCOy3B+0HzAZ+FzW/0+1WSg2ObMdHkXUNwn67GsaY6ZHljzLG5BljftZJef8K7Bv5jMR+p/kcFdsich7wF6AQ+01WjyqlcrrYByMj5R0T2Re/wg42dwD9sPf7w1Hpf4f9yMjjItvwCfCOUqogMv9q4ARgFjA6sq2tL6uI7I/XI/n3B47HfgXqOV2UsZVSahbwZGQ9JcA1wNNKqX3iWX47+/oi4DKgGPtZ/a9HbVdXeW7CPukNRfLMM8Y8GpVkCfZ3UghAAnQ6OAT4ELgc+6H35UqpP7QL1KOVUnXRH+zabyulVBb2AXXrQfZB4Fi1bSecayPLbwBOBE41xmxzLdsYUwu8gh3AiJTnPOChqDQPGmOqjTEhY8wzwOLI9uyIc4FFxph7jTF+Y8xG4Ba6fq5wP6AhjrxvNsbURE6I/gcEjDH3G2OCxpg3sJ9fvGdU+jDwO2OMN9J8fjv2yQmw3e0+B1hpjLnFGNMU2ZaYloOuKKUc2Pv5OmPMRmNME/Z3YzKwd1TS/xpjPjfGhIH7sAP1+C6y9gI3RsqzCPuk7GtjzJfGmBDwBDBOKVUYSf8T4DZjzPJIa85NQAg70IL9f7nNGLPSGOMFfgtEP3f4/4DnjDGvRPbTcuwTiXifE30+8IIx5o3I/+k14CXggjiX78qDxph5xhg/cBv2vjkhAfk2YAd9IQAJ0H2eMabKGHONMWYGdg3nSuB6IoExYo0xpij6g30AjHY6kId9oAW79lIJtK+l/SmSxwBjzCxjzJwuivcwcGakefewSPleBDuQKKVuUkp9F2mCrAOmY9eWdsRoYP92JyEPYdfeOlMLbLfmg32Nf6vmduNbp+VHjVcYY5qjxtdiv9Ywnu0ehf0Six3VH8gE1mydYIxpBCqA4VHpNkfNb4oMRm9DexWRYL5V+/2wdXu35jG8XRnC2PthaxmGRcajy1ARld9o4Ix2/88/Yl96iUfM+iNWEbsPdtTarQPGfplBGZH/704qwO7/IQQgATqtGGOajTGPYNfI9ujm4j/Hvp68VCm1BbuG3I/OO4vF4x3Ah93Eez7wTKS2BPZr7X6G3XzcL3LSsIjOO7d5gNx204ZEDa8D3m13IlIY6dDWmQXADjWpb8eAds3Fo7D3J2x/u9fSdU12e2+3qcTe56O2TlBK5WG/WrA33/G7vl0ZHJHxrWXY2G5+LrEnZ+uAh9r9PwuMMVN3ZP0RY6LWv73vE3S+r6PLrbAvZ2z9/8bkq5RyYe/7rbp6A9w07O+kEIAE6D5N2Z2VblF25yh3pGPOqdg/9E+6kc8U7OuKJ2MH9q2fvbFroMftSPkiTZ+PAZcCpxDVvI1dWwhiBxSHUuoC7JpkZ+YBM5RSMyPb+UvsWtZWjwGWUuoCpVRWpKY6Ril1TBd5vgwc0e0N2z4HcJtSKlspNQa7+XbrtcbtbfcTwERldzLLUUplKKWiy7iFLgJ4pKb6GHCzUmpI5EThL8ByYG6Cti8ejwBXKqUmRPorXIv9etvXIvMfB36nlBqrlMrGvgwQfTz6N/BjpdTsqO/2FKXUwXGu/1HgVKXU0Uopp1LqWOzv4NZLOAuxT6ROiHxXTgYOapdHZ/v6AqXUjEjL0O+AnKjtmgccruwOkZnAn4DojopbsDuJRX93UUrlY//eXo1z+8QuQAJ03+bHPjt/EbtprBK4DrjUGNOdl83/AphvjJljjNkS9VmM/dL6TjuLxeFh4GDsZvboAPEodmerldi1qSl0cVJhjPkQuAt4E7tpdSDwWdT8LcCh2D2z12I3X7+EXWvqzOPA9EgQTaR12DWqNdjb+CZ2AILtbHekI9Eh2B3cNmAf0KM7mF0L3KTsntH3drL+KwCN3Su4DLtZ+AeRE6becgf2rUNvA+XYlziOivRWBrt/wFvAl9j7qQx7vwFgjFmKfV33cuz/dwV20I/rEogx5jPsa/F3Yn8XbgfONsZ8GZm/Cruj133Yv51jgBfaZdPZvr4P+Eck3x8Bxxtj6iPznsQOsvOxm9TLsP/PW8v1PfAfYG6k6X5rp7czgA+MMSvi2T6xa5D3QYtdmlLqImB/Y0xcvYPjyO987A5acj9rGlJKrcX+/z6xvbTdyDMTWIp9EvVtovIVfZ8r2QUQIpmMMfcA9yS7HGLXFenl3lW/A7GLkiZuIYQQIgVJE7cQQgiRgqQGLYQQQqQgCdBCCCFECpIALYQQQqQgCdBCCCFECpIALYQQQqQgCdBCCCFECvr/ffJzDGJ28XsAAAAASUVORK5CYII=\\n\",\n \"text/plain\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" importance stddev p_value n p99_high p99_low\\n\",\n \"Capital Gain 0.043333 0.009074 0.007151 3 0.095327 -0.008660\\n\",\n \"Education-Num 0.041333 0.006110 0.003603 3 0.076345 0.006322\\n\",\n \"Relationship 0.035000 0.008544 0.009646 3 0.083958 -0.013958\\n\",\n \"Age 0.016333 0.011015 0.062011 3 0.079451 -0.046785\\n\",\n \"Capital Loss 0.012667 0.002309 0.005450 3 0.025900 -0.000566\\n\",\n \"Marital Status 0.008667 0.002517 0.013487 3 0.023087 -0.005754\\n\",\n \"Hours per week 0.005667 0.007095 0.150357 3 0.046320 -0.034986\\n\",\n \"Country 0.003333 0.002517 0.074372 3 0.017754 -0.011087\\n\",\n \"Workclass 0.003000 0.004583 0.187228 3 0.029259 -0.023259\\n\",\n \"Occupation 0.002000 0.005292 0.289958 3 0.032321 -0.028321\\n\",\n \"Race 0.001333 0.002517 0.227834 3 0.015754 -0.013087\\n\",\n \"Sex -0.000333 0.004933 0.541239 3 0.027933 -0.028599\"\n ]\n },\n \"execution_count\": 13,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"val_data[label] = y_valid # add labels to validation DataFrame\\n\",\n \"predictor.feature_importance(val_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Features with near zero or negative importance score above hardly contribute at all to AutoGluon's overall accuracy on the validation data, whereas features near the top of this list contain the most predictive signal.\"\n ]\n }\n ],\n \"metadata\": {\n \"anaconda-cloud\": {},\n \"kernelspec\": {\n \"display_name\": \"multilabel\",\n \"language\": \"python\",\n \"name\": \"multilabel\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.7.4\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 1\n}\n"
},
{
"path": "examples/tabular/interpret/SHAP with AutoGluon-Tabular Diabetes regression.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Explaining AutoGluon-Tabular Predictions with KernSHAP for Regression\\n\",\n \"\\n\",\n \"This example uses a small diabetes dataset where the goal is to predict a continuous target variable (i.e. a regression problem). We train an AutoGluon regressor and then explain each of its predictions via [Shapely values](https://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions.pdf) that quantify how much each feature contributed to a particular AutoGluon-prediction deviating from some \\\"baseline\\\" value. We use the [Kernel SHAP variant](https://shap.readthedocs.io/en/latest/generated/shap.KernelExplainer.html) which is appropriate for explaining arbitrary black-box models like the potentially heterogeneous ensemble of many models that AutoGluon-Tabular uses to make its predictions.\\n\",\n \"\\n\",\n \"You must first install the [SHAP package](https://github.com/slundberg/shap/) (`pip install shap`).\\n\",\n \"\\n\",\n \"**References:** This notebook is derived from a [similar notebook](https://shap.readthedocs.io/en/latest/example_notebooks/kernel_explainer/Diabetes%20regression.html) that demonstrates how to use Kernel SHAP with sklearn regression models. For more Kernel SHAP examples, you may refer to [this article](https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a). Note that this notebook only demonstrates data that have been preprocessed to only contain numerical features; handling of categorical features is demonstrated in the notebook: \\\"SHAP with AutoGluon-Tabular\\\", which we recommend trying out first.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Converting `np.inexact` or `np.floating` to a dtype is deprecated. The current result is `float64` which is not strictly correct.\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"| \\n\",\n \" | importance | \\n\",\n \"stddev | \\n\",\n \"p_value | \\n\",\n \"n | \\n\",\n \"p99_high | \\n\",\n \"p99_low | \\n\",\n \"
|---|---|---|---|---|---|---|
| Capital Gain | \\n\",\n \"0.043333 | \\n\",\n \"0.009074 | \\n\",\n \"0.007151 | \\n\",\n \"3 | \\n\",\n \"0.095327 | \\n\",\n \"-0.008660 | \\n\",\n \"
| Education-Num | \\n\",\n \"0.041333 | \\n\",\n \"0.006110 | \\n\",\n \"0.003603 | \\n\",\n \"3 | \\n\",\n \"0.076345 | \\n\",\n \"0.006322 | \\n\",\n \"
| Relationship | \\n\",\n \"0.035000 | \\n\",\n \"0.008544 | \\n\",\n \"0.009646 | \\n\",\n \"3 | \\n\",\n \"0.083958 | \\n\",\n \"-0.013958 | \\n\",\n \"
| Age | \\n\",\n \"0.016333 | \\n\",\n \"0.011015 | \\n\",\n \"0.062011 | \\n\",\n \"3 | \\n\",\n \"0.079451 | \\n\",\n \"-0.046785 | \\n\",\n \"
| Capital Loss | \\n\",\n \"0.012667 | \\n\",\n \"0.002309 | \\n\",\n \"0.005450 | \\n\",\n \"3 | \\n\",\n \"0.025900 | \\n\",\n \"-0.000566 | \\n\",\n \"
| Marital Status | \\n\",\n \"0.008667 | \\n\",\n \"0.002517 | \\n\",\n \"0.013487 | \\n\",\n \"3 | \\n\",\n \"0.023087 | \\n\",\n \"-0.005754 | \\n\",\n \"
| Hours per week | \\n\",\n \"0.005667 | \\n\",\n \"0.007095 | \\n\",\n \"0.150357 | \\n\",\n \"3 | \\n\",\n \"0.046320 | \\n\",\n \"-0.034986 | \\n\",\n \"
| Country | \\n\",\n \"0.003333 | \\n\",\n \"0.002517 | \\n\",\n \"0.074372 | \\n\",\n \"3 | \\n\",\n \"0.017754 | \\n\",\n \"-0.011087 | \\n\",\n \"
| Workclass | \\n\",\n \"0.003000 | \\n\",\n \"0.004583 | \\n\",\n \"0.187228 | \\n\",\n \"3 | \\n\",\n \"0.029259 | \\n\",\n \"-0.023259 | \\n\",\n \"
| Occupation | \\n\",\n \"0.002000 | \\n\",\n \"0.005292 | \\n\",\n \"0.289958 | \\n\",\n \"3 | \\n\",\n \"0.032321 | \\n\",\n \"-0.028321 | \\n\",\n \"
| Race | \\n\",\n \"0.001333 | \\n\",\n \"0.002517 | \\n\",\n \"0.227834 | \\n\",\n \"3 | \\n\",\n \"0.015754 | \\n\",\n \"-0.013087 | \\n\",\n \"
| Sex | \\n\",\n \"-0.000333 | \\n\",\n \"0.004933 | \\n\",\n \"0.541239 | \\n\",\n \"3 | \\n\",\n \"0.027933 | \\n\",\n \"-0.028599 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ],\n \"text/plain\": [\n \" age sex bmi bp s1 s2 s3 \\\\\\n\",\n \"74 0.012648 0.050680 0.002417 0.056301 0.027326 0.017162 0.041277 \\n\",\n \"26 -0.107226 -0.044642 -0.077342 -0.026328 -0.089630 -0.096198 0.026550 \\n\",\n \"45 0.027178 0.050680 -0.035307 0.032201 -0.011201 0.001504 -0.010266 \\n\",\n \"389 -0.005515 0.050680 0.001339 -0.084857 -0.011201 -0.016658 0.048640 \\n\",\n \"154 0.067136 0.050680 0.020739 -0.005671 0.020446 0.026243 -0.002903 \\n\",\n \"\\n\",\n \" s4 s5 s6 label \\n\",\n \"74 -0.039493 0.003712 0.073480 85.0 \\n\",\n \"26 -0.076395 -0.042572 -0.005220 137.0 \\n\",\n \"45 -0.002592 -0.014956 -0.050783 53.0 \\n\",\n \"389 -0.039493 -0.041180 -0.088062 51.0 \\n\",\n \"154 -0.002592 0.008640 0.003064 197.0 \"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"label = 'label'\\n\",\n \"feature_names = X_train.columns\\n\",\n \"train_data = X_train.copy()\\n\",\n \"train_data[label] = y_train\\n\",\n \"val_data = X_valid.copy()\\n\",\n \"\\n\",\n \"display(train_data.head())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We then train an AutoGluon classifier:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"No path specified. Models will be saved in: AutogluonModels/ag-20201021_071251/\\n\",\n \"Beginning AutoGluon training ... Time limit = 20s\\n\",\n \"AutoGluon will save models to AutogluonModels/ag-20201021_071251/\\n\",\n \"AutoGluon Version: 0.0.15b20201020\\n\",\n \"Train Data Rows: 353\\n\",\n \"Train Data Columns: 10\\n\",\n \"Preprocessing data ...\\n\",\n \"Using Feature Generators to preprocess the data ...\\n\",\n \"Fitting AutoMLPipelineFeatureGenerator...\\n\",\n \"\\tAvailable Memory: 5113.28 MB\\n\",\n \"\\tTrain Data (Original) Memory Usage: 0.03 MB (0.0% of available memory)\\n\",\n \"\\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\\n\",\n \"\\tStage 1 Generators:\\n\",\n \"\\t\\tFitting AsTypeFeatureGenerator...\\n\",\n \"\\tStage 2 Generators:\\n\",\n \"\\t\\tFitting FillNaFeatureGenerator...\\n\",\n \"\\tStage 3 Generators:\\n\",\n \"\\t\\tFitting IdentityFeatureGenerator...\\n\",\n \"\\tStage 4 Generators:\\n\",\n \"\\t\\tFitting DropUniqueFeatureGenerator...\\n\",\n \"\\tTypes of features in original data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('float', []) : 10 | ['age', 'sex', 'bmi', 'bp', 's1', ...]\\n\",\n \"\\tTypes of features in processed data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('float', []) : 10 | ['age', 'sex', 'bmi', 'bp', 's1', ...]\\n\",\n \"\\t0.1s = Fit runtime\\n\",\n \"\\t10 features in original data used to generate 10 features in processed data.\\n\",\n \"\\tTrain Data (Processed) Memory Usage: 0.03 MB (0.0% of available memory)\\n\",\n \"Data preprocessing and feature engineering runtime = 0.08s ...\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'root_mean_squared_error'\\n\",\n \"\\tTo change this, specify the eval_metric argument of fit()\\n\",\n \"AutoGluon will early stop models using evaluation metric: 'root_mean_squared_error'\\n\",\n \"Fitting model: RandomForestRegressorMSE ... Training model for up to 19.92s of the 19.92s of remaining time.\\n\",\n \"NumExpr defaulting to 4 threads.\\n\",\n \"\\t-61.2732\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.55s\\t = Training runtime\\n\",\n \"\\t0.13s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesRegressorMSE ... Training model for up to 19.21s of the 19.21s of remaining time.\\n\",\n \"\\t-60.308\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.32s\\t = Training runtime\\n\",\n \"\\t0.11s\\t = Validation runtime\\n\",\n \"Fitting model: KNeighborsRegressorUnif ... Training model for up to 18.74s of the 18.74s of remaining time.\\n\",\n \"\\t-63.4875\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.0s\\t = Training runtime\\n\",\n \"\\t0.11s\\t = Validation runtime\\n\",\n \"Fitting model: KNeighborsRegressorDist ... Training model for up to 18.62s of the 18.62s of remaining time.\\n\",\n \"\\t-64.222\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.0s\\t = Training runtime\\n\",\n \"\\t0.1s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMRegressor ... Training model for up to 18.51s of the 18.51s of remaining time.\\n\",\n \"\\t-59.3265\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.15s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMRegressorXT ... Training model for up to 18.36s of the 18.36s of remaining time.\\n\",\n \"\\t-57.1053\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.2s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: CatboostRegressor ... Training model for up to 18.15s of the 18.15s of remaining time.\\n\",\n \"\\t-58.6823\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.3s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: NeuralNetRegressor ... Training model for up to 17.84s of the 17.83s of remaining time.\\n\",\n \"\\t-65.6087\\t = Validation root_mean_squared_error score\\n\",\n \"\\t2.31s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMRegressorCustom ... Training model for up to 15.49s of the 15.49s of remaining time.\\n\",\n \"\\t-65.0328\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.24s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: weighted_ensemble_k0_l1 ... Training model for up to 19.92s of the 14.7s of remaining time.\\n\",\n \"\\t-57.0979\\t = Validation root_mean_squared_error score\\n\",\n \"\\t0.37s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"AutoGluon training complete, total runtime = 5.7s ...\\n\"\n ]\n }\n ],\n \"source\": [\n \"predictor = TabularPredictor(label=label, problem_type='regression').fit(train_data, time_limit=20)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Explain predictions\\n\",\n \"\\n\",\n \"SHAP is intended to explain how much each feature contributes to a particular prediction. In this regression context, this corresponds to how much a predicted value differs from a baseline reference value. We first create a wrapper class around AutoGluon to allow it to be called for prediction inside of the `shap` package:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"class AutogluonWrapper:\\n\",\n \" def __init__(self, predictor, feature_names):\\n\",\n \" self.ag_model = predictor\\n\",\n \" self.feature_names = feature_names\\n\",\n \" \\n\",\n \" def predict(self, X):\\n\",\n \" if isinstance(X, pd.Series):\\n\",\n \" X = X.values.reshape(1,-1)\\n\",\n \" if not isinstance(X, pd.DataFrame):\\n\",\n \" X = pd.DataFrame(X, columns=self.feature_names)\\n\",\n \" return self.ag_model.predict(X)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next, we define the baseline reference value of the features (AutoGluon predictions will be interpreted in terms of their difference from the prediction for the baseline feature-values). Rather than use the whole training set to estimate average column-values as our reference, we summarize with a set of weighted kmeans, each weighted by the number of points they represent.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Baseline feature-values: \\n\",\n \" | \\n\",\n \" | age | \\n\",\n \"sex | \\n\",\n \"bmi | \\n\",\n \"bp | \\n\",\n \"s1 | \\n\",\n \"s2 | \\n\",\n \"s3 | \\n\",\n \"s4 | \\n\",\n \"s5 | \\n\",\n \"s6 | \\n\",\n \"label | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|
| 74 | \\n\",\n \"0.012648 | \\n\",\n \"0.050680 | \\n\",\n \"0.002417 | \\n\",\n \"0.056301 | \\n\",\n \"0.027326 | \\n\",\n \"0.017162 | \\n\",\n \"0.041277 | \\n\",\n \"-0.039493 | \\n\",\n \"0.003712 | \\n\",\n \"0.073480 | \\n\",\n \"85.0 | \\n\",\n \"
| 26 | \\n\",\n \"-0.107226 | \\n\",\n \"-0.044642 | \\n\",\n \"-0.077342 | \\n\",\n \"-0.026328 | \\n\",\n \"-0.089630 | \\n\",\n \"-0.096198 | \\n\",\n \"0.026550 | \\n\",\n \"-0.076395 | \\n\",\n \"-0.042572 | \\n\",\n \"-0.005220 | \\n\",\n \"137.0 | \\n\",\n \"
| 45 | \\n\",\n \"0.027178 | \\n\",\n \"0.050680 | \\n\",\n \"-0.035307 | \\n\",\n \"0.032201 | \\n\",\n \"-0.011201 | \\n\",\n \"0.001504 | \\n\",\n \"-0.010266 | \\n\",\n \"-0.002592 | \\n\",\n \"-0.014956 | \\n\",\n \"-0.050783 | \\n\",\n \"53.0 | \\n\",\n \"
| 389 | \\n\",\n \"-0.005515 | \\n\",\n \"0.050680 | \\n\",\n \"0.001339 | \\n\",\n \"-0.084857 | \\n\",\n \"-0.011201 | \\n\",\n \"-0.016658 | \\n\",\n \"0.048640 | \\n\",\n \"-0.039493 | \\n\",\n \"-0.041180 | \\n\",\n \"-0.088062 | \\n\",\n \"51.0 | \\n\",\n \"
| 154 | \\n\",\n \"0.067136 | \\n\",\n \"0.050680 | \\n\",\n \"0.020739 | \\n\",\n \"-0.005671 | \\n\",\n \"0.020446 | \\n\",\n \"0.026243 | \\n\",\n \"-0.002903 | \\n\",\n \"-0.002592 | \\n\",\n \"0.008640 | \\n\",\n \"0.003064 | \\n\",\n \"197.0 | \\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\"\n ]\n },\n \"metadata\": {\n \"needs_background\": \"light\"\n },\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"shap.summary_plot(shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A dependence plot can be used to visualize how the BMI influences predicted outcomes.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAe4AAAFACAYAAAB6AZ/IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxyElEQVR4nO3deZhcVZnH8e/bHbITICEhBBISJUGUNRwEZFE2UQQBBQFJAmEkBB1mHAdnGEYBwQUFl0GCEFBkUQbDEhSRbWQREeWVRPY9G4SQhZAA2bvO/HFvJ5WqTnd1d1XXrerf53nuU1X3nlv1VtLJ2+85555rMUZERESkNjRUOwAREREpnRK3iIhIDVHiFhERqSFK3CIiIjVEiVtERKSGKHGLiIjUkB7VDiCLzj777PjTn/602mGIiNQaK/87fm7ja5bj7eX/jBqjirsF77zzTrVDEBERaZEqbhERybBuX2AXUeIWEZEMU+IupMQtIiIZpsRdSGPcIiKSYVawlXCG2Rgz+4uZvZQ+jm6hTaOZTTGzV83sFTP7Ut6xIWb2ezN7ysyeN7MrzSwzha4St4iI1JurgCkxxjHAFODqFtqcAuwIjAb2Ay40s5HpsfOA52OMuwG7AXsBn6t00KVS4hYRkQxrX8VtZkOAscDN6a6bgbFmNrig6YnANTHGXIxxETAdOCE9FoHNzawB6AX0BN7o3PcoHyVuERHJsHZ3lQ8H3ogxNgGkj/PT/flGAHPyXs/Na3MxMAZ4E1gA3Btj/HMHv0DZKXGLiEiGbZy4zWySmXneNqkCH3oC8BSwLbAdcJCZHV+Bz+mQzAy2i4hI9cRZi1l32f1Y/940nvcpbIs+1Q4ptXGVHWOcCkxt5YR5wHZm1hhjbDKzRmBYuj/fXGAH4In0dX4FfjZweowxBywzszuBg4FbO/NNykWJW0Skm4u5HGsO/QnMWpK8fvEtNps+ubpBrde+y8FijAvNbCZwMnBT+jgjHcfONw04w8xuBwYBxwIHpsdmAZ8C/mZmPYHDgNs7+AXKTl3lIiLd3Xur1ydtgNxTr1cxmELtvxwMmAycbWYvkVTPkwHM7G4zC2mbG4HXgJeBx4GLYoyz0mNfBQ40s6eBmcBLwDWd/SbloopbRKSbswF9aPjsbuR++xQAjeP3qXJEG8SCZF1K6o4xvgAUfYkY45F5z5uAszZx/qvA4e0KtAspcYuICD1um0Tu3uewzXvTcFDReiWSIUrcIiKC9Wik8TO7VjuMFmjJ00JK3CIiklkd6Sqvd0rcIiKSYUrVhTSrXEREpIbUTcUdQugN/JjkertVwF/cfVIIYQxwPcl1ekuACe7+cvUiFRGR0qniLlRPFfcPSBL2GHffFfhmuv8qYIq7t3aXGBERyaCIbbRJnSTuEEJ/YALwTXePAO7+VgihxbvEhBAK7xIjIiKZ1KEFWOpavXSVf5CkG/yCEMLBwHvAN4CVwBvu3gTg7k0hhOa7xBQufyciIhkTqx1ABtVL4m4EPgDMcPevhxD2AX7HhnurtimEMAmYBDB06NCKBCkiIu2lKrtQXXSVk9zlZR1pl7i7/xVYTFJxbxdCaARIH1u6SwzuPtXdg7uHrbbaqssCFxGRTdMYd7G6SNzuvhh4kHRt2XQm+RCSheFnktwdhvRxhrurm1xEpCZojLtQXSTu1GTgvBDC08D/AuPd/Z10/9khhI3uEiMiItmnirtYvYxx4+6vAZ9oYX+Ld4kREZFaoGRdqG4St4iI1B9V2cXqqatcRESk7qniFhGRzFLFXUyJW0REMkyJu5ASt4iIZJYq7mJK3CIikmFK3IWUuEVEJLNUcRfTrHIREZEaosQtIiJSQ9RVLiIimaWu8mJK3CIikmFK3IXUVS4iknFx1VpiLlftMKpCNxkppsQtIpJh6/77Ttb0/VfWDP4Pco+9Wu1wupwSdzElbhGRjIrz3qbpu/dAjPD2+6z7rzurHZJkgMa4RUSyqmcPaGyApqSb3Pr1rHJAXU9VdjFV3CIiGWXbDKDHNafAyEHYvqPocfkXqh1SFVjBVsIZZmPM7C9m9lL6OLqFNo1mNsXMXjWzV8zsSwXHv2BmT5vZM+njNuX4NuWgiltEJMMaJ36Mxokfq3YYVdPBivsqYEqM8SYzGwdcDRxS0OYUYEdgNDAImGFmD8QYZ5tZAC4EDokxLjCzLYDVHf0O5aaKW0REMqu9k9PMbAgwFrg53XUzMNbMBhc0PRG4JsaYizEuAqYDJ6TH/g24LMa4ACDGuCzGuKrz36Y8lLhFRCTD2t1VPhx4I8bYBJA+zk/35xsBzMl7PTevzYeBD5jZI2b2pJl9w8wyM9iuxC0iIpkVCzYzm2RmnrdNqsDHNgK7AYcDHwc+DYyvwOd0iMa4RUQkswq7x2OMU4GprZwyD9jOzBpjjE1m1ggMS/fnmwvsADyRvs6vwOcCt8YYVwOrzexO4KPADZ35LuWiiltERDKsfV3lMcaFwEzg5HTXycCMdBw73zTgDDNrSMe/jwVuTY/9GvikJTYDDgX+0bnvUT5K3CIiklkdXDltMnC2mb0EnJ2+xszuTmeMA9wIvAa8DDwOXBRjnJUe+19gIfAcyS8BzwI/L8PXKQt1lYuISGZ15HKwGOMLwD4t7D8y73kTcNYmzs8BX0u3zFHFLSIiUkOUuEVERGqIuspFRCSztFZ5MSVuERHJLCXuYkrcIiKSYUrchZS4RUQks2K1A8ggJW4REcksdZUXU+IWEZHMUuIupsQtIiIZpsRdSIlbREQySxV3MSVuERHJLE1OK6bELSIimaWKu5gSt4iIZJgSdyElbhERySxV3MWUuEVEJLM0xl1MiVtERDJLFXcxJW4REcksJe5iuh+3iIhIDVHFLSIiGaaKu5ASt4iIZJYmpxVT4hYRkczSGHcxJW4REcksJe5iStwiIpJZStzF6i5xhxAuAC4EdnX3Z0II+wJXA32A2cA4d19YvQhFRKRUGuMuVleXg4UQxgL7AnPS1w3ATcBX3H0M8AhwSfUiFBGR9rGCTeomcYcQegFTgLPydu8FrHL3R9PXVwFf6OrYRESkYyK20SZ1lLiBi4Cb3H123r4RpNU3gLsvBhpCCAO7ODYREekAJe5idTHGHULYDwjAuZ14j0nAJIChQ4eWKTIREekMjXEXq5eK++PAzsCsEMJsYHvgXmBHYIfmRiGErYGcu79d+AbuPtXdg7uHrbbaqmuiFpENlr0Pp18BB58Pdzxe7WgkI1RxF6uLitvdLyFv0lmavI8CngMmhRAOSMe5JwPTqhKkiLTuP2+E6/6YPP/zC/DKFBgxuLoxSdUpWRerl4q7Re6eA8YDPwshvExSmXe4O11EKuj1JRuer10HC5dVL5Y6sGpFE88+9g7zX1tR7VA6JRZspTCzMWb2FzN7KX0c3UKbRjObYmavmtkrZvalFtrsZGYrzOyyzn2L8qqLiruQu4/Me/4YsGv1ohGRkvzb0fDgM7BiNRw5FvYcVe2IataaVTmuPuclFs5dhTXAF84ZyW4H1eYQYAcr7quAKTHGm8xsHMlaHocUtDmFZDh1NDAImGFmD8QYZ0OS2NPzpncs8sqp64pbRGrIobvB7Kvg6R/D786DxsZqR1Sz3nxtBQvnrgIg5uAfDy+tckQd194xbjMbAowFbk533QyMNbPCcZcTgWtijLkY4yKSBH1C3vFzgbuAlzr5FcpOiVtEsmPwFrDLDtCg/5o6Y6uhvejZZ8Of4baj+lQxms4p7Co3s0lm5nnbpIJThgNvxBibANLH+en+fBtdLgzMbW5jZrsDRwA/Lvf3KYe67CoXEenOBgzcjNO/vSN/v38JA4f2Yv/jhlQ7pA4rrLJjjFOBqZX6PDPbLH3/iTHGJrPsTY5T4hYRqUPDd+rH8J36VTuMapgHbGdmjWnibQSGpfvzzSW5XPiJ9HVzBb4t8EHg7jRpbwmYmQ2IMRZW91Wh/igREcms9o5xxxgXAjOBk9NdJwMz0nHsfNOAM8ysIR3/Pha4NcY4N8a4dYxxZIxxJPATkrHwTCRtKKHiDiH0AGYAe7v7qsqHJCIikujgymmTgevN7HxgKTABwMzuBs6PMTpwI7AP8HJ6zkUxxlmdjbcrtJm43X1dCGFLtPKciIh0sY5cDhZjfIEkKRfuPzLveRMb35RqU+91YbsDqLBSu8r/B/hOWn2LiIh0Ed3Ws1CpifhMYCRwVgjhTSDXfCC9z7WIiEjZacnTYqUm7m9XNAoREZEWaIy2WEmJ292vr3QgIlLf4pJ34cnZsMtwbNstqx2O1AhV3MVKHrMOIewNnE6yssw84Bfu/kTrZ4mIQHzjbfjohTB/KWzRl/joN7BdCheyEimmirtYSZPTQgjHAo8AW5BcGjYAeDiEcFzlQhORuvG7GUnSBli2Am75a3XjkZqRwzbapPSK+wLg8+5+d/OOEMKnSe6BfUclAhOROrLTtq2/LrB89ntYA2w+on8Fg5JaoK7yYqUm7pHAPQX77mXD3VdERDbJDv4w8YYz4fcz4WOjsXH7b7LtzMufZ8ZPngNgr6/vwm5n7dRFUUoWqau8WKnXcc8BDivYdyjJWq8iIm2y8Qdg//vP2L8c0Wq7p372QovPpXtq75Kn3UGpFffFwJ0hhFuBWSQV+OeBUysUl4h0U32H9uHdOe8D0G9Y3ypHI9WmZF2spIrb3W8jqbBXAAFYCRzu7rdWMDYR6YYOm/oxhh8ylOGHbcshV+5b7XCkygrvxy2tVNwhhNvc/fPp84nufh3wWJdFJiLd0pajBzD6GwEMttihV7XDkSpTxV2sta7yQ/Oe/w9wXYVjERHhlpuX8IffvwPA0cdsyedPGFTyuTPmN9GvpzFma92xuF4ocRdrLXE/G0K4GXga6BlCOK+lRu7+3YpEJiLd0v33Ldvw/N5lJSfuL92+ip/7Oszgp0f15Cv79axUiCJV1dqvpeOA5cDBQCNweAtb4UxzEZFOGTJkQz2xzTablXTO8lWRn/s6AGKEH/95bUVik66nMe5im6y43X0WyV3BCCHMdPeDuywqEem2vvrv23L7rW9jBp87fmBJ5/TdDIb0Mxa+n/zXPmpgyzVJzEVyb75Lw+B+WM/GssUslaOu8mKl3mRkjwrHISICwJAhmzH5y9u065wejcY9E3tz0R/XsHkv4/tHFHeTx5VrWXL4Taz58zwad9iCQQ+fSo8dtixT1FIpqrKLlXyTERGRLNtzWCN3jOuzyeMrp7/Imj/PA6BpzjLev+IJtrj08K4KTzpIFXcxJW4RqVsL3o2cc+9a3lkV+Q69GJx3rGHQppO8ZIcSdzElbhGpWxPvWMM9r+QA+Euf7Xnl/INYe/sLbLb3MPp/VYu71IJctQPIICVuEal5C95uYtn7OcZs3wOzDRXanGUbRkjfXgm5cz/OkG99ousDlA6LDaq4C5WUuEMIjcB/kaxNPsTdtwghHAGMcverKhmgiEhr7nliJRdcv5ymHBy+Vy++909brj92zv49OOPOteQiTNyzkUF9lQRqTdRfWZFSlxe6GPgs8J9smOT3EunlYiIi1fLr/1tBU9qfev/fV7Pg7ab1x04f24NXv9qLf3y5F784Tguy1KLYYBttUnri/iJwjLvfzoYhh9kkdwkTEamaoYM2XI/dr7exeUFVPXKrBnYbqiVQa1Vs2HiT0se4+wILC/b1BFaVNxwRkfY57+QB9O31Lm+/m2PiEf3o11v/u9eT2Kgqu1CpiftJYCJwbd6+LwJ/K3tEIiLtsGX/Bi6csEW1w5AKyal7vEipifsc4KEQwklA3xDC70juy61lUEVEpGLUPV6spD8Sd38G2Bn4A0nV/Qiwh7u/UMHYRESkm9PktGIlX8ft7ouAH1YwFhERkY3ocrBipV7H3eK9uEH34xYRkcrpSJVtZmOA64FBwBJgQozx5YI2jcDlwKdILnO+JMZ4bXrsm8BJQBOwFjgvxnhvJ75GWZVacReuxD8MGAU8Cihxi4hIllwFTIkx3mRm44CrgUMK2pwC7AiMJknwM8zsgRjjbJKJ1z+MMa4ws92Bh81s2xjjyq77CptW6m09iyahhRD+GTZas19ERKSscu0suM1sCDCWDQXnzcAVZjY4xrgor+mJwDUxxhywyMymAycAlxZU108BRpLcX+/Idyi3zszX+xkwuVyBiIiIFOrA5LThwBsxxiaA9HF+uj/fCGBO3uu5LbQBmAC8GmPMRNKGziXu3UH3WxMRkcqJtvFmZpPMzPO2SZX6bDP7OMmS3ydX6jM6otTJafezYY1ygH4kXRE/qkRQIiIiANE2rg9jjFOBqa2cMg/YzswaY4xN6SS0Yen+fHOBHYAn0tcbVeBmth9wE3BMjPHFTn2JMit1ctqjBa/fBc5z94fLHI+IdJEYI03zltMwqA8N/XQDDsmm9o5xxxgXmtlMkir5pvRxRsH4NsA04Awzu51k/PpY4EAAM9sbuAU4Psb4ZCfCr4hSJ6d9q71vHEI4x90vS5/rcjKRDIm5yKLjb2PlHS/RsFVvhtxzEr0+OqzaYYkU6eCiK5OB683sfGApyTg1ZnY3cH6M0YEbgX2A5svELooxzkqfXwn0Aa7Ou7/7+Bjj0x36EmW2ycQdQijpX7G7z9/EoUOAy9LnhZeTNYvocjKRLrfmr2+w8o6XAMgtXcXyH/yFwbd+vspRiRTryAIsMcYXSJJy4f4j8543AWdt4vy92/+pXae1ivt1Nh7XLmTp8caWDrr7kXnPtaa5SIY0DOoDDQa55J94w9Z9qxyRSMsKx7il9cQ9qsuiEJEutdmYQQy69kjeveLv9NhxK7b83icAWLsmx5N/WkaPHsaeB2xBg26pKFXW3jHu7mCTidvd52zqWHuFEIYC3yK5o9jmBZ8zpgzvP4hkvOKDwBqSMYsz3X1RCGFfklVz+gCzgXHuXnhvcZFup//E3ek/cfeN9v3ikrm8OPM9AF5+5j1O+sr21QhNZL16rLjNbF/gNGB7kt7tX8YYHy/1/JJvMhJC+BDwCZLV0tb/Sbr7RSWc/qv08VpgRamf2Q4R+IG7PwQQQrgUuCSEcAbJrMLT3P3REMI3gEuA0ysQg0hNa1oX1ydtgOf83SpGI5Kot5uMmNmpJAuY3QbMAEYC/2dmX44xXl/Ke5R6HffJwC9Jln7bLX3cneT2niW9BbCNu68qsX27uPvbwEN5ux4nmXSwF7DK3ZsvZ7uKpOpW4hYp0NjDGDG6D3NfTpZjHrVzvypHJAK5+qu4vwEcHWP8v+YdZnYdcA3JjVHaVOrKaf8NjHf3vYEV6eNkoNTr214EtiqxbaeEEBpIkvZvKbig3t0XAw0hhIFdEYtIrTnz/JEc+cUhfPa0oYz7qrrJpfoKV06rA0OABwv2PQRsXeoblNpVPoLkYvV8N5CsRPMfJZx/BvCzEMINwIL8A+7+WIkxlOqnwHvAFcBxpZ4UQpgETAIYOnRomUMSqQ19+jVy2PFDqh2GyHp1OMZ9J8kNTm7O23cCML3UNyg1cb8DbJE+vhVC2JnkHqel9qXtDBwKfLZg/yYvJ+uIEMJlJLdoO9rdcyGE5iXtmo9vDeTSrvWNuPv6ZfTGjx/f2mVwIiLSReowcTcAvzSzySRDtyOBfYHfmNn6pVxjjJtcg73UxP0ASfV6HfCb9PVa4A8lnn8pcA5wg7tX5H6mIYTvkoxpf8bdV6e7/w70CSEckI5zT6a450BERDKqTrrH860Ffp33+rV0A9islDcodcnT/MlcFwAvAAMocSAd6O/uV5fYtt1CCB8B/gt4CXgshAAwy92PCyGMB64OIfQmvRysUnGI1JPYlKPpv+8k9/gsGo7bgx7/eki1QxKpeTHGiWbWHziKDZeD/T7GWPJlHKXOKh/h7nMB3D2y8W8Lpbg9hPApd7+nneeVxN2fZRO3GE3H0HetxOeK1LPczx6h6fv3AdD08Ms0fGRbGg7bucpRSXfTwbXKM8vMAvB7YCXJPLERwOVmdmS6hnqbSu0qfy2E8CDwc+COvK7oUm0G3BZC+CPwZv4Bd6/YvVRFpOPi60sLXr9TnUCkW6vDMe4rgR/FGL/fvMPM/oPk2u6S1kgv9XKw0cBjwPeAN0MIV4a0P7pETSRj44tJknj+JiIZ1PBP+8M2AwCwj2xLw7G7t3GGSPnFBttoqwM7Az8s2Pcj4EOlvkGpY9yzSMa2LwghHAqcCjwYQnjN3dv81+zuE0MIRX367lqaSSSrGkYPoedLFxJnL8HGbIP11u/ZUgX1V3HPBHZJH5vtWvC6VSUveZrnIZKJacOBg0o5Ia3Oi/r0QwhHuntJffoi0vVsQB9sNy3EItVTD1W2mX0x7+V9wF1mdi3JAmEjSVbznNrCqS1qz1rluwETgS8Cq0kWYDmjxNOvBH7k7uv79EMI7erTFxGR7qdOxri/U/B6LUnPdbN1JPn14lLerNRZ5TOAnUiWEZ0A3O/uuVLOTW2qT/+b7XgPERHpZqKVOhUru2KMZb1NdqkV9zXAr939nQ5+zkw62acvIiLdTz10lZdbqZPTrmzvG4cQivr0Qwgd7tMXEZHup066ysuqI5PTSlXWPn0REemGlLeLVCxxu3tZ+/RFRKT7UcVdrJIVt4iISKdojLtYm4k7hLAjyUSyf7j7a221FxERKRdV3MVanWcfQvgc8DxwG/BcCOHILolKRESEJHHnb9L2WuXfAM4DNidZ8vS8ikckIiKSUuIu1lbiHgX80N3fJ1kwZcfKhyQinbV0eRNX/vodLr9xKYveXlftcEQ6TIm7WFuJu7F5hTR3Xwv0rHxIItJZ3/7Z2/z2wfe5+5EVfPPyJdUOR0TKqK3JaT1DCPnd470LXuPu3y1/WCLSGXPmr13/fO6b64gxYqpWpAapyi7WVuJ+HDg87/VfC15HQIlbJGOOOKAft933HgCHf6yvkrbULCXuYq0mbnf/RBfFISJldMYJW7Df7r1Z1wS7f0gjXFK7OnIdt5mNAa4HBgFLgAkxxpcL2jQClwOfIilCL4kxXtvWsSzo0AIsIQQDjgTOdPfPljckESmHXcb0qnYIIp3WwYr7KmBKjPEmMxsHXA0cUtDmFJIJ16NJEvwMM3sgxji7jWNV1677pYUQhoUQzgdmA3cAyysRlIiICLR/VrmZDQHGAjenu24GxprZ4IKmJwLXxBhzMcZFwHTghBKOVV0pK6cZ8GngzPRxMbAlsJe7P13R6EREpFvrQMU9HHgjxtgEEGNsMrP56f5Fee1GkNytstnctE1bx6qurZXTvgnMIvltIwKfJ/lCy4C3Kh2ciIh0b4UVt5lNMjPP2yZVO8au1lbF/S2Sgf1j3f3u5p0hhIoGJSIiAsUVd4xxKjC1lVPmAduZWWNabTcCw9L9+eYCOwBPpK/zq+zWjlVdW4l7PDAJ+F0I4SngF8CvSKpvERGRimpvV3mMcaGZzQROBm5KH2ekY9X5pgFnmNntJBPQjgUOLOFY1bXaVe7uv3L3jwO7AA+RrFf+BrA1oLJbREQqKtrGW4kmA2eb2UvA2elrzOxuM2vOXTcCrwEvk6xZclGMcVYJx6qupMvB3P154N9CCOcCXwDOAO4KIbi7f7SSAYrIBm+vjPxtQeTDg4wRA7QwhdS/jlwOFmN8Adinhf1H5j1vAs7axPmbPJYF7bqO291Xk/wmcmMI4cMk3egi0gXeej+y901NzHsX+m0GD53YSBiq5C31TSunFevQAiwA7v4c8NXyhSIirbl3dmTeu8nz99fCr5/PEYY2VjcokQrLKXEXaTVxhxBepo2JaO4+pqwRiUiLdhpoGBv+Qe48SP+hSf2L6Oe8UFsV97fznhswBfhy5cIRkU3ZZ1vjN0c3cMcrkb2HGl/aVf+hSf1TV3mxtm4ycn3+6xDCjwr3iUjXOX6nBo7fqdpRiHQdJe5iHR7jFhERqTQl7mLtusmIiIiIVJcqbhERyax2LLrSbbR3VvmAEMJL+W00q1xERCpFl4MVa8+schGpgnd/+yq5ZavZ/PgxNPRRJ5l0LxrjLtauWeUi0rUWnvsnlnzfAegz9Wl2eOQLmP4jk25EibtYW13lPQBz97V5+04D9gAecffbKxqdSDf37h2vrn++8tH5NC1eSY/BfasYkUjXUld5sbZmld8CTGx+EUL4Bsl9UA8AfhVC+FIFYxPp9vrsO3T9856jt6RxYO8qRiPS9Tp4d7C61lbiDsBdea/PBr7k7gEYR4bvniJSD4ZefRhDfnAgg877KCMeOgFr1BWc0r1EbKNN2p6ctpW7zwcIIewMbAH8Jj02naT6FpEKaejdg0FfD203FKlT6iov1tav7++HEPqnzwPwjLuvSl8bug5cREQqKJpttEnbiftPwMUhhA8BZwL35B3bCXizUoGJiIgocRdrK3H/J/Ap4DlgAPCjvGOnAI9WKC4RERFytvEmbV/HPQvYOYQw0N3fLjj8A2BNxSITEZFuT1V2sZLGqFtI2rj7O2WPRkREJE9OM8mLdIvJZSGEMcD1wCBgCTDB3V+ublQiItIWVdzFustFoVcBU9IbokwBrq5yPCIbaVq8grWvvVPtMEQyR2Pcxeo+cYcQhgBjgZvTXTcDY0MIg6sXlcgG701/mdnbX8WcD17Dwkn3VjsckUzJmW20STdI3MBw4A13bwJIH+en+0Wqbul3HyeubgJg+TVPse71d6sckYhkWbcY4y5FCGESMAlg6NChbbQWKZ/Gof3WP7e+m9EwoGcVoxHJFo1xF+sOiXsesF0IodHdm0IIjcCwdP967j6VdAnX8ePHx64PU7qrIVd9kkU9HqBp4Qq2+uZ+NAzoVe2QRDJD49rF6j5xu/vCEMJM4GTgpvRxhrsvqmpgIqkew/qz7e3HVjsMkUzSjUWK1X3iTk0Grg8hnA8sBSZUOR4RESmBJqQV6xaJ291fAPapdhwiItI+StzFukXiFhGR2qQx7mLd4XIwERGpUTlso60czKyvmd1iZq+Y2QtmdlQrbc9I271qZleYWUO6/xgz+7uZPWNmz5rZv5cluBIocYuISGZV6Lae5wDLY4w7AkcD15pZ/8JGZjYKuADYDxidbuPSwwuAo2OMuwAfA84yswPLFWBrlLhFRCSzKrTk6YmkS1/HGF8GHPh0C+2OB6bHGBfFGHPANem5xBj/GmOcnz5fBjwP7FC2CFuhxC0iIplVuOSpmU0yM8/bJnXgbUcAc/Jez6Xl1TRLamdmHwL2Bf7YgVjaTZPTREQkswrHtWOM6xfL2hQze5Ik6bZkm/JEtv6ztgXuBL7cXIFXmhK3iIhkVlMHusdjjGNbO25mc0m6tZsX4hoBPNhC0+Z25LVbv+qmmQ0BHgB+EGOc1v5IO0Zd5SIiklkVujvYNOBMADMbDewN3NNCu9uAY81scDqb/AzgN+l5g4D7gStijD8vV2ClUOIWEZHMqtDktEuBLc3sFeAuYFKM8V0AM7vIzCYDxBhfAy4GHgdeBl4jWTob4FxgDHCmmc1Mt4lli7AV6ioXEZHMKte12/lijO8DJ2zi2PkFr68mnYFesP/rwNfLHlwJlLhFRCSzmrTkaRF1lYuIiNQQVdwiIpJZWqu8mBK3iIhkVpPux11EiVtERDKrI9dx1zslbhERySzdj7uYEreIiGSWZpUXU+IWEZHMWlftADJIiVtERDJLFXcxJW4REcmsdcrbRZS4RUQks9bpcrAiStwiIpJZa5W3iyhxi4hIZq3VGHcRJW6pSy/+aTFP37OQrUf25cDTRtC4mZblF6lFa6sdQAYpcUvdWTJ3BdMvfIGYg1cee5vGzYwDT9uh2mGJSAesUMVdRGWI1JUYI/c/s4a5ffuu3/fO/FVVjEhEOmOlbbyJKm6pMxNvW831T/aAXcdwyLwFHLl4Ibt/Zmi1wxKRDlqjWeVFlLilbqxZF7n+yQ3rLD0zehum/2g4mw/uVcWoRKRTlLeLqKtc6kbPHsYHBm74V/6RYT2UtEWk7qjilrpy38Q+fOehNfTuARce2rPa4YhIZ2lyWhElbqkrHxzUwC8+37vaYYiIVIwSt4iIZJcq7iJK3CIikl3K20WUuKVT7pmV429vwpa9I1v1Mj432ujXs7L/0mKMmH4LF+km9G+9kBK3dNhvX8lxzPRc3p7IFTPgsS820tiw6X9s63KR7/ypiWcX5ZiwWyNHjWks6fMWzFvFz787h6WL1/KJz27NUeN1fbZI3VPeLqLLwaTDHp4Xi/b9bQG88V7r533/z01c+PA6pj2X47jfrOXFxbnWT0jd/au3WPLWWnJN8Mc7FrNgrlZEE6l7VrCJErds2uq1kRmvrOHNJU0tHv/kSKOwsB6xOWzTt8Xm6724ZEOiXpeD15YW/wLQksYeG39YQw/9Kxapf8rchdRVXkZrV67jnTkr2GL7vvTsX9t/tGvWRs788VKemb2Wnj3gh5O3ZL8Pb7yYyRGjGnjkJOPx+TkWvA+NDXDW7g30aiOhTty9kWnP5Vi1Dj4y2DhgRGm/Px41bihLFqxh6aK1fOKYrRkyTIuriNQ95eoitZ1dMmTl0jXcetrjLJu3gr5b9+L46/ZhwHZtlJ4Z9szstTwzO7mh3pp1cPujK4sSN8D+2xn7b1faGHWzg0c18vyXjdeWRj66XQP9S5zMNmhoT7522Y7t+iwRqXXK3IXUVV4mr9y/gGXzVgCwYvFqXrhrfpUj6pyhAxvZLO/XuuGD25ec2zJyywYOGdVYctIWkW6qAj3lZtbXzG4xs1fM7AUzO6qVtmek7V41syvMrKHgeG8ze9bMvDzRtU2Ju0z6D914ta7+29T26l3DBjVy2ZlbcvDuvRh/WF/OPKp/tUMSke6oMkPc5wDLY4w7AkcD15pZ0X9yZjYKuADYDxidbuMKmn0HeLxskZVAXeVlMuqgIez/tZ2Y86dFbLvnVux8zHbVDqnT9v9IL/b/iMaRRaSaKtIrdyJwKkCM8eW0Wv40MK2g3fHA9BjjIgAzuwaYCNyQvj6QJJn/CNi9EoG2RIm7jPYcN4o9x42qdhgiIvWjMqNpI4A5ea/nAsPb087M+gE/AT5Lkry7jBK3iIhkV8EqiWY2CZiUt2tqjHFqQZsnSZJuS7YpU2SXAlNijG+YmRK3iIhIS9IkPbWNNmNbO25mc4EdgEXprhHAgy00bW5HXrt56fMDgCPN7HygN7CVmT0VY9ytzS/RSZqcJpWxcjWcfgXs+e9w2fRqRyMikm8acCZAWi3vDdzTQrvbgGPNbHA6m/wM4DcAMcbdYowjY4wjgZOAp7siaYMqbqmUH0yH6/6YPJ85C/YZAwd+uKohiUgNqswY96XAL83sFaAJmBRjfBfAzC4C5scYr4oxvmZmF7Nh1vh9wE0Viagdaj5xhxCmAIcCq4H3gH91d0+PbQPcCIwEVgKT3P2vVQq1e1m0fOPXi5e33E5EpFXlz9wxxveBEzZx7PyC11cDV7fxfg8BoVzxtaUeusr/AOzq7rsD3wNuyTv2PeARdx8DfAW4KYSgFT+6wr98BoYNTJ4f9GE4cq/qxiMitUlLlRep+Yrb3e/Ke/kXYPsQQoO754AvkFTbuPujIYTVJL8VPdHlgXY3Y4bBq1fCwmWw/SBoqIffEUWkyylZF6m3/03/Gfi9u+dCCIMAc/fFecc3da2eVELvnjBisJK2iHSCSu5Cma+4QwitXo/n7k1pu5OALwIHdfBz1l8bOHTo0I68Reatnvceyx97i/57bU2fHbeodjgiIm1Tri6S+cTt7q1ejwcQQjiOZL3YQ939rfS8JSEEQghb51Xd+dfgFX7O+msDx48fX9oNomvIyleX84+972Td0tU09Glktz8dTf+9tq52WCIi0k4134cZQjiKZJ3YI9x9dsHhacDktN0BQB/g710aYEYsved11i1dDUBuZRNL7pzTxhkiIhmgnvIima+4S3AdsAa4NYT1s/EPdfclwLkkM8lPJbkcbHw6aa3b6b/HwOSHPu1L6LfnoKrGIyJSElO2LlTzidvdB7dybAFwWBeGk1kD9h/KznceztK75zHgwKFsfdzIaodUdrm3VxDfWUXjBwZWOxQRkYqp+cQtpRt09A4MOnqHthvWoDX3vMSyz90MK9fSa9zuDLixxbUVRKTWqOAuUvNj3CIAKy55BFauBWD1Tf9g3UuL2zhDRGqDBrkLqeKWutCwTf8NL3r1oGHL3tULRkTKR7m6iBK31IX+Pz2K2BTJzV9O33MPomFI/7ZPEhGpQUrcUhcahvRni1tPrnYYIlJuqriLKHF3Q/c/9j5vvNXEIfv2YcS2m1U7HBERaQcl7m7mtvve5ZppyS0273roPa69eBu2HNBY5ahERDZB13EX0azybubZV9asf/7eisjcN9dVMRoREWkvJe5uZt/dN8y2HjywkQ8MV1e5iGSYrgYroq7ybuaT+/dj8MBG5i9sYt/de9O/r353ExGpJUrc3dCeO/dmz52rHYWISAlUZRdR4hYRkQxT5i6kxC0iItmlvF1EA5wiIiI1RBW3iIhklyruIqq4RUREaogqbhERyS5V3EVUcYuIiNQQVdwiIpJdWqu8iBK3iIhkl/J2EYsxVjuGzAkhLALmVOGjtwYWV+FzS6X4Oi/rMSq+zst6jJWMb7G7f6pC7y3NYozaMrLttddeXu0YFF/3jlHx1X+MWY9PW9ubJqeJiIjUECVuERGRGqLEnS1Tqx1AGxRf52U9RsXXeVmPMevxSRs0OU1ERKSGqOIWERGpIbqOu8JCCH2B64C9gHXAOe5+1ybangH8J8mVi38A/sXdcyGEfwFOz2v6AeBad/9aCOETwN3AS+mx1e6+TxfH12oMIYRvAqelL3/p7hd3cXzHAOcDvdJjv3D3H6bnnAb8BJidvs0sdz+uhLjGANcDg4AlwAR3f7mgTSNwOfApIAKXuPu1nTlWqjLE903gJKAJWAuc5+73psd+CRzGhkuKprn7d9oTX5livBD4MjA/bf5nd/9Keqzkn5sKxncDsFte892AY939t63FXub4Pgl8F9gV+Km7n1Ni7J3+GZTKUcVdeecAy919R+Bo4NoQQv/CRiGEUcAFwH7A6HQbB+Dul7v7Hu6+B7A3sAr4dd7pzzUfb0/SLld8rcUQQjgIOAHYJd1OSPd1ZXwLgKPdfRfgY8BZIYQD805/IC/2NpN26ipgiruPAaYAV7fQ5hRgxzSW/YALQwgjO3msVJ2N72/A3u6+G8kvjbeEEPrknXtJ3p9Zu5N2mWIEuCEvjvzEV9LPTSXjc/cJef9uTwWWAveWEHs543sN+BJwaXtib+OYVJkSd+WdSPoPKv1t2IFPt9DueGC6uy9y9xxwTXpuoaOBN93dMxpfS+9/g7uvdPeVwA0lnle2+Nz9r+4+P32+DHge2KEdMWwkhDAEGAvcnO66GRgbQhjcQuzXuHvO3RcB00l+ienMsS6Jz93vdfcVabunSHoqBpUaQ1fE2IZSf266Kr5/An7l7qtLjaEc8bn7K+4+k6TXoVDFfgalspS4K28EG6/CNhcY3ol2p5N0AeYbE0J4MoTw1xDCqVWKb1MxlPr+lY4PgBDCh4B9gT/m7f54CGFmCOGREMJnSohpOPCGuzcBpI/zW/i81mLq6LFSlCO+fBOAV9399bx9XwshPB1CmB5C2LkdsZU7xpNCCE+FEO4LIezXjvO6Kj5CCD2BLwK/KDh3U7GXM77WVPJnUCpIY9ydFEJ4kuSHvCXblPmztgUOYcN4McCTwHB3X5Z2Fz8QQnjD3R/owvhajaE1VfjzuxP4cnMFDtwF3OLuK0MIewJ/CCEc7O7Pl/Oza1UI4ePAxcDhebv/m6TXJxdCmADcE0L4QHMS6UJXAd9x97UhhMOBO0MIO7v7ki6Ooy3HAnPTyrdZrcQuGaTE3UnuPra14yGEuSTdsovSXSOAB1to2tyOvHbzCtqcCtzt7uvXGXb35XnPZ4UQpgP7Aw90VXxtxNDq9+qqP7+0a/EB4AfuPi3v8/P/LGeEEP4MfJSkO31T5gHbhRAa3b0pncgzjOK/r+aYnsiLaU4nj5WiHPGRVoE3Ace4+4vN+939jbznN4QQfgxs39UxuvuCvDjuDyHMI5lH8XDeeW393FQsvjynU1BttxF7OeNrTSV/BqWC1FVeedOAMwFCCKNJJpfd00K724BjQwiDQwgNwBnAbwraTKTgP4AQwrYhBEufDwQ+CczsyvjaiGEaMCGE0Ced3DShhe9V6fgGAfcDV7j7z/NPCiFsl/d8B5Ju9KdaC8jdF6bf7+R018nAjHQssDD2M0IIDenY47HArZ081qZyxBdC2Bu4BTje3Z/MP6ngz+wIkpnnb9AOZYoxP449gJHAi3nnlfJzU7H40s/eHjgQ+FX+SW3EXs74WlOxn0GpLFXclXcp8MsQwisk/8FNcvd3AUIIFwHz3f0qd38thHAx8Hh63n0k1Q5p2/2B/mw8KxXg8ySzpNeS/H1e7+53dnF8m4zB3R8KIdwOPJu2vcHdS60qyhXfucAY4MwQwpnpvv9x9+uAr4TkcrHmyTvnufuMEuKaDFwfQjifZLbwhDSmu4Hz08mDNwL7AM2X6Fzk7rPS5x09VqrOxncl0Ae4OoTQ/J7j3f3p9H23AXLAcuCz7t7S5KdKx/jdEMJeJD8Xa9L4mivZTf7cdGF8kPSS/c7dlxa8d2uxly2+EMIBwP8CAwALIZwE/JMnl/ZV+mdQKkQrp4mIiNQQdZWLiIjUECVuERGRGqLELSIiUkOUuEVERGqIEreIiEgNUeIWqbIQwuwQwri2W5b8fn8IIfxHud5PRLJF13GL1Bl3L/lmGiJSe1Rxi4iI1BBV3CLZ8IEQwqPAHsALwFnu/kQI4ZdAI7AW+BzwPsm9pp8nuXXph0huWXlK841TQggPkdxj/Ntd/B1EpAuo4hbJhsnAvwIDSdaEvjuEMCA9djzJWuwDSe7UdQ1wEXAcyR3UIvCtrg5YRKpDiVskG37u7n939zXA94GVwFHpsT+6++/dPQfcAPQDbnT31919BUmiDy2+q4jUHSVukWyY3fzE3SPJbRW3T3e9mXdsReE+YAWweYXjE5GMUOIWyYaRzU/SW6SOAF6vWjQiklmanCaSDaeHEO4Angb+DegL/J7k3uYiIuup4hbJhqnA5ST3VT4R+Iy7L6tuSCKSRboft4iISA1RxS0iIlJDlLhFRERqiBK3iIhIDVHiFhERqSFK3CIiIjVEiVtERKSGKHGLiIjUECVuERGRGqLELSIiUkP+Hz4zDwxRuYabAAAAAElFTkSuQmCC\\n\",\n \"text/plain\": [\n \"
\"\n ]\n },\n \"metadata\": {\n \"needs_background\": \"light\"\n },\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"shap.dependence_plot(\\\"bmi\\\", shap_values, X_valid.iloc[0:N_VAL,:])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Scoring Overall Feature Importance via Permutation Shuffling\\n\",\n \"\\n\",\n \"Note that if you'd like to understand how much each feature contributes to AutoGluon's general predictive accuracy (rather than explaining individual predictions), AutoGluon offers a built-in method for this based on [permutation-shuffling](https://explained.ai/rf-importance/):\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 12,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Computing raw permutation importance for 10 features on weighted_ensemble_k0_l1 ...\\n\",\n \"\\t1.76s\\t= Expected runtime\\n\",\n \"\\t1.54s\\t= Actual runtime\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"s5 11.321709\\n\",\n \"bmi 4.601622\\n\",\n \"age 1.945009\\n\",\n \"s4 0.802883\\n\",\n \"s3 0.736253\\n\",\n \"sex 0.549610\\n\",\n \"s6 -0.172562\\n\",\n \"s1 -0.242301\\n\",\n \"bp -0.295671\\n\",\n \"s2 -0.673330\\n\",\n \"dtype: float64\"\n ]\n },\n \"execution_count\": 12,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"val_data[label] = y_valid # add labels to validation DataFrame\\n\",\n \"predictor.feature_importance(val_data)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Features with near zero or negative importance score above hardly contribute at all to AutoGluon's overall accuracy on the validation data, whereas features near the top of this list contain the most predictive signal.\"\n ]\n }\n ],\n \"metadata\": {\n \"anaconda-cloud\": {},\n \"kernelspec\": {\n \"display_name\": \"featgen\",\n \"language\": \"python\",\n \"name\": \"featgen\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.7.4\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 1\n}\n"
},
{
"path": "examples/tabular/interpret/SHAP with AutoGluon-Tabular.ipynb",
"content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:36.140712Z\",\n \"start_time\": \"2026-01-07T01:36:36.134197Z\"\n }\n },\n \"source\": [\n \"from autogluon.tabular import TabularDataset, TabularPredictor\\n\",\n \"import pandas as pd\\n\",\n \"import numpy as np\\n\",\n \"import sklearn\\n\",\n \"import shap\\n\",\n \"\\n\",\n \"shap.initjs()\\n\",\n \"\\n\",\n \"import warnings\\n\",\n \"warnings.filterwarnings('ignore')\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"\"\n ],\n \"text/html\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 18\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Explaining AutoGluon-Tabular Predictions with Kernel SHAP\\n\",\n \"\\n\",\n \"We recommend first starting with this tutorial before the other SHAP tutorials. This example uses the original version of adult census income dataset, which is a binary classification task with categorical & numerical features. The code shown here should also work if you have text columns in your dataset and for regression or multiclass classification problems. We train an AutoGluon classifier and then explain each of its predictions via [Shapely values](https://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions.pdf) that quantify how much each feature contributed to a particular AutoGluon-prediction deviating from some \\\"baseline\\\" value. We use the [Kernel SHAP variant](https://shap.readthedocs.io/en/latest/generated/shap.KernelExplainer.html) which is appropriate for explaining arbitrary black-box models like the potentially heterogeneous ensemble of many models that AutoGluon-Tabular uses to make its predictions.\\n\",\n \"\\n\",\n \"You must first install the [SHAP package](https://github.com/slundberg/shap/) (`pip install shap`).\\n\",\n \"\\n\",\n \"**Note:** Unlike the [similar notebooks](https://shap.readthedocs.io/en/latest/example_notebooks/kernel_explainer/Census%20income%20classification%20with%20scikit-learn.html) from the SHAP package, this example shows you how to apply Kernel SHAP to data with categorical features. In the second half of this notebook, we also demonstrate Shapely values for interpreting predictions in multiclass classification tasks with more than 2 classes.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Load the raw census data\\n\",\n \"\\n\",\n \"Note these data contain categorical as well as numerical features. We first consider predicting the `class` column, a binary classification task. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:36.551005Z\",\n \"start_time\": \"2026-01-07T01:36:36.157964Z\"\n }\n },\n \"source\": [\n \"N_SUBSAMPLE = 500 # subsample datasets for faster demo\\n\",\n \"N_TEST = 50\\n\",\n \"\\n\",\n \"train_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame\\n\",\n \"train_data = train_data.sample(N_SUBSAMPLE, random_state=0)\\n\",\n \"test_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame\\n\",\n \"test_data = test_data.sample(N_TEST, random_state=0)\\n\",\n \"\\n\",\n \"label = 'class'\\n\",\n \"\\n\",\n \"y_train = train_data[label]\\n\",\n \"y_test = test_data[label]\\n\",\n \"X_train = pd.DataFrame(train_data.drop(columns=[label]))\\n\",\n \"X_test = pd.DataFrame(test_data.drop(columns=[label]))\\n\",\n \"\\n\",\n \"display(train_data.head())\"\n ],\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Loaded data from: https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv | Columns = 15 / 15 | Rows = 39073 -> 39073\\n\",\n \"Loaded data from: https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv | Columns = 15 / 15 | Rows = 9769 -> 9769\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \" age workclass fnlwgt education education-num \\\\\\n\",\n \"6118 51 Private 39264 Some-college 10 \\n\",\n \"23204 58 Private 51662 10th 6 \\n\",\n \"29590 40 Private 326310 Some-college 10 \\n\",\n \"18116 37 Private 222450 HS-grad 9 \\n\",\n \"33964 62 Private 109190 Bachelors 13 \\n\",\n \"\\n\",\n \" marital-status occupation relationship race sex \\\\\\n\",\n \"6118 Married-civ-spouse Exec-managerial Wife White Female \\n\",\n \"23204 Married-civ-spouse Other-service Wife White Female \\n\",\n \"29590 Married-civ-spouse Craft-repair Husband White Male \\n\",\n \"18116 Never-married Sales Not-in-family White Male \\n\",\n \"33964 Married-civ-spouse Exec-managerial Husband White Male \\n\",\n \"\\n\",\n \" capital-gain capital-loss hours-per-week native-country class \\n\",\n \"6118 0 0 40 United-States >50K \\n\",\n \"23204 0 0 8 United-States <=50K \\n\",\n \"29590 0 0 44 United-States <=50K \\n\",\n \"18116 0 2339 40 El-Salvador <=50K \\n\",\n \"33964 15024 0 40 United-States >50K \"\n ],\n \"text/html\": [\n \"\"\n ],\n \"text/html\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 25\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.716775Z\",\n \"start_time\": \"2026-01-07T01:36:45.656767Z\"\n }\n },\n \"source\": [\n \"ROW_INDEX = 0 # index of an example datapoint\\n\",\n \"single_datapoint = X_train.iloc[[ROW_INDEX]]\\n\",\n \"single_prediction = ag_wrapper.predict_proba(single_datapoint)\\n\",\n \"\\n\",\n \"shap_values_single = explainer.shap_values(single_datapoint, nsamples=NSHAP_SAMPLES)\\n\",\n \"\\n\",\n \"\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values_single, X_train.iloc[ROW_INDEX,:])\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" 0%| | 0/1 [00:00\"\n ],\n \"text/html\": [\n \"\\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 19\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Train binary classifier using AutoGluon\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.340927Z\",\n \"start_time\": \"2026-01-07T01:36:36.556718Z\"\n }\n },\n \"source\": \"predictor = TabularPredictor(label=label, problem_type='binary').fit(train_data, time_limit=20)\",\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"No path specified. Models will be saved in: \\\"AutogluonModels/ag-20260107_013636\\\"\\n\",\n \"Verbosity: 2 (Standard Logging)\\n\",\n \"=================== System Info ===================\\n\",\n \"AutoGluon Version: 1.4.1b20251106\\n\",\n \"Python Version: 3.12.6\\n\",\n \"Operating System: Linux\\n\",\n \"Platform Machine: x86_64\\n\",\n \"Platform Version: #26~22.04.1-Ubuntu SMP Wed Feb 19 06:54:57 UTC 2025\\n\",\n \"CPU Count: 96\\n\",\n \"Pytorch Version: 2.7.1+cu126\\n\",\n \"CUDA Version: 12.6\\n\",\n \"GPU Memory: GPU 0: 22.05/22.05 GB | GPU 1: 22.05/22.05 GB | GPU 2: 22.05/22.05 GB | GPU 3: 22.05/22.05 GB\\n\",\n \"Total GPU Memory: Free: 88.18 GB, Allocated: 0.00 GB, Total: 88.18 GB\\n\",\n \"GPU Count: 4\\n\",\n \"Memory Avail: 318.83 GB / 363.85 GB (87.6%)\\n\",\n \"Disk Space Avail: 176.64 GB / 1938.16 GB (9.1%)\\n\",\n \"===================================================\\n\",\n \"No presets specified! To achieve strong results with AutoGluon, it is recommended to use the available presets. Defaulting to `'medium'`...\\n\",\n \"\\tRecommended Presets (For more details refer to https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets):\\n\",\n \"\\tpresets='extreme' : New in v1.5: The state-of-the-art for tabular data. Massively better than 'best' on datasets <100000 samples by using new Tabular Foundation Models (TFMs) meta-learned on https://tabarena.ai: TabPFNv2, TabICL, Mitra, TabDPT, and TabM. Requires a GPU and `pip install autogluon.tabular[tabarena]` to install TabPFN, TabICL, and TabDPT.\\n\",\n \"\\tpresets='best' : Maximize accuracy. Recommended for most users. Use in competitions and benchmarks.\\n\",\n \"\\tpresets='best_v150': New in v1.5: Better quality than 'best' and 5x+ faster to train. Give it a try!\\n\",\n \"\\tpresets='high' : Strong accuracy with fast inference speed.\\n\",\n \"\\tpresets='high_v150': New in v1.5: Better quality than 'high' and 5x+ faster to train. Give it a try!\\n\",\n \"\\tpresets='good' : Good accuracy with very fast inference speed.\\n\",\n \"\\tpresets='medium' : Fast training time, ideal for initial prototyping.\\n\",\n \"Using hyperparameters preset: hyperparameters='default'\\n\",\n \"Beginning AutoGluon training ... Time limit = 20s\\n\",\n \"AutoGluon will save models to \\\"/home/ubuntu/workspace/code/autogluon/examples/tabular/interpret/AutogluonModels/ag-20260107_013636\\\"\\n\",\n \"Train Data Rows: 500\\n\",\n \"Train Data Columns: 14\\n\",\n \"Label Column: class\\n\",\n \"Problem Type: binary\\n\",\n \"Preprocessing data ...\\n\",\n \"Selected class <--> label mapping: class 1 = >50K, class 0 = <=50K\\n\",\n \"\\tNote: For your binary classification, AutoGluon arbitrarily selected which label-value represents positive ( >50K) vs negative ( <=50K) class.\\n\",\n \"\\tTo explicitly set the positive_class, either rename classes to 1 and 0, or specify positive_class in Predictor init.\\n\",\n \"Using Feature Generators to preprocess the data ...\\n\",\n \"Fitting AutoMLPipelineFeatureGenerator...\\n\",\n \"\\tAvailable Memory: 326473.57 MB\\n\",\n \"\\tTrain Data (Original) Memory Usage: 0.25 MB (0.0% of available memory)\\n\",\n \"\\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\\n\",\n \"\\tStage 1 Generators:\\n\",\n \"\\t\\tFitting AsTypeFeatureGenerator...\\n\",\n \"\\t\\t\\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\\n\",\n \"\\tStage 2 Generators:\\n\",\n \"\\t\\tFitting FillNaFeatureGenerator...\\n\",\n \"\\tStage 3 Generators:\\n\",\n \"\\t\\tFitting IdentityFeatureGenerator...\\n\",\n \"\\t\\tFitting CategoryFeatureGenerator...\\n\",\n \"\\t\\t\\tFitting CategoryMemoryMinimizeFeatureGenerator...\\n\",\n \"\\tStage 4 Generators:\\n\",\n \"\\t\\tFitting DropUniqueFeatureGenerator...\\n\",\n \"\\tStage 5 Generators:\\n\",\n \"\\t\\tFitting DropDuplicatesFeatureGenerator...\\n\",\n \"\\tTypes of features in original data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('int', []) : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\\n\",\n \"\\t\\t('object', []) : 8 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\\n\",\n \"\\tTypes of features in processed data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('category', []) : 7 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\\n\",\n \"\\t\\t('int', []) : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\\n\",\n \"\\t\\t('int', ['bool']) : 1 | ['sex']\\n\",\n \"\\t0.0s = Fit runtime\\n\",\n \"\\t14 features in original data used to generate 14 features in processed data.\\n\",\n \"\\tTrain Data (Processed) Memory Usage: 0.03 MB (0.0% of available memory)\\n\",\n \"Data preprocessing and feature engineering runtime = 0.06s ...\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\\n\",\n \"\\tTo change this, specify the eval_metric parameter of Predictor()\\n\",\n \"Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 400, Val Rows: 100\\n\",\n \"User-specified model hyperparameters to be fit:\\n\",\n \"{\\n\",\n \"\\t'NN_TORCH': [{}],\\n\",\n \"\\t'GBM': [{'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}}, {}, {'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'ag_args': {'name_suffix': 'Large', 'priority': 0, 'hyperparameter_tune_kwargs': None}}],\\n\",\n \"\\t'CAT': [{}],\\n\",\n \"\\t'XGB': [{}],\\n\",\n \"\\t'FASTAI': [{}],\\n\",\n \"\\t'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\\n\",\n \"\\t'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\\n\",\n \"}\\n\",\n \"Fitting 11 L1 models, fit_strategy=\\\"sequential\\\" ...\\n\",\n \"Fitting model: LightGBMXT ... Training model for up to 19.94s of the 19.94s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.83\\t = Validation score (accuracy)\\n\",\n \"\\t0.38s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBM ... Training model for up to 19.55s of the 19.55s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.85\\t = Validation score (accuracy)\\n\",\n \"\\t0.53s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestGini ... Training model for up to 19.01s of the 19.01s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0\\n\",\n \"\\t0.84\\t = Validation score (accuracy)\\n\",\n \"\\t0.38s\\t = Training runtime\\n\",\n \"\\t0.06s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestEntr ... Training model for up to 18.55s of the 18.55s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0\\n\",\n \"\\t0.83\\t = Validation score (accuracy)\\n\",\n \"\\t0.38s\\t = Training runtime\\n\",\n \"\\t0.05s\\t = Validation runtime\\n\",\n \"Fitting model: CatBoost ... Training model for up to 18.11s of the 18.11s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0\\n\",\n \"\\t0.85\\t = Validation score (accuracy)\\n\",\n \"\\t1.89s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesGini ... Training model for up to 16.21s of the 16.21s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0\\n\",\n \"\\t0.82\\t = Validation score (accuracy)\\n\",\n \"\\t0.39s\\t = Training runtime\\n\",\n \"\\t0.06s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesEntr ... Training model for up to 15.74s of the 15.74s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0\\n\",\n \"\\t0.81\\t = Validation score (accuracy)\\n\",\n \"\\t0.38s\\t = Training runtime\\n\",\n \"\\t0.06s\\t = Validation runtime\\n\",\n \"Fitting model: NeuralNetFastAI ... Training model for up to 15.28s of the 15.28s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.83\\t = Validation score (accuracy)\\n\",\n \"\\t0.38s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: XGBoost ... Training model for up to 14.88s of the 14.88s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0\\n\",\n \"\\t0.85\\t = Validation score (accuracy)\\n\",\n \"\\t0.28s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: NeuralNetTorch ... Training model for up to 14.60s of the 14.59s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.83\\t = Validation score (accuracy)\\n\",\n \"\\t0.98s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMLarge ... Training model for up to 13.60s of the 13.60s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.1/318.8 GB\\n\",\n \"\\t0.83\\t = Validation score (accuracy)\\n\",\n \"\\t1.82s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: WeightedEnsemble_L2 ... Training model for up to 19.94s of the 11.77s of remaining time.\\n\",\n \"\\tFitting 1 model on all data | Fitting with cpus=96, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\tEnsemble Weights: {'LightGBM': 1.0}\\n\",\n \"\\t0.85\\t = Validation score (accuracy)\\n\",\n \"\\t0.05s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"AutoGluon training complete, total runtime = 8.3s ... Best model: WeightedEnsemble_L2 | Estimated inference throughput: 35812.0 rows/s (100 batch size)\\n\",\n \"Disabling decision threshold calibration for metric `accuracy` due to having fewer than 10000 rows of validation data for calibration, to avoid overfitting (100 rows).\\n\",\n \"\\t`accuracy` is generally not improved through threshold calibration. Force calibration via specifying `calibrate_decision_threshold=True`.\\n\",\n \"TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\\\"/home/ubuntu/workspace/code/autogluon/examples/tabular/interpret/AutogluonModels/ag-20260107_013636\\\")\\n\"\n ]\n }\n ],\n \"execution_count\": 20\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Explain predictions\\n\",\n \"\\n\",\n \"SHAP is intended to explain how much each feature contributes to a particular prediction. For classification, \\\"how much\\\" is quantified in terms of the deviation between predicted probability of a particular class from a baseline reference value. In this example, we will explain predictions of the \\\" >50K\\\" class.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.390035Z\",\n \"start_time\": \"2026-01-07T01:36:45.388241Z\"\n }\n },\n \"source\": [\n \"target_class = \\\" >50K\\\" # can be any possible value of the label column\"\n ],\n \"outputs\": [],\n \"execution_count\": 21\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next we create a wrapper class around AutoGluon to allow it to be called for prediction inside of the `shap` package:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.442380Z\",\n \"start_time\": \"2026-01-07T01:36:45.439640Z\"\n }\n },\n \"source\": [\n \"class AutogluonWrapper:\\n\",\n \" def __init__(self, predictor, feature_names, target_class=None):\\n\",\n \" self.ag_model = predictor\\n\",\n \" self.feature_names = feature_names\\n\",\n \" self.target_class = target_class\\n\",\n \" if target_class is None and predictor.problem_type != 'regression':\\n\",\n \" print(\\\"Since target_class not specified, SHAP will explain predictions for each class\\\")\\n\",\n \" \\n\",\n \" def predict_proba(self, X):\\n\",\n \" if isinstance(X, pd.Series):\\n\",\n \" X = X.values.reshape(1,-1)\\n\",\n \" if not isinstance(X, pd.DataFrame):\\n\",\n \" X = pd.DataFrame(X, columns=self.feature_names)\\n\",\n \" preds = self.ag_model.predict_proba(X)\\n\",\n \" if predictor.problem_type == \\\"regression\\\" or self.target_class is None:\\n\",\n \" return preds\\n\",\n \" else:\\n\",\n \" return preds[self.target_class] \"\n ],\n \"outputs\": [],\n \"execution_count\": 22\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next, we must define the baseline reference value for SHAP. In this example, we aggregate all of the training examples with label \\\" <50K\\\" (the negative class in our binary classification task) as our baseline (and take a random subsample of these for efficiency).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.496908Z\",\n \"start_time\": \"2026-01-07T01:36:45.489375Z\"\n }\n },\n \"source\": [\n \"negative_class = \\\" <=50K\\\"\\n\",\n \"baseline = X_train[y_train==negative_class].sample(50, random_state=0)\\n\",\n \"display(baseline.head())\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" age workclass fnlwgt education education-num marital-status \\\\\\n\",\n \"26097 23 ? 146399 Bachelors 13 Never-married \\n\",\n \"32812 44 ? 256211 Assoc-voc 11 Married-civ-spouse \\n\",\n \"9888 27 Private 169557 HS-grad 9 Divorced \\n\",\n \"9538 45 Private 204057 Bachelors 13 Divorced \\n\",\n \"21531 28 Private 257283 HS-grad 9 Never-married \\n\",\n \"\\n\",\n \" occupation relationship race sex \\\\\\n\",\n \"26097 ? Not-in-family White Male \\n\",\n \"32812 ? Husband Asian-Pac-Islander Male \\n\",\n \"9888 Machine-op-inspct Not-in-family White Male \\n\",\n \"9538 Adm-clerical Unmarried White Female \\n\",\n \"21531 Transport-moving Own-child White Male \\n\",\n \"\\n\",\n \" capital-gain capital-loss hours-per-week native-country \\n\",\n \"26097 0 0 55 United-States \\n\",\n \"32812 0 2129 40 Poland \\n\",\n \"9888 6849 0 40 United-States \\n\",\n \"9538 0 0 40 Germany \\n\",\n \"21531 0 0 40 United-States \"\n ],\n \"text/html\": [\n \"| \\n\",\n \" | age | \\n\",\n \"workclass | \\n\",\n \"fnlwgt | \\n\",\n \"education | \\n\",\n \"education-num | \\n\",\n \"marital-status | \\n\",\n \"occupation | \\n\",\n \"relationship | \\n\",\n \"race | \\n\",\n \"sex | \\n\",\n \"capital-gain | \\n\",\n \"capital-loss | \\n\",\n \"hours-per-week | \\n\",\n \"native-country | \\n\",\n \"class | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 6118 | \\n\",\n \"51 | \\n\",\n \"Private | \\n\",\n \"39264 | \\n\",\n \"Some-college | \\n\",\n \"10 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Exec-managerial | \\n\",\n \"Wife | \\n\",\n \"White | \\n\",\n \"Female | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \">50K | \\n\",\n \"
| 23204 | \\n\",\n \"58 | \\n\",\n \"Private | \\n\",\n \"51662 | \\n\",\n \"10th | \\n\",\n \"6 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Other-service | \\n\",\n \"Wife | \\n\",\n \"White | \\n\",\n \"Female | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"8 | \\n\",\n \"United-States | \\n\",\n \"<=50K | \\n\",\n \"
| 29590 | \\n\",\n \"40 | \\n\",\n \"Private | \\n\",\n \"326310 | \\n\",\n \"Some-college | \\n\",\n \"10 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Craft-repair | \\n\",\n \"Husband | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"44 | \\n\",\n \"United-States | \\n\",\n \"<=50K | \\n\",\n \"
| 18116 | \\n\",\n \"37 | \\n\",\n \"Private | \\n\",\n \"222450 | \\n\",\n \"HS-grad | \\n\",\n \"9 | \\n\",\n \"Never-married | \\n\",\n \"Sales | \\n\",\n \"Not-in-family | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"2339 | \\n\",\n \"40 | \\n\",\n \"El-Salvador | \\n\",\n \"<=50K | \\n\",\n \"
| 33964 | \\n\",\n \"62 | \\n\",\n \"Private | \\n\",\n \"109190 | \\n\",\n \"Bachelors | \\n\",\n \"13 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Exec-managerial | \\n\",\n \"Husband | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"15024 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \">50K | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 23\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"AutoGluon predictions will be interpreted in terms of their difference from the average prediction across the baseline feature-values (i.e. the prediction for a new datapoint will explained by quantifying how much each feature contributes to this prediction differing from the average prediction over the rows in our baseline DataFrame). With the target class and baseline we chose in this example, SHAP will indicate **why** a particular predicted probability of the \\\" >50K\\\" class deviated from the average predicted probability of the \\\" <=50K\\\" class across the training data. The **why** is reported in terms of how each individual feature contributed to the overall deviation of this prediction from the baseline value (ie. the Shapely values).\\n\",\n \"\\n\",\n \"We can now create a `KernelExplainer` which will return Kernel SHAP values to explain particular AutoGluon predictions.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.564358Z\",\n \"start_time\": \"2026-01-07T01:36:45.542921Z\"\n }\n },\n \"source\": [\n \"ag_wrapper = AutogluonWrapper(predictor, X_train.columns, target_class)\\n\",\n \"explainer = shap.KernelExplainer(ag_wrapper.predict_proba, baseline)\\n\",\n \"print(\\\"Baseline prediction: \\\", np.mean(ag_wrapper.predict_proba(baseline))) # this is the same as explainer.expected_value\\n\",\n \"\\n\",\n \"NSHAP_SAMPLES = 100 # how many samples to use to approximate each Shapely value, larger values will be slower\"\n ],\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Baseline prediction: 0.11633540893904865\\n\"\n ]\n }\n ],\n \"execution_count\": 24\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Let's first explain a prediction for a single datapoint from the training data.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:45.603900Z\",\n \"start_time\": \"2026-01-07T01:36:45.598903Z\"\n }\n },\n \"source\": [\n \"shap.initjs()\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"| \\n\",\n \" | age | \\n\",\n \"workclass | \\n\",\n \"fnlwgt | \\n\",\n \"education | \\n\",\n \"education-num | \\n\",\n \"marital-status | \\n\",\n \"occupation | \\n\",\n \"relationship | \\n\",\n \"race | \\n\",\n \"sex | \\n\",\n \"capital-gain | \\n\",\n \"capital-loss | \\n\",\n \"hours-per-week | \\n\",\n \"native-country | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 26097 | \\n\",\n \"23 | \\n\",\n \"? | \\n\",\n \"146399 | \\n\",\n \"Bachelors | \\n\",\n \"13 | \\n\",\n \"Never-married | \\n\",\n \"? | \\n\",\n \"Not-in-family | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"55 | \\n\",\n \"United-States | \\n\",\n \"
| 32812 | \\n\",\n \"44 | \\n\",\n \"? | \\n\",\n \"256211 | \\n\",\n \"Assoc-voc | \\n\",\n \"11 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"? | \\n\",\n \"Husband | \\n\",\n \"Asian-Pac-Islander | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"2129 | \\n\",\n \"40 | \\n\",\n \"Poland | \\n\",\n \"
| 9888 | \\n\",\n \"27 | \\n\",\n \"Private | \\n\",\n \"169557 | \\n\",\n \"HS-grad | \\n\",\n \"9 | \\n\",\n \"Divorced | \\n\",\n \"Machine-op-inspct | \\n\",\n \"Not-in-family | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"6849 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \"
| 9538 | \\n\",\n \"45 | \\n\",\n \"Private | \\n\",\n \"204057 | \\n\",\n \"Bachelors | \\n\",\n \"13 | \\n\",\n \"Divorced | \\n\",\n \"Adm-clerical | \\n\",\n \"Unmarried | \\n\",\n \"White | \\n\",\n \"Female | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"Germany | \\n\",\n \"
| 21531 | \\n\",\n \"28 | \\n\",\n \"Private | \\n\",\n \"257283 | \\n\",\n \"HS-grad | \\n\",\n \"9 | \\n\",\n \"Never-married | \\n\",\n \"Transport-moving | \\n\",\n \"Own-child | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ]\n },\n \"execution_count\": 26,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 26\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"**Note:** If you ever see message \\\"Visualization omitted, Javascript library not loaded\\\", simply re-running the corresponding cells should show the plots. If you don't have Javascript, you may stil produce plots by adding argument: `force_plot(..., matplotlib=True)`.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We can also plot Kernel SHAP explanations aggregated across many predictions, say over all the datapoints in the test data.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:47.886616Z\",\n \"start_time\": \"2026-01-07T01:36:45.724249Z\"\n }\n },\n \"source\": [\n \"shap_values = explainer.shap_values(X_test, nsamples=NSHAP_SAMPLES)\\n\",\n \"shap.force_plot(explainer.expected_value, shap_values, X_test) # if you do not have add argument \"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" 0%| | 0/50 [00:00\"\n ],\n \"text/html\": [\n \"\\n\",\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ]\n },\n \"execution_count\": 27,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 27\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A summary plot is an even better way to see the relative impact of all features over many datapoints. Features are sorted by the sum of their SHAP value magnitudes across all samples.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:48.155319Z\",\n \"start_time\": \"2026-01-07T01:36:47.934133Z\"\n }\n },\n \"source\": [\n \"shap.summary_plot(shap_values, X_test)\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\"\n ],\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAv4AAAK8CAYAAACX9bveAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAA+zlJREFUeJzs3Xd4FNX6wPHvbEvvtBQCoXcRg4AVxUJXOmJBQIqC117utWG5v3vxWrAgIhcpKlcBkaaAiIBggVAF6YQQSkJJ35St8/tjySab3YQ0Ut/P8+R5MmfOzDkz2945c84ZRVVVFSGEEEIIIUSdpqnuCgghhBBCCCGuPgn8hRBCCCGEqAck8BdCCCGEEKIekMBfCCGEEEKIekACfyGEEEIIIeoBCfyFEEIIIYSoByTwF0IIIYQQoh6QwF8IIYQQQoh6QAJ/IYQQQggh6gEJ/IUQQgghRL0zffp0/P39r7guISEBRVFYtmxZmfZf3u2uJl11V0AIIYQQQoiaKjw8nN9//502bdpUd1UqTAJ/IYQQQgghiuHl5UXPnj2ruxqVQrr6CCGEEEIIUQxPXXbMZjN/+9vfCA0NJTg4mMmTJ7N48WIURSEhIcFl+7y8PKZNm0ZISAjh4eE8++yzWK3WKj4KBwn8hRBCCCFEvWW1Wt3+7HZ7idu8+OKLzJkzhxdeeIFvvvkGu93Oiy++6DHvSy+9hEajYcmSJUyZMoV3332X//73v1fjUK5IuvoIIYQQQoh6KTs7G71e73Gdn5+fx/TU1FRmz57Nyy+/zAsvvADA3XffzR133MHp06fd8vfo0YMPP/wQgDvvvJNNmzaxbNkypkyZUklHUXoS+Ash6j2LxcL8+fMBGDduXLE/AkIIIWoYZWjJ69XlJa728fHhl19+cUv/7LPPWLx4scdt9u/fT15eHoMHD3ZJv+eee9i4caNb/rvuustluUOHDvz8888l1/sqkcBfCCGEEELUSxqNhtjYWLf0NWvWFLtNUlISAA0bNnRJb9Sokcf8wcHBLssGg4G8vLwy1rRySB9/IYQQQghRSylX+Kt84eHhAFy8eNEl/cKFC1elvMokgb8QQgghhBCl1KlTJ7y9vVm5cqVL+ooVK6qnQmUgXX2EEEIIIYQopbCwMB599FH++c9/4u3tTdeuXVm6dClHjx4FHN2HaqqaWzMhhBBCCCFKVPVdfQD+/e9/M2nSJP71r38xYsQILBaLczrPoKCgq1ZuRSmqqqrVXQkhhKhOMquPEELUUsrwktery0peX4kefPBBtm3bxsmTJ6uszLKSrj5CCCGEEKKWunqt+iXZsmULv/76K9dddx12u501a9bw1Vdf8d5771VLfUpLAn8hhBBCCCHKwN/fnzVr1jBjxgxyc3OJiYnhvffe48knn6zuqpVIAn8hhBBCCCHK4LrrruO3336r7mqUmQT+QgghhBCilqqerj61lczqI4QQQgghRD0ggb8QQgghhBD1gAT+QgghhBBC1AMS+AshhBBCCFEPyOBeIYQQQghRS8ng3rKQFn8hhBBCCCHqAWnxF0IIIYQQtZS0+JeFtPgLIYQQQghRD0jgL4QQQgghRD0gXX2EEEIIIUQtJV19ykJa/IUQQgghhKgHpMVfCCGEEELUUtLiXxbS4i+EEEIIIUQ9IC3+QgghhBCilpIW/7KQFn8hhBBCCCHqAQn8hRBCCCGEqAekq48QQgghhKilpKtPWUiLvxBCCCGEEPWAtPgLIYQQQohaSb1Ci7/cD3AlLf5CCCGEEELUAxL4C1GHGY1Gjh8/jtFovOK6suStSsWVXZ11EjVTWd8TFX0PyXtQCFHbSFcfIeqo7du3s2rVKmw2G1qtlsGDB9OjRw+P6zp37sz+/ftLlbfwuuo6huqsk6iZyvqeqOh7SN6DQojaSFr8haiDjEajMygBsNlsrF69GqPR6HHd3r17S503f111HcP58+errU6iZirr+7Si7+vq/FwIIURFSOAvRB2UnJzsDEryWa1WkpOTPa4rqqS8+euutuLKPnz4cLXVSdRMZX2fVvR9XZ2fCyFEUcoV/kRh9S7wP3fuHLGxscyZM+eq7H/QoEFMmjTpquxbiNJq0qQJWq3WJU2n0xEeHu5xXVEl5c1fd7UVV3b79u2rrU6iZirr+7Si7+vq/FwIIURF1LvA35OsrCzmzJnDzp07q7sqHs2ZM4fNmzdXeD+LFy9m9erVFa+QqPH8/f0ZPHgwOp1jGI9Op2PQoEH4+fl5XNe1a9dS581fV13H0KhRo2qrk6iZyvo+rej7ujo/F0IIURGKqqpqdVeiKqmqitlsRqvVOr+0z507x+DBg5k4cSKTJ0+u0P4HDRpEeHg4n332WWVUF4DY2FgGDhzI9OnTK7Sfq1E3UbMZjUaSk5Np0qQJ/v7+Ja4rS96qVFzZlVkni8XC/PnzARg3bhx6vb5C+xPVo6zviYq+h6rzcyGEcLArY0tcr1EXVlFNaod6M6tPdnY2fn5+KIqCl5dXdVdHiCrh7+9Pq1atSrWuLHmrUnFlV2edRM1U1vdERd9D8h4UQtQ2VRL4r169mtdff51PPvmEffv2sXLlStLS0mjVqhXPPvssnTt3ZteuXXzyySccOXIEPz8/RowYwSOPPOLcxx9//MHKlSs5ePAgly5dQq/X07FjR8aPH891113nUt6kSZNISkpi9uzZfPjhh+zcuZPMzEx27tzp1rq/c+dOpkyZAsDcuXOZO3cuAOHh4c5uMUuXLmXz5s3Ex8eTlpZGUFAQ119/PY8++igRERHlPi8mk4kFCxawfv16zp8/j16vp3Hjxtxwww088cQTzroCrFmzhjVr1ji3ze+W9OOPP7J27VqOHj1Kamoqvr6+dO3alSlTptC6dWtn/tjYWACSkpKc/wOsWrWKiIiIYu8q5L92n376qXO7jIwM/vvf//LLL79w8eJFfHx8CA8P56677uKhhx4q9/kQBS2IAQEBZGVlFduSWJqWxrLmAdzK9pSWv6/StnYWd0xXSvdUVkXqUV+VdD6vdBelpGXgineMKuu1Ket7rbg6lKbO5a1PWe6glXRMV+P9bDQaOXnyJAAxMTEVOnYhah4ZwFsWVdri//HHH2Oz2Rg9ejRWq5Uvv/ySadOm8frrr/Pmm28yZMgQ+vXrx4YNG/j000+JiIigf//+gCMAzcjIoH///jRu3JgLFy6wcuVKHnvsMT799FOuvfZal7JycnKYPHkyXbp04bHHHiM1NdVjnWJiYnj66ad57733uO2227jtttsA8PX1deb58ssv6dSpE6NGjSIoKIgTJ06wYsUK4uLi+PrrrwkODi7X+ZgxYwarVq1iwIAB3H///dhsNk6fPk1cXBwAISEhvPHGG7z66qtce+21DBkyxG0fS5YsISgoiCFDhtCgQQPOnDnDd999x4QJE/jyyy+Jjo4G4I033uC9994jODiY8ePHO7cPCQkpc71ffPFFdu/ezbBhw2jdujUmk4mTJ0+ya9cuCfwroPC84Pk8zQ9emvnDy5pHo9GgqiqFe/4pioKiKNjtdrf6AKWaw7y4Yyr83IDi0j2VVd561Fee3gfg+Zxd6dkOhZcLvzdK87pV5LUp7Xz5VzrW0tS5NPUrzTm90r6r8vkU27dvZ+XKlc7PsaIoXHPNNeU6diFE7Velgb/NZmPBggXO/rMxMTE888wzvPDCC8yfP58OHToAcM899zBw4ECWLl3qDPxffvllfHx8XPY3bNgwRo4cyfz5890C/4yMDIYNG8Zjjz1WYp3CwsLo3bs37733Hq1atXKWV9jXX3/tVvYtt9zCY489xsqVKxk7tuT+ZcXZvHkzN9xwA6+//rrH9T4+PvTv359XX32VyMhIj3X76KOP3Oo2YMAAxowZw+LFi3nxxRcB6N+/P7NnzyY0NNTjfkrLaDQSFxfH8OHDef7558u9H+Gq6Lzg+fLnB+/YsaOzlc7T/OH56z3tqzR5Cgf3+YpeCOTva9WqVaiq6tzG0/6vdEx79+51K69oen5Z+f97qkfhdcXVo77y9D4o7rVr3ry5x2c75Cu6XPi9UZrXrbyvTWney6U91ivVuTT1K+05LWnfxR2Tp9egou/n/LIKf75VVS3XsQsh6oYqndVn+PDhLoPm8oP1Tp06OYN+wNmNJzEx0ZlWOLjNyckhPT0drVZLp06d+OuvvzyW9+CDD1ZKvfPLttvtGI1G0tPTadOmDf7+/hw4cKDc+/X39yc+Pp7jx49XuG6qqjrrFhISQrNmzSpUt+J4eXlhMBg4cOAA586dq/T9V6bU1FRMJpNz2Wg0kpWV5Vw2m82kpKS4bJOUlFTicnJyskswXFlllDS3vtVq5fTp087yPc0ffvToUZc6XmmO8YMHD15xLv/i2Gw2twsFq9XKwYMHXdIqUkbhsorbh6d1hc8VlO/1OH/+vMvy1XrNr3YZf/31l8cLJk+vnadnI1REca/NyZMny3QcpXkvp6amcvr06VIda0kK77e418NTfUpTTuHPR3HHtGfPnhKPtTzvq9J+BvPLqSnvXSmjdpdR9WQe/7Ko0hb/yMhIl+XAwEAAj/3kAwMDycjIcC6fOXOGWbNm8ccff7i8acFx67KokJAQAgICKqPaxMXFMXfuXP766y+3N3fRuhRlNBrJy8tzq5tWq+Xpp5/mtddeY/To0URGRhIbG8vNN9/MLbfcgkZTumuyw4cP8+mnn7Jr1y5yc3Nd1hU935VBr9fz9NNP8+677zJ48GBatGhBbGwsvXv35vrrr6/08ioiNDTUZbloa5bBYCAsLMwlreg83EWX8/sJV3YZRqMRrVbr8Udap9M5u2zlzx9eOJ9Op6Nt27YudfSUp3C5HTp0YOXKleUK9rRarUsLZ/7+O3bs6JKvImUULgvcW/yLW1f4XEHpXw+LxeJcbty4scv6q/WaX+0yOnbs6LGLlKfXrn379vz444+VFvwX99q0aNHCZcrLKx1Had7LoaGhGAwGt3yejrUkhfdb3OvhqT6lKafw56O4Y+rWrRtbt24t9ljL874q7Wcwv5ya8t6VMmp3GaJmq9IW/+KC2Ss9TCgnJ4eJEyfy+++/M3r0aGbMmMHHH3/MrFmz6N69u1t3BABvb+9KqfNff/3FtGnTSElJYdq0abz77rvOsoOCgq74o/LOO+/Qt29fl7/8FsXevXuzatUq3njjDbp3705cXBzPPvsskydPdglEipOcnMykSZM4cuQIEyZM4J133nHWrUWLFmVq7SqOpx+M4cOHs3r1al5++WXatm3Lxo0beeyxx/j73/9e4fLqq6LzgucrOj94aeYPL08ejUbj9vn0lKbT6Rg8eDD33HPPFecwL+mYCj83oLj0/LKK20fRdTKXuitP74PiXjtPz0Yo+loUXi783ijN61be16a08+WX5livVOfS1K+057SkfVfl8ynyyyr8OdZoNOU6diFqKhWlxD/hqlZM57ljxw4uXrzIq6++6hxIlW/27NkV3r+nOwb51q1bh81m48MPP3RpQc/Nzb1iaz/AQw89RL9+/VzSCl9dBwUF0b9/f/r374+qqnz00UcsWrSILVu2cMcdd5S4702bNpGTk8N7773nMlMPOMY4GAyGUh9nUFCQyx2WfGfPnvWYv0GDBtx7773ce++92Gw2Xn31VdavX88DDzzg1vIrSqdHjx507NjxirP6FM5X3Iwc5ckDZZvV50r7v9IxlWVWn5LOS2nqUV8V9z7wlOYpb0Vn9amM16Y07+XSHmtp6lxZ57SkfRe3j9Iea1nk71Nm9RFCQC0J/PPvCBRt2f/jjz8qpR97fj/5zMzMUpf9+eefl6pFvUWLFrRo0cIt3WazkZOT49IdSVEUZ5eNwkG4r6+vx6A8vxWnaN2+++47UlJS3G7P+fj4eDxGgOjoaPbv309eXp7zbklmZqZzkF6+/G5Lhe+oaLVaWrduzfr164vdvyidwvOCF+1yUly+iuZp0qSJMwDwVHZx9SntHObFHVNp0ourZ+FAReZSL1lx57M0z0a40vKVngNRWa9Ned5rxaWV9tkVFS3nSvuuyudT+Pv707lz56tejhDVQ1r1y6JWBP5du3YlLCyMmTNnkpSURKNGjTh69Cg//PADrVq1qtDgWIDg4GCaNm3Kjz/+SFRUFKGhofj4+HDLLbfQu3dvFi9ezBNPPMGQIUPQ6/Vs376d48ePl3saT3B0X+rbty+33HILbdu2JSQkhHPnzrFs2TICAwO55ZZbnHk7derEjh07WLBgAU2aNEFRFO6++25uvPFGPvroI1599VVGjhxJQEAA+/bt47fffiMqKsqtm07nzp1ZuXIls2fPJiYmBkVRuOWWW/Dx8WHkyJG88sorTJkyhf79+5OVlcWKFSsIDw93Gfhz6tQpJk2axG233UbLli0JCAggISGBZcuWERkZ6Ta7kqjZrsb0gVdDbamnEEIIUZPVisA/ICCAjz/+mA8//JBvvvkGm81Gu3bt+OCDD1i5cmWFA3+AN998k/fee49Zs2aRl5dHeHg4t9xyC127duXtt9/mv//9L59++ileXl5cf/31fPbZZ0ycOLHc5Xl7e3PfffexY8cOduzYQU5ODg0aNOCWW25h3LhxNGzY0Jn3xRdfZMaMGcyfP5/s7GwA7r77bqKiovjwww+ZNWsW8+fPR6PRcM011zBnzhzefvttt5H4jz32GBkZGSxdupSsrCxUVWXVqlX4+PjQr18/Ll68yJIlS3j//feJjIzkkUceQaPRuNxVady4MYMHD2bXrl1s3rwZi8VCw4YNGTJkCGPHjq20sRXi6ivtVInVrbbUUwghhKjpFNXTyFghRJ13/Phx/vvf/7qlP/LIIzWqC0BV1NNisTB//nwAxo0b5zLtsBBCiJrLqkwocb1OnVdFNakdqnRWHyFEzZE/rWBhRadKrAlqSz2FEEKImk4CfyHqqdJOlVjdaks9hRBCVAd5gFdZ1Io+/kKIq+NqTB94NdSWegohhBA1mQT+QtRztWVav9pSTyGEEKKmksBfCCGEEELUSvJ03rKRPv5CCCGEEELUA9LiL4QQQgghailp8S8LafEXQgghhBCiHpDAXwghhBBCiHpAuvoIIYQQQohaSQb3lo20+AshhBBCCFEPSIu/EEIIIYSopaTFvyykxV8IIYQQQoh6QAJ/IYQQQggh6gHp6iOEEEIIIWolGdxbNtLiL4QQQgghRD0ggb8QQgghhBD1gAT+QgghhBBC1APSx18IIYQQQtRK0se/bCTwF0LUeQd/S+fEnkxCw7247u4GePtpq7tKQgghRJWTwF8IUaf9tPAsW75Jdi7v3nCJRz9sj04vPR2FEELUL/LLJ4SosywmO7+tuOCSduFUHof/yKimGgkhhKhcyhX+RGES+Ash6iyL2Y7FZHdLz8m0VkNthBBCiOolgb8Qos7yDdAR2cbXJU1nUGjXM6iaaiSEEKIyqSgl/glXEvgLIeqs9AtmzifkuqTFdA4gMMxQTTUSQgghqo8E/kKIOuuvbWlYzapLWvy+LGxWtZgthBBCiLqrRgT+kyZNYtCgQdVdjVJbvXo1sbGx7Ny5s7qrIkStYjPbyMswO5dzs6we++CXlsVkJ9fo2l/faFbJNDkCe52+4DavxmZHa7Gh1StYcizYrXZyU03lLltcffaMPNQc85UzCiHqMRncWxYynWcxdu7cya5duxgzZgwBAQHVXR0har09nx9n19zjWLKtNLw2lIxGoST8lY3eS8MNQxpxx0ORZdrfhgVn+X3lBSwmO62vC+TeZ2J4Pk7h8/0qdhVGtFX4+KZQ/L9OInjXOSJOp6C1qRgDvVlww2k03jpsZjthbQK49Y3OV+moRXnYM/PIHPsdppWHwUuL75Tu+L/XF0WRH3EhhKiIGtHiXxPt2rWLuXPnkpWV5bauf//+/Prrr3Tr1q0aaiZE7XNuZwp/zDyMJdvROn80wU7CX9mAo9V+y9fJHPo9vdT7O7AtjV+WJDvvFhzblclT89KZs0/FYgebCl8fVnnvgMKI/oE0TbiE1ua4C+CVY0ZVFGxmx7YpR7P4+R/7KvFoRUUZX9qIacUhUFXIs5Iz83fyFu6t7moJIWogGdxbNhL4l4NWq8XLywuNRk6fEKWR+KvrXPomb/fBtcd2lX5u/eO7Mt3Sfk13fxrvugSV3LiLLml5fjoo0nKcHp+NmiWf55rCvO6YW5rJQ5oQQoiyqfAvndls5vPPP2fkyJHccMMN9O7dm6eeeorDhw+75c3MzOStt96iT58+3HTTTUyaNIlDhw553G9sbCzTp093Sy+uf73RaGTWrFkMHz6cG264gT59+jBhwgTWr1/vzJOQkMC///1vRo4cyS233MKNN97IAw88wIoVK1z2NX36dObOnQvA4MGDiY2NJTY2ljlz5pRYh/T0dGbMmMGAAQPo2bMnAwYMYMaMGaSnp3s8hri4OL744gvuueceevXqxdChQ1mzZo3H8+HJnDlziI2NJSEhgVmzZtG/f3969erFfffdx7Zt21zy7ty5k9jYWFavXu22n+nTpxMbG+uSlj/u4ty5czz77LP07t2b2267jenTp5OTk4Pdbufzzz9n8ODB3HDDDdx///3s3bu31HUXNcO2MyorjtkxmitvsKvZprI23s6GBDs2u4rVrnLG2zGlpl2jkOvjjcbm3q9fo4G/fk0jN8uKzapybGcGx3dnYrvcUp9+ysiJH8+RlZRLaLiX2/aROptbWkOLBXu4o+ysAG9SGgag2MAr14qP0Yxid+xb76PBOx4MxzSoqudzYTyaQdLyU+Sezi722LN/PUfGihPYsirWL92eZSL74+0Y39qM7Wwmqqpi/iWBvFWHsV/IwrZiL7Zv92BbthvTkn3kfXcQe0ZesfuznUonb+kBTBuOk7f0ALbEdM8ZVRV++QtW7YAcD+MfzlyCZb/BkbOlO5CkVEf+g6dLzKZeMmL7djf2PY582lahbnk0oT5YU3LJ+PY4ObsuuK2vcrtOwLe/Q4r7XeGaTLXZsf90CNsPB1BNluqujhCiilWoj7/VauXxxx/nzz//pH///owcORKj0ch3333HhAkTmDt3Lh06dHDmnTZtGgcPHqR///507tyZo0eP8thjjxEUVLE5tbOyspgwYQLx8fH06dOH4cOHY7PZOHLkCNu2bePuu+8GHMHv7t27uemmm4iIiCAvL4+ffvqJt956i7S0NMaNGwfA0KFDyc7OZtOmTTz99NMEBwcD0Lp162LrYDQaGT9+PKdPn2bw4MG0a9eOI0eOsGzZMuLi4li4cCF+fn4u28yaNQuTycTQoUMxGAwsW7aM6dOnExUVRdeuXUt9/NOnT0en0/HAAw9gsVj43//+x7PPPsvy5cuJiIgo28ksJDc3l0cffZRu3bo5X7tVq1ZhMpkIDg7mwIEDjBw5EqvVypdffsnTTz/N6tWr3Y5T1Dx5VpUBy+38nOgIckO8Yf0wLd3DK3ZbNDFTpfc3Nk5ebrxvHQKY7CRcaMjTDUIwGAyonu6U2e1sX3OJ7WsuofdS8PbTkZXqCEoaNPUmto2GPxedABUUrULXSW3Q6hVsloIgfWzDPOLSzGR5Oe4m+FisNFsbz5JsE/43t8ao16PY7LTfl0iT045gzaZRSG3kS9ixbDT7HBcTcX/8RI+1d6Lz1zv3feTVPZx4+wDgKL/DzOtpNrFNQfVNVhIGrca44XLgGuJF8x8G49czvMzn0LL9NKm3zgeTo1tU9is/o20Zgu1EmqN8RcVfTUOLDStajIQACkqgF8Fr7sdwc3OX/eV8+AdZT60Fe6ELGq2GgJn98J3WoyAtOw/ufgN+vdxo0zAQNrwG18Q4luesh6lzIf+i7e9D4f8eKP5AFm2CCZ+A9fIF2RMDYOYEt2y2VfuwjpoHeY7XW/NgD7zHXod5/XEoVOX02Xs49d8E1MuvedCo1kQv7ouiqeJb+aoKY96Hry83rvgYYNlz0P+6qq1HOagpRiy3zUTdf/nCLToUw6YnUVo0rN6KCSGqTIUC/2+++YZdu3bx0Ucf0atXL2f68OHDGTVqFDNnzuSzzz4DYNWqVRw8eJCJEycyefJkZ96YmBjee+89wsPL/gOZb9asWcTHx/OPf/yDoUOHuqyz2wtaFgcMGMDw4cNd1o8ZM4YpU6awYMECHnzwQXQ6HV26dKFVq1Zs2rSJ3r17lyp4XrhwIYmJibzwwguMGDHCmd6mTRvefvttFi1axKOPPuqyjdlsZtGiRej1jgCjT58+3HPPPSxZsqRMgX9wcDDvv/++c+BbbGwsY8eOZfny5UybNq3U+ykqPT2dhx56iIceesiZlpWVxU8//US7du2YP38+Op3jLRQTE8MzzzzDunXrGDZsWLnLFFVj0V+qM+gHSMuDZ7bY+GV0xcb7v/6b3Rn0AxxLA6yAVoMxKICQ4loYC10MWEwqlkL50k4a+fP3gu46qk1l79yj2Bs1ctnu5I8XmGK7yOGGQdgVhXYXM/C5HHQaL3/GGiZnEJSe49xGa1dpkJzt0gs0Iy6FxP8eo8WTjkaL7OOZnPjPAZfyD7+wk4iRzdEHOS4y0r864gz6AWxpJpKe3kqr30aWfMI8yJy82hn0O/d3OegHUFWFXPzxJwMdNgzkYcYHNdNE1t9+IGzPY8689pQcsp7/0TXoB7DZyXr+R7zv74ImxMeR9t+fCoJ+gIuZ8Nwi+PE1yMyBZxYWBP0A//4OHr4d2nj4fswxwROfFwT9AB98D+NuL7iQAFS7HevUb5xBP4D9i+3kHc1zCfoBrOidQT9AxjfHyBrbnsB+zd3Lv5rW7SkI+gFyzTDtv3Cim1sXsprG9u5PBUE/QGIq1tfWoP9iXPVVSghRpSrU1Wft2rU0b96c9u3bk56e7vyzWq306NGDffv2kZfnuP28efNmtFot999/v8s+hg8fXqEWYrvdzo8//khMTIxb0A+49MP38fFx/m8ymUhPTyczM5OePXuSnZ1NQkJCueuxefNmQkJCGDJkiEv60KFDCQkJYdOmTW7bjBgxwhn0AzRq1Ijo6GhOny75tnhRo0ePdpntomPHjvj6+pKYmFjGo3Cl1WoZNWqUS1rXrl1RVZVhw4Y5g36Aa6+9FqDMdb+aUlNTMZkKuisYjUaXwdpms5mUlBSXbZKSkkpcTk5OdukKUlvL2J7o3l1l34WCMstbxv5LHrrJKAoGu734oP8KdBarW5pqVdFZXdOtZhUvm51rktO4NinVGfQX5md07w7jKVTL2l8QaCduO+kWhNpybOTEO86P0Wgkc6fruQDI3XfJ+X9ZXnPrYdcxCZ7YKBjPoKXgOK1/nncpw3r0kttFREEFLeT+VVAPy+4Tblns+046/jmR7LgjUJiqkvbLXs/HcfoSpLu/x/J2HHV9X53PgDNpbvmsR1Pc0jS4dw/L23ep6j+D+0+51YOT50k95dr9qSZ8zouWYdqd4FZ1696C7+ya+n0lZdSuMqqaDO4tmwo17508eRKTycQdd9xRbJ709HSaNGnC2bNnadCgAf7+/i7rDQYDkZGRHmfPKY384L3wHYfi5OTk8Nlnn7FhwwbOnz/vtj4z033AYGmdO3eO9u3buwTDADqdjujoaI9jHiIj3acvDAoKIjk52bmckZGBxeIaMDVo0MBlOSoqyuN+MjJKP1jSkwYNGuDl5dqXOjAwEMDtLkh+ekXLrEyhoa79hD2998LCwlzSit55KrrcpEmTOlHGXa39+PyIayB1c1TBRXJ5y7gp0kZccpEo2a5i1mpJ9vWmSU7x/dCLYzHoQaO4tFprvLVYdXqXfF6+Gkw5JT8TIDPYlyZn3QPNokJuaOT8P+au1iTo9qMWeuiXPtSAf/tgwHGubH2akzn7oMs+/G4u+IyU5TXX94zCssVDcFmIjoJg3lroa1x/czOXMtRAM0qgF2qm+4+yEuyNb7eC7w79bZ1h0RaXPJpbOjr+aR8FYQGu/dl1WkL69XDJ7zyOFo0hPASSCp1rRcH79mvwLjQ9sld4COZ2TVAPJ7vsR399FKb1rhciNg/tVH43R+BX1Z/Bm9q71YMuzQht7vo9XBM+50XL8LqtPbb1R1zSdLcWdFmrqd9XUkbtKkPUbBWex79Vq1Y89dRTxa4PCQmpaBEubDb3VrzSeumll9i2bRtDhgyhW7duBAUFodFo+PXXX1m8eLFLt6CqUNysQIWvpJ977jl2797tsr7ooOLS7Kek+a+LO6clzVpUmjJFzTWircLGRIV5l+e87xAGH95e8VltXumlYfd5G1vOOJbvbg52k8KGkyprWkXywPFEvHMuX8iqKigKOosVg8lErr8vKgrBjfT4Bes5e9TRJafl9SF069KYP949iCXHhleQnlte6cIfv2VzZIfjQrNZR38aRHqx60f3lmIffy2+QTpSzppIaRTI+bRsGp1LRwHsWgWNTQWtAjYVVVEJH9mcqLEtndt7R/jS8YMeHHphJzajFX2ogS7/vRGtd0Gre+DQloRO6UzqZwfAruLVPoTIj3uX6xwG/vdeUm+eh5psdCR4adHf1AzLzydBVdF6gY/Jsc6CDguO7kba1mEEznZ9EKLiZyBwwRAyJ6xETcstSA/1IfDze1F8C82u9OCtsOkAfLHF8dpc0xzeffjySTDAwsdh7EeO4N/PG94fB5GuAYOTXgeL/uboC38x09EPfsaD0LKJW1bdwrFYhn3maPk36NC+1JeACTdjG/wV1t0FLY3ePiomXy+sKSYUvYaGz1+H343lH8NUbje0g5eHw4wVYLFCdAP4vPxdKquS9m+3of4Wj33VnwAoN7ZE9/rAaq6VEKIqVSjwb9q0KWlpaXTv3v2KU1tGRkayfft2jEajyxWn2Wzm7NmzzhbjfMW1WJ8963o7NTg4mMDAQI4dK3mqt6ysLLZt20b//v35xz/+4bJux44dbvnL+qCYyMhITp06hdVqdWn1t1qtJCYmemzdL42nnnqqQnci8uUPoC7NORV1n0ZR+OwuLdNvUEnLcwT+lfFwpBBvhc2jdRxLU9EpEBPs2Gd8mopN1dMyqDPJJ3OY//ej5GU7LrTtCpi8veg3qSkx1wTQqJkPGo1Cyrk8FEVxzuDTpn8kmaezCW7uj85bS6u7IC3ZhM2qEhruxb/HuM/F33tMOD0HNeTdh/c7EhSF+HYRnI5pSOjFVAJCdQz5IBZDC18Wv/8FqpfKXU/3QqNz/T6LntCa8BHNyD1pxK9tkEvQ79itQtTs22j86vXYUvLw6hha7vOpaxVGw3PPYdl+BjXLhOGW5iheOmxnMlAzTWjbN0T96xz4GNBn5eHt74Oaa0PXqZHHMr2HdMCrb2usRy6haeyP/bwRXdsGKD6ud0zQamHh3+CfYyAzFzo0dV0/IBbOzIXDZx0t+oG+JR/IHdc48h88A80bQbDnLp2a65tjOPkm6l9JKFHBKGGO34ewXY9iPXQRdApqtgVty1AifPXk/ZWKPsIPXQMfj/urEm+OgScGwtkU6BTtOHe1gOJjQL/yUdSEFFSzFU2bxtVdJSEqgXTnKYsKNfENGDCAlJQUvvrqK4/rC/cbu/XWW7HZbG55ly1bRna2e1/Q6Oho9u/f7xwjAI6uOKtWrXI9AI2Gu+++m/j4eLdpOaGgBTr/wqRoi/SlS5c8bufr6+ssszRuvfVW0tLS3Pa1YsUK0tLSuO2220q1n6Lat29Pjx49XP7KIyIiAq1W63aRs2/fPvbv31+ufYraL8JfoWMDpdKfiNo6RHEG/QAtQhRah2rQaBUaRvtgyi24u2bX6bDpdNisKk1ifNFcnqUlLMLbZdpOg5+OBu2C0BUKukOaeNEgyhu7XSUv2/3OlcFLg6o6BgwXZvHSk+fnRV6mhYAuIWi9tVibqNhKuEGpDzQQeE2oW9DvkifcD+9OYRU+n4qiYOjZFK87W6F4ORoStFFB6Do4gntNp0g0LRui6doUXasG6Ds3LrFMxUePvms42vAA9F3D3YP+wqIauAf9+bwN0DXmykF/PoPekb+YoN9ZP50WzTVRzqA/n659Q3StG6DvGo4mwAtFq8GnS4PqDfrzNQh0DFSuJUF/YUrzMAn6hainKtTif99997F9+3Y++OAD4uLi6N69O35+fiQnJxMXF4fBYHDOfT948GC+++475s6dy9mzZ+nSpQtHjhzhp59+Iioqyq27yciRI3nllVeYMmUK/fv3JysrixUrVhAeHu42EOXRRx8lLi6Ot956i+3bt3PNNdcAcOTIEaxWK2+++SZ+fn707NmTtWvX4uXlRceOHUlKSmL58uVERka6tYR36tQJgA8//JB+/fphMBho2bIlrVq18nguxo4dy8aNG3n77bc5cuQIbdu25ciRI6xcuZJmzZq5zIxTHXx9fRk0aBArVqzgH//4B9dddx2nT59m9erVtG7dmqNHj1Zr/UT9oTdoaHt9EIf/KPjMaXUK7XsFl3ufOr2Gdj2CXZ7+q9UptL7Gn32fHcVXYyPHXhCgKXYV71wTLfuGV/pFjxBCiKojA3jLpkKBv06nY+bMmSxbtowffvjBGeQ3bNiQjh07MnBgQd9BvV7PrFmz+OCDD9iyZQs///wzHTp0YNasWcycOdNt1Hi/fv24ePEiS5Ys4f333ycyMpJHHnkEjUbDgQMHXPIGBgYyf/58Pv/8czZt2sSmTZvw8/MjJibGZVaaN998k48++oitW7fy/fff07RpUx577DF0Oh2vv/66yz67du3K448/zvLly3nrrbew2WxMnDix2MDf39+fefPmMWfOHH755RdWrVpFWFgYw4YNY/LkyTVibvunn34aVVXZvHkzW7ZsoX379rz33nt89913EviLKjXkqeZ8/+lpjsZlENTQwJ0PRRAW6V2hfd77ZDMM3hqOXN7nHQ9FsOuDgyRuvUCARoM9OBCTjxdeGpXgXCPtB0Vw4/MdK+mIhBBCiJpPUWU0phCiDsq5lMfC239yS+/zr2tpM8B1zI3FYmH+/PkAjBs3zmWaXSGEEDVXlvJ0iesD1PeqqCa1Q8Wn8RBCiBpIo9OgaN1vAeu85GtPCCFE/SS/gEKIOsk72ODWsh/Y1JdmtzQqZgshhBCibqvwPP5CCFFT3fpaFxq0C+TM9hRCYvy45qEWaA21bxYWIYQQojJI4C+EqLO0eg1dHmhBlwdaVHdVhBBCiGongb8QQgghhKiVZDrPspE+/kIIIYQQQtQDEvgLIYQQQghRD0hXHyGEEEIIUUtJV5+ykBZ/IYQQQggh6gFp8RdCCCGEELWSDO4tG2nxF0IIIYQQoh6QwF8IIYQQQoh6QLr6CCGEEEKIWkm6+pSNtPgLIYQQQghRD0iLvxBCCCGEqKWkxb8spMVfCCGEEEKIekACfyGEEEIIIeoB6eojhBBCCCFqJbW6K1DLSIu/EEIIIYQQ9YC0+AshhBBCiFpJpvMsG2nxF0IIIYQQoh6QFn8hhBBCCFFLSYt/WUiLvxBCCCGEEPWABP5CiCpxMUflUk7B/At2u8qpNDsmqyMtyaiSnld98zN4Z1jR59jc0jMumDDnuqbbkrOwp+V63pHFCgkXwOa+r1JJM8L5dNe0ixlwKdOx+zNZ2LPM5du3EEKIek26+gghrqoci8pDP9hZfkxFUWBEG4VHO9oZvySP+FQ7wb4K4VE+HEpX0Gtg8jUKH9yuQaNUze1b+yUj5hELGbL5BDYtWI6uQPfxMFKTTHz75jHOx+eg99Zw85hIet0dSuboJZh/PAE6Dd7juxHwyUAU7eU2lO/+gClz4EIGRIXB/GlwxzWlq4jVBo/OgQWbHP/3vRbmTYXHP4fvdqAqkBMcwbnUSPAxEPJcLA1ev/HqnRghhKgFZHBv2SiqqsoUqEKIq+bVbTbe/KPQ14yqEpydS3p+63+AAXz0Ltss6qfhwY5Vc0Myd9z/sC6Ic0nzXngfi/f6knggyyX9kZCzaJfsc0kLmDMYn0mxkJoFURMht1BrfFgAnJkL3oYrV2T2OnjsM9e061rBrpMuSRdpShqRAET+OAy/O5tfed9CCFFHpSj/KHF9mPp/VVST2kG6+gghrqqNiUXaFmxqQdAPYNBeeZuryLbxmFua6adjbkE/gG3LSbc088Z4xz+/H3EN+gFSsmCv+zYebfzTPe2vM25JvmQ4/8/ZmFi6fQshRB2lopT4J1xJ4C+EuKrahhb54tUo6AvH+jb3IL9d0W2uIk3bRm5p+vaNCAn3cktXW4S5pWnbNXD80yYCinZP0uugRePSVaRtpHtaw0C3JAs+zv8N7UJLt28hhBACCfyFEFfZSz01RPgXLEcFKbxyh3dBjGw0o1cKgv9ODWDKNVUX+Hv9X38I8nYuKx2bYJhyA3dOboZWX1CPmGsDafThXSghBYG3tm0DfB/v4VhoHQFPDnTd+asjoFFw6Sry5EDHxUO+sACYPRHCQ5xJFrxIxZHH+8YIAka3K92+hRBCCKSPvxCiCmSbVVadUNEoMKilgq9e4ehFG1vibXRsrKFdYy1r4lWCvKB/jIJeW7W3Z83nM9j84mdYfDTc+c5UDL6OC4HMS2ZOxKUT2NBAi+uCUBQFe3ouptVHUPwMeA1sg2IoMkfCzuOw5yT0bAOdm5WtIiYLrNnp6DI0KBaC/MCYC6t3gUbBen0HsjedQRfuj+/dzVE0chtbCFG/XVJeKnF9A/WfVVST2kECfyFEvWexWJg/fz4A48aNQ6/XX2ELIYQQNYEE/mUj03nWUdnZ2SxcuJDt27dz5swZcnJyaNy4MX369GHixIl4exd0bUhPT+eDDz7gl19+wWw207FjR5588knee+89kpKSWL16tcu+Dx48yOeff86ePXvIyckhPDycAQMGMHbsWHQ6eUvVZN8dtfP9CTvNgxSmdNXQwLf4FmNVVVly0M6PJ+y0DlV4oLOGZYfs7L+gcmNTDQ9do0FXihbn74/YWHHYRmSAwpTuOlKy7cyLM2OxwdAuen48o7DupJ0wb4Vnu2vo2+LKPRAvZKt8utfO6SyVQS01DG5d/DYn9maxf1savoE6QpsYOH04B4O/lkyDFzlmiI31JdegsPXotfhnaPC1nafvkIYEB+s4tDyRS4czadwlmLb3NMWUYWb/nMNk/XqGpoE2Wjx+DfobmwNw/LcUtvyYximLgej2fgzqG0TjBsV/Hszbz5DzxX6sKJyIjCBN602rm8Lw0akcX38Or0ADHUdEExDhy4mVpzn5/iG0qLR8pCUBx5Kwnckg0DcdQ3Y6xLaE8X3cZg9K3JvO4Y0X8QrQ0XVQOEHh3mQn53L4f/HkppiI0ecScPoi2hYhZN/YmqMzD6KcySYkxg8fPw3erYNoOKUjulBvzwdRRPrpHA6sOIMl10bbvuFEdAku1XaqqnJq9RmStybj18CbAKMJS1IOIX2jaDC8Ran2UZWMP8STveI42nA/gh/tiq6JX3VXSYh6Swbwlo20+NdRCQkJTJ48mdtvv53o6Gi0Wi27d+9m48aN9OjRg48//hgAs9nMww8/zNGjRxk0aBAdO3bk2LFj/PTTTwQGBmK1Wl0C/23btvHcc8/RtGlT+vXrR2BgIPv37+eHH37gtttuY8aMGdV1yOIK3vjVxmu/2p3LLYNh78M6/A2evzSf32DlP78XPITKRwe51oL1D3bRsOjeklvG3/vVwjPrCzZq7KuSkWkhLz9JAUJ9XAbFfnSHhmnd3Gf6yZdhUrlmvpVTmQVp/7pFw4s93bfZ/VMK3870PPONVaNwLjAQVaOQpUCLrGznz4fBR0NXryzSdl505m/WuzGX9qeRnVIwc0/XpJNc99lt7DUHs2TReY42KRgo7O+rMPO1JjTyEPzn/XCMlEFfg93x9WvRaFh/7XVk+PnhlZOHweQowztYT+ehzUh75g+0l/OqQLDeSHvLHnxJL9hp32th7SvOxYMbLrDmrcPOZe8AHaNmdGTjI9vIvWiiXUoSrdPPF9RJo+eMvRHq5aFfGuwYsOPVJogOe0ei8Sn5oj41IZtvxm3HfPkhaIoGBvz7Glre6j54uqjd/9zHoTlHUewqEeez0FsL3qdRL3Wl2Vvdr7iPqpI2cxcXn9rsXNZF+dNs70Now3yK30gIcdVcVF4ucX1D9a0qqkntIM2zdVRkZCTff/+9Swv8yJEjmT17NvPmzePAgQN06tSJlStXcvToUR599FEmTJjgzNuqVStmzJhBeHi4M81kMvHmm2/SqVMnZs+e7dz3sGHDaN26Ne+//z47d+4kNja26g5UlIrFpvJOnN0l7UQ6LD2iMq6ze+BvNKt8FOf65NnCQT/Al3/a+b/bVaICi29t+fc2143Op9ugcDW8dW4z4fzrD3uJgf//Dtpdgn6At3fYeb6H+0O/tiw9T3F0dhU/sxmjtxdNs3Nd2ozMuXaOJllpWCjt1Gb3fR1qGEmHGVvY3qgbZ0JcA1xjjsr6X4w8ODTYbbust39zBv0AerudtufOsKN1W8zeBmfgn5du4dTMg4QUyqsAqVp/fC3prjtdtwf2nYRrYgDYvvi0y+q8LCu/v3uQ3IsmNHY7MRkXXdZ72y34k0sWjtZrOxrs2DEdzSBteTxh97dxO47C/lx22hn0A6h22PVlwhUDf0u2laMLjgPgm2txCfoBzs08QNNXuqHxKv49UZVS/7XDZdl6xkjml4cIeaJbNdVICCFKT2b1qaP0er0zMLdarWRmZpKens71118PwIEDBwDYunUrWq2W++67z2X7e++9F39/f5e07du3k5KSwqBBgzAajaSnpzv/brzxRmeemiI1NRWTyeRcNhqNZGUVzM1uNptJSUlx2SYpKanE5eTkZArfJKstZZw+l0y2BTfpearHMi5lZBe0yhdDBTJNBfV0P65kMvI8bVWIh6fzZhRUw+O5SjfhJsvsiKOLHkeOseSD0Fw+zxoPNz7tmit/PVo0OqypOZiyrdg85M++/LyCosdhvuj+jAC91VFXVVFczpJisrnlRS3mYis9x/lvTqb7iTJnOd4EGlVFq9rd1mtwTwOwpTsuREp675o8nOucdNMV37uXki5iMznK1dg9vA45VuyFzkF1fwbtGe7n1ZKSXSM+51KGlFETyqhqMo9/2UhXnzps6dKlfPvtt8THx2O3u/6gT548mYkTJzJs2DDy8vL4/vvv3bYfM2YMWVlZzq4+Cxcu5KOPPiqxzMGDB/Pqq69W3kGISjNshZXlRws+7l5aOPKIjmZBnr8Y7/7SzI/xxX89XNNYYe/kkp9IO3a5mUV7C4I2rWrDVjiQ1SoQ7O1yAfBIF4W5fYu/GXk8TaXj51bMhXZzX3uFxYPct/l+7hl+W3nRLR0clyDnggKwarVgtdIoz/WHq2V2KpqkglsLAZG+GJNyKBwvN087T++JzfhZG8WaXRbOhgQ71ynA/73QiI5t3J8HkPX2r2S+sNElbXPHTpwNa4DOZMYnJ8+5kxadQtAvPuqS1+BtpkfeZvQUqnOzhnD8E9A5Wsa3zDnp2uqvwIAnYtj+4k5Um8r1SfE0zik4PjsKCYRjI79lXcULGxofHZ2O3Ych0rUhoKiE3y6x8qk9Lmm9Jrfk+vFX7qO/8f5fSN56Hq3VTmRyJppCb7uQQdF0WHX3FfdRVZLHrSNzwV8FCToNzfY9hFcH92c8CCGuvgvKKyWub6S+WUU1qR2kq08d9eWXXzJz5kx69uzJ6NGjadCgAXq9nosXLzJ9+nS3C4HSyL9GfOKJJ2jTxvNt/4YNG3pMF9Xv835aQr1t/BCv0ixQ4a2bNcUG/QBfDdXzzI9WNsTbaRWqML6rlv8dsPHnBZWbmmp4964rf318MlCPvwFWXh7cO/02H+IvWfnoVzNWO/Rpq2drqsLxVNBr4P4OCjP7lNylo1WIwuqhWl7dVjC49z+9PbfO3/1wBBqNwv5taXj7aQkI1nE+0YTeV0uajxcBdh3XXefLBW8dcWvP0yjbQkiwnnvua0TLps34/b2DzsG9NzzbgbSTRna8uZfspByiLJl0fyAK75du506TimZ2POt2Z3He25fgUD33DQ/2GPQD+D97A2qelZwF+7Cg4WjTKDIaNuGanqH42S2c/CkJryAD3Sa0IuaOJmzVQO6qU2jsKgG9wmiWlsyl090I80lCn5uOEtsS3n3YGfQD3DShGShw6KcLePnr6DGmKR3uaIS/v5a9sw5zJKwNvtaLBJ5LQdsqlHPtY8hamYxXugm9vxYfrYpP2yAi3upxxaAfoPkNDbjj5Q7sWXwKS66Ndv0jiB0bc8XtAG78qAe739xH8tbzmFr7E5Keh+1iHiF9o4h5t2ep9lFVGn3cB8VP7xjcG+FP2Gu9JOgXolpJq35ZSIt/HTVmzBiMRiMrVqxAU6gLwm+//cbf/vY3Jk6cyOTJk3n88cfZsWMHmzZtwtfX15nParVyxx13EBAQ4Gzx//nnn3n++ed58cUXGT58eJUfkxBXi0znKYQQtdMFpeReBo3UN6qoJrWD9PGvo7RaLYqiuPTDs1qtLFiwwCXfzTffjM1m43//+59L+nfffYfRaHRJ69WrF6GhoSxYsICMjAy3MvPy8sjOzq68gxB1QrZZ5bzRc/tCUpZKrqVmtz3YzDaM53NLnV+9mIWa5Ta4oVJYTDayLlVfX1ohhBC1m3T1qaP69OnDxx9/zN/+9jduu+02srOzWb9+vds8+/feey/Lly9n9uzZnDlzxmU6z6ZNm2KzFZrO0ceH119/nWeffZZhw4YxePBgmjZtSlZWFgkJCWzatIn//Oc/MquPcHp1o4V3frOSa4GbojV8M9JARKBCfKqdUUvM7DynEuAFr9yq47mbal4r+8Flp/hj5mFMmRZCWgZw54xrCWsT6DGvesmIdcx/UTccAi8dmsduRfvuCBQPA5jLY+eys2z7PAFzto1Grf0Z/Fo7Qpv6XnlDIYSow2p201HNIy3+ddSDDz7I1KlTOXv2LO+++y5Lly6lR48evP766y75DAYDs2fPZuDAgWzZsoWZM2dy6tQpPvnkE/z9/fHycu2j3KtXLxYuXEivXr1Yu3YtM2bM4MsvvyQhIYH777+f1q1bV+Vhihrsh6M23tziCPoBtiXamfq9Y3aY8Sss7Dzn+LrOMsHzP1r5LdHD7DXVKO2kkS1v7seU6TiAtBNZ/PTinmLz255d5gj6AUxW7O9vxP51XKXUJfloFj9/dAJztuMcXThm5If/O1Ip+xZCCFF/SIt/HaXVahk3bhzjxo1zW7dz506X5ZCQEKZPn+6SZrPZOHPmDJ06dXLbvlWrVrz5poySFyXbcMI9kN9wwo7FprIlwX1w+U8n7NwQXTPmagc4u/2SW1NS6vEsci7l4dvA/Um29vygvxB1wyG47/oK1+XUznS3tHMHszDn2jD41JxzJoQQVU2m7CwbafEX5OW590f+9ttvycrKokePHtVQI1EXtAlz/3ppHaqg1yrEhLh/UbcOq1lf3kHN/NzSvIP1eAV5nsJUaeP+oCqlTeNKqUtIlPtTYf0bGtB7yVe4EEKI0pMWf8E///lPTCYTXbp0wWAwsH//ftatW0fTpk0ZMmRIdVdP1FIPddXy+W6rs0uPtw7+c7ejH/+7d+sZtdSM5fJNgVubaxjWoWa1XEf1bEDz3o1JuPzEXkUDPZ9qj1bvOdjW/t+9WO/6AIyOwbdKh3A0k2+ulLq0uiGM5rHBJFxu+Ve0cNtjLVA0NetiSQghqpq0+JeNTOcpWLNmDUuXLiUxMZGcnBzCwsK48cYbmTJlCmFhMj+1KD+rTeX7o3YuZKsMbKslPKDgCzox3c7aY3aaBin0baVBU41BbHHTeaqqytntl8hIzCGyRxjBzUqez169mIV91T6UYF+UQV1QDJXXtqLaVU7GpZF5Po+Y7qEEhbt3NxJCiPomSZle4vpwteT19Y0E/kKIek/m8RdCiNpJAv+yka4+QgghhBCilpKuPmUhI8OEEEIIIYSoB6TFXwghhBBC1EoyuLdspMVfCCGEEEKIekACfyGEEEIIIeoB6eojhBBCCCFqJZmasmykxV8IIYQQQoh6QFr8hRBCCCFErSSDe8tGWvyFEEIIIYSoByTwF0IIIYQQoh6Qrj5CCCGEEKKWkq4+ZSEt/kIIIYQQQtQD0uIvhBBCCCFqJRncWzbS4i+EEEIIIUQ9IIG/EEIIIYQQ9YB09RFCCCGEELWSPLm3bKTFXwghhBBCiHpAWvyFEEIIIUStJIN7y0Za/IUQQgghhKgHpMVfCFGrGI1GkpOTadKkCf7+/i7pJ0+eBCAmJsZlnac8jRo1IisriyZNmuDl5VXh8subryKuVhlVUXchhKgM0uJfNhL4CyFqje3bt7Nq1SpsNhtarZbBgwfTo0cPtm/fzsqVK7Hb7QAoisK9995Ljx49XLZdsWIFquo6FEyr1TJgwIAKlV/efBVxtcqoiroLIYSoHtLVRwhRKxiNRmdACmCz2Vi9ejXnz59n1apVzqAfQFVVVq1ahdFodG67cuVKt6A/fz8//PCDc79lLT+/jLLmq4irVUZV1F0IIUT1kcBfCFErJCcnuwXnVquVw4cPewzabTYbycnJzm0LXxgUZbVaMZvN5So/v4yy5quIq1VGVdRdCCEql3KFP1GYBP71zJw5c4iNjeXcuXPVXRUhyqRJkyZotVqXNJ1OR/v27d3SwdGFJzw83LmtRlP8151Op8NgMJSr/PwyypqvIq5WGVVRdyGEENVHAv86aPPmzcyZM6e6qyFEpfL392fw4MHodI6hSTqdjkGDBtGoUSMGDx7sEthrNBoGDx6Mn5+fc9t77rnHY/Cv0+no37+/x4uH0pSfX0ZZ81XE1SqjKuouhBCVSb3Cn3ClqJ46vYpabfr06axZs4adO3e6rbNardhsNgwGA4oit8BE7XO1ZvWZP38+AOPGjUOv15e5/PLmqwiZ1UcIUd+dVP5d4voY9cUqqkntILP61DM6nc7ZmidEbeTv70+rVq08pnfu3PmK2xbO07hxYwAsFkuFyy9vvoq4WmVURd2FEEJUvToZAaanpzNnzhx++eUXUlJSCAsL45ZbbmHy5MkEBwe75LVYLCxevJj169dz6tQpdDod0dHRDBw4kFGjRjnzGY1GFi5cyKZNmzh37hw+Pj40b96ckSNHcvfddwMwadIkkpKSWL16tUsZ586dY/DgwUycOJHJkycDsHPnTqZMmcJrr71GdnY2S5YscbawjRw5ktGjR7vs48CBAyxbtow///yT8+fPo9VqadWqFQ8++CC33XabM9+kSZPYvXs3ALGxsc701157jUGDBjFnzhzmzp3LqlWriIiIcKnj7Nmz2b59O1lZWTRq1Ii77rqLCRMm4O3t7cyXv/2yZcv4/vvv+f7770lLS6N58+ZMnTqVm266qTwvmRCAa0szUGzLfnJyMlqtlsTERKKjozEajeTl5QHg7e3tbM0PCAhwtuoDLq39Fy5ccP6flpbmHNR64sQJoqKiPJZZeD95eXnFluXv7+/cpmh6ccedv8+ix1Dc8Xs6pvx65B9b/p2Poi34V2rRlxZ/IURtIfP4l02dC/yNRiPjx4/n9OnTDB48mHbt2nHkyBGWLVtGXFwcCxcudPZXtVgsTJs2jV27dtGzZ0/69euHwWDg+PHjbNq0yRn4Z2VlMWHCBOLj4+nTpw/Dhw/HZrNx5MgRtm3b5gz8y+Obb74hJSWFoUOH4uvry/r163nnnXfIzMxk0qRJznybN28mISGBO+64g/DwcDIyMlizZg3PPfccb731Fn379gVg/PjxqKrKnj17eOONN5zbd+nSpdg6JCUlMXbsWIxGI8OHDyc6Oppdu3Yxf/589u3bxyeffOJ2l2D69OnodDoeeOABLBYL//vf/3j22WdZvny5ywWFEKVVeP54RVFQFAW73e42X3/h6SZLK79bW2l6Ni5YsKDYMjUaTYmzA4FjUHHnzp3Zv3+/Sz2LmxO/uOcLeNqu6Dm60jFpNBq6dOnirEvRunmqk8zjL4QQdVedC/wXLlxIYmIiL7zwAiNGjHCmt2nThrfffptFixbx6KOPArB48WJ27drFuHHjmDp1qst+Cv+4z5o1i/j4eP7xj38wdOjQYvOVR2JiIkuXLnV2ORg5ciQTJkxg3rx53HPPPc70CRMmMG3aNJdtR48ezZgxY5g3b54z8O/Zsyfr1q1jz5499O/fv1R1mDVrFmlpacycOdPZYj9ixAg++OADvvjiC9asWcO9997rsk1wcDDvv/++M/iIjY1l7NixLF++3K2eQlxJ0fnjVVV1BrT5c8k3b968XEF//v7KorgyS/N5t9ls7N27t9h9duzY0dmKXtLzBa5Ul9Ick91ud6lL0boVrVNx8/gXrrMQQtQk0uJfNnVuVp/NmzcTEhLCkCFDXNKHDh1KSEgImzZtcqatW7eOwMBAHnnkEbf95M/+Ybfb+fHHH4mJiXEL+gvnK6++ffs6g3sAvV7PmDFjsNlsbN261Znu4+Pj/D8vL4/09HTy8vLo3r07J0+eLPcDdux2O7/88gtt27Z166bz8MMPo9Fo2Lx5s9t2o0ePdhkc3LFjR3x9fUlMTCxXPa6G1NRUTCaTc9loNJKVleVcNpvNpKSkuGyTlJRU4nJycrJLwCVlVE4ZnuaPL6yk+fqvlqtRZv6c+Pnn6krPFyi83b59+67K8efXyWg0Eh8fX+I8/rXtfSVlSBlSRtWXIWq2Otfif+7cOdq3b+/WNSW/7/7hw4edaYmJibRt2xYvL69i95eenk5mZia9evW6KvWNiYlxS2vRogUAZ8+edaalpqYye/ZstmzZQmpqqts2RqOxXC1yaWlp5OTkOMssLCgoiAYNGrjUI19UVJTH/BkZGWWuw9USGhrqslz0/BgMBsLCwlzSis5X7mmOdimj8svInz++uMA2f77+H3/8scqC/6tRZv6c+PndDfOfL3Cl4F+n09G1a1e2bNlS6cdfuE4tWrRwex0Kz+Nf295XUoaUIWVUfRmiZqtzLf7VqbjpMSv6Q62qKtOmTWPNmjUMGDCAf/3rX3z00UfMmjXL2cWnol2Oyqq4Ox0yO6woj6Lzx2s0Gud7rOh8/eWZlUqj0ZRp+triyizNfvKDdE+ND0XnxC/p+QKVUReNRuNSl6J1K1onmcdfCFHbqCgl/glXda7FPzIyklOnTmG1Wl1+eK1WK4mJiURGRjrTmjVrRkJCAmazudindgYHBxMYGMixY8euWHZgYKDLHYV8nlrM8+XPyFFYfHy881gAjh07xtGjR11mBcq3YsUKt+3LEuCEhITg5+fnLLOwzMxMLl26RJs2bUq9PyHKq0ePHnTs2LHEWX0K59FqtZw+fZqmTZtWeFaf9PR0fv75ZwBuv/12IiMjPZZ5NWb1yd9/aWb1Ka4u+cdRGbP6FC1D+vYLIUTdUecC/1tvvZX58+ezYsUKhg8f7kxfsWIFaWlpLv30+/bty4cffsi8efOcA37zqaqKoihoNBruvvtuli5dyooVK9wGuebnA8eFxKZNmzhw4ACdOnUCHC3xixcvLra+69atY8KECS7ziS9evBitVuvsc5/fGli0Nf348eMe+9/njwfIyMggKCio2LLz933zzTezbt06fvvtN2644QbnugULFmC32+ndu3eJ+xCishSdP764+frz0z11lcuX/5kqPIYmfw5/o9GIj4+PM7ANDQ1ly5YtgKOrXdEHeBWtV9HnBXgqq/A2hdM9Ke4ZBJ62K6kunurhaZsrzdMv8/gLIWoL6WdQNnUu8B87diwbN27k7bff5siRI7Rt25YjR46wcuVKmjVrxkMPPeTMe99997F161bmzZvHwYMH6dGjB15eXsTHx3Pq1Ck++eQTAB599FHi4uJ466232L59O9dccw0AR44cwWq18uabbwIwZMgQvvzyS5577jlGjx6NXq9n48aNJXb1iY6O5uGHH2bYsGH4+vqybt06Dh48yCOPPOJs0YuJiaFFixYsWrSIvLw8mjVrRmJiIsuXL6dVq1YcOnTIZZ+dO3dmyZIl/Pvf/+amm25Cp9PRqVMnl7sdhU2dOpXt27fz7LPPMnz4cJo2bcru3bvZsGED3bp1Y+DAgeV/QYSoYTxNV9mtW7fqrpYQQghx1dW5wN/f35958+Y5H+C1atUqwsLCGDZsGJMnT3bpq6rX6/n444/58ssvWb9+PZ988gkGg4Ho6GgGDRrkzBcYGMj8+fP5/PPP2bRpE5s2bcLPz4+YmBiXh3xFRkbyzjvv8Mknn/Dpp58SFBRE//79GTx4sMvdh8JGjRpFdnY233zzjfPW+jPPPMN9993nzKPVavnggw+YOXMma9asITc3l5YtWzJ9+nSOHj3qFvjffffdHDlyhB9//JGNGzdit9t57bXXig38w8PDWbBgAZ9++ilr164lKyuLxo0bM27cOCZMmCBP+hV1RnHTVbZt27aaayaEEEJcfYoqozGrReEn9xa+yBBCXD3Hjx/nv//9r1v6uHHjnF19xo0b59bVRwghRM10THmnxPWt1WerqCa1g8zqI4SoN/KnDS1Mp9NdsQ++EEIIURdI4C+EqDdkukohhKhbZDrPspHO20KIesXTdJUWi6W6qyWEEEJcdRL4V5PY2Fh27txZ3dUQol6S6SqFEKJukFb9spGuPkIIIYQQQtQDEvgLIYQQQghRD0hXHyGEEEIIUSvJnPRlIy3+QgghhBBC1APS4i+EEEIIIWolGdxbNtLiL4QQQgghRD0ggb8QQgghhBD1gHT1EUIIIYQQtZJ09SkbafEXQgghhBCiHpAWfyGEEEIIUSvJdJ5lIy3+QgghhBBC1AMS+AshhBBCCFEPSFcfIYQQQghRK8ng3rKRFn8hhBBCCCHqAWnxF0IIwJTlg0Znq+5qCCGEKANp8S8bCfyFEPVa+iUL//3nSZJOdQVU/mc5y/1PNkOrkx8TIYQQdYt09RFC1GurFyWRdMp0eUlh329Z7Pg5rVrrJIQQQlwNEvgLIeq1k4dy3NISDmdXQ02EEEKUlXqFP+FKAn8hRL3WMMLgltYk2qsaaiKEEEJcXdLHXwhRr+k89OXX6aVNRAghagMZ3Fs28usmhKjXkhJNbmlH9xlJv2Qh9YKZ82fyqqFWQgghROWTFn8hRL3WKMJARorFJe3gziwO7jzsXI5u7cP4vzcjIFhf1dUTQghRAmnxLxtp8RdC1GuaUkzbmXgslx++Ol8FtRFCCCGunjIH/qtXryY2NpadO3dejfqIWmDSpEkMGjSouqsh6pG0DBsJZyzY7ZU/R0Oyh64+nsQfzCYzzULSqTxUVSUny8rpEzmcPp6DMdOK6Vga5vgM140On4HEi87FpIRc/tqZSV6uPChMCCFE1ZOuPkKIGu2zr9NZ87MRux0iGut4ZWoYTSMqr8tNRDNvt64+/pnZGAP9XDOq8ObEw9jt4BeoJTfHht3qWKWoKl2OJNDt0En8+zUj+uMb0dz3Duw4BoB1SC8+iBzAubOOgF+jgSGPhHND3waVdhxCCFEfyZSdZSNdfYQQNVbcn7ms+skR9AOcO29l1peV+3CtAQ82wS9Q61z2M+bQPP4cPtkFg3p1qp1LyWZnPbIzC4J+AFVR2NcuhvOhgRjXnsI0/GNn0A+g++53Wvzyq3PZbofl/00iM831gkMIIYS4mupM4J+Xl4fVar1yxhqmtta7PkrLVdmXZMNiK337QnK2yoGLKqpaN9okci0qey+oGM3ux5NejvNzJX8dM7ulHfSQdiZL5XBK+coNb+bNwLGNnMt+xly0Nhtd9h+n9eEEYo6fIfh86S42UsN80WDDFH+ec4GNsRcadBaTkojebME72wSqimqH4z9fJOdMNna7StKpPLLSXS8EVJsdy5/nsae4P2TMue7S5XUmC+w7CZmOZdWuYjlwAdt5Y+lOxMnzcDypdHkrmfVEKtZ4eVqyEKLsVJQS/4Srcnf1UVWVL774gmXLlnHhwgXCw8MZP348AwcOdMm3YsUKli5dSkJCAjqdjk6dOjFx4kS6du3qzHPu3DkGDx7MxIkTmTx5ssv2c+bMYe7cuaxatYqIiAgApk+fzpo1a9iwYQMffvghv/76K2lpaaxcuZKIiAjWrFnDkiVLSExMxGq1EhYWRufOnXnmmWcICQkp8bh27tzJlClTeO2118jOzmbJkiUkJyfTpEkTRo4cyejRo922SUxMZO7cuezYsYOMjAwaNmzIHXfcwaRJk/Dx8XHmu1K9i/rss8/47LPPWLlyJZGRkQBcunSJvn37oigKP/30E0FBQQCcPHmSESNGMG3aNB5++GHnPrZv386iRYv466+/MJvNREdHM3z4cIYPH+5W3sGDB/n888/Zs2cPOTk5hIeHM2DAAMaOHYtOV/JbJT09naeeeor4+Hj+85//cP3115eYv7Z5d5uFlzeYybNCeIDCN6O9uLm5ttj8qqry+EY7n+5TsanQLhRWDdHSOqT2fgl9d8zOhPV20vIgwACz+mh4sKOj7eC9bRZeunx+mvgrfD3ai1tjij8/pRUT5d6lp0mDgv1a7SoPr7Wz+JCKCnRrDKuHaInwL/15zkixsGn5Jeeyl9VKx7Pn8bLZsCdd4HxQAH+2aVaqfXXO2EULzSmWtxxAXLNrCclO4+Ht3xCZkcRxXWu88yxkBfoCEJyaRcKkAySqKplNg/kzJhJFr+GmfmHcMz4Cc9xZ0oYtwX46Ewxa/F+6mYBXbwXAsjuJtKHfYDuVAXoNQcOb4rthM1zKBD9vLM+OJO2r09iOp4JOg9/j1xP43t2eK52dByPfgR92O5Z7d4LvXoBgP8/5K5E900Ta0G8wbzwJgNfdLQleNhKNv/tD1YQQQlRcuVv8Z82axQ8//MDQoUP529/+hqIoTJ8+nb179zrzfPjhh7z11lvodDoee+wxHnjgAU6ePMnkyZPZtm1bhSs/depULl26xIQJE5g6dSq+vr58//33TJ8+HS8vL6ZMmcIzzzxDv379OHXqFKmpqaXe9zfffMPChQvp168fU6dOxd/fn3feeYfPPvvMJd+hQ4d48MEH2bNnD0OHDuWFF17gpptu4uuvv2bq1KkeW/M91duT7t27AxAXF+dM27FjBxqNBlVVXQZY5+fJ3wZg+fLlTJs2jdzcXMaPH89TTz1FVFQU//73v/nggw9cytq2bRsTJkwgMTGRBx54gGeffZYuXbowZ84cXnrppRLP1dmzZxk/fjxJSUl89tlndS7oP3jBzrNrHUEtQFKWykPLTCUONF15XGXWXkfQD3A4FR77yV4Ftb06jGaVsWsdQT9Alhkm/mjnYo7K4Yt2nil0fpKNVz4/pdUyQovBVjAQVlFVtFkF+/58v8pXl4N+gN3n4bktZTvPqxclceFsQUt7tyMJeF0uUwOEZ2QRnJFFQEbJLefXnN5P2wvHMdgtDNu7Bj9TNml+IXx93b2ca30t8eHNnEE/QHpoAEkRoQAEnk4n/Fwqdhv8siaFQ7syyXh4pSPoBzDbML62GfOOs45tx610BP0AFite//vBEfQDZOeR8fpvjqAfwGon+/0/yFtb0PXIxburCoJ+gM0H4M0lpThzFWf85y/OoB/AtP4E2TMq/tsghBDCs3K3+JvNZhYtWoRe72iR69OnD/fccw9Lliyha9euJCQk8MUXX3DNNdfw6aefOvPde++9jBgxghkzZtCrVy+02vK3CrZs2ZI333zTJW3z5s34+fkxe/Zsl1bqKVOmlGnfiYmJLF26lMaNGwMwcuRIJkyYwLx587jnnnuc6W+88QYNGjRg0aJF+PkVtJBdf/31PPfcc6xdu9ZtBhxP9fakc+fOeHt7s3PnTu69917AEeC3adMGk8lEXFwcffr0cab7+/vTrl07wHFn4J133uGuu+7in//8p3OfI0aM4J133uGrr75i2LBhREVFYTKZePPNN+nUqZPLeRs2bBitW7fm/fffZ+fOncTGxrrV8fDhwzzxxBP4+/vz+eefe7xzUdttS3CfgSUhTeV0hkqzYlrwt511D3q3nqm93X3+vOgI9gsz2SAuWSUp1f38JKarnEpXiQmt2B2O+BMmgiwWLFYrdkVBb7eTZ4KUS1YaNtJXynmOP1jQjUZrsRKame2Wp0FmNvoTZ8n29eZky0hsevevzn6HfnbeVNbbrTRNO8vhJm1ICmrChQ53kHnM/e5FZpAvkWdTAAhKz+ZclGOwb/yeTIIPXnTLb96WiK59A6x/FkwtqsOEFtcGBgvujQnmbYl492vtfgK2HXJP2+oh7SowbztdqjQhhChe7b2TXh3K3eI/YsQIZzAP0KhRI6Kjozl92vGlvWXLFlRV5aGHHnLJ17BhQwYNGkRSUhJHjhypQNXhgQcecEvz9/cnLy+Pbdu2Vahfdd++fZ3BPYBer2fMmDHYbDa2bt0KwPHjxzl27Bh9+/bFYrGQnp7u/OvatSs+Pj788ccfpaq3Jzqdjq5du7q07O/atYvu3bvTvXt3duzYATi6lezevZtu3bo5L6R++uknzGYz99xzj0u90tPTufnmm7Hb7c7tt2/fTkpKCoMGDcJoNLrkvfHGG515itq+fTuTJ08mIiKCefPm1bigPzU1FZOpYKpGo9FIVlaWc9lsNpOSkuKyTVJSkttylybuH5OGfo4uP8WV0bmB+xdRl4bFl1FYcnKyy3u3so6jImU097dg0Lh+njSKSscwhc6N3c9PA1/Q5F6o8HF4+zhasfWqipfdjgbw99cQHOIIvJt7ubfCdworKLM0ZQQWym/TadnYoxN2xfX10+fZsGk0nI1u7DHo9zHnEpyb6Vy2o5Ac6Bg3EKRY0H72O77Z7k8A9iuUlu3v7fw/IEJFExXoll/fuRGKvwFNs6CCOmPAXuSrXId7WdlNvTy/Hp2j3fLSpVmVvK80HdxnNTK3cO1iVBs+H1KGlCFllG5aZFH9yt3in9/nvLCgoCCSk5MBR799cLRuF5WfdvbsWTp06FDeKtCsmXu/23HjxrF7926effZZgoKC6NatGzfeeCN33nmns0U+JyeHnBzXwXJBQUEuFygxMTFu+27RooWz3uDoVw+OcQhz5szxWEdP3YuK1vvSpUsuy3q93tl3PzY2lj/++IOTJ0+i1+s5d+4c3bt3x2QysWTJEi5cuEBaWhoZGRku3XwSEhIAeOyxxzzWq3Dd8o/jjTfeKDZv0S+G1NRUnnjiCVq0aMHs2bPx9vYuZsvqExoa6rLs7+/vsmwwGAgLC3NJCw8Pd1sOByZ11/FZnKNV1aCFDwd6YdApxZZxX3uVxYcVfkxwfDkGecH7t2mLLaOwJk2aXJXjqEgZEUEGZtxq55nNduyqo33l1V5amgUpNAvSMuV6HZ/ucJwfvRY+GOhFs0jXAK48x3F9jygOH7zApo2OoFqrhTEPNUSvdwTmz90cyI/JNrZf/l1q5Atv31rwtVaaMvz8vYHLFxCKQmJEI440S6N9guNznqX3wowWnVUh19fz+7xp2hm0dsedD1VR+LFdb9J9g9FrVG7fFkeYOYvOR0/yW2BHzF6O75mA3Dwizjg+V9mBPpyNdATB7a8L4Ma7orDM6k/aqGXk96Hyvr8zhjtaoCgKQZ8MIG34Esi1oqIlNzYW3z07UWyObk6Bo2JI+yEV9fJtGq8BrQl55AaUQhc0ztfj+SGwdg8cOuNYjmkMr46skvdV0Gu9SdmUgO2EY2Cvtk0YYW+5jkWoDZ8PKUPKkDKqjwzgLZtyB/4ajeebBeVpZVeU4l80m634B914Cjajo6NZunQpO3bsIC4ujt27d/PWW285BwlHRUXxxRdfMHfuXJftPv30U49dWUqSf6wPPPAAvXr18pgnMNC91a5ovfv27euy3K1bN+dYgsL9/A0GAzqdjmuvvRaLxYJGo2HHjh2kp6e75C1ct9dff50GDTzPFZ5/8Zaf94knnqBNmzYe8zZs2NDtuNq1a8e2bdtYu3YtQ4YM8bhdXTHnXi+m9tRz5KKdW2K0NL7C4FGDVmH9cC2/nVU5n6PSJ1oh0Kt2fzk9eZ2Ge1spxCWrdG2kuAxUnn2PF4/10HP4op2bm2toElB5E4Y9NK4Rt90RRPI5C23aehMUXPC1FWBQ+H2Mli2nVTLNcEczBV992c5z0in3lqq/WkQRfDELq0aDSecI1P08tNjnO9q4Nf931xM0TT9H58ld6HhrDOHnzTSYtQnNeUdjSOyFeFpuSCaxdTTN5t1Nqy7+XPolHJ2vDr9rG9D5r2yCQvQ0a+vopqMd3JbGp5/CtDkBXYsQ9N0Kfny9+7d2rNuUgLZ5MIbYCDhzCX4/Cp2i8WofRaP0PEw/n0QbEYChZ1TxJ6BxMPz5Pvy8H6w2uKMLGCrvOQkl0UYF0vDQVEw/xaNoFAx9WqDo6sxkc0IIUeNctQd45QeVJ06cICrK9UcnPj7eJU9+cJyZmUlR+a3rZWEwGLjpppu46aabAMfA1SeffJKvvvqKF154gQEDBrjMKgS4Bbz5reAl1Ts62nGLXKPR0KNHjzLXM9+sWbNclgtfLLRr1w5/f3/i4uLQ6/V06tQJHx8ffHx8aNu2LXFxcWRmZhIaGupyd6Vp06YABAcHX7Fu+cfh4+NT6uPQ6XT85z//4e9//zv/93//h9VqZcSIEaXatrbq0kTjsdtPSW6IVKhL/Q+bByk0D/J8PJ2baOhcxvNTWk2betG0qZfHdYqi0Du6/Oe4YYTB7QFeqqKQbXAtzxLmQ1EGbw3mPEcre5pfCGl+IdxyQzOatvShaUsfjFsbkLW4IH+IJYcmLeyE9gwGILxvwXdjl55BFKVp4IvPcM93RTVhRdZFNYARBRf5mmBvfIa293zQRem0cFfX0uWtZIpe63nsgRBCiEp31ZpWbrnlFhRF4YsvvnCZ2ebSpUusXr2a8PBw2rZtC4Cfnx9hYWHExcW53DE4c+YMmzdvLlO5+a3fheUPeM3IcMyCERUVRY8ePVz+irbMr1u3jvPnCwbQWSwWFi9ejFardV5QtG3blpYtW/Ltt99y5swZt3KtVquzzJIUrUv79gU/1lqtlm7durF7925n//58sbGxzrsa1113ncudkzvvvBODwcCcOXPIy3NvqTQajZjNjm4AvXr1IjQ0lAULFnisb15eHtnZ7gMedTod//rXv+jTpw8zZszgf//73xWPVYiaRufhDoHR35vUkIJb4lathnNdmuIfVNBW0qyND0MeCUerK9j++j4hxLQr6OLk+2h39N0Lxr5oGvsR8O8+lX0IQghRb6lX+BOurlqLf/PmzXnwwQdZtGgREydO5M477yQnJ4fvvvuOnJwc3nzzTZcZfUaOHMns2bP529/+xq233sqlS5f49ttvadmyJQcPHix1uVOnTiUgIIBrr72Wxo0bk5WVxerVq1EUhf79+5d6P9HR0Tz88MMMGzYMX19f1q1bx8GDB3nkkUec/dsUReGNN97g0Ucf5b777mPw4MG0aNGCvLw8zpw5w88//8y0adPcZvUpq+7du/PLL78AuHRH6t69O1988YVbOkDjxo158cUXeeuttxgxYgT9+/cnPDyctLQ0jh8/zubNm1m6dCkRERH4+Pjw+uuv8+yzzzJs2DAGDx5M06ZNycrKIiEhgU2bNvGf//zHY1conU7HP//5T3Q6He+++y42m63Ug5eFqAk8dfUJDffiL00MARnZGMxW0kP8sdm1vPyflpw5kYuPn5aWHf1QFIX23QI48Vc2jSK8iIhxvSugCfQi7I9HMG9OQM0y4XVXSxSfqulGI4QQQhR11QJ/gL/97W80bdqUpUuX8vHHH6PX6+nYsSNvvfUW1157rUvesWPHYjQa+eGHH9i1axcxMTG88sorHDp0qEyB//Dhw9mwYQPLly8nIyODoKAg2rZty/PPP1+mPvyjRo0iOzubb775xvkAr2eeeYb77rvPJV/btm356quvmD9/Pr/88gvffvstfn5+hIeHM2jQIJcW+vLK34eXlxddunRxpl977bXodDqsVqvHcgYPHkx0dDRffvkly5cvJysri+DgYJo1a8ajjz7qMqCnV69eLFy4kIULF7J27VrS0tIIDAwkKiqK+++/n9ati78Vr9VqeeONN9DpdMycOROLxcK4ceMqfNxCVIXwZl5uXX2yMxxji7KCClrvg8L0BIXoCenh+nCpgGA9XW8MLnb/ikbB63b3yQKEEEJUnAzuLRtFrcicl3VQ4Sf3VrSlXghR851LyGXWK/HkZbs++MvHT0Pu5TSdXuGBp5vSuYd7P3whhBDVJ075tMT13dWyPceprruqLf5CCFHTRTT34aZ+ofy0zHVa3dxsO1P/2YKsNCstO/q59O8XQgghaiP5JRNC1HsRLdxnDAprbCCmnW+J0w0LIYSoXtJtpWxkwmQhRL3X7lp//BsXPKROb1AY8ki4BP1CCCHqFGnxLyI2NpadO3dWdzWEEFVIo1GIvPYouen+3NTzbtp2DcI/UL4ehRCiprPL4N4ykV82IYS4zCfYSJdegej18tUohBCi7pFfNyGEEEIIUSvJdJ5lI338hRBCCCGEqAck8BdCCCGEEKIekK4+QgghhBCiVpLpPMtGWvyFEEIIIYSoB6TFXwghhBBC1EoyuLdspMVfCCGEEEKIekACfyGEEEIIIeoB6eojhBBCCCFqJenqUzbS4i+EEEIIIUQ9IC3+QgghhBCiVpLpPMtGWvyFEEIIIYSoByTwF0IIIYQQoh6Qrj5CCCGEEKJWksG9ZSMt/kIIIYQQQtQD0uIvhKj3zl+ycTwpBp3WSnaunWC9ex5rfBq53x5CE+SFz+hOaAK9qr6iQgghXEiLf9koqqrKgGghRL315+E8Xpt5CYvVsdwwVMN7LzUmJEjrzJP3UzwpA/4HZhsA2ubBNNwxAW1Dv+qoshBCiMu2KJ+XuP5WdXwV1aR2kK4+Qoh67auVmc6gH+Biqp0fNhtd8mS9utkZ9APYEtLJnr2zimoohBBCVA7p6iOEqNcupNjc0i4WSbMmZrjlsSVmXrU6CSGEKB3ptlI20uIvhKjXrr/G2y2te5E07wGt3fJ4D3RPE0IIIWoyafEXQtRZF7NVvvnLht1ip9OlNA7EmzG1DmHYTX7EBMHa43YuRPgT0zybZqsPYvbSEvNoJ268zpc9Z6ysO2ShRZiGe/99B9bUPI5vukCevw+tRrXA55628ONecjYd4ahPOKYbOtH1xiC8fLRXrthl1jQTF7+Jx55toUFwFl7nL8BtHaFXm6t4VoQQou6Qwb1lI4F/NTt79izvvfce+/btIz09nYEDBzJ9+vRSbz99+nTWrFnDzp3S31iIwg5fsnPTfDOZRjuT9h7BaMwFwM5phm9uSci1ofx8wn45d0MmBPvzwo+/ojt9nM/9RzBhY8G+ekcrTNJHcKlZAAB//ZzHAwPeI3jtr/gCXYFtMdfzft8hPP7vlvgFXvmr1XTayL6eqzCfywHgFFY6s4sAvoa3RsFLQyvvZAghhBBIV59q9/rrr7N7927Gjh3LG2+8wdChtefHfufOncyZM4esrKzqrooQbmb8aiMlF665kErE5aAfHF96tx8+zc9nXPMv6HktF/x9sSZkcuTt3S7r8nZc4NKRgvd5UE46wWt/dclzw8k4bCcu8MeG1FLV7+zMA86gH8COjkRaOhb++R1k5hSzpRBCiHzqFf6EK2nxr0Zms5k9e/YwcuRIHnzwwequTpnt2rWLuXPnMmjQIAICAqq7OkK4OJXh+MoPzjO7rQvONYPi+pNg02o4H+hPI2MOISmus/qEFtlHgNl1PYAGleCcDNIuWEpVP9Mp932YuDy2INcMFzIh0LdU+xJCCCFKQ1r8q1FqaiqqqhIYGFjdVRGizhnY2vH1diQsyG3d0YbBFH1GV+MMI+2TLwKQ1KO5y7pDRfaR5N8Ys5frAOAsLz8SQyJpH1u6i+DQgdFuaSE4yqddBLRsXKr9CCGEEKUlLf7VJL9vPsDcuXOZO3cuAK+99hqvv/46EydOpEOHDsydO5fjx48TEBBA//79mTp1Kjpd8S/bmjVrmD59Op9++imxsbEAWK1WbrvtNnJzc/nyyy9p164dANnZ2dx+++3ce++9/P3vf3fuY9myZSxevJikpCSaNGnC6NGj8fX15fXXX3fut3D9Bw8e7Nx24sSJTJ48uXJPlqh14lPsfPeXlVBfhRGddfh7KRxKUVl9QiXSH4a1UfDWeR6QlWNRWXpE5WIu3BwBcYk2zmeraHUKrcM0DGuvcW676ZSdP86pdFBMNExMxy9YT/ubQzmYpSEPhbtbadh80o9v20QTmZFNWK4JuwbO9IzgX2FGFh7TcEjjRaQplwd2/MlvHVpw572NeGxMK/SvHiYvOZdjrRrxwJRoOu3OI+6LBEwmlUbXNeLQwElELlrBYTWaLH0AF5o04q6xkbRraSBt9j5sJhtJrcJJz4FGLX05l2Ql71AaLUw5BF8XSrpFxW9AU3I3nkU122igvUi0JR57i3CUvrHw/T7o3wVF494+Y800c3HJSWxZFvzubsrxBBPWQ0l0NJ4ksHtT6HsteNiutsj44wJpm5Pxax9Eg4FNUbS191iEEFeXDO4tGwn8q8nQoUNp06YN7733Hrfddhu33XYbANHRjlbAX3/9lWXLljFs2DAGDx7Mli1b+OKLLwgICGD8+OKfQpcf7MfFxTn/P3DgALm5uWg0Gnbu3OkM/Pfs2YPNZqN79+7O7RcsWMDHH39Mu3btmDp1Knl5eXzxxReEhIS41T87O5tNmzbx9NNPExwcDEDr1jLFYX3341ErgxbmOp939a/NCi/c6cOkn1Tsl3vXvLsTtt2nxVfv+oWdYVLp9ZWNQ/nd5FUVci4/XUtRABvv/q6wbZyBl3+x836cnTaXMhi9/wS6yw8h/37RWd7s2I48vQ6sCr5GC0l6b5IaOFroLUDmSSs7k/Vk+TvSTun8mH9jN9qlZbMhF+4aupFrsx3de3qcPo93ahKsPUisxU56oC+/hgWz/7QvXi1HYDY47h3oLVZ67k3gxGN7sFzIYXP3TlwIK3geQLaXgTyDgdCUXDrM2nb5eMA7QOGmlP34WyzY8UMbn446cwPM3ABDr0P59nGXc2Q+n8uenqvJSzCS561n5+pMzF6OOvxsbcDD780jckBrWPxUhV7H6pLwrz+J/8cu53LYwKZcs/qOaqyREELUHdKMUk26dOlC7969AWjVqhX9+/enf//+hIaGAhAfH8/ChQuZMmUKw4cP58MPP6RFixZ88803Je63SZMmNG3alLi4OGdaXFwcwcHB9OzZkx07drikK4rivEDIyMhg7ty5tGrVinnz5vHAAw/wyCOPsGDBAhITE93q36pVKwB69+7trL8E/uLlH02FH3LLsUsqz22xO4N+gD0X4H+H3YddzT+gFgT94AiOfXTOIBlgT7LKJzttfLDTMSNPn/izzqAfwHYhj25nLzkWzDYCckwuZegBrUYhy8/LJf2cvzc5Og0Xs+Cv8IJuNhq7SubhDNK8vVGAYzGRWLRaAnJynUE/gEWv49e1KdguZJPcMIQLYcEu+/cxmUFVaXw+1eV48nS+xPs3c5SF1WUblu9C3X7CtZ6fHCIvwTE+4HTzBs6gH8Cs82JLq5vgf1thTzy1jTXLQsKbe13SUtacJu2X5OqpkBCixlNRSvwTriTwr6F69+5NRESEczk/QE9JSSEnp+TZPmJjYzl48KAzX37r//XXX8/evXuxWh3Bxc6dO2nVqpWztX779u2YTCaGDx+Ol1dBUNSgQQP69etXyUd49aWmpmIyFQR9RqPRZQYis9lMSkqKyzZJSUklLicnJ6MWCjKlDPflhDT3gD7dfXwtxy6Z3co4fD7XPaOH7+2/LhZcSHgavBt0OU2xg8bDtA6KVuMSfOfL0zrm4M/2cb0o0NjsmAyOG6Q5l9epHuqVrTdc3t79oWAaQFFVvEzu9c3RelPsDeuESy6vR37QD5DrU3SkAqR7O8YMpe454pJeG95X5gu52HPdn6ScV2ggdG04DilDyqjPZYiaTQL/GioyMtItLSjIMcAwIyOjxG27d++O1Wplz5495OXlceDAAWJjY4mNjSUnJ4cDBw6QkZHBsWPHnK39AOfOnQOgWbNmbvv0lFbThYaGulzA+Pv7u8w+ZDAYCAsLc9kmPDy8xOUmTZqgFAoYpQz35YHt3B9gdV1D95D23rZebmUM6+Djlg+re+Q+vquWJn6O/496GLx7rIEjTdUrmHWuX3MqYLPYwG53SdfZ7QSZHTPyRF1Mc1lnMegIznJcSDe+vM6qcT/OKHsOoNDkYhqK3bXeVo0GVaMhNdR9MH9jUyp4ap3y1kOfDi6vR+iAKOfqBhfdp9JtfekE+HkTOuRml/Ta8L7yaRGAbzvX11MxaAi9o6ARpDYch5QhZdTnMkTNJoF/DaUpYWBe4SttT2JjY1EUhZ07d7Jv3z7MZjPdu3enbdu2BAUFERcXx65du7Db7S79+4WoDO8O8GZIRx0aBYK84Z93G1g1TMtdzR0/FA18YFYfDT0j3C8G7myu4T+3agjxdjT0N/NV0edZ0Fx+zzfwhY/76bg5WsuKYVo6NoAf2jTldHgQKODtr+XaB5oS3tURPDYK1tAkxhez7nKQrgGjAqFecP2FdIIvB/phGhtdLmViUG0M6qHnpq6OCqiAOdiHHi91onG/pvwZ1YgDTULRRnlj8vcmKDsHrc2Gxm6njS2Lu7+8idAnryUQKz0PHsNX47i48A4xkO3rjdZmxyvXRoCvFkWnoPVSaKkkoc8zcV7bEBPeqPlTeMY0RFk6FaWB6yxBjUa2oNmrXdEG6AlPSqetwYxeD1rVRtezf9Lbfhy+fQ5C/Cv5lb36FEWh07LbCIhtAIB3M386fdMbr3CZ1lQI4Zn9Cn/ClQzurYNCQ0Np0aIFO3bsQKvV0rhxY2eL/XXXXUdcXBxpaWlotVq6devm3C7/qv7UqVNuFwSnTp1yK0fx0FVCiBBfheUP+pBrUdFrQKd1vE/WD9eSY1Hx0jr62Bfn2e4anrxOwWwDX72CyapDwdHwX3jbHhEaDjyiwWjW4W9oj8VkR6tT0GgVBgLZZhUfPWgUPTkWL2xWFT8vhRwL+BnAZrs8dtiiMv+fZzlxLg8FSFibRZsRrdhhaoA5x4ZNq+GvbXaU4TfzVXRBN50xY7x4aYQf2FXsuRYMwY7uPb7v96bRjJtpo6rcpddiNdsxeGuxmO2O3kWmNuj89djybJxflcifY3/lktbR4hZwTXd6/tIXjWoDP69iP2PNX+9Gs1e6YrfY0frosFnt2O2gN18DfuM9dmOqLfw7htA9bhBWowWtn06+Z4QQohJJi38d1b17d44dO8amTZtcuvN0796d/fv389tvv9GuXTv8/QtaBXv06IHBYGDZsmUu/fUuXbrE2rVr3crw9XW0wmVmZl7FIxG1lY9ecQb9+Xz1SolBfz6dRnHO+OOlUzDolGK39Tc40vReGjSFyvMzKGguB42+eoUAHw0ajYK/l4KiKOh0Cnq9wsl9Rk4eyXP0wcdxMfDTdynkmlRsOi0oCuczVRbvcO3DuvgPE4lpKjovrTPoz6cYtGi8dGg0CgZvx90GvUGDTq9B5+/ol6/11nJs+j5UW8EdvKz96SR9nYDi733FgFfRadD6ONputDoNeoMG/H1qddBfmM5fL0G/EOKKVI1S4p9wJYF/HRUbG4vdbndrve/evTsWi4UzZ864teoHBwczceJEjh8/zoQJE/jqq6+YN28eDz/8sPOOQeEf4k6dOgHw4Ycfsnr1atavX8/x48er4OiEqDypF9wH29osrt3pcjUaj0NvT6e6D0Qti9wE96f35px0TxNCCCEqgwT+ddR1112H9vIMJYVb/Js3b07Dhg3d0vONGzeO559/npycHD7++GPWrFnDgw8+SP/+/QFcBvB07dqVxx9/nLNnz/LWW2/x0ksvsXHjxqt5WEJUuvbd/N0ayYMa6F1azoOsVgINRfL4KHSPcZ9Vpywa9nUfxN+ov3uaEEIIURkU9UojRYUA3n77bZYsWcK6deto0KBBdVdHiEq1fsUFVi25iN2mpVVLPQ9MjeSHVens+DUTjVbhlj5BNL8hmOe/yebYeRstGml5e5QfN7Q2XHnnJTAl5/LnhF+59FMS+lAvWr3UmebT2lfSUQkhRN23TreoxPV9rQ9VUU1qBxncK1yYTCa3abkuXbrE999/T8uWLSXoF3XOkcO5fLsqC4vGCzSQnKvFJ0DPuEeb8MCERqCAXu+4ObrlJQNZuXb8vZVK6X/u1cSH7t/fgTXbgsZLi0YnN2GFEKK+O3v2LL/88gsXLlxg2LBhREVFYbPZyMjIICgoyNmjozwk8Bcudu3axQcffMDtt99Oo0aNOHfuHCtWrCA3N5fHH3+8uqsnRKX7blkKFnPBjc9LF638/FMG9w4NdQyYLSLAp/KDc51fxboMCSFEfVWXBvCqqsozzzzDxx9/jNVqRVEUOnfuTFRUFEajkebNm/PGG2/w5JNPlrsMCfyFi6ZNmxIVFcV3331HRkYGBoOBDh068PDDD9OjR4/qrp4Qle7SRauHNEs11EQIIUR99p///IcPPviAF154gT59+nDnnXc61wUFBTF06FC+/fZbCfxF5WnatCnvvvtudVdDiCrTpasvmza6Tknb5Rp5YJQQQoiqNXfuXB566CH+7//+j5SUFLf1Xbp08Ti9ellIh1IhRL02fFQY3WJ9ARWN1sqAQUFc3zPgitsJIYSofqqm5L/a5PTp09xwww3Frvfz86vws5Nq2SkRQojK5eurZcrUhlx/4xa699rKPUODq7tKQggh6qFGjRpx+vTpYtfv2rWL6OjoCpUhgb8QQgBarR2NRmY3FkKI2kTVKiX+1SZDhw7l008/JT4+3pmWP4Pcjz/+yIIFCxgxYkSFypDAXwghhBBCiGr2+uuvEx4eTteuXXnooYdQFIUZM2Zw00030a9fP7p06cI//vGPCpUhgb8QQgghhKiV7BqlxL/aJCgoiD/++IPnn3+es2fP4u3tzZYtW0hPT+e1115j69at+PpWbPIJmdVHCCGEEEKIGsDHx4eXX36Zl19++arsX1r8hRBCCCGEqAekxV8IIYQQQtRKtW3KzpKMHz/+inkURWHevHnlLkMCfyGEEEIIIarZzz//7JzFJ5/NZiMpKQmbzUbDhg3x8/OrUBkS+AshhBBCiFpJrWUDeEuSkJDgMd1isTBnzhxmzpzJhg0bKlRGHbpBIoQQQgghRN2i1+uZNm0ad911F9OmTavQviTwF0IIIYQQooa75ppr+OWXXyq0D+nqI4QQgEnVocVe3dUQQghRBmrd6elzRRs2bJB5/IUQoiIyTCoPfg9rsh5Eh43Tv8G/bq3uWgkhhKhv3njjDY/p6enp/PLLL+zevZsXX3yxQmUoqqqqFdqDEELUYiNX2Vh61PVrcEFfhbGdtNVUIyGEEKX1XYP/lbh+yKX7qqgmFafReO6BHxISQsuWLXnkkUeYOHGi28w/ZSEt/kKIem3lUTuowOUvUo3dzn/iNIztVL31EkIIUb/Y7Ve/u6kM7hVC1GsGk9UZ9APYNRoyL5qrsUZCCCHE1SEt/kKIek3jobejX64ZqNgAKiGEEFefvRYP7k1MTCzXdtHR0eUuUwJ/IUS9FpOaxb6IMJe0yIwcILha6iOEEKJ+aN68ebn669tstnKXKYG/EKJeSwxyf/z5BW99NdRECCFEWdXmJ/d+/vnnFRqoWx4S+Ash6rV0Hy+3tEsBPtVQEyGEEPXJww8/XOVlyuBeIUS9pnr4Fsz2NlR9RYQQQoirTFr8hRD1nPttVqsibSJCCFEb1MUn9/7666/s3r2bjIwMtyk+FUXhlVdeKfe+JfAvp0mTJpGUlMTq1asrfd9z5sxh7ty5rFq1ioiIiErff2mtXr2a119/nU8//ZTY2Nhqq4cQVa0O/o4IIYSo4VJTUxkwYAA7duxAVVUURSH/Obv5/1c08JdmrUq0evVqFi9eXN3VEEJUlET+QghRK6iKUuJfbfLcc8/x559/snjxYuLj41FVlfXr13P06FGmTJlC165dOXfuXIXKkMC/nGbNmsW3337rkrZ69Wr+97+SHx1dm/Tv359ff/2Vbt26VXdVhBBCCCHqtB9++IHJkyczatQoAgICANBoNLRq1YpZs2bRvHlznnzyyQqVIYF/Oen1egyGuj0AUKvV4uXlhUYjbxNRN/11yf3hXQCokJKrMnuvnY9220nOLiafEEKIamVXSv6rTdLT0+nYsSMA/v7+ABiNRuf6u+66i/Xr11eojFrbx99isbB48WLWr1/PqVOn0Ol0REdHM3DgQEaNGgXAxYsX+fLLL4mLiyMpKQmTyURkZCQDBgzgwQcfRKvVOveX35991qxZ7N27l9WrV5OSkkKzZs0YN24cd999t0v5Rfv4Dxo0iKSkJACX/vD5/eMPHDjAsmXL+PPPPzl//jxarZZWrVrx4IMPctttt1XoXNhsNubPn8+KFStITU0lOjqa8ePHc/LkSbexAgkJCXz99dfs3r2b5ORkbDYbMTExDB8+nHvvvddlv576+OenzZ49m8OHD7Ns2TIuXLhAeHg448ePZ+DAgRU6FiGqyoztNv6+1XNAb7JB+Gwblstjql7cCt0bg69eYVwnhfh0WHvSTkyQwt97aGgTWst+XYSojU4kw7++hWNJcFdXePYe8JJnbnh06Az8ezmcuggDr4MnB4FWA5+uh2W/Q8NAeO5euK5ldddUFBIREUFycjIAXl5eNGrUiH379nHPPfcAcPbs2QrP+18rA3+LxcK0adPYtWsXPXv2pF+/fhgMBo4fP86mTZucgf+xY8fYtGkTvXv3JioqCqvVyu+//87HH3/M2bNneemll9z2/dFHH5Gbm8vw4cMBR6D70ksvYTabGTRoULF1euaZZ/j4449JT0/n6aefdqbHxMQAsHnzZhISErjjjjsIDw8nIyODNWvW8Nxzz/HWW2/Rt2/fcp+Pt99+m2+//ZbY2FgeeOAB0tPTmTFjhseBwTt37mT37t3cdNNNREREkJeXx08//cRbb71FWloa48aNK1WZs2bNwmQyMXToUAwGA8uWLWP69OlERUXRtWvXch+LEFXhvNHO37eqFNeOb1Wh8MocC2w5A6Cy9mTBii1nVNbE2zgyXkuojwT/Qlw1mTlw0z8gOd2x/MtBOHIWFj1RrdWqkS6kO85V6uWW4i1/QcJFaBQEr31dkG/1Ttj3HrQKr5ZqCne33HILGzZscMano0aN4u2330ar1WK325k5c6ZbQ3RZ1crAf/HixezatYtx48YxdepUl3WFpz3q1q0bK1eudLk6GjNmDK+88gorV65k8uTJNGjQwGX79PR0vv76a+ctluHDhzN69Gjef/997rzzTry9vT3WqXfv3ixevBiTyUT//v3d1k+YMIFp06a5pI0ePZoxY8Ywb968cgf+J06c4Ntvv6VXr1588MEHzm45d9xxB2PGjHHLP2DAAOdFTb4xY8YwZcoUFixYwIMPPohOd+W3hdlsZtGiRej1jtaWPn36cM8997BkyRIJ/EWNt/hw8UF/WV3KhaVHVSZfI4G/EFfNiu0FQX++xVvho0fAw9O367Vvfi0I+vPN2wjBvq5pOSZYuAnedI8VapPa/OTeop5++mk2bNiAyWTCy8uL6dOn89dffzln8bnlllv46KOPKlRGrey8vW7dOgIDA3nkkUfc1hXuj+7t7e0M+i0WCxkZGaSnp9OrVy/sdjsHDx5023748OHOoB8cfayGDRtGZmYmu3btKnedfXwKngSal5dHeno6eXl5dO/enZMnT7r04SqLrVu3Ao6LiMLH3qpVK3r27FliPUwmE+np6WRmZtKzZ0+ys7NJSEgoVbkjRoxwBv0AjRo1Ijo6mtOnT5frOK6G1NRUTCaTc9loNJKVleVcNpvNpKSkuGyT312ruOXk5GTn1FpSRu0tQ1fJPxR5eXluZZRU79p0rqQMKaNmlEGxatdxVO+5Kspms1X6cYjy69y5M08//TReXo4nyoeEhPDTTz+RmppKRkYGmzdvJjy8YndoamWLf2JiIm3btnWemOJYrVYWLFjADz/8wOnTp13eqACZmZlu2zRv3twtLb+7ztmzZ8td59TUVGbPns2WLVtITU11W280Gl0uOArLv2gpzNfXF19fX+e0Ts2aNXPbrlmzZvz2228uaTk5OXz22Wds2LCB8+fPu23j6Zx4EhkZ6ZYWFBTk7JtWE4SGhrosFz2/BoOBsLAwl7SiH6iiy02aNJEy6kAZI71VXvkVMirhtyrMBx7oUnBBXdfOlZQhZdSIMoYEwgtfwPn0ghWjb4IgP4r+ctbo46iKMkbdCNO/gbRCDYrjb3d09Zn+TUGajwHtuD6VfhxVrS49wOvgwYN06NDBLT04OLjSyqiVgX9pvf/++3zzzTfceeedjB8/npCQEHQ6HYcPH+ajjz5yuxC4WlRVZdq0aZw8eZLRo0fToUMH/P390Wg0rF69mnXr1rk9ma2wffv2MWXKFJe0iRMnMnny5DLX5aWXXmLbtm0MGTKEbt26ERQUhEaj4ddff2Xx4sUl1qOw4mb6qapzKkRFNPZT+O0+LcNW2Tjsfh2Orw6uawItgxSe665wKhNm7VWx2mF8J4X4DPjhpJ0Wlwf3hkn/fiGurkBf+PX/4P/yB/de4xicKtw1DoZt/yw0uDcWnro8uLdhICz9HRpdHtzbuvoeEircderUiU6dOjF69GhGjhxJq1atKr2MWhn4N2vWjISEBMxmc4lTav7www9069aNf/3rXy7pJXVH8dTV5eTJk4DnVu7CihtpfezYMY4ePeoxWF+xYkWJ+wRo06YNs2bNcknLr0v+AN5Tp04RFRXlkufUqVMuy1lZWWzbto3+/fvzj3/8w2Xdjh07rlgPIeqSDg0Ussye1/UKh59G6QrlhX4tXPO82KNW9pQUovZq2QTmTb1yPgEdmnoe+PxYP8efqJFmz57NkiVLePXVV3nllVfo2rWr8yLAU8+O8qiVv1x9+/YlMzOTefPmua0r3OKs0WjcWqBzc3NLfLrusmXLXPrbG41Gvv32WwICArjuuutKrJevry+ZmZluZea3jhdNP378OJs3by5xnwCBgYH06NHD5S8/yL/55psB+Prrr11a648fP84ff/xRqnpcunSpVBcgQtQ1xbXTW+TGlRBC1Ap16cm9kydPZuPGjZw9e5YPPvgAPz8/XnzxRVq0aOGcxKWiT+6tlS3+9913H1u3bmXevHkcPHiQHj164OXlRXx8PKdOneKTTz4BHDPNLF++nL///e9cf/31pKSksHr1aoKCgordd3BwMGPHjnVO3bl69WqSk5N5+eWXi53RJ1+nTp3YunUrb7/9Nl26dEGj0dC9e3diYmJo0aIFixYtIi8vj2bNmpGYmMjy5ctp1aoVhw4dKve5aNmyJUOGDOG7777jscceo3fv3qSnp7N06VLatm3LoUOHnHci/Pz86NmzJ2vXrsXLy4uOHTuSlJTE8uXLiYyMdBtHIERd19gXzngYV38myz1NCCGEqAqNGzdm2rRpTJs2jbNnz7J06VKWLFnCM888w7PPPovFYin3vmtl4K/X6/n444/58ssvWb9+PZ988gkGg4Ho6GiXufaffvpp/Pz82LBhA1u2bKFx48YMGTKEDh068Nhjj3nc9+OPP87evXtZunSp82FYpZ1n//777+fs2bNs3LiRb7/9Frvd7nz41QcffMDMmTNZs2YNubm5tGzZkunTp3P06NEKBf4AL774Ig0bNmTlypV88MEHNGvWjBdffJG//vqLQ4cOuQyCfvPNN/noo4/YunUr33//PU2bNuWxxx5Dp9Px+uuvV6geQtQ2xTXs+5gs1NKvRyGEqFdq29N5yyo8PJyOHTvSvn17Dhw4QHZ2doX2p6gyGhPw/JTa2u6pp54iLi6OLVu2uDylWAjh8LeNNj7a4/4V2NaSzeG/F39nUAghRM3wRbOlJa5/8NSIKqpJ5VFVlc2bN/PNN9/w3XffcenSJUJCQhg6dCijRo2iT58+5d63NGnVAXl5eW7dkI4dO8Zvv/3GDTfcIEG/EMWYfoOGj3ZboUg/0FSNvpgthBBCiKtj69atLFmyhGXLlnHhwgUCAwO59957GTVqFHfccUepHrB6JRL41wFr1qzhhx9+4MYbbyQkJISEhAS+++47dDpduab8FKK+CPVRPI7wzfOSwF8IIWqD2jaAtyS33nor/v7+DBo0iFGjRtG3b98SZ68sDwn864B27do5bwllZGTg5+dHbGwskyZNol27dtVdPSFqOPcfDXsd+iERQghROyxdupQBAwZccTKZipA+/kKIek15x+qW5qcF41PSLiKEEDXdwphlJa4fe3J4FdWkdqiV8/gLIcRVJQ3+Qggh6iAJ/IUQogidfDMKIYSog+RethCiXtNrwPL/7d13XFPX/z/wVxI2YQgiKIiiiAOxqChO1GrVquCi1rqoteK2dbWO1lFtbW3dRcXWba1WHIAb/eBsxb3qLuAEFzLCJrm/P/yRryEBCTvk9Xw88qg59+Se97kE+s7JOecqVMucLMsnFiIi0g7XZGmH41pEpNd6u6qXTfXi/0iIiKjy4Yg/Eem137tJkJ6dg/0xAgyhwGQvCT5tzD+NRES6QOA4jVY44k9Ees3KWIQ9fsByi61YZrEV89uWd0RERKSvkpOT8eOPP6Jbt25o2rQpzp07BwBISEjAkiVLcP/+/WKdn8NaREQATETq23oSEVHFVplu4PX48WN06NABjx49Qr169XD79m3IZDIAgI2NDYKDg/HgwQMsX768yG0w8SciIiIiKmfTpk1DSkoKrly5gmrVqqFatWoqx/v06YN9+/YVqw1O9SEiIiIiKmdHjhzBxIkT0ahRI4g0fJNRp04dPHr0qFhtcMSfiIiIiHRSZZrqk56eDjs7u3yPp6SkFLsNjvgTEREREZWzRo0a4eTJk/ke37t3L5o2bVqsNpj4ExEREZFOEkQFP3TJl19+ie3bt+Onn35CUlISAEChUOD+/fsYOnQo/vnnH0yaNKlYbXCqDxERERFRORsyZAgePHiAb775BrNmzQIAdO/eHYIgQCwW44cffkCfPn2K1QYTfyIiAHK5HFlZWUhNTYW1tXV5h0NERHpo1qxZGDp0KHbt2oX79+9DoVCgbt266NevH+rUqVPs84sEQRBKIE4iIp31999/IywsDAAgkUjg5+cHb2/vco6KiIjeZW2DvQUeD7zdp0ziKK60tDS0b98eI0eOxOjRo0utHc7xJyK9JpPJsH//fuVzuVyO8PBw5U1TiIiISpuZmRliYmI0buNZkpj4E5Fei4+Ph1wuVynLyclBfHx8OUVERESFJYhEBT50Sffu3XH48OFSbYOJPxHpNQsLC43llpaWZRwJERHps2+//RZ3797F0KFDcfr0aTx58gQJCQlqj+Lg4l4i0mv53RAlOTlZ7XbpREREpcXd3R0AcPPmTWzbti3fenm/pdYGE38i0msc8Sci0l2CWLem8xRk9uzZpT7Hn4k/Eem158+fayx/9uwZR/yJiKjMzJ07t9TbYOJPRKRBaY+6EBFRCeDfaq0w8ScivZbfqD5H+4mIqCx9991376wjEonw7bffFrkNJv5EpNe4uJeIiCqCgqb6iEQiCIJQ7MSf23kSkV7j4l4iIt0liEUFPnSJQqFQe+Tk5OC///7DpEmT4OXlle+6tMJi4k9EeunZs2c4ceIE7t69q/H41atX1e7eK5PJcP/+fd7Vl4iIyoRYLIaLiwt++eUX1KtXDxMmTCjW+TjVh4j0zvbt23HlypUC6xw7dgzHjx+Hn58fvL29ERUVhbCwMMjlckgkEmU5ERGVH127O29x+Pj44Ouvvy7WOZj464HMzExs3LgRhw8fxrNnz2BoaAh7e3u0adMGX3zxhbJeVFQUNm/ejH///RdZWVlwdnaGv78//P39lXVmzJiBY8eOYdWqVfDy8lKW//PPP5g4cSI+/PDDQi1OISotz549w+3bt+Hs7KwcmZdKpXj48CGcnZ3x8uXLdyb9ueRyOfbs2QOZTIb//e9/ypumyOVyhIaGQiwWw9nZGSkpKXBwcIBUKtV4HplMhvj4+ALrEBERFeTChQsQi4s3WUckCIJQQvFQBfXdd98hLCwMPXv2RJMmTSCXy/Ho0SNcvnwZW7duBQDs3r0bCxcuhIeHBzp27AhTU1NERUXh+PHjGDp0qPIDgkwmw+DBg5GdnY1t27bB2toaL1++xKBBgyCVSrF161aYmZmVZ3dJjxVmJL+05PctAL8pICIqPUGeBws8Pu7Kh2UUSfFt3rxZY3liYiJOnjyJ3bt34/PPP8fatWuL3AYTfz3w/vvvo3HjxlixYoXG4y9fvoSfnx86deqE77//XuXYL7/8gr/++gu7d++Gk5MTAODGjRv4/PPP0bp1ayxevBjjx4/H5cuXsWHDBjRo0KDU+0OkybNnz7B06dJyjcHAwADTp09XjurLZDIsXLhQ5fbqeesQEVHRVabEv6DR/KpVq+Lzzz/H7NmzYWJiUvQ2ivxK0hlSqRTR0dG4f/++xuNHjx5FVlYWevfujcTERJVH+/btoVAocO7cOWX9xo0bY8yYMTh16hRGjhyJc+fOYfz48RUu6U9ISEBmZqbyuUwmU9m6MSsrC69evVJ5TVxcXIHP4+Pj8fZnZbZRcdq4ffs2yltOTg4eP36s7Ed8fLxK0p9b5+bNmypllfHnwTbYBtvQzzao6GJiYtQesbGxSEpKwvPnz/HDDz8UK+kHOOKvF44fP445c+YgNTUVjo6O8PLyQvv27eHj4wOxWIwff/wRISEhBZ5j9OjR+Pzzz5XPBUFAYGAgLl++jFatWmHlypW80ymVq4oy4j9jxgyYm5sDyH/E/+06RERUdL82O1Tg8fGXupdRJMX38OFD2NnZwdTUVOPx9PR0vHjxAs7OzkVug4t79UDHjh0RFhaGM2fO4NKlSzh37hxCQ0PRtGlTrFq1SvnJfd68eahatarGczg6Oqo8f/r0Ke7duwcAePToEdLS0pjIULmyt7eHp6dnuc3xNzAwgK+vr8rvgVQqhZ+fH8LDw5GTk6OxDhEREQC4uLhgy5YtGDRokMbjYWFhGDRokNo3ydpg4q8nrKys0KNHD/To0QOCIGDlypXYvHkzTpw4gZo1awIArK2tC7XoMCcnB7NmzYJcLsfUqVOxePFi/Pjjj5g/f35pd4OoQAMHDkSnTp1w584d1KxZU2VXn0ePHqFmzZrKRbY3b95EWloaUlNTNU4Tat++PaysrJCUlIRq1arB2dlZeeOUatWqqfy7oF19vL294e7uzl19iIhKQWXazvNdk3Cys7OLvasPE/9KTi6XIy0tTeXupCKRCPXr1wcAJCUl4YMPPsCqVasQHByM5s2bq80fk8lkMDIygpGREQBg9erVuHHjBubMmQNfX188e/YMW7Zsgbe3N3r16lV2nSPSwN7eHvb29mrlLi4uGp/nbv+ZV4sWLVCtWjW1c7/r35pIpVK4urq+O3giItIrycnJSExMVD5/9eoVHj58qFYvMTER27dvR/Xq1YvVHhP/Si4tLQ3du3eHj48P6tevjypVquDp06cICQmBpaUlfHx8YGdnh+nTp2PBggX46KOP0KNHD1SvXh2vX7/G/fv3cfz4cezcuRM1atTA2bNnsXnzZnTv3h2+vr4AgHHjxuHixYtYtGgRmjRpUqy5Z0RlLb/bnz979kwt8SciIipJS5cuVd7/SCQS4csvv8SXX36psa4gCFiwYEGx2mPiX8mZmJjgk08+wblz53Du3DmkpaWhatWq8PHxwfDhw2FnZwcA8PPzg7OzM7Zu3Yrdu3cjJSUF1tbWqFWrFsaMGQNbW1skJCRgzpw5cHR0xIwZM5RtGBgY4IcffsDgwYMxa9YsrF+/HoaGhuXVZaISwcXqREQ6QMf/VHft2hVSqRSCIOCrr77CJ598gmbNmqnUEYlEMDc3R/PmzVVunloUTPwrOUNDQ4wfP75QdT09PeHp6ZnvcWNjYxw+fFjjMScnJ5w4caIoIRKVq/xG9TnaT0REpa1169Zo3bo1ACA1NRX9+/dH48aNS609Jv5EpNfe3tP6bcnJyUz+iYgquMq0uHfOnDml3gYTfyLSa28vfH+bpaVlGUdCREQE5fbrSUlJUCgUKsdEIhG+/fbbIp+biT8R6TUu7iUiooogISEBPXv2xLlz5yAIAkQikXKLz9x/FzfxL95moERElRQX9xIRVXyCWFTgQ5dMmzYN165dw7Zt2xAdHQ1BEHD48GHcvXsXo0ePhqenJ54+fVqsNpj4E5Fe4+JeIiKqCA4cOIBRo0bh448/Vk5DFYvFcHV1RVBQEGrXrp3vVp+FxcSfiPRaQYt7iYioYhNEogIfuiQxMRHu7u4AoLzLe+4d6IE3W3/mt7tiYTHxJyK95uDgAIlEolJmYGBQ7LsjEhERaaNGjRqIj48H8GYL9WrVquHq1avK40+ePCn2NFQu7iUivSaVStGzZ0+Eh4dDEAQYGBjA19cX5ubm5R0aERHpER8fH0RERGDWrFkAgI8//hiLFi2CRCKBQqHAsmXL0K1bt2K1wcSfiPReixYtcPXqVWRlZSEgIADW1tblHRIRERWCrk3nKcjkyZMRERGBzMxMGBsbY+7cufj333+Vu/j4+Phg5cqVxWqDiT8REQCJRAJTU1OO9BMRUbnw8PCAh4eH8nmVKlVw9OhRJCYmQiKR5HvfGW0w8SciIiIinVSZRvzzU5LfQnNxLxERERFRBfDw4UOMHj0a9evXh42NDU6ePAkAePnyJSZOnIjLly8X6/wc8SciIiIinVSZRvxv3ryJ9u3bQ6FQwNvbG/fv30dOTg4AoGrVqjh9+jRSU1Oxbt26IrfBxJ+IiIiIqJx99dVXsLa2xtmzZyESidRuJNmzZ0/s2LGjWG1wqg8RERERUTk7efIkxowZAzs7O4379Ts7O+PJkyfFaoMj/kRERESkkyrTVB+FQgEzM7N8j7948QLGxsbFaoMj/kREAORyOdLT05GamlreoRARkR5q1qwZ9u/fr/FYTk4Otm/fjlatWhWrDSb+RKT3zp8/jwcPHiAuLg6LFi1CVFRUeYdERESFIIhEBT50yYwZM3Do0CGMGTMGN27cAAA8e/YMR48eRdeuXXHr1i1Mnz69WG2IBEEQSiJYIiJdJJPJ8P333+PtP4VisRgzZ86EVCotx8iIiOhdfm5/osDj0051KKNISsaWLVvwxRdfICkpCYIgQCQSQRAEWFpaYvXq1fjkk0+KdX7O8ScivRYTE4O84x8KhQIxMTEqd1AkIiIqbUOHDkW/fv1w5MgR3L9/HwqFAnXr1kW3bt14514iouJKTEzUWJ6cnFy2gRARkdYEsW5N58lr5syZGDhwIJo0aaIsMzc3R9++fUulPc7xJyK9lpSUpFU5ERFRSfnxxx+V8/kB4NWrV5BIJPjf//5XKu1xxJ+I9JqpqalW5UREVHHo2gLewijN5bcc8ScivZaRkaGxPC0trYwjISIiKl1M/IlIr1lZWWkst7a2LttAiIiIShmn+hARacCdjomIKr7KMNUnNjYWly5dAvB/68vu3buX7wBUs2bNitxWpUj8AwMDERcXh/Dw8PIOhYh0TH6LeLmrDxERlYVvv/0W3377rUrZ2LFj1erl7usvl8uL3JbOJP7btm2DhYUFfH19yzuUSoHXk+gNExMTrcqJiKji0PUR/w0bNpRpezqT+P/555+oXr26xkQ1KCiIX8trqaDrSaRP8lvcm185ERFRSQkICCjT9irF4l5DQ0MYGRmVdxiVWmpqanmHQFQqLC0tNZbnt+iXiIhIV5V44h8eHg4vLy+cP38eW7ZsQe/evdG6dWv069cP+/btU6l75MgRTJo0CT179kTr1q3RuXNnTJkyBffu3VOp5+Xlhbi4OFy6dAleXl7Kx9OnTwG8meP/9sj1jBkz0KpVK4135IyNjYWXlxcWL16sFsuIESPg4+ODtm3bIiAgAEePHtWq7zKZDEFBQfD390ebNm3QuXNnjBgxAocPH1apd+/ePUydOhWdO3dGmzZt8NFHH2HTpk1qc7by9ivX06dP4eXlheDgYGXZhQsX4OXlhfDwcISFhWHAgAFo3bo1evXqhU2bNqm8/l3X09fXF4GBgbh9+zbGjx+PDh064JNPPkFkZCS8vLywZ88ejf0fMGAA+vTpw29fqNTIZDLcv38fMpmsxM4pyudr4ocPH+LZs2dq7eWNQVNMpREnERGpE0SiAh+kqtSm+gQFBSEzMxP9+vWDkZERQkJCMHfuXDg5OcHT0xMA8Ndff8HKygp9+/ZF1apV8fjxY+zZswcjRozA1q1b4ezsDAD47rvvsGTJElhbW+Ozzz5TtlGlShWNbffs2RMRERE4fPgwPv74Y5Vj+/fvV9bJtWrVKqxfvx5t2rTB6NGjIRaLERkZienTp+Orr77CgAED3tnflJQUjBgxAtHR0ejcuTP8/f0hl8tx584dnD59Gt26dQMA3Lx5E4GBgTAwMMBHH30EW1tbnDp1CitXrsS9e/ewYMGCwl9kDXbt2oWEhAT4+fnBwsICBw8exMqVK2Fvb4/u3bsDKNz1fPbsGcaMGYMuXbrg/fffR1paGtq3bw9bW1uEhYWp3Ur6+vXriI6OxtixY/NNpIiKIyoqCmFhYZDL5ZBIJPDz84O3t3exz3v79m2N5VevXsXVq1cBQNkeAJUYPDw8cP36dZWY8tYpqTiJiIiKq9QS/6ysLGzevBmGhoYAgM6dO6N3797466+/lIn/ypUr1e6O2bNnTwwaNAjbtm3D9OnTAQA9evTA6tWrYWNjgx49eryz7datW8PW1hb79+9XSfwFQcDBgwfh6uqKBg0aAHjzP/3169dj+PDhGDdunLLuwIEDMWXKFAQFBaFnz54wNzcvsM2goCBER0dj5syZ6Nevn8oxhUKh/Pcvv/yC7OxsbNiwAfXq1QMAfPzxx5gxYwYOHToEPz8/tGzZ8p19zE98fDxCQkIglUoBAL1790avXr2wY8cOZeJfmOv55MkTfPPNN+jTp49KuZ+fHzZs2IDo6GjUqVNHWR4aGgqJRMI1A1QqZDKZMpkGALlcjvDwcLi7uyvf60U973///ffOenK5HGFhYRAEQfn7LJfLceXKFbU6uf8uyTiJiEgzjuprp9Tm+H/00UfKpB8AqlWrBmdnZzx69EhZlpv0C4IAmUyGxMREVKlSBbVq1cKNGzeK3LZEIsGHH36ImzdvIjY2Vll+8eJFxMfHo1evXsqygwcPQiQSoWfPnkhMTFR5+Pj4IDU1FdevXy+wPYVCgSNHjsDFxUUt6QcAsfjNZU5ISMC1a9fg4+OjTPqBN1MNckfeIyMji9xv4M00nbcTDBMTE3h4eODhw4dancfKykpjEt+nTx+IRCKEhoYqy9LT0xEREYE2bdrAzs6u6MGXsISEBGRmZiqfy2QypKSkKJ9nZWXh1atXKq+Ji4sr8Hl8fLzKVCa2UTZtxMfHq02Fy8nJwa1bt4rVRt4YCyKXy1U+xOdXR1OcN2/e1Niv/OKo6D8PtsE22AbbyK8NqthKbcTf0dFRrczKygrx8fHK57dv38aaNWtw8eJFpKenv/P12ujVqxe2bt2K/fv3K0fy9+/fD4lEohz5BoCYmBgIggB/f/98z5X7S5GUlITs7GyVY1WrVkViYiKSk5PRunXrAmPKnUP/9kh5LhcXF4jFYjx58qRwHcxHftc9v73KCzqPRCLRWN6yZUscOHAAEyZMgIGBASIiIpCamorevXsXOe7SYGNjo/I874irkZERbG1tVcqqV69e4HMHBwe2UQ5tODg4QCKRqCTVBgYGaNSokcrUMm3bcHBwgFgsfmdCD7wZUHh7xD+/OgDU4nR3d9fYr7fjeFtF/3mwDbbBNthGfm2UNY74a6fUEv/cUe68cj8lxsfHIzAwEObm5hgxYgRq164NExMTiEQiLF68WO2DgLZcXV3h5uaGgwcPYuzYscjMzMT//vc/eHt7o2rVqip1RSIRVqxYkW/MdevWBQBMmzZNeWe1XBcuXChWnAXJb658QTdu0JSsF0VBe5j37dsX06dPx4kTJ9C5c2eEhobC1tYW7dq1K5G2ifKSSqXw8/NDeHg4cnJyYGBgAF9f33dOwSvMed3d3d/5rV5uewBUYmjcuDFu3LihElPeOiURJxERUUkot338IyMjkZaWhiVLlsDLy0vlWFJSktr2nEVZMNqrVy8sWbIEFy5cwMuXL5GamqoyzQcAatasib///hsODg5wcXEp8HyTJk3SeDdPa2trWFpaqu1GlFeNGjUAANHR0WrHYmNjoVAoVEbsLS0tNS48LO63AkDRrmeujh07wsbGBqGhoahbty6uXr2KgIAAGBjozG0hSAd5e3vD3d0d8fHxcHBwKLE58/ndEr1evXro1asXUlJSVNrLG4NMJlOLqTTiJCIiKq5y28c/d3Q97/zaPXv2qM03A96sB9CUdBeke/fukEgk2L9/P/bv3w+pVIoOHTqo1Mld3BoUFKRxJP3tWBo2bAhvb2+VR25funXrhujoaOzdu1ftHLl9tLGxQZMmTXDy5Encv39f5Xjunds6deqkLK9VqxZSU1NV1jsoFAps27ZNq+ugSVGuZy4DAwP06tULZ8+exW+//QYAFW6aD1VOUqkUrq6uJZpMGxsbayx3cXGBvb29Wnt5Y9AUU2nESURE6gRRwQ9SVW5DtG3btsXKlSsxe/ZsDBgwABYWFrh69Sr+/vtvODk5qSXhHh4eCA0NxerVq+Hi4gKRSAQfHx+1XYHeZmNjgzZt2uDYsWPIysqCn5+f2v/k3d3dERgYiLVr12LQoEHo0qUL7Ozs8PLlS9y6dQtnzpzB2bNn39mfMWPG4Pz581iwYAGioqLw3nvvAQDu3LmDnJwczJ8/HwAwdepUBAYGYuTIkcrtPE+fPo1//vkH3bt3V9nRp2/fvti6dSumTZuGgQMHwtDQEMeOHStwqk9hFeV6vq1v377YsmULDh8+jGbNmim3XiXSNfktSivudEMiIqKKptwSfycnJ6xYsQJBQUHYsGEDxGIx3nvvPQQHB2PRokVqq8jHjh2LpKQk7Ny5EykpKRAEAWFhYe9MVHv16oVTp04BUN27/22BgYFo1KgRtm/fjj///BPp6emwsbFB3bp1MXXq1EL1x9LSEhs2bMD69esRGRmJyMhImJubw8XFRWVL0UaNGmH9+vUIDg5GSEgI0tPT4ejoiAkTJmDIkCEq53R0dMQvv/yCVatWYc2aNbCyskKPHj3g5+dX4GLkwijq9cxVs2ZN5Y3aONpPuiy/O/TmNwWIiIgqDi7u1Y5I4G1WqYgmTpyI69ev4+DBgwUuBiaqyE6dOqW8sd/bfH190bZt23KIiIiICuu7blEFHp99mDdQfFu5zfEn3fbo0SOcPXsWH374IZN+0mn5rXVJTEws20CIiIhKGbdhIa3cuHEDMTEx2L59OwwNDdWmJxHpmvw+uBZ22hsREZUfTvXRDhN/0kpISAj2798PR0dHzJ8/X7lFKZGuysjI0KqciIhIVzHxJ63MnTsXc+fOLe8wiEqMpaWlxvL8Fv0SEVHFoeCIv1Y4x5+I9Fp+N7PjvgdERFTZMPEnIr2WlJSkVTkREZGu4lQfItJr1apV01hub29fxpEQEZG2BHCqjzY44k9Eeq1hw4ZqZSKRSGM5ERGRLmPiT0R6TSqVws/PT/lcLBajT58+MDc3L8eoiIioMASRqMAHqeJUHyLSey1atMDVq1eRlZWFgIAAWFtbl3dIREREJY4j/kREACQSCUxNTTnST0RElRZH/ImIiIhIJ3E6j3Y44k9EREREpAc44k9EREREOokj/trhiD8RERERkR7giD8RERER6SSBA/5a4Yg/EREREZEeYOJPRARALpcjPT0dqamp5R0KERFRqWDiT0R67/z583jw4AHi4uKwaNEiREVFlXdIRERUCAqRqMAHqWLiT0R6TSaTITw8XPlcoVBg7969kMlk5RgVERFRyWPiT0R67datWxAEQaVMEATcunWrnCIiIqLCEkSiAh+kiok/Eem158+fa1VORESkq5j4E5Fes7Ky0qqciIhIV3EffyIiDUT8ipiIqMLjdB7tcMSfiPRaUlKSxvLExMSyDYSIiKiUccSfiPRaflN6rK2tyzYQIiLSGrfs1A5H/IlIr+Xd0YeIiKiyYuJfjp4+fQovLy8EBweX2DkDAwPh6+tbYucjquySk5M1lnOqDxERVTac6kNEeo1TfYiIdJfAmT5a4Yg/Eek1TvUhIiJ9wRH/cpCRkQEDA156ooqAU32IiHSXAA75a0Nvs8+4uDj4+vpi5MiRGDVqlLJ8/PjxOHv2LCZNmoTBgwcrywMCApCamoqQkBAAwL179xAcHIzLly8jPT0djo6O6NWrF4YMGQKJRKJ83dy5c7Fv3z5ERERgxYoVOHPmDF6/fo3Q0NB8Y/vnn3/w9ddfw83NDUuWLIGlpSUA4NGjR1i/fj2ioqKQkJAAa2trNGrUCCNHjkTDhg3zPd+NGzcQEhKCa9eu4dmzZ5BIJHB1dcXQoUPRqVMnlbrx8fEIDg7G+fPn8erVK0ilUtSsWRP9+vVDr169AAAKhQLbt29HWFgYnj59CpFIBFtbW3h6emLmzJn8UEMVlkwmQ3x8PBwcHCCVSiGTyfDo0SONdV+9eoXr16/DxcVFWTc+Ph4WFhZISUmBg4MDAKicj4iIqCLT2wytevXqcHR0xPnz55WJf3Z2Nq5cuQKxWIwLFy4oE3+ZTIbbt2+jX79+AICbN28iMDAQBgYG+Oijj2Bra4tTp05h5cqVuHfvHhYsWKDW3rhx42Bra4sRI0YgPT0dZmZmSEtLU6u3b98+zJ8/Hz4+PliwYAGMjY2VbY4ZMwY5OTno3bs36tati+TkZFy6dAlXr14tMPE/fvw4YmNj0aVLF1SvXh1JSUnYt28fpk2bhgULFqB79+4AgJycHIwbNw4vXryAv78/nJ2dIZPJcP/+fVy+fFmZ+K9fvx5r1qxB+/bt0b9/f4jFYjx9+hQnT55EVlYWE3+qkKKiohAWFga5XA6JRAIPDw9cv34dcrlcY/2bN2/i5s2bEIvFaNKkiVpdsVgMQRAgCAIkEgn8/Pzg7e1dVt0hIiLSml5naC1atMC+ffuQkZEBExMTXL9+HRkZGfjwww9x8uRJ5OTkwMDAAJcuXYJcLoeXlxcA4JdffkF2djY2bNiAevXqAQA+/vhjzJgxA4cOHYKfnx9atmyp0lbdunUxf/58lbK8if+GDRsQFBQEf39/fPXVVxCL3yzBEAQBc+fORXZ2NjZt2qRsEwCGDx8OhUJRYD9HjBiB8ePHq5QNHDgQgwYNwrp165SJf0xMDB48eIAJEyYgICAg3/NFRkbCxcUFS5cuVSmfMGFCgXEQlReZTKZM+gFALpfjypUrhXqtQqHQWPft3zu5XI7w8HC4u7tz5J+IqAxxH3/t6PXiXi8vL+Tk5ODy5csAgPPnz8PGxgaffPIJUlNTcfPmTQDAhQsXIBKJ4OXlhYSEBFy7dg0+Pj4qCbhIJMJnn30G4E1inNeQIUPyjUMQBPz0008ICgrC6NGjMX36dGXSDwB37txBdHQ0fH19VdrM9XZdTUxNTZX/zsjIQGJiIjIyMtCiRQvExMRAJpMBgDJhuXjxIhISEvI9n1QqxfPnzwudOJWXhIQEZGZmKp/LZDKkpKQon2dlZeHVq1cqr4mLiyvweXx8vMpiULahG23Ex8fnO7JfUnJychAfH6/z14ptsA22wTaK0wZVbCJBj7e0ePXqFbp164Zhw4Zh4sSJGDlyJKpWrYrvv/8eXbp0weDBgzFixAgMGjQIALBt2zbcuHEDn376KUaMGIExY8aonC87Oxtt27ZFq1atsGLFCgD/N8f/9OnTMDExUan/9OlT+Pn5wdzcHKmpqRg7dqzyw8PbIiIiMGPGDMycOVM53Sg/gYGBiIuLQ3h4uLIsISEBq1evxokTJzQm9Pv27VPOV161ahU2btwIAHBzc0OLFi3QpUsXuLu7K+tfu3YNU6dORUJCAuzs7NC8eXO0a9cOnTt3hqGhYYHxEZUHmUyGhQsXlmryb2BggBkzZsDc3LzU2iAiIlVf+N8s8PjykEZlFIlu0OsRf1tbW9SpUwcXLlxARkYGbty4gRYtWkAsFqNZs2Y4f/48EhMTce/ePbRo0aJYbeVN+t/WsmVLWFtbY8+ePXj8+HGx2slLEASMHz8e+/btQ8+ePbFw4UKsXLkSQUFByik+b09ZGDt2LHbv3o3JkyfDyckJoaGhCAgIUH6QAYAmTZpg7969+Omnn9CxY0fcvXsX33zzDQYNGoSkpKQSjZ+oJEilUvj5+SnXnxgYGMDT07NQ61HEYrHGumKxWPltm4GBAXx9fZn0ExFRhabXc/yBN9N9QkJCcPLkSWRnZyvn5rdo0QLLly/H33//DUEQlIl/jRo1AADR0dFq54qNjYVCoYCjo6NWMbi6umL06NEYM2YMRo0ahdWrV8PZ2Vl5PPffd+/e1bp/9+7dw927d9V2LwKAvXv3anyNk5MTBg4ciIEDByIzMxMTJkzA5s2bMWTIENjY2AAAzMzM0LlzZ3Tu3BkAsHPnTvz0008IDQ3FsGHDtI6TqLR5e3vD3d1dbVefiIgIREVFqdVv1KgRmjZtyl19iIgqMIFz/LWi1yP+wJsEX6FQ4LfffoODgwOcnJyU5VlZWdi4cSMkEgmaNm0KALCxsUGTJk1w8uRJ3L9/X3keQRCwYcMGAFDbIrMw6tati7Vr10IulyMwMBCxsbHKY25ubqhTpw7CwsLw33//qb22oNlaby8Qftv9+/dx/PhxlTKZTIacnByVMmNjY9SuXRvA/+13rml/8wYNGqjUIaqIpFIpXF1dlUm6VCqFnZ2dxrp169aFh4eHSl1XV1fY29srz5H3fERERBWZ3o/4N2/eHGKxGDExMfD19VWW16lTB7a2toiOjoaHh4fKV/hTp05FYGAgRo4cqdzO8/Tp0/jnn3/QvXt3tR19Cqt27dpYu3atcuR/1apVqFu3LkQiEebMmYOxY8ciICBAuZ1nSkoKLl26hNatW2PgwIEaz+ni4oI6depg8+bNyMjIQK1atfDw4UPs3r0brq6uuHXrlrLuhQsX8P333+P9999HrVq1YGZmhlu3biE0NBSNGzdWfgDw9/eHh4cH3N3dYWdnh5cvX2LPnj0wNDRE165di9R3ovKS3/Q0TlsjIqLKRu8Tf0tLS7i5ueH27dvK7TpztWjRAocOHVIrb9SoEdavX4/g4GCEhIQob+A1YcKEAnfvKQxnZ2cEBwdjzJgxGD16NFatWoV69erB3d0dmzZtwrp163D06FHs2rUL1tbWcHd3h6enZ77nk0gkWL58OZYtW4Z9+/YhPT0ddevWxdy5c3H37l2VxL9evXro1KkTLl68iEOHDkEul8PBwQHDhw9X6deQIUNw5swZ7NixAzKZDDY2NmjcuDGGDx8ONze3YvWfqKzl3iAvLysrqzKOhIiItKXgTB+t6PWuPkREp0+fxr59+9TKe/XqhXbt2pVDREREVFjjB9wq8Pivf+V/g1N9pPcj/kSk3zjVh4hId3Fxr3b0fnEvEem3/Kb0WFtbl20gREREpYyJPxERERGRHuBUHyLSa5zqQ0SkuxTgVB9tcMSfiPRaflN9uKsPERFVNhzxJyK9ZmxsrFU5ERFVHFzcqx2O+BORXjMxMdGqnIiISFcx8Scivebi4gJRnhEjsVgMFxeXcoqIiIiodDDxJyK9JpVK4evrq3wukUjQu3dvmJubl2NURERUGApRwQ9SxTn+RKT3WrRogatXryIrKwsBAQHcw5+IiColJv5ERHgz0m9qasqRfiIiHaLg4l6tcKoPEREREZEeYOJPRERERKQHONWHiIiIiHQS9/HXDkf8iYiIiIj0AEf8iYiIiEgncctO7XDEn4iIiIhIDzDxJyICkJWVhcTERLx48aK8QyEiokISICrwQaqY+BOR3tu5cyceP36MhIQErFixAtu3by/vkIiIiEocE38i0mvPnj3DtWvXVMquXLmCZ8+elVNEREREpYOJPxHptbxJ/7vKiYio4lCIRAU+SBUTfyLSa2Kx5j+DEomkjCMhIiIqXdzOk4j0WkZGhsby9PT0Mo6EiIi0xVF97XDEn4j0mpWVlVblREREuoqJPxGRBiKOIhERUSXDqT5EpNeSkpK0KiciooqDd+7VDkf8iUivGRsbayw3MTEp40iIiIhKFxP/YgoMDISvr295h1Fo4eHh8PLywoULF8o7FKIKITMzU2M5F/cSEVV8CogKfJAqJv6V0IULFxAcHIyUlJTyDoWowuPiXiIi0hdM/Cuhixcv4rffftOY+Pfo0QNnzpxBs2bNyiEyIt3Bxb1ERFTZcHGvnpFIJLwxEdFbuLiXiEh3CRyk0YpeJ/5ZWVnYunUrDh06hMePH8PIyAhNmzbFqFGj0KBBA5W6ycnJWLFiBSIjI5GZmYlGjRph0qRJGs/r5eWFXr16Ye7cuSrl4eHhmDdvHtasWQMvLy9luUwmw6ZNmxAZGYmnT5/C1NQUtWvXxoABA9CtWzcAQGxsLLZv345Lly4hPj4ecrkcLi4u8Pf3R58+fZTnmjt3Lvbt2wcA8PPzU5aPHDkSo0aNyjeGxMREBAcH4+TJk3j16hVsbW3h4+ODUaNGwdraWq0Pq1evxu3btxESEoLnz5+jevXq+Oyzz9CrVy+tfgZE5S2/O/fGx8fj+vXrqFatGlJSUmBhYYHnz58DAFxcXJR1HBwcIJVKyyxeIiKiotLbxD8nJwcTJkzAtWvX0KNHDwwYMAAymQx79uzBiBEj8Ntvv6FRo0bKuuPHj8fNmzfRo0cPeHh44O7duxg7dmyx5wGnpKRgxIgRiI6ORufOneHv7w+5XI47d+7g9OnTysT/woULuHTpEtq1a4caNWogIyMDR48exYIFC/D69WsMHz4cANCvXz+kpqYiMjISkydPVibt9erVyzcGmUyGzz77DI8ePYKfnx8aNGiAO3fuICQkBOfPn8emTZtgbm6u8pqgoCBkZmaiX79+MDIyQkhICObOnQsnJyd4enoW65oQlZWoqCicOHFC47G7d+/i7t27Go+JRCKIRCIoFApIJBL4+fnB29u7NEMlIiINuJ2ndvQ28d+xYwcuXryIlStXonXr1spyf39/fPzxx1i2bBnWrl0LAAgLC8PNmzeVo+a5XFxcsGTJElSvXr3IcQQFBSE6OhozZ85Ev379VI4pFArlv3v27Al/f3+V44MGDcLo0aOxceNGDB06FAYGBmjSpAlcXV0RGRmJjh07okaNGu+MYdOmTXj48CG+/vprfPTRR8pyNzc3LFq0CJs3b8aYMWNUXpOVlYXNmzfD0NAQANC5c2f07t0bf/31FxN/0gkymQxhYWFFeq0gCBAEAQAgl8sRHh4Od3d3jvwTEVGFpreLew8ePIjatWujYcOGSExMVD5ycnLg7e2Nq1evIiMjAwBw/PhxSCQSDB48WOUc/v7+aiPh2lAoFDhy5AhcXFzUkn5AdQqCqamp8t+ZmZlITExEcnIyWrVqhdTUVMTGxhY5juPHj6NKlSro27evSnm/fv1QpUoVREZGqr3mo48+Uib9AFCtWjU4Ozvj0aNHRY6jpCUkJKhs1SiTyVQWPGdlZeHVq1cqr4mLiyvweXx8vDLhYxu63UbulLmSkJOTg/j4+HLpB9tgG2yDbVSkNqhi09sR/5iYGGRmZqJLly751klMTISDgwOePHmCqlWrqo3mGRkZwdHRscjbZuYm729/45CftLQ0rF27FhEREXj27Jna8eTk5CLFAABPnz5Fw4YNYWCg+nYwMDCAs7Mzbt++rfYaR0dHtTIrKytl8lMR2NjYqDzX9POztbVVKcv77U3e5w4ODmyjkrTh4OAAiURSIsm/gYGBMrbKeK3YBttgG2yjsG2UNQUX92pFbxN/AHB1dc13gS4AVKlSpUTbK06CMWvWLJw+fRp9+/ZFs2bNYGVlBbFYjDNnzmDbtm0q04LKQn4LIt8eBSCqyKRSKfz8/LB3716t37dvz/E3MDCAr69vsb79IyIiKgt6m/jXrFkTr1+/RosWLfJNYnM5OjoiKioKMplM5dNyVlYWnjx5AktLS5X6VlZWGrcCfPLkicpza2trWFpa4t69ewW2n5KSgtOnT6NHjx6YOXOmyrFz586p1dd2/3FHR0c8ePAAOTk5KqP+OTk5ePjwocbRfaLKwNvbG0+fPkVUVJTasTp16qB169bc1YeIqALj3Xm1o7dz/Hv27IlXr17hjz/+0Hj87TlvHTp0gFwuV6sbEhKC1NRUtdc6Ozvj+vXryjUCwJupOHkXEorFYnTr1g3R0dHYu3ev2nlyRyFzP5jkHZV8+fKlxteZmZkp2yyMDh064PXr12rn2rt3L16/fo1OnToV6jxEusjOzk5jubu7Ozw8PGBvbw9XV1fY29vDw8MDHh4ekEqlkEqlcHV1ZdJPREQ6Q29H/D/55BNERUVh+fLlOH/+PFq0aAFzc3PEx8fj/PnzMDIyQnBwMIA3++Hv2bMHv/32G548eYImTZrgzp07OHr0KJycnNSm8AwYMADffvstRo8ejR49eiAlJQV79+5F9erV1RbRjBkzBufPn8eCBQsQFRWF9957DwBw584d5OTkYP78+TA3N0erVq1w8OBBGBsbw93dHXFxcdi9ezccHR3Vvl1o3LgxAGDFihX48MMPYWRkhLp168LV1VXjtQgICMCxY8ewaNEi3LlzB/Xr18edO3cQGhqKWrVqYdiwYSVyzYmIiIhKkpwD/lrR28TfwMAAy5YtQ0hICA4cOKBM8u3s7ODu7q5yIypDQ0MEBQVh+fLlOHHiBP73v/+hUaNGCAoKwrJly9RWvH/44Yd48eIF/vrrLyxduhSOjo74/PPPIRaLcePGDZW6lpaW2LBhA9avX4/IyEhERkbC3NwcLi4u+Pjjj5X15s+fj5UrV+LUqVPYv38/atasibFjx8LAwADz5s1TOaenpycmTJiA3bt3Y8GCBZDL5Rg5cmS+ib9UKsW6deuUN/AKCwuDra0t+vfvj1GjRnHuMlVqvHMvERHpC5HA1ZhEpMeOHTuGiIgItfKuXbvi/fffL4eIiIiosPxGFLyNeNi6mmUUiW7Q2xF/IiIAKmtxClNOREQVB7fz1I7eLu4lIgLe7MKlTTkREZGu4og/EREREekkBQf8tcIRfyLSa1zcS0RE+oKJPxHpNRMTE63KiYiIdBWn+hCRXlMoFFqVExFRxcE792qHI/5EpNeaNGmisTz3ZnpERESVBRN/ItJr9vb2asm/p6cn7OzsyikiIiIqLLlIVOCDVHGqDxHpvY8++ggJCQlIS0vDkCFDUKNGjfIOiYiIqMQx8SciAmBkZAQjIyOO9BMRUaXFxJ+IiIiIdBL38dcO5/gTEREREekBjvgTERERkU6ScztPrXDEn4iIiIhIDzDxJyIiIiLSA5zqQ0REREQ6Sc6ZPlrhiD8REQC5XI709HSkpqaWdyhERESlgok/Eem98+fP48GDB4iLi8PPP/+MqKio8g6JiIgKQSESFfggVUz8iUivyWQy7Nu3T/lcLpcjNDQUMpmsHKMiIiIqeUz8iUivxcTEQKFQqJQpFArExMSUU0RERFRYcpGowAepYuJPRHotIyNDq3IiIiJdxcSfiPRaZmamxvKsrKwyjoSIiKh0cTtPItJrSUlJWpUTEVHFkVPeAegYjvgTkV6zsrLSqpyIiEhXccSfiPSaIAjlHQIRERURF/BqhyP+RKTXkpOTNZYnJiaWbSBERESljIk/Eem1/Kb0WFtbl20gREREpUyvE38vLy/MnTu3VM4dGBgIX1/fUjl3UWnT3+DgYHh5eeHp06elGxQRERFREeWICn6QKr1O/Itr27ZtCA8PL+8wiKgYuKsPERHpCy7uLYY///wT1atX1ziyHxQUpNOLBkeMGIFPP/0URkZG5R0KUamytLTUWM5dfYiIKr4ccFhfG5Um8ZfL5cjOzoaJiUl5hwIAMDQ0LO8QisXAwAAGBpXm7UGUr/xu1KXLH9yJiIg00cmpPuHh4fDy8kJUVBR+//139O7dG23atEFERAQEQUBISAiGDBmCtm3bon379hg1ahQuXLhQqHMfOXIEkyZNQs+ePdG6dWt07twZU6ZMwb1791TqeXl5IS4uDpcuXYKXl5fykTsnPr85/pcuXcLYsWPRoUMHtG3bFoMHD8bevXvV6uW+/sWLF5g5cyY6deqEtm3bYvz48Xjw4IFK3czMTAQHB6Nfv35o27YtOnbsiI8//hjLly/X2Mdr164hMDAQ7dq1Q+fOnTF//nykpaWp1NE0xz+37L///sPPP/+Mbt26oW3btggICMC5c+cKdX2JypJMJsP9+/chk8k0Ho+KikJERITGY3n/ZrzrXERERBWdTg/pLl++HDk5Oejbty/Mzc1Rq1YtzJ49G4cPH0bnzp3h6+uL7OxsHDx4EOPGjcOiRYvQoUOHAs/5119/wcrKCn379kXVqlXx+PFj7NmzByNGjMDWrVvh7OwMAPjuu++wZMkSWFtb47PPPlO+vkqVKvme++TJk5g2bRpsbW0xZMgQmJmZ4ciRI1iwYAGePHmCcePGqdRPT0/HyJEj4eHhgXHjxuHJkyfYvn07pkyZgh07dkAikQAAfvrpJ4SFhaFnz54YPHgw5HI5Hj16hPPnz6vFcPfuXUyaNAm+vr7o1q0bLl68iNDQUIjFYsyaNatQ133OnDkQi8UYNmwY0tLSsHv3bkyYMAErVqyAt7d3oc5BVNqioqIQFhYGuVwOiUQCPz8/lfenTCbT+KE7V3x8PH744QfMnDnzneciIqLykc2ZPlrR6cQ/IyMD27ZtU07viYyMxMGDBzFz5kz069dPWW/gwIEYPnw4Fi9eDB8fH4gKuNnDypUrYWpqqlLWs2dPDBo0CNu2bcP06dMBAD169MDq1athY2ODHj16vDNWuVyORYsWwdTUFJs2bYKdnR0AYMCAARg1ahQ2bdoEX19f5QcL4M0+4kOHDkVAQICyrEqVKlixYgXOnTuH1q1bAwCOHz+ONm3aYN68ee+M4969e9iwYQMaN24MAOjfvz9SU1MRFhaGSZMmwczM7J3nkEgk+P3335XTmfz8/ODv74+ff/4ZISEh73w9UWmTyWTKRB148/sXHh4Od3d3SKVSAMCtW7feOZ0nOTkZly5deue5iIiIdIFOTvXJ5e/vrzKn/8CBAzA3N0fHjh2RmJiofMhkMrRv3x5Pnz7Fw4cPCzxnbtIvCAJkMhkSExNRpUoV1KpVCzdu3ChyrLdu3UJ8fDz8/PyUST/wZi3AsGHDoFAocOLECZXXiMViDBw4UKWsRYsWAKDSD6lUiujoaNy/f/+dcXh4eCiT/rfPKZfLC71156BBg1TWMNjb26N79+6IjY1FTExMoc5RFhISEpCZmal8LpPJkJKSonyelZWFV69eqbwmLi6uwOfx8fEqySLbqJhtxMfHKxP1XDk5OXj8+LGyjcLeoOvkyZMazxUfH1/q/cjFNtgG22AbutJGWcsWiQp8kCqdHvF/e3QcAGJjY5GamoquXbvm+5qEhATUqlUr3+O3b9/GmjVrcPHiRaSnp6scc3R0LHKsuUl1nTp11I7VrVsXAPDkyROVcjs7OxgbG6uU5e408vZWg5MnT8acOXMwcOBAODo6wsvLC+3bt4ePjw/EYtXPdpr6oOmcBXFxcVEry+3XkydPNB4vDzY2NirP847OGhkZwdbWVqWsevXqBT53cHBgGzrQhoODAyQSiUrCbmBggJo1a8Lc3BwA0KRJExw7dgzv0rFjR+zcuVPtXLmx6Pq1Yhtsg22wjZJsgyo2nU788+7gIwgCqlSpggULFuT7mtwkW5P4+HgEBgbC3NwcI0aMQO3atWFiYgKRSITFixerfRAobXmT9re9/Wm7Y8eOCAsLw5kzZ3Dp0iWcO3cOoaGhaNq0KVatWqUyOp+7LuBd5yTSZVKpFH5+fggPD0dOTg4MDAzg6+urTPqBN99UeXp64sqVK/mex9raGp6ensjMzCzwXERERLpApxP/vGrWrImHDx/Cw8OjUHPV84qMjERaWhqWLFkCLy8vlWNJSUlqe9oXtFYgr9yR9ujoaLVjuWXF+UbBysoKPXr0QI8ePSAIAlauXInNmzfjxIkT6NKlS5HPq0lMTAzc3NxUykqiD0QlydvbG+7u7oiPj4eDg4PG+fgDBw6EWCzGpUuX1I41atQIw4YNK/S5iIio7GWXdwA6Rqfn+OfVs2dPKBQK/PrrrxqP553HllfuCHveke89e/ZofK2pqSmSk5MLFVuDBg3g4OCA8PBwvHz5Ulmek5ODLVu2QCQSvXPHIU3kcrnKfD3gzQeS+vXrAyidu49u27YN2dn/96v27NkzHD58GLVq1aow03yIgDcj/66urgUm6jVq1NBYnvfbwcKci4iIqCKrVCP+Xbp0ga+vL/766y/cvn0b7du3h7W1NZ4/f45r167h8ePHCA0Nzff1bdu2xcqVKzF79mwMGDAAFhYWuHr1Kv7++284OTmpLfDz8PBAaGgoVq9eDRcXF4hEIvj4+KjtCgS8mWLz1VdfYdq0aQgICEDfvn1hZmaGiIgIXL9+HcOHD1dbs1AYaWlp6N69O3x8fFC/fn1UqVIFT58+RUhICCwtLeHj46P1Od9FLpfj888/R7du3ZCWloZdu3YhMzMT06ZNK/G2iIiIiPKTxgW8WqlUiT/wZo95Ly8v7NmzBxs3bkR2djZsbW3RoEEDtX3y83JycsKKFSsQFBSEDRs2QCwW47333kNwcDAWLVqktrJ97NixSEpKws6dO5GSkgJBEBAWFqYx8QcAHx8frFq1CuvWrcOWLVuQnZ2N2rVr45tvvkGfPn2K1F8TExN88sknOHfuHM6dO4e0tDRUrVoVPj4+GD58uMoOQiVl3rx52LVrFzZt2oSUlBS4urpizpw5aNWqVYm3RVTa8vtWrLC7/hAREekKkcAVnVRIwcHB+O233xAWFpbv9AgiXXP69Gns27dPrdzX1xdt27Yth4iIiKiwjL8seBp35jLbAo/rm0o1x5+ISFsc+yAiIn3BxJ+I9Bqn+hARkb6odHP8iYi0YW1trbG8SpUqZRsIERFpLQtc3KsNjvhToY0aNQoXLlzg/H6qVPKb6sMpQEREVNlwxJ+I9Bqn+hAR6TAO+GuFI/5EpNfs7e01ljs4OJRxJERERKWLiT8R6bWGDRtClOcGMCKRCA0aNCiniIiIiEoHE38i0mtSqRS+vr7K5xKJBH369IG5uXk5RkVERIUiEhX8IBWc409Eeq9Fixa4evUqsrKyEBAQkO9OP0RERLqMI/5ERHgz0m9qasqRfiIiqrSY+BMRERER6QEm/kREREREeoBz/ImIiIhIN3EBr1Y44k9EREREpAc44k9EREREuokD/lrhiD8RERERkR5g4k9EBEAulyM9PR2pqanlHQoREVGpYOJPRHrv/PnzePDgAeLi4rBo0SJERUWVd0hERFQoonc86G1M/IlIr8lkMoSHhyufKxQK7N27FzKZrByjIiIiKnlM/IlIr926dQuCIKiUCYKAW7dulVNERERUaBzw1woTfyLSa8+ePdNY/vz58zKOhIiIqHRxO08i0mtWVlYay62trcs2ECIi0h5H9bXCEX8i0muifO76mHf6DxERka5j4k9Eei0pKUljeXJychlHQkREVLo41YeI9JqJiYlW5UREVJFwro82OOJPRHotIyNDY3l6enoZR0JERFS6OOJPRHqNi3uJiHQYB/y1whF/IiIiIiI9wMSfiPRafot7ExMTyzYQIiKiUsbEvxKSy+X5zlsmIlX5LeI1NTUt40iIiEh7vHWvNjjHX8eFh4dj3rx5CAoKwvXr1xEeHo74+Hh88803sLOzQ2hoKG7evImXL1/C0NAQ7u7u+Oyzz9C8eXO1cz169Ajr169HVFQUEhISYG1tjUaNGmHkyJFo2LChst7Nmzexfv16XL58GWlpaahevTp69uyJgIAAGBjwLUUVj0wmQ3x8PBwcHCCVSlWO5fch+dGjR5DJZEhNTcXt27fRoEED2Nvbl0W4REREpYJZWiWxfPly5OTkoG/fvjA3N0etWrWwY8cOJCUloUePHrC3t8fz588RGhqKsWPHYs2aNWjatKny9Tdv3sSYMWOQk5OD3r17o27dukhOTsalS5dw9epVZeJ/+vRpTJs2DTVr1sSQIUNgaWmJ69evIzg4GHfv3sVPP/1UXpeASKOoqCiEhYVBLpdDIpHAz88P3t7eyuPXrl3T+Lpbt25hwYIFyucHDx6Ep6cnBg4cWOoxExFRIXFQXytM/CuJjIwMbNu2TWXaQr169dSmK/Tv3x8DBgzAhg0blIm/IAiYO3cusrOzsWnTJtSrV09Zf/jw4VAoFACAzMxMzJ8/H40bN8bq1auVo/v9+/dHvXr1sHTpUly4cAFeXl6l3V2iQpHJZMqkH3gzDS48PBzu7u6QSqWIiYnJd46/JleuXEGnTp048k9ERDqJc/wrCX9/f7W5ym8n/WlpaUhMTIREIkHjxo3x77//Ko/duXMH0dHR8PX1VUn6c4nFb94mUVFRePXqFXx9fSGTyZCYmKh8tG3bVlmnokhISEBmZqbyuUwmQ0pKivJ5VlYWXr16pfKauLi4Ap/Hx8dDEAS2oSNtxMTEKJP+XDk5OYiPjwcAXLp0Cdq6fPlypbxWbINtsA22URJtUMUmEt7+6ZHOyZ3jv2zZMrRr107l2OPHjxEUFISzZ8+q/KIDgEgkwvnz5wEAERERmDFjBmbOnIl+/frl29amTZuwcuXKAuPx8/PD7Nmzi9gbopIlk8mwcOFCleTfwMAAM2bMgLm5OWJiYhAcHKzVOadMmQI7O7uSDpWIiIpANCOlwOPCQosyikQ3cKpPJZF3tD8tLQ0jR45Eeno6PvnkE7i6usLc3BwikQgbN25UJv3ayP2M+MUXX8DNzU1jHSZEVJFIpVL4+fkhPDwcOTk5MDAwgK+vL8zNzQEALi4ukEqlkMlkhTqfp6cn3+NERKSzmPhXUufOncOLFy8we/Zs+Pn5qRxbvXq1ynNnZ2cAwN27dws8Z249U1NTlcWRRBWZt7c33N3d893Vp2PHjti3b5/a65o3b44PP/wQqampuHPnDurXr8+5/UREFQ5X92qDc/wrKYlEAgDIO5Pr7NmzuHHjhkqZm5sb6tSpg7CwMPz3339q58o9R+vWrWFjY4ONGzdqXBCZkZGB1NTUkuoCUYmRSqVwdXVVS/oLUqNGDUilUtjb28PHx4dJPxER6TyO+FdSnp6esLW1xbJlyxAXF4dq1arh7t27OHDgAFxdXXH//n1lXZFIhDlz5mDs2LEICAhQbueZkpKCS5cuoXXr1hg4cCBMTU0xb948TJ06Ff3794efnx9q1qyJlJQUxMbGIjIyEj///DN39SGdkt+uPtrs9kNERKQLmPhXUhYWFvj111+xYsUK7NixA3K5HA0aNMDy5csRGhqqkvgDgLu7OzZt2oR169bh6NGj2LVrF6ytreHu7g5PT09lvdatW2PTpk3YtGkTDh48iNevX8PS0hJOTk4YPHiwxl2BiCqy/O7cm185ERFVIJzpoxXu6kNEem3fvn04ffq0WrmPjw969OhRDhEREVFhiWYWvDmD8EPhp3jqA474E5Fes7Ky0qqciIgqEBGH/LXBxb1EpNdE+fxPg1+GEhFRZcPEn4j0Ghf3EhGRvmDiT0R6Lb9FvKampmUcCRERUeli4k9Eei0jI0OrciIiIl3Fxb1EpNcsLS01lnNxLxGRDuDaXq1wxJ+I9Fp+U32MjIzKOBIiIqLSxRF/ItJrnONPRKTLOOSvDY74E5Fec3FxgVis+qdQLBbDxcWlnCIiIiIqHUz8iUivSaVS9OrVS/lcIpGgd+/eMDc3L8eoiIiISh6n+hCR3mvRogWuXr2KrKwsBAQEwNraurxDIiKiwuBMH61wxJ+ICG9G+k1NTTnST0RElRZH/ImIiIhIN3HEXysc8SciIiIi0gNM/ImIiIiI9ACn+hARERGRjuJcH20w8SciApAqGMEQ8vIOg4iIqNQw8ScivfY6Q8CAUOBoymBIoMD9k8DS9wWIRBxFIiKq8PinWiuc409Eeu3Tg3IcfSQCIIIcEiy/LMLv1xXlHRYREVGJY+JPRHrtQLR62ZILQtkHQkREVMqY+BORXsvRkOM/Tin7OIiIiEobE38iojwUHPAnIqJKiIt7iYjyYN5PRKQjuLhXKxzxJyLKg/8fISKiyogj/kRERESkm7j1slY44k9ElAfn+BMRUWXExJ+IKA8xB5CIiKgSYuJfQgIDA+Hr61sq5w4ODoaXlxeePn1aqPq+vr4IDAwslViIiIiISDcx8S9F4eHh2LZtW3mHQURERETExb0lJSgoCIKgOjE4PDwccXFxGDRoUDlFRURFIXCOPxGRbuDUTK0w8S8hhoaG5R0CERWFIKjtCiESMfMnIqLKp9Ik/tnZ2di2bRsOHz6MBw8ewMDAAM7OzujVqxc+/vhjAMCLFy+wdetWnD9/HnFxccjMzISjoyN69uyJoUOHQiKRKM8XHh6OefPmISgoCFeuXEF4eDhevXqFWrVqYfjw4ejWrZtK+4GBgYiLi0N4eDiAN/Ps4+LiAABeXl7KemvWrIGXlxdu3LiBkJAQXLt2Dc+ePYNEIoGrqyuGDh2KTp06lco1On78ODZv3oy7d+9CJBKhXr16GDZsGDp27KhS7+rVq1i3bh3u3LmDlJQUWFlZoV69ehg5ciQ8PDwAAElJSfj9999x8uRJvHjxAqampqhevTq6du2KYcOGlUr8RMWVmCFgYZQCZ+MEvGcngoWR5nrpWQKsV+bg/ZoiLPQRo74Nh5SIiEj3VYrEPzs7G+PHj8fFixfRqlUrfPjhhzAyMsL9+/cRGRmpTPzv3buHyMhIdOzYEU5OTsjJycE///yDX3/9FU+ePMGsWbPUzr1y5Uqkp6fD398fwJsPBLNmzUJWVlaBi3mnTJmCX3/9FYmJiZg8ebKy3MXFBcCbJDw2NhZdunRB9erVkZSUhH379mHatGlYsGABunfvXpKXCDt37sRPP/2E2rVr4/PPPwcA7Nu3D1OnTsXMmTPRr18/AEBsbCzGjRsHW1tbDBw4EDY2NkhISMCVK1dw9+5dZeI/ffp0XLp0Cf3790e9evWQmZmJmJgYXLx4kYk/VVi+e+Q4/eTNv08+fjOqL83IgMzUVKWeIBYjKRPYc1/Aicdy3PlMgqpmTP6JiCoe/m3WRqVI/Ldt24aLFy9i+PDhGDdunMoxhUKh/HezZs0QGhoK0Vtf6w8aNAjffvstQkNDMWrUKFStWlXl9YmJidi+fTukUikAwN/fHwMHDsTSpUvxwQcfwMTERGNMHTt2xLZt25CZmYkePXqoHR8xYgTGjx+vUjZw4EAMGjQI69atK9HEPzk5GStWrICTkxM2btyo0pfBgwdj2bJl+OCDD2BhYYGzZ88iIyMD33//PRo3bqzxfDKZDOfPn4e/vz+++uqrEouTqDTdeCEok/63VU1LVkv835aQAey8K2CMJ//nQkREuq1S7Opz6NAhWFpaKkey3yYW/18XTUxMlEl/dnY2kpKSkJiYiNatW0OhUODmzZtqr/f391cmygAglUrRv39/JCcn4+LFi0WO2fStRCMjIwOJiYnIyMhAixYtEBMTA5lMVuRz5xUVFYX09HQMHDhQrS8DBw5EWloaoqKilGUAcOLECWRmZmo8n7GxMYyMjHDjxo1CbzFaHhISElT6IJPJkJKSonyelZWFV69eqbwmd3pWfs/j4+NVFnGzDd1pI79Z+wlmFvkc+T+KCtQPtsE22AbbqMhtlDnROx6kolKM+D98+BD169eHsbFxgfVycnKwceNGHDhwAI8ePVLbhSc5OVntNbVr11Yry52u8+SJhuHDQkpISMDq1atx4sQJJCQkqB2XyWQqSfrbcj+0vM3MzAxmZmYa6+fGWadOHbVjuWW5dbp27YoDBw5gw4YN2LZtGzw8PNCqVSt069YN1atXB/BmIfPkyZOxePFi+Pn5oU6dOvDy8kLHjh3RsmXLQl6B0mdjY6PyPO/1NDIygq2trUpZbh/ze+7g4MA2dLQNDzsR2tQA/s7zWdXtZRwu1KyL/FQxBgbU/78BhPLuB9tgG2yDbVTkNqhiqxSJf2EtXboUO3bswAcffIDPPvsMVapUgYGBAW7fvo2VK1eqfRAoLYIgYPz48YiJicHAgQPRqFEjSKVSiMVihIeH49ChQypTlPK6evUqRo8erVI2cuRIjBo1qtixGRkZYdWqVbhx4wbOnj2LS5cuITg4GL/99hsWLFigXHjs7++Pjh074vTp07h48SKOHTuGv/76Cx988AEWLlxY7DiISsO+fhJ8f/atxb2GQPKZe6qJvyDAIj0VImspOtYU4UcfMew4v5+IiCqBSpH416pVC7GxscjKyoKRUT7bdAA4cOAAmjVrppaYPnr0KN/XxMbGqpXFxMQAABwdHQuMSyTSnCzcu3cPd+/e1Zis7927t8BzAoCbmxuCgoJUygqKxcnJCQAQHR2tNiKfX18aN26snOMfHx+PwYMHY/Xq1So7DlWtWhV9+vRBnz59IJfLMXv2bBw+fBhDhgyBu7v7O/tBVNaqmIjwS0eJSpn9/1qpVhKJYCfKxn8TK8WfRyIiIqVKMce/e/fuSE5Oxrp169SOvT2KLxaL1Ub109PTC7y7bkhIiMp8e5lMhl27dsHCwgLNmzcvMC4zMzMkJyertZm77iBv+f3793H8+PECzwkAlpaW8Pb2VnnkJveaeHt7w9TUFDt27EBqaqqyPDU1FTt27ICZmRlatXqT/CQmJqq93t7eHlWqVFFOL8rIyEBGRoZKHYlEgnr16gHQPGWKqKJ6LrVSK4u3qFIOkRAREZWuSjGk9cknn+DUqVNYt24dbt68CW9vbxgbGyM6OhoPHjzAqlWrAACdO3fG7t27MWPGDLRs2RKvXr1CeHg4rKzU/8efy9raGgEBAcqtO8PDwxEfH49vvvkm3x19cjVu3BinTp3CokWL0KRJE4jFYrRo0QIuLi6oU6cONm/ejIyMDNSqVQsPHz7E7t274erqilu3bpXcxQFgYWGBiRMn4qeffsKnn36KXr16AXizneejR48wc+ZM5by/devW4ezZs2jXrh0cHR0hCAJOnTqF2NhY5TadDx48QGBgIDp16oS6devCwsICsbGxCAkJgaOjI5o2bVqi8ROVKg3fzPH2XUREOoIzMbVSKRJ/Q0ND/Prrr9i6dSsOHz6MVatWwcjICM7Ozip77U+ePBnm5uaIiIjAiRMnYG9vj759+6JRo0YYO3asxnNPmDABV65cwc6dO5GQkABnZ+dC77M/ePBgPHnyBMeOHcOuXbugUCiUN/Bavnw5li1bhn379iE9PR1169bF3Llzcffu3RJP/AHgo48+QtWqVbFlyxb89ttvAN5MGfrll19UbuDVoUMHvHz5EkePHkVCQgKMjY1Rs2ZNfPPNN+jduzeAN98A+Pn54eLFizh+/Diys7NhZ2eHvn37IiAg4J0fiIiIiIio7ImEslrRqmNy79ybm6gTUeUk+iVHrczMAEj9slKMixARVWqiBQVvJSp8U/COj/qmUszxJyIiIiKigjHxJyLKg1+DEhFRZcTvsomIiIhIN+WzdTppxsQ/H76+vioLg4lIj3DIn4iIKiEm/kREeXAAiYhIR/DvtVY4x5+IKA8x/0dCRESVEBN/ItJrEg1JvqO07OMgIiIqbUz8iUiv9aijnvlPas4hfyIiqnyY+BORXtvYXYxOTm9W80ogx/j3BAS+xz+NRERU+XBxLxHpNRtTEQ73B1au+wMGIjnGdBwKEVf3EhHpBv651goTfyIiAFJxwbd9JyIi0nX8PpuIiIiISA9wxJ+IiIiIdBTn+miDI/5ERERERHqAI/5EREREpJs44K8VjvgTEREREekBJv5ERERERHqAiT8RERERkR5g4k9EREREpAe4uJeIiIiIdBMX92qFI/5ERERERHqAiT8RERERkR5g4k9EREREpAeY+BMRERER6QEu7iUiIiIi3cTFvVrhiD8RERERkR5g4k9EREREpAeY+BMRERER6QEm/kRERESkd+bOnQupVFreYZQpLu4lIiIiIt0k4upebXDEn4iIiIhIDzDxJyIiIiLdJHrHoxiuX7+Obt26wdzcHFZWVvD398fDhw+Vx0eMGIH27dsrn798+RJisRgtWrRQlslkMhgaGmLnzp3FC6aEMPEnIiIiInrLo0eP4OPjg1evXmHr1q1Ys2YNLl26hA4dOiAlJQUA4OPjg/PnzyMjIwMAcPLkSRgbG+Py5cvKOn///TdycnLg4+NTbn15G+f4U6UkCILyl47oXbKzs5Geng4ASE5OhqGhYTlHRESkmywsLCCqBPPuly5diuzsbBw5cgQ2NjYAgKZNm6JRo0bYuHEjJkyYAB8fH2RmZiIqKgodOnTAyZMn0bdvXxw5cgRnzpxB9+7dcfLkSbi5ucHe3r6ce/QGE3+qlFJSUmBlZVXeYZAO+vLLL8s7BCIinZWUlARLS8sya0+YWjqp7KlTp/D+++8rk34AaNCgAd577z2cPn0aEyZMgIuLC5ycnHDy5Ell4j969Gikp6fjxIkTysS/ooz2A0z8qZKysLBAUlJSoerKZDL07NkT+/fv17ttvXLxGvAaALwG+t5/gNcA4DUobv8tLCxKIaqy9/r1a3h6eqqV29vbIyEhQfk8N+FPTk7G1atX4ePjg9TUVISEhCAzMxPnzp3DyJEjyzDygjHxp0pJJBIVesRBLBZDIpHA0tJSL//IA7wGAK8BwGug7/0HeA0AXgN9738uGxsbPH/+XK382bNncHNzUz738fHB5MmTcfz4cVStWhUNGjRAamoqvv76a0RGRiIzM1NlAXB54+JeIiIiIqK3tGvXDseOHcPr16+VZXfu3MG1a9fQrl07ZVnuCP+SJUuUU3o8PT1hamqKH3/8ETVr1kTt2rXLOvx8ccSfiIiIiPSSXC5HSEiIWvkXX3yBDRs2oGvXrpg1axYyMjLwzTffwNnZGZ9++qmyXoMGDVCtWjWcOHECK1asAABIJBK0bdsWBw8exODBg8uqK4XCxJ/0npGREUaOHAkjI6PyDqXc8BrwGgC8Bvref4DXAOA10Lf+Z2Rk4KOPPlIr37JlC06cOIGpU6di8ODBkEgk+OCDD7BkyRK1dQw+Pj4ICQlRWcTboUMHHDx4sEIt7AUAkSAIQnkHQUREREREpYtz/ImIiIiI9AATfyIiIiIiPcA5/qS3Tp48idWrV+PBgwdwcHDAp59+Cj8/vwJf8++//yIkJASXL1/GixcvUK1aNXTu3BkjRoyAqalpGUVeMorS/+zsbKxatQo3btzArVu3kJGRgaNHj8La2rpsgi6i2NhYLFq0CNeuXYO5uTl69OiBsWPHvvMOvYIgYNOmTdi5cycSExPh5uaGyZMnw8PDo4wiLzlFvQY7d+7EmTNncOPGDSQmJuLHH39Ely5dyijqklOU/r98+RJ//PEHoqKi8PjxY0ilUjRt2hTjx49H9erVyzD6klHU98C3336LGzdu4MWLFzA0NISrqytGjBiBVq1alVHkJaeo1+Bt27Ztw5IlS9CuXTssW7as9IItBUXtv6+vL+Li4tTKz5w5A2Nj49IKl0oBE3/SS1euXMG0adPQu3dvTJkyBefPn8f8+fNhZmZWYFITERGBR48eYdiwYXB2dkZ0dDSCg4Nx48YNrFmzpgx7UDxF7X9GRgb27t2LRo0aoWnTpvjnn3/KMOqiSU5OxujRo+Hs7Iyff/4Zz58/x9KlS5GRkYGvv/66wNdu2rQJwcHBGD9+POrVq4edO3di/Pjx+OOPP+Dk5FRGPSi+4lyD/fv3AwDatm2r/LeuKWr/b926hcjISPj5+cHDwwOJiYn4/fffERAQgB07dqBKlSpl2IviKc57IDs7G4MHD0bNmjWRlZWF0NBQfPHFF1izZg2aNm1aRj0ovuJcg1wvX77Eb7/9pnI3V11R3P537twZQ4YMUSnTlwXAlYpApIfGjRsnDB8+XKVs5syZgr+/f4GvS0hIUCs7ePCg0Lx5c+HmzZslGmNpKmr/BUEQFAqFIAiCEBYWJjRv3lx4/fp1aYRYYtavXy+0a9dOSExMVJbt2rVLaNmypfD8+fN8X5eRkSH4+PgIv/76q7IsKytL6NWrl7Bw4cJSjbmkFfUaCIIgyOVyQRAE4cmTJ0Lz5s2FiIiIUo21NBS1/8nJyUJ2drZKWXx8vODl5SVs2bKl1OItDcV5D+SVk5Mj9OjRQ1iwYEFJh1mqSuIafPvtt8Ls2bOFkSNHCl988UUpRVo6itP/Xr16CT/++GNph0hlgHP8Se9kZWXhwoULaiPbXbt2RUxMDJ4+fZrvazWN8NWvXx8A8OLFi5INtJQUp//Am7si65K///4bLVu2hJWVlbLsgw8+gEKhwNmzZ/N93bVr15CamqpynQwNDdGpUyecOXOmVGMuaUW9BsCbu3jquqL238LCAgYGql+M29vbo0qVKjrz+56rOO+BvCQSCSwsLJCdnV3SYZaq4l6DK1eu4MSJE5gwYUJphllqSvI9QLpL9/+iE2np8ePHyMnJUbuTnouLC4A3cyC1ceXKFQCoUHfmK0hJ97+ii42NVeurhYUFqlatWmBfc49puk7x8fHIyMgo2UBLUVGvQWVRkv1/8OABEhISlL8vuqK410AQBOTk5CAxMRFbtmzBo0eP0K9fv9IJtpQU5xrI5XIsWrQIw4cPR9WqVUsvyFJU3PfAoUOH0Lp1a7Rv3x4TJ07E/fv3SydQKlWc4096Jzk5GQDUbsBhaWmpcrwwEhMTsXbtWnTo0AHOzs4lF2QpKsn+64Lk5GS1vgJv+l9QX5OTk2FkZKS2cM3CwgKCICAlJQUmJiYlHm9pKOo1qCxKqv+CIOCXX36BnZ0dunXrVpIhlrriXoPQ0FAsWLAAAGBmZoYffvgBTZo0KfE4S1NxrsHOnTuRnp5e4e7Cqo3i9N/HxweNGzeGg4MDnjx5gvXr12PEiBE6t96JmPhTJSGTyfDy5ct31nN0dCyxNnNycjBz5kwAwIwZM0rsvEVRHv0n0jdr167FuXPnsHLlSp3bxau4OnbsCDc3NyQmJuLo0aOYMWMGfv75Z7Rt27a8Qyt1CQkJCA4Oxrx587Ta/acymTZtmvLfTZs2RatWrdC/f39s3boV06dPL8fISFtM/KlSOHr0qHI0qiAhISHKkW2ZTKZyLHfEI/d4QQRBwLx58/Dvv//it99+K/evfsu6/7rE0tJSra8AkJKSUmBfLS0tkZWVhczMTJVR/5SUFIhEIo0jZxVVUa9BZVES/d+zZw9+++03fPvtt2jZsmVJh1jqinsNrK2tldv2tmnTBsnJyVi+fLlOJf5FvQZr1qxBvXr10LRpU6SkpAB4M/VHLpcjJSUFpqamamtBKqKS/DtQtWpVeHp64tatWyUVHpWRiv9OJSqEPn36oE+fPoWqm5WVBQMDA8TGxqJ169bK8vzmdGuybNkyHD16FMuXL4ebm1sRIi5ZZd1/XVK7dm21+au535AU1NfcYw8ePFD5GcfGxsLBwUFnpvkARb8GlUVx+x8ZGYkff/wRo0ePRu/evUsnyFJW0u+BBg0a4O+//y6Z4MpIUa9BbGwsLl26hE6dOqkd69SpE1asWIE2bdqUcLQlT9//DtAbXNxLesfIyAheXl44duyYSnlERARcXFxQo0aNAl+/ceNGbNu2DXPmzNHJkb/i9l/XtGnTBufOnVOO1AFvviERi8UF3oCoSZMmMDc3x9GjR5VlOTk5iIyM1KlRTqDo16CyKE7/L1y4gFmzZqFPnz74/PPPSzvUUlPS74GrV6/q3NTBol6DKVOmYM2aNSoPNzc3eHh4YM2aNXB3dy+L8IutJN8DL168wJUrV9CoUaOSDpNKGUf8SS99/vnnGDVqlPIupBcvXsShQ4ewcOFClXre3t7o2bMnZs+eDeDNrga//vorPvzwQzg6OuL69evKuk5OTjpzQ5+i9h94c6fG9PR03Lx5E8CbOwCbmZmhTp06qFOnTpn2ozD69++PHTt2YMqUKfjss8/w/PlzLF++HP369YOdnZ2y3pgxYxAXF4e9e/cCAIyNjTF8+HCsXbsWVapUgaurK3bu3ImkpCS1m9hUdEW9BgBw8+ZNPH36FImJiQCAGzduAHiztW3z5s3LshtFVtT+x8TEYOrUqahZsyZ69Oih8vtepUoVnVrUWNRrcPr0aezfvx/t2rWDvb09kpOTcejQIfzzzz/4/vvvy6k3RVPUa5C7ZfPbpFIpzMzM4OXlVVbhF1tR+3/o0CGcPn0abdu2hZ2dHR4/foyNGzdCIpHo3N9CYuJPesrT0xOLFi3C6tWrERoaCgcHB3zzzTdqe9vL5XIoFArl89y9jg8ePIiDBw+q1J0zZw58fX1LP/gSUNT+A8CPP/6ocuv27777DgAwcuRIjBo1qvSD15KlpSVWr16Nn3/+GVOmTIG5uTn69OmDsWPHqtTLnbP7toCAAAiCgK1bt+L169dwc3PDypUrdSrhA4p3Df766y/s27dP+Xzr1q0AgGbNmmHt2rWlH3wJKGr/b9y4AZlMBplMhhEjRqjU7dWrF+bOnVsW4ZeIol4DJycnZGVl4ddff0ViYiKsra1Rr149BAcH68wHv1zF+T2oDIraf0dHR7x48QKLFy9GSkoKLCws0KJFC4waNUrnvvUhQCQIglDeQRARERERUeniHH8iIiIiIj3AxJ+IiIiISA8w8SciIiIi0gNM/ImIiIiI9AATfyIiIiIiPcDEn4iIiIhIDzDxJyIiIiLSA0z8iYiIiIj0ABN/IqICfPrppxCJROUdBoA3d5I1MDBARESEsuz48eMQiUTYuHFj+QVGFcLGjRshEolw/PjxIr2e7yXNrly5ArFYjBMnTpR3KETFxsSfSA9FR0cjMDAQDRo0gJmZGapUqYKGDRsiICAAkZGRKnVr166Nxo0b53uu3MT45cuXGo/funULIpEIIpEIp06dyvc8uXVyHyYmJqhXrx4mT56MhISEonW0kpk8eTLatm2LDz74oLxDKROxsbGYO3curly5Ut6hUBlJTEzE3Llzi/zhpagKeq95enqiT58+mDJlCgRBKNO4iEqaQXkHQERl68KFC+jQoQMMDQ0xbNgwuLu7Iz09Hffu3cORI0dgYWGBTp06lVh769atg4WFBUxNTbF+/Xq0b98+37qenp6YMmUKACAhIQEHDhzA0qVLERERgYsXL8LIyKjE4tI1//zzDyIiIrB3716Vch8fH6Snp8PQ0LB8AitFsbGxmDdvHmrXrg1PT8/yDofKQGJiIubNmwcA6NixY5m1+6732pdffokOHTrgwIED6NmzZ5nFRVTSmPgT6Zl58+YhLS0NV65cwXvvvad2PD4+vsTays7OxpYtW/DRRx/BysoKa9euxYoVK2BhYaGxvqOjI4YMGaJ8PnHiRPj6+mLfvn0IDQ3FRx99VGKx6ZpVq1ahatWq6NGjh0q5WCyGiYlJOUVFpB/at2+P2rVrY82aNUz8Sadxqg+Rnrl37x5sbW01Jv0A4ODgUGJthYeH4/nz5wgICMCnn36K1NRU7NixQ6tzdOvWDQBw//79fOusXr0aIpEIYWFhascUCgWcnJxURvGOHDmCjz/+GHXq1IGpqSmsra3RtWvXQs/h7dixI2rXrq1WHhsbC5FIhLlz56qUC4KA1atXo3nz5jAzM4NUKkWnTp3UplXlJycnB3v37kWXLl3URvY1zct+u2zVqlWoX78+TExM4OHhgX379gEArl+/ju7du8PS0hK2traYOHEisrOzNfYzOjoavXv3hpWVFSwtLdG3b19ER0er1FUoFPj+++/h4+MDBwcHGBkZwdnZGWPGjMGrV6809mvXrl3o2LEjrK2tYWZmhvr162PixInIysrCxo0bld88DR8+XDkFrDCjwLGxsRg6dCjs7e1hbGyMunXrYubMmUhLS1OpN3fuXIhEIty5cwczZ86Ek5MTjI2N8d577+HAgQPvbAf4v3n1x44dw3fffYdatWrB1NQU3t7eOHv2LADgxIkTaNeuHczNzVG9enXMnz9f47n27t2Ltm3bwtzcHFKpFG3btkVoaKjGur/99hsaNGgAY2NjuLq6YtmyZflOQ0lKSsLXX38NV1dXGBsbw87ODp988onaz1Bbhb3OBa2TEYlE+PTTTwG8ed+6uLgAeDNAkfszz/1de/v3688//0STJk1gYmICZ2dnzJ07Fzk5OSrnLuzvaWHeayKRCN26dcOhQ4cgk8m0vFJEFQdH/In0TN26dXHnzh3s3r0b/fr1K9Rr5HJ5vnP4MzMz833dunXr4OLigvbt20MkEqFp06ZYv349Pv/880LHe+/ePQBA1apV860zcOBATJo0CZs3b4afn5/KsWPHjuHJkyfKKUTAm//RJyQkYNiwYXBycsKTJ0/w+++/o3PnzoiMjCxwOlJRDB06FH/++Sf8/f0xfPhwZGZm4o8//sAHH3yA3bt3q8Wc18WLFyGTydCyZUut2g0KCsLr16/x+eefw8TEBCtWrEDfvn2xc+dOjBw5Ep988gn69OmDI0eOYOXKlahWrRq++eYblXOkpqaiY8eO8Pb2xsKFC3Hv3j2sWrUKZ8+exeXLl5UfFLOysvDzzz+jf//+6N27N8zNzXH+/HmsW7cOp0+fVpuqNWvWLPzwww9o1KgRJk2ahOrVq+O///7Drl278N1338HHxwczZ87EDz/8gMDAQOXPxN7evsA+P3jwAC1btkRSUhLGjh2LevXq4fjx41i4cCHOnDmDY8eOwcBA9X99AQEBMDQ0xNSpU5GVlYVly5ahT58+uHv3rsbEUZPp06dDLpfjiy++QFZWFhYvXoyuXbti8+bNGDFiBAIDAzF48GD89ddfmD17NlxcXFS+3Vq1ahXGjRuHBg0aYPbs2QDevE/79OmD4OBgBAYGKusuW7YMkyZNwnvvvYcffvgBaWlp+OWXX1CtWjW1uJKSktCmTRs8fPgQn332Gdzd3REXF4dVq1bB29sbFy5cQK1atQrVx+Je53dp2LAhli5dikmTJqFv377Kv09SqVSlXlhYGKKjozFu3Dg4ODggLCwM8+bNw4MHD7Bhwwat+1LY91rr1q0RHByM06dPo3v37lq3Q1QhCESkV/7++2/B0NBQACDUq1dPGD58uLBq1Srh5s2bGuvXqlVLAPDOx4sXL1Re9+TJE0EikQhz5sxRli1btkwAoLEtAELXrl2FFy9eCC9evBDu3r0rLFmyRDA0NBSsrKyEZ8+eFdgvf39/wdjYWEhISFApHzJkiGBgYKDyeplMpvb6+Ph4wdbWVvjwww9VygMCAoS8fyo7dOgg1KpVS+0cMTExAgCVPu/evVsAIAQHB6vUzc7OFpo3by7Url1bUCgUBfZt/fr1AgAhNDRU7VhkZKQAQNiwYYNaWY0aNYTExERl+dWrVwUAgkgkEnbt2qVynmbNmgkODg5q/QQgfPHFFyrluX0aNWqUskyhUAhpaWlq8f3+++8CAGHHjh3KsqioKAGA0KlTJyE9PV2lvkKhUF4PTX17l0GDBgkAhP3796uUT506VQAg/P7778qyOXPmCACEnj17qvwMzp07JwAQpk+f/s72NmzYIAAQmjZtKmRmZirLQ0NDBQCCgYGBcP78eWV5Zmam4ODgILRq1UpZlpCQIJibmwt169YVkpKSlOVJSUlCnTp1BKlUKrx+/VoQBEF4/fq1YGZmJjRs2FBITU1V1n306JFgbm4uABAiIyOV5RMnThRMTEyEK1euqMQdGxsrWFhYCAEBAcoyba63NtdZ0+9QLgAqMWj6Hcp7TCwWCxcvXlSWKxQKoU+fPgIA4Z9//lGWa/N7Wpi+nzp1SgAg/PLLL/nWIaroONWHSM+0bt0aFy9eREBAAJKSkrBhwwaMHTsWjRo1go+Pj8av/2vXro2IiAiNj65du2psZ+PGjVAoFBg2bJiybPDgwTA0NMT69es1vubIkSOws7ODnZ0d3NzcMHnyZDRq1AhHjhzROJr5toCAAGRmZqpMJZLJZNizZw+6d++u8npzc3OVOq9evYJEIoG3tzeioqIKbEdbW7duhYWFBfr06YOXL18qH4mJifD19UVsbKzyW438vHjxAgBgY2OjVduffvoprKyslM+bNGkCS0tL1KhRQ+3bnnbt2iE+Pl7jNIbp06erPO/bty/q16+vstBYJBLB1NQUwJtviBITE/Hy5Uu8//77AKByXf/44w8AwMKFC9XWJ+ROsygKhUKBsLAwNG3aVG0txIwZMyAWi7Fnzx61133xxRcqbbZo0QJSqfSdP5e3jRkzRuUbjdxRY29vb3h5eSnLjYyM0LJlS5VzR0REIDU1FRMnToSlpaWy3NLSEhMnToRMJsPRo0cBvPkdSUtLw7hx42BmZqas6+TkhMGDB6vEJAgC/vjjD/j4+MDR0VHl/Wdubo5WrVrhyJEjhe5jrqJe55LywQcfoFmzZsrnIpEIX331FQCUaru2trYAgOfPn5daG0SljVN9iPSQh4eHck74gwcPcOLECfz+++84deoUevfurTYtw9zcHF26dNF4rq1bt6qVCYKA9evXo0mTJlAoFCrz89u2bYstW7Zg4cKFalMBvL29sWDBAgCAsbExatWqBWdn50L1KTe537x5M0aPHg3gzRzy1NRUlQ8fAPDff/9h1qxZOHz4MBITE1WOlfSe/bdu3UJKSkqBU1SePXsGNze3fI/nxiRouZVgnTp11MqqVKmCmjVraiwHgFevXqlMrbC2tta47qNhw4bYu3cvUlNTlR+k/vrrLyxevBiXL19WWy/w+vVr5b/v3bsHkUiU7zqTonrx4gVkMhnc3d3VjtnY2KB69eoaP9hquk62trb5rk3QJO85cq9n7pz1vMfePndMTAwAaIw7tyw37tz/NmjQQK1uo0aNVJ6/ePECr169Un6g1kQs1n78r6jXuaQ0bNhQrSy376XZbu7vX0W5rwdRUTDxJ9JztWrVwrBhwzB06FC0b98eZ86cwblz59CuXbsin/PEiRP477//AAD16tXTWGffvn3o06ePSlnVqlXz/YDxLgYGBhg0aBCWLVuG+/fvw9XVFZs3b0aVKlVU5tDLZDL4+PggNTUVX375JTw8PGBhYQGxWIyFCxfif//73zvbyu9//HkXFwJvkgU7Ozts27Yt3/MVdJ8EAMqkTdv7GUgkEq3KAe0/XOTavXs3Pv74Y7Rs2RLLly9HzZo1YWJiArlcju7du0OhUKjUL87IfknL73pocy2Kcq1LW278Xbp0wddff11ucWjz+1KR2839/cvvQxSRLmDiT0QA3vxP0tvbG2fOnMGTJ0+Kda7169fD2NgYmzdv1jiiOGrUKKxbt04t8S+ugIAALFu2DJs3b8bIkSNx/PhxBAYGwtjYWFnn2LFjePr0KdavX4/hw4ervD7vwtb82NjY4OLFi2rlmkYb69Wrh7t376JVq1ZqixQLK/eDgTZTT0pKYmIi4uPj1Ub9b926hWrVqilH+7ds2QITExNERkaqTEG5ffu22jnd3Nxw8OBBXL16tcAFy9p+MLCzs4OFhQX+/fdftWOvX79GXFxchbwfQO63Bf/++y86d+6scuzmzZsqdXL/e/v27Xzr5rKzs4O1tTWSk5OL/IFaE22vc+4UtYSEBJXpapp+XwrzM79165ZaWd7rlNtuYX9PC9Nu7jeX7/qgTlSRcY4/kZ6JiIjQOOKVnp6unO+bd8qANpKSkhASEoKuXbtiwIAB8Pf3V3v4+fnh4MGDiIuLK3I7mnh6eqJJkybYunUrtmzZAoVCgYCAAJU6uSOweUdzjxw5Uuj5/W5ubkhJScG5c+eUZQqFAkuXLlWrO2zYMCgUCsyYMUPjuZ49e/bO9po2bQpLS0vl9pBl7ccff1R5vmfPHty5c0flg5tEIoFIJFIZ2RcEQTl1622DBg0CAMycORNZWVlqx3N/NrkflAr7TYdYLIavry8uX76MQ4cOqfVBoVCgb9++hTpXWfrggw9gbm6OlStXIiUlRVmekpKClStXQiqVKu/W/MEHH8DU1BRBQUEq22Y+fvxY7VslsViMwYMH49y5cwgJCdHYdlHmq2t7nXOnseWuU8i1ePFitXMX5mceERGBS5cuKZ8LgoBFixYBgMp7Upvf08K0e/bsWRgYGKBt27b51iGq6DjiT6RnJk2ahFevXsHPzw8eHh4wMzPDo0ePsG3bNty9exfDhg2Dh4dHkc//559/Ij09Hf3798+3Tv/+/bFx40Zs2rRJbeFocQUEBGDKlCn46aef4ObmhlatWqkcb9euHRwcHDBlyhTExsbCyckJV65cwZYtW+Dh4YHr16+/s43AwEAsXrwYffv2xRdffAEjIyOEhIRo/ECVu4Xnr7/+ikuXLqFXr16oWrUqHj9+jH/++Qf3799/57xkiUSCfv36Ye/evcjMzFT5BqO0Va1aFbt378bTp0/RsWNH5Xae9vb2Kvcr8Pf3x65du/D+++9j2LBhyM7Oxt69e9X2dAeAli1b4uuvv8ZPP/2EZs2a4eOPP4aDgwNiYmIQEhKCc+fOwdraGo0aNYKFhQVWrVoFMzMzWFtbo1q1asoFw5r88MMPiIiIQJ8+fTB27Fi4urri5MmT2LFjB3x8fNQ+CFYE1tbWWLRoEcaNGwdvb2/lvvYbN27E/fv3ERwcrFykXaVKFcyfPx9Tp05FmzZtMGzYMKSlpWHNmjWoV68eLl++rHLu77//HmfOnMGAAQMwYMAAtGrVCkZGRnjw4AEOHDiA5s2bq9wDorC0uc6ffPIJZs6cicDAQNy+fRs2NjY4dOiQxi2CbW1t4erqiu3bt6Nu3bqwt7eHubk5fH19lXXee+89vP/++xg3bhyqV6+O0NBQHD16FEOHDkXr1q2V9bT5PX3Xe00QBBw6dAjdu3cv8jd3RBVCuewlRETl5vDhw8LYsWOFJk2aCLa2toJEIhFsbGyEjh07CuvWrRPkcrlK/Vq1agnu7u75ni93q77c7Ty9vLwEAwMDtW0135aRkSFYWFgIbm5uyjL8/20Viys+Pl4wMDAQAAgLFizQWOfq1atCt27dBGtra0EqlQodOnQQTp48qXHbwfy2Ity/f7/w3nvvCUZGRkL16tWFr776Srh9+3a+WxFu3rxZaNeunWBhYSEYGxsLtWrVEvr27Sts3769UP3K3QIzJCREpbyg7Tw1bU1Yq1YtoUOHDmrluVtbxsTEKMtyt0P877//BD8/P8HCwkKQSqWCn5+fcO/ePbVzrF27VmjYsKFgbGwsODg4CCNHjhRevXqltmVjrm3btglt2rQRpFKpYGZmJtSvX1/44osvVLbF3L9/v9C0aVPB2NhYAKAx9ryio6OFIUOGCHZ2doKhoaHg4uIizJgxQ2X7y/z6/K7rlFfudp5vb6GZK79+5/ee2r17t9C6dWvBzMxMMDMzE1q3bi3s2bNHY7tr1qwR3NzcBCMjI6Fu3brC0qVLldu+5o0lNTVV+O6774TGjRsLJiYmglQqFRo0aCB8/vnnwtmzZ5X1tN0+tbDXWRAE4ezZs0KbNm0EY2NjwdbWVhg5cqTw+vVrjdcoKipKaNOmjWBmZiYAUG7J+fY2nNu2bRM8PDwEIyMjwcnJSfj222+FrKwstXa1+T0t6L12/PhxAYCwb9++Ql0boopKJAhFXMlFRERlqnv37khNTcWpU6fKpL2OHTsiNjYWsbGxZdIeUUFiY2Ph4uKCOXPmqN0du7T17dsXjx49wvnz5yvMonSiouAcfyIiHbF48WL8888/Rdp7nYiK5vLlywgNDcXixYuZ9JPO4xx/IiId4e7uXupbIBKRqqZNm6ptR0ukqzjiT0RERESkBzjHn4iIiIhID3DEn4iIiIhIDzDxJyIiIiLSA0z8iYiIiIj0ABN/IiIiIiI9wMSfiIiIiEgPMPEnIiIiItIDTPyJiIiIiPQAE38iIiIiIj3AxJ+IiIiISA/8P3V73Na2269ZAAAAAElFTkSuQmCC\"\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 28\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"A dependence plot can be used to visualize how the number of years of education increases the chance of making over 50K annually.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"scrolled\": true,\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:48.270461Z\",\n \"start_time\": \"2026-01-07T01:36:48.161843Z\"\n }\n },\n \"source\": [\n \"shap.dependence_plot(\\\"education-num\\\", shap_values, X_test)\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"
\"\n ],\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAsQAAAHFCAYAAAD14e+7AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAaT5JREFUeJzt3Xd8jef/x/HXyUI2jdqSqFU1as8Sq1qJ0aCLKqkWqQ5UdVdVW0SplhpRStGhNrWithLar11aJYhRM5ITIuv8/vBz9OQkZJ/IeT8fjzy+ua/7vq/7c5/68s6V675ug8lkMiEiIiIiYqccbF2AiIiIiIgtKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxawrEIiIiImLXFIjvUSaTCaPRiN6rIiIiIpIzCsT3qPj4eAICAoiPj7d1KSIiIpIZhm4WX5MnT6ZGjRrUqFGDyZMn27o6u2bQq5vvTUajkYCAADZu3Ii7u7utyxEREZG7MXSz3DYttE0dYsXJ1gUURFFRUYwdO5Z9+/bh5uZGx44dCQ0NxdnZ+Y7nvf/++xw4cIALFy7g7OxM5cqVeeGFF2jSpInFcUajkfHjx7Nx40aSk5Np0qQJb775Jj4+Pnl5WyIiIiKSDgXiNGJjYxkwYAAVK1YkLCyM8+fPM2HCBBISEhg+fPgdz01KSqJnz55UqFCBxMREli5dymuvvcbUqVOpW7eu+bi3336bY8eO8fbbb+Pi4sLXX3/Nq6++ypw5c3By0n8SERERkfyk9JXGwoULiY+PJywsDC8vLwBSUlIYM2YMISEhlCxZMsNzR48ebbHdrFkzOnfuzC+//GIOxPv27eO3335j0qRJ5pFjX19fevTowYYNG2jfvn0e3ZmIiIiIpEcP1aWxfft2GjVqZA7DAO3btyc1NZUdO3ZkqS9HR0c8PDxISkqy6N/Dw4PGjRub2/z8/KhatSrbtm3L+Q2IiIiISJYoEKcRFRWFn5+fRZuHhwc+Pj5ERUXd9XyTyURycjIxMTF89913nDp1iuDgYIv+fX19MRgMFuf5+/tnqn8RERERyV2aMpFGbGwsHh4eVu0eHh7Exsbe9fylS5cyatQoAFxdXfn000+pXbt2jvtPTEwkMTHRvK3l1kRERERyhwJxLgsICKBq1arExMQQERHB22+/TVhYGM2bN89Rv7NmzSI8PDyXqhQRERGRWxSI0/D09MRoNFq1x8XF4enpedfzvb298fb2Bm4+VBcbG8vEiRPNgdjT05N///03y/337duXnj17mrfj4+MJDAy8az0iIiIicmeaQ5yGn5+f1Vxeo9HIxYsXreYWZ0b16tWJjo626P/EiRNWr1xOb+7yf7m4uODu7m7+cnNzy3ItIiIiImJNgTiNZs2aERkZSVxcnLktIiICBwcHqxdsZMbevXspV66cRf+xsbFERkaa206cOMGRI0dyPK1CRERERLJOUybS6NatGz/++CNDhw4lJCSE8+fPM3HiRIKDgy3WIB44cCBnz55lyZIlAGzdupWVK1fSokULSpUqRWxsLKtXr+a3337jk08+MZ9Xu3ZtmjZtysiRIxk8eLD5xRxVqlShdevW+X27IiIiInZPgTgNT09PpkyZQlhYGEOHDsXNzY2uXbsSGhpqcVxKSgopKSnm7fLly5OYmMikSZOIiYnB29ubKlWqMG3aNOrXr29x7meffcb48eP55JNPSElJoXHjxrz55pt6S52IiIiIDRhMaSezyj3BaDQSEBDAxo0bcXd3t3U5IiJyNyt2w5ZDULMiPNUcXJxtXZHkN0M3y23TQtvUIVY0JCkiIpLXBs+EL1bc3p67GdZ8YLt6RMSCHqoTERHJS+djYNIqy7a1e2Dbn7aoRkTSoUAsIiKSly7GQXKKdfvZK/lfi4ikS4FYREQkLz1YHqqXs2zzKAbt69imHhGxokAsIiKSlwwGWPo2tKkFTo5Q1x9WvANeesGSSEGhh+pERETyWtWysP4jW1chIhnQCLGIiIiI2DUFYhERERGxawrEIiIiImLXFIhFRERExK4pEIuIiIiIXVMgFhERERG7pkAsIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrCsQiIiIiYtcUiEVERETErikQi4iIiIhdUyAWEREREbumQCwiIiIidk2BWERERETsmgKxiIiIiNg1BWIRERERsWsKxCIiIiJi1xSIRURERMSuKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxa062LqAgioqKYuzYsezbtw83Nzc6duxIaGgozs7OGZ5z8eJF5s2bx86dO4mOjsbd3Z26desyaNAgypQpYz5u9+7dDBgwwOr89u3b89lnn+XJ/YiIiIhIxhSI04iNjWXAgAFUrFiRsLAwzp8/z4QJE0hISGD48OEZnvfnn3+yYcMGOnfuTK1atYiJiWHGjBk8//zz/PjjjxQvXtzi+A8//BA/Pz/ztre3dx7dkYiIiIjciQJxGgsXLiQ+Pp6wsDC8vLwASElJYcyYMYSEhFCyZMl0z3v44Yf5+eefcXK6/ZHWrl2boKAgVq5cSa9evSyOf+CBB6hRo0be3YiIiIiIZIrmEKexfft2GjVqZA7DcHM6Q2pqKjt27MjwPA8PD4swDFCqVCmKFy/OhQsX8qxeEREREckZjRCnERUVRefOnS3aPDw88PHxISoqKkt9nThxgsuXL+Pv72+177XXXuPq1av4+PjQoUMH+vfvT9GiRTPsKzExkcTERPN2fHx8lmoRERERkfQpEKcRGxuLh4eHVbuHhwexsbGZ7sdkMjFu3DhKlixJhw4dzO3u7u707t2bevXqUaRIEXbt2sXcuXM5fvw4X3zxRYb9zZo1i/Dw8Czdi4iIiIjcnQJxHpk+fTqRkZF89dVXFCtWzNxevXp1qlevbt5u2LAhPj4+jB07lgMHDlCzZs10++vbty89e/Y0b8fHxxMYGJh3NyAiIiJiJzSHOA1PT0+MRqNVe1xcHJ6enpnqY/HixYSHh/POO+/QqFGjux7fvn17AA4fPpzhMS4uLri7u5u/3NzcMlWLiIiIiNyZAnEafn5+VnOFjUYjFy9etFgmLSMbNmxg9OjRDBgwgC5duuRNkSIiIiKSaxSI02jWrBmRkZHExcWZ2yIiInBwcKBJkyZ3PHf37t28++67dO3alX79+mX6mmvWrAHQMmwiIiIiNqA5xGl069aNH3/8kaFDhxISEsL58+eZOHEiwcHBFmsQDxw4kLNnz7JkyRIAjh8/zhtvvEGFChXo2LEj+/fvNx9bvHhxypcvD8D7779P+fLlqV69uvmhuvnz5xMQEKBALCIiImIDCsRpeHp6MmXKFMLCwhg6dChubm507dqV0NBQi+NSUlJISUkxbx84cACj0YjRaOSFF16wODYoKIgRI0YAUKlSJVatWsW8efNITEykbNmy9O3bl759++b5vYmIiIiINYPJZDLZugjJOqPRSEBAABs3bsTd3d3W5YiIiMjdGLpZbpsW2qYOsaI5xCIiIiJi1xSIRURERMSuKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxawrEIiIiImLXFIhFRERExK4pEIuIiIiIXVMgFhERERG7pkAsIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrCsQiIiIiYtcUiEVERETErikQi4iI2LtrN25+idgpBWIRERF7lZQML00B7+dufoVMgsQkW1clku8UiEVEROzVhOUQvu5mME5Khlm/QthSW1clku8UiEVEROzV6v9Zt636I//rELExBWIRERF75V/Kuq1SOm0ihZwCsYiIiL16Oxju97q9fZ8HvNPNdvWI2IiTrQsQERERG6lcBg5/BQt/g1QTdG8KJTxsXZVIvlMgFhERsWfF3aFfe1tXIWJTmjIhIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrCsQiIiIiBVyfPn0wGAw0adLEap/JZKJChQoYDAZGjBiRpX43btyIwWBg9+7duVTpvUmrTKQjKiqKsWPHsm/fPtzc3OjYsSOhoaE4OztneM7FixeZN28eO3fuJDo6Gnd3d+rWrcugQYMoU6aMxbEXLlxg7Nix7Ny5EycnJ1q3bs3gwYNxd3fP61sTERGRe5S7uzs7d+7k+PHj+Pv7m9u3bNnCv//+S5EiRWxY3b1NI8RpxMbGMmDAAJKTkwkLCyM0NJTFixczfvz4O573559/smHDBtq1a8fnn3/O4MGDOXr0KM8//zxXrlwxH5ecnMygQYM4efIko0aN4q233mLHjh289957eX1rIiIicg/z9fWlTp06/PDDDxbt33//PR06dKBo0aI2quzep0CcxsKFC4mPjycsLIymTZvSpUsXXn31VRYtWsSFCxcyPO/hhx/m559/JiQkhIYNG9K+fXu+/PJLrly5wsqVK83HRUREcOzYMcaMGUPLli159NFHef/999m6dSsHDhzIj1sUERGRe9QzzzzD999/b95OTk7m559/5tlnn7U69rfffqNz586ULVsWNzc3Hn74Yb777ru7XsNkMjFu3DiqVq1KkSJFqFSpEhMmTMjV+yhoFIjT2L59O40aNcLL6/arLNu3b09qaio7duzI8DwPDw+cnCxnoJQqVYrixYtbBOnt27dTpUoV/Pz8zG2NGzfGy8uLbdu25d6NiIiISKHz9NNPc+DAAQ4dOgTA2rVruX79Op07d7Y69sSJEzRv3pwZM2awfPlyunXrxgsvvMDs2bPveI3XXnuNDz74gOeff56VK1fSp08fhg8fztSpU/PkngoCzSFOIyoqyuoPlYeHBz4+PkRFRWWprxMnTnD58mWLeT5RUVH4+vpaHGcwGPD19c1y/yIiImJffH19adq0Kd9//z0ff/wx33//PZ07d8bNzc3q2Kefftr8vclkomXLlkRHRzNt2jSef/75dPv/559/mDRpElOnTuWll14CoF27dly7do2PPvqIl156CQeHwjeeWvjuKIdiY2Px8LB+j7uHhwexsbGZ7ufWrxtKlixJhw4dzO1xcXHp9u/p6XnH/hMTEzEajeav+Pj4TNciIiIihcczzzzDDz/8wPXr11m6dCnPPPNMusdduXKFV199FV9fX5ydnXF2dmb69On89ddfGfYdEREBQLdu3UhOTjZ/tWvXjnPnznHq1Kk8uSdb0whxHpk+fTqRkZF89dVXFCtWLMf9zZo1i/Dw8FyoTERERO5lPXr04PXXX+eDDz7A2dmZxx57LN3j+vTpw/bt2/nggw946KGH8PT0ZMqUKfz4448Z9n3x4kVMJhM+Pj7p7j916pTVb7oLAwXiNDw9PTEajVbtcXFxeHp6ZqqPxYsXEx4ezvvvv0+jRo0s9nl4eKTbf2xsLKVKlcqwz759+9KzZ0/zdnx8PIGBgZmqR0RERAqPUqVK0aZNG8aPH88LL7yQ7rKwCQkJrFixgvHjx/PKK6+Y21NTU+/Yd4kSJTAYDGzduhUXFxer/dWqVcv5DRRACsRp+Pn5Wc3lNRqNXLx40eJBuIxs2LCB0aNHM2DAALp06ZJu/0ePHrVoM5lMnDhxgsaNG2fYr4uLS7p/MEVERMT+vPrqq7i6uvLiiy+mu//GjRukpqZaZIe4uDiWLVt2x37btm0LwKVLl+jUqVPuFVzAKRCn0axZM2bNmmUx1zciIgIHB4d03w7zX7t37+bdd9+la9eu9OvXL8P+V61axcmTJ6lYsSIAkZGRXL16lebNm+fuzYiIiEihFBQURFBQUIb7vby8aNiwIaNHj6ZkyZI4OTkxevRovLy8OH/+fIbnVa1alZdffpnnnnuOYcOG0bhxY5KSkvjrr7/YsGEDS5YsyYO7sT0F4jS6devGjz/+yNChQwkJCeH8+fNMnDiR4OBgSpYsaT5u4MCBnD171vwH4/jx47zxxhtUqFCBjh07sn//fvOxxYsXp3z58sDNJzVnzZrFm2++ycsvv0xCQgJffPEFLVq0oGbNmvl6ryIiIlJ4zZ8/n/79+/P8889z33338eqrr2I0Ghk3btwdz/vyyy+pVq0a06ZNY+TIkbi7u1OtWjV69OiRT5XnP4PJZDLZuoiC5vjx44SFhbF3717c3NwIDAy0enXzSy+9xNmzZ1m+fDkAy5cv56OPPkq3v6CgIIt3i58/f56wsDB27tyJo6MjrVu3ZsiQIVl6dbPRaCQgIICNGzfqlc8iIiL3AkM3y23TQtvUIVYUiO9RCsQiIiL3GAXiAkvrEIuIiIiIXVMgFhERERG7pkAsIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrCsQiIiIiYtcUiEVERETErikQi4iIiIhdUyAWEREREbumQCwiIiIidk2BWERERETsmgKxiIiIiNg1BWIRERERsWsKxCIiIiJi1xSIRURERMSuKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxa1kOxNeuXeOjjz4iIiIiL+oREZF7welLMGs9rN8HJpOtqxERyRGnrJ7g6urK2rVrqVOnTl7UIyIiBd3iHfDUeEhKvrnd4WFY8S44Odq0LBGR7MpyIAaoVKkSZ8+eze1aALh+/TqrV6/m5MmTXL16FVOakQeDwcAHH3yQJ9cWEZG7MJlg8KzbYRhgzR5YGgndmtqsLBGRnMhWIO7duzejR4+mY8eO+Pr65loxe/fuZciQIcTGxmZ4jAKxiIgNJSTCiQvW7YdP538tIiK5JFuBOCoqilKlSvH000/TokULKlasSNGiRS2OMRgM9OvXL0v9hoWF4eDgwOeff07dunXx8PDITnkiIpJXihWBJlVhx1+W7W1r2aae9JhMkJKqKRwikmnZCsTTp083f79x48Z0j8lOID5+/Dj9+/enZcuW2SlLRETyw6xB8OTnsP8EuBWF97tDk2q2ruqmsCUwehHEXofuTWHaAPB0tXVVIlLAZSsQL1u2LLfrAMDHxwcnp2yVJCIi+aV6edg3AU6ch/s8wL2YrSu6acVueHPO7e0ftoJnMZg20HY1icg9IVvps0yZMrldBwBdunRh9erVPPXUUzg66lddIiIFmu/9tq7A0vJd1m3LdikQi8hd5Xg4NiYmhjNnzgBQtmxZvL29s91X3759uXDhAn379qV79+6ULVsWBwfrpZLr1auX7WuIiEghZUywbktIyv86ROSek+1A/NdffzFu3Dj27Nlj0f7www8zbNgwqlSpkuU+b9y4wdWrVzl8+DCjRo2y2m8ymTAYDERGRma3bBERKaz2n7Rui7ue/3WIyD0nW4H46NGj9OvXjxs3btCqVSsqVaoEwLFjx9i8eTP9+vVj5syZPPDAA1nqd8yYMURERNCqVSvq1q2Lp6dndsrLsaioKMaOHcu+fftwc3OjY8eOhIaG4uzsfMfzFixYwLZt2zhw4AAxMTGMHj2adu3aWRyze/duBgwYYHVu+/bt+eyzz3L1PkRE7IpTOi9fNRjyvw4RuedkKxBPmzYNJycnvvnmG6uR4KNHj/LSSy8xdepUwsLCstTvpk2b6Ny5M++99152ysoVsbGxDBgwgIoVKxIWFsb58+eZMGECCQkJDB8+/I7nrly5EoDmzZubv8/Ihx9+iJ+fn3k7J1NNREQEeLsbPDnOsu1RvVVVRO4uW4H4jz/+oEePHulOi6hcuTLdu3dn4cKFWe7XZDJRo0aN7JSUaxYuXEh8fDxhYWF4eXkBkJKSwpgxYwgJCaFkyZIZnjtz5kwcHBw4c+bMXQPxAw88YPN7FREpVHo0g8kvwsc/w/UbEFgf5r5u66pE5B6Qzu+X7i4hIYH77rsvw/0+Pj4kJKTzcMNd1K9fnwMHDmSnpFyzfft2GjVqZA7DcHM6Q2pqKjt27Ljjuek9ACgiIvko9HE4+w3EzIV5gzVlQkQyJVsJrly5cmzdujXD/Vu3bqVcuXJZ7nfo0KH8/vvvzJ07l6Qk2zwZHBUVZTGVAcDDwwMfHx+ioqJy7TqvvfYajRo1omPHjkycOPGuP0AkJiZiNBrNX/Hx8blWi4iI5DGTCRb+Bq9/AzPW3XwFtogUGNmaMtGxY0cmT57Mu+++S0hIiDlAHj9+nG+//ZYdO3YwaNCgLPc7YMAArl+/zpdffsmkSZPw8fFJdz3ipUuXZqfsTImNjU33ldEeHh7ExsbmuH93d3d69+5NvXr1KFKkCLt27WLu3LkcP36cL774IsPzZs2aRXh4eI6vLyIiNjAoHL5efXt7/hb4daTt6hERC9kKxM899xxHjhxh7dq1rFu3DsP//0rKZDJhMplo164dvXr1ynK/pUqVMvdVWFWvXp3q1aubtxs2bIiPjw9jx47lwIED1KxZM93z+vbtS8+ePc3b8fHxBAYG5nm9IiKSQ//GwLS1lm0bDsCWQ/CIniURKQiyFYgdHR357LPP6NKlC5s2beL06dPAzakUAQEBNG7cOFvFTJ8+PVvn5SZPT0+MRqNVe1xcXJ4tA9e+fXvGjh3L4cOHMwzELi4uuLi45Mn1RUQkD102Qkqqdfv5q/lfi4ikK1OB+KOPPqJbt27msPbHH3/g7+9PkyZNaNKkSZ4WmN/8/Pys5gobjUYuXrxoNbdYRETkrh4sDw9VgIOnbrd5uUJ7LQknUlBk6qG6FStWEB0dbd4eMGAAO3fuzLOibKlZs2ZERkYSFxdnbouIiMDBwSHPwv+aNWsAtAybiEhhtexteLweuBWFJlVh1fvg6WrrqkTk/2VqhNjb25tLly6Zt00mU54U07Bhw7vOITYYDHkaxrt168aPP/7I0KFDCQkJ4fz580ycOJHg4GCLNYgHDhzI2bNnWbJkibnt0KFDnDlzhpiYGADzEnLFixenfv36ALz//vuUL1+e6tWrmx+qmz9/PgEBAQrEIiKFVaXS8IvtXjolIneWqUBcu3ZtZs6cyblz58zzaH/99VdOnTqV4TkGg4F+/fplqZjAwECrQJycnMzp06c5cOAAlStXplq1alnqM6s8PT2ZMmUKYWFhDB06FDc3N7p27UpoaKjFcSkpKaSkpFi0/fTTT6xYscK8PXfuXADq1atnnh9dqVIlVq1axbx580hMTKRs2bL07duXvn375ul9iYiIiEj6DKZMDPeeOXOGESNGsGfPHkwmEwaD4a6jxAaDgcjIyFwrdO/evQwZMoQvvviCWrVq5Vq/9yqj0UhAQAAbN27E3d3d1uWIiIjI3Ri6WW6bsv5WX8kbmRohLlu2LNOnTycpKYlLly7RqVMnhg4dSqtWrfK6PrM6derQuXNnvvrqqwKxGoWIiIiIFA5ZWnbN2dmZ0qVLExQURM2aNSlTpkxe1ZWuChUq8PPPP+frNUVERESkcMvWOsQffvhhbteRKb///jtFihSxybVFREREpHDKViDOK/99IO2/YmNjiYyMZPv27XTp0iWfqxIRERGRwqxABeKPPvoowwf2HB0d6dKlC0OGDLFBZSIiIiJSWBWoQDx16lSrNoPBgKenJ+XKlaNYsWI2qEpERKQQ23EEpq0FE/BiO2j+oK0rEsl3BSoQ33p5hYiIiOSDTQeh7YeQknpz+7tNsOZ9aKfXSot9ydSrm0VERKQQ+nLl7TAMkJoKE1farh4RG8nxCPGpU6e4dOkSlStXzpUXRJw7d46FCxdy6tQprl69ajWf2GAwMGXKlBxfR0RExO5du5G5NpFCLtuBeMuWLYwbN46zZ88CMHnyZBo2bMjly5cJCQlh0KBBtGvXLkt9btu2jWHDhpGUlISrqyteXl7ZLU9ERETupk9rWP0/6zYRO5OtQLx7927eeOMNqlWrRlBQkMWb40qUKEH58uVZu3ZtlgPx5MmT8fb2Zty4cdSoUSM7pYmIiEhmPdXi5ojwlDWQaoL+j8JzAbauSiTfZSsQz5gxg6pVq/Ltt98SGxtr9SrlWrVq8csvv2S536ioKAYOHKgwLCIikl/6tr35JWLHsvVQ3aFDh3jsscdwcEj/9FKlSnHx4sUs91u8eHGcnZ2zU5KIiIiISLZkKxCnpqbi4uKS4f6YmJhsBduOHTvy66+/ZqckEREREZFsyVYg9vf353//+1+G+7ds2ULVqlWz3G9QUBBJSUkMGTKEXbt2cfr0ac6dO2f1JSIiIiKSW7I1h7hLly6EhYWxZMkSWrVqBdxcDi0hIYGvvvqK/fv389FHH2W53+7du5tf3bx169YMj4uMjMxO2SIiIiIiVrIViLt3787evXv55JNP+OKLLzAYDLz77rvExMSQmppKp06dePzxx7Pcb79+/TAYDNkpSUREREQkWwymtG++yIINGzbwyy+/cOLECUwmExUqVCAwMJC2bfW0al4zGo0EBASwcePGXHkhioiIiOQxQzfLbdNC29QhVnL0prrWrVvTurUW8BYRERGRe1e2HqrLL5cuXaJRo0bs2rXL1qWIiIiISCGVrRHi8PDwux5jMBjo169fdrq3kIMZHSIiIiIid5WtQJz2zXT/dWuViNwKxCIiIpLH4hPAZAL3YrauRMQmshWIly1bZtWWkpJCdHQ08+fPx2g0MmLEiJzWJiIiInkpKRkGToM5m24G4p4tYdoAKKK3xop9ydYc4jJlylh9lS9fniZNmjBx4kQcHBxYvnx5jotzdnamXr16eHh45LgvERERSWPCcvhm/c1gnJwCszdA2BJbVyWS73L9oTqDwUDbtm1ZuXJljvvy9PRk2rRpVK9ePRcqExEREQur03nrbHptIoVcnqwykZSUxNWrV/OiaxEREcktlUpZtz1QOv/rELGxHK1DnJ5Dhw7xww8/4Ofnl63z9+3bx08//cTJkyczDNVLly7NQYUiIiICwFvBsHw3nP//f299POHtYNvWJGID2QrEXbp0Sbf96tWrXLt2DUdHR957770s97tixQpGjhyJk5MTFStWpHRp/ZQqIiKSZyqXgSOTYOFvkGqCbk2ghJ7bEfuTrUBcqlQpDAaDRZvBYKBatWr4+vryxBNPULZs2Sz3O3PmTHx9ffn6668pWbJkdkoTERGRrPB2gxfa2boKEZvK9XWIc+LcuXO8+uqrCsMiIiIikm8K1Kub77//fpKSkmxdhoiIiIjcY5KTk7N9boEKxN26dWPVqlWkpKTYtI6oqChCQ0Np0aIFHTp0YOLEiZkK6gsWLOD111+nXbt2NGjQgIiIiHSPu3DhAsOGDaNly5a0adOGjz/+GKPRmNu3ISIiIlIo9OnTh2vXrmW4/+jRozRr1izb/WdqykTDhg2t5gzfjcFgYOfOnVk658EHH+TXX3/l+eefp0ePHpQrVw4HB+vMXq9evSz1mxWxsbEMGDCAihUrEhYWxvnz55kwYQIJCQkMHz78jufeWnu5efPmGa7DnJyczKBBgwAYNWoUCQkJTJw4kffee48vvvgiV+9FRETkXvX3FRPvbkll30UTzcoa+PQRB0q7ZS2LSOHx3XffsXPnTn744Qfq1KljtW/QoEFZzqr/lalAHBgYmKOLZNbAgQPN348aNcrqmiaTCYPBQGRkZJ7VsHDhQuLj4wkLC8PLywu4+VrqMWPGEBIScsf5zTNnzsTBwYEzZ85kGIgjIiI4duwYCxYsMC9N5+npyaBBgzhw4AA1a9bM9XsSERG5l9xINtH2pxROxd3cPnLZxP4LKex6LtdXi5V7xC+//ELv3r1p0qQJY8aM4dVXXyU+Pp6BAwcyb948GjRowPfff5/t/jP1J2vEiBHZvkBWfPjhh/lynTvZvn07jRo1ModhgPbt2/PZZ5+xY8cOOnXqlOG56Y1mp9d/lSpVLNZpbty4MV5eXmzbtk2BWERE7N6vJ03mMHzL7n/h4EUTD/lolNgedejQgX379tGrVy8GDx7MqlWrOHr0KMeOHWPo0KF8+umnODll/wemAvWjVlBQkK1LICoqis6dO1u0eXh44OPjQ1RUVK707+vra9FmMBjw9fXNlf5FRETuda7O6Yde1wKVWiS/lSpVirVr19KyZUvWrFmDwWDgyy+/5OWXX85x3wXqobqCIDY2Fg8P60XJPTw8iI2NzXH/cXFx6fbv6el5x/4TExMxGo3mr/j4+BzXIiIiUhC1LA8N0rxV+okqBvy9NTpszy5fvkzXrl3Zvn07jzzyCJ6enrz99tt89913Oe472z9r7dmzh2+//ZYDBw4QFxeHyWSy2J+dh+oArl+/zpw5c9iwYQOnT58GoFy5crRu3ZrevXtTrFix7JZ8T5s1axbh4eG2LkNERCTPGQwGIp505Ks/TOw5b6JFeQMD6ygM27PNmzfTq1cv/v33X0aPHs2wYcM4fvw4zzzzDH369GHdunVMmTIFNze3bPWfrUD8xx9/EBoairu7OzVr1mTbtm00bNiQa9eucfDgQSpXrkz16tWz3O/Vq1d58cUXOX78OMWLF6datWoAnDx5khkzZrB+/XrCw8Mt5vfmNk9Pz3SXQIuLi8PT0zPH/Xt4eKTbf2xsLKVKlUrnjJv69u1Lz549zdvx8fEEBgbmuB4REZGCyKuIgfeaKgTLTW3atMHX15ctW7bQqFEjAPz9/dm2bRvvvvsuYWFh7Ny5kyNHjmSr/2wF4pkzZ+Lj48N3332HwWCgffv29O3bl4YNG7Jjxw6GDx9+1yXK0jN16lSioqJ48803CQ4OxtHREbi5ysPixYsJCwtj+vTpDBs2LDtlZ4qfn5/VXF6j0cjFixctHoTLSf9Hjx61aDOZTJw4cYLGjRtneJ6LiwsuLi45vr6IiIjIvaZ79+5Mnz7danDS0dGR0aNH07ZtW55//vls95+tOcQHDx6ka9euFC9e3Lw0WmpqKgBNmjShY8eOTJ06Ncv9bt68mS5dutCjRw9zGIabN9u9e3c6d+7Mxo0bs1NypjVr1ozIyEji4m4/3hoREYGDgwNNmjTJlf7//vtvTp48aW6LjIzk6tWrNG/ePMf9i4iIiBQ2P/zwwx1/U9++fXv27t2b7f6zFYgTExPN6/HeGrX879tDqlatyp9//pnlfi9fvmyeJpGeatWqcfny5Sz3mxXdunXD1dWVoUOHsmPHDpYtW8bEiRMJDg62WIN44MCBdO3a1eLcQ4cOERERwfbt2wE4cOAAERER/P777+Zj2rVrR6VKlXjzzTfZsmUL69atY+TIkbRo0UJLromIiIjcQVRUFDNmzOCTTz4x/0Y/MTGRkydP5mhKbbamTPj4+HD+/HkAihUrhoeHB//88w+tW7cG4Pz589laC65EiRJ3nPtx5MgRSpQokZ2SM83T05MpU6YQFhbG0KFDcXNzo2vXroSGhlocl5KSYvWK6Z9++okVK1aYt+fOnQvcfLPe9OnTAXBycuKrr74iLCyMd999F0dHR1q3bs2QIUPy9L5ERERE7mXDhw9n/PjxpKSkYDAYaNq0KX5+fiQkJFCjRg1GjRrF66+/nq2+sxWIa9SoYTEs3bhxY+bPn0/p0qUxmUz89NNPPPTQQ1nut2XLlixatIjq1avzxBNPmF90kZqaypIlS1i2bBnBwcHZKTlL/P39+frrr+94zK2A+18jRozI1EtM7r//fsLCwrJbnoiIiNyDDGE/WmxPmjyZyZMnA/Dyyy/nynq6hdW0adMICwvj1VdfJSgoiEcffdS8z9PTk86dO7N8+fJsB2KDKe16aZmwY8cOVqxYwXvvvUfRokWJjo7mxRdf5OLFiwDcd999TJo0icqVK2ep35iYGEJCQoiOjqZ48eLmF1icOHGCK1euUL58eWbOnIm3t3dWSy50jEYjAQEBbNy4EXd3d1uXIyJSMCyNhDGL4YoRnnkE3ukGTo53P08kHxjGJVtsm97Qm0Yyq06dOlSuXJmFCxdy6dIlSpYsSUREBG3atAFg9OjRTJo0iejo6Gz1n63/Ek2aNLF4wKx8+fIsWrSIyMhIHB0defjhh7MV0ry9vZkzZw6zZ89m06ZNHDx4ELi5DnGXLl3o3bu3wp+IiKRvxxEIHgv//5A3H/4AySkw8hnb1iUiOfbXX38xcODADPeXLFnSPDCbHbn2o0mxYsVo1apVjvtxd3fXrw1ERCTr5m6+HYZvmb1BgVikEChatOgd39J74sSJHM0gyNYqEz179uSHH34gJiYm2xcWERHJVa7prNVexDn/6xCRXNeoUSMWL16c7r6EhAS+++67HC1fm60R4itXrvD5558zceJEmjVrRmBgIC1btszyyhJ//PEHcHMVhv9u382t40VERMxKeFi3uRfN/zpEJNcNGzaMDh068NxzzxESEgLAuXPnWLNmDR9++CHR0dHMnz8/2/1n66E6k8nEzp07WblyJZs2bSIhIQEPDw8effRRgoKCMr3CRMOGDTEYDGzbtg1nZ2fz9p2uazAYiIyMzGrJhY4eqhMRSaPfZPhmvWXb/V7w7yzb1COShh6qy5np06fz2muvkZiYaM6EcPOdGFOmTKFPnz7Z7jtb/yUMBoP5wbrr16+zfv16Vq5cyaJFi1i4cCEVK1YkKCjoroV98MEHGAwG88jyrW0REZEsq1sJSBOI6/rbpBQRyX0vvfQSnTt3ZsGCBRw+fBiTyUSVKlV48sknKVeuXI76ztYIcUb+/fdffvnlF2bPns3169fZuXNnbnUtaWiEWEQkjes3oNNnsH7fze0yxWHtB1DT17Z1ifw/jRDnrqSkJJYtW8bly5fp1KkTpUuXznZf2XqoLj3R0dEsWbKEpUuXEh8fb36pRlaEh4dz9OjRDPf/888/hIeH56RMEREprIoVgYgRsHMMrH4fjk9VGBYpJN58800aNmxo3jaZTLRv354nn3yS/v37U6tWLf75559s95+jQGw0Glm0aBEvvPACwcHBzJgxA1dXV15//XVWrlyZ5f6mT5+uQCwiIjnTqAp0qKsVJkQKkdWrV/PII4+Yt5cvX87mzZsZNmyY+WG60aNHZ7v/bI3Vb9myhZUrV7JlyxYSExMpUaIETz/9NEFBQVStWjXbxdxNYmIijo5645CIiIiIPTl16hRVqlQxby9fvhx/f39zCD548CDz5s3Ldv/ZCsRDhgzBxcWFRx55hKCgIJo2bZrtoGo0GjEajebtmJgYzp07Z3Xc1atXWbVqFaVKlcrWdURERETk3pSYmGixvO+GDRto166debtSpUqcPXs22/1nKxAPHz6cDh064OGRzpqPWTR//nxmzJgB3Fy9Yvz48YwfPz7dY00mE6+++mqOrykiIiIi944KFSrw22+/8eKLL3Lw4EGOHTvGyJEjzfvPnz+fo0UGshWIu3fvnu0LplW/fn3gZtidMWMGAQEBFkPicDMoFytWjFq1alGnTp1cu7aIiIiIFHxPP/00H3/8MefPn+fgwYN4enrSsWNH8/7//e9/PPDAA9nu3+brfdSvX98cis+dO0e3bt2oWbOmjasSERERkYLi7bff5tSpUyxZsgQvLy/mzJmDt7c3cHNa7bJlyxg8eHC2+8/VdYgl/2gdYhERa8mpJlYdNxGTAIGVDJQoppc9ScGhdYjzRmpqKnFxcbi6uuLsnL3VZQrkf4mUlBSioqKIi4sjNTXVan+9evVsUJWIiBRksTdMBPyYwv/O39z2cIG13R1pUlahWKQwc3BwwMvLK0d9FLhA/O233zJ79mzi4+MzPCYyMjIfKxIRkXtB+D6TOQwDxCXCW5tT2Ph0wfmn7vJ1E95FwcGgkC5SkOTam+pyw5IlS5g8eTJVq1Zl4MCBmEwmnnnmGZ577jk8PT158MEH+eCDD2xdpoiIFECHL1vPAPzzsg0KScfucyYempXMfZNT8J2ewtKj1r/9FBHbyVIgTk5OZv369Xz77bcsWbKEmJiYXC1m4cKF1KpVi2nTpvHEE08A0KJFC1555RV++OEHzp49S0pKSq5eU0RECoc2Fa1HXdum05bfUlJNdFuawqFLN7ej4+Cp5amcj9cjPCIFRaYDcWxsLM899xxvv/02kydP5tNPP6Vbt278+eefuVbM8ePHadu2LXBzqTXAHIB9fHx44okn+OGHH3LteiIiUng8Xd3Aq/UMOP//v2wtysH41rb/Reifl+BknGXbjRTYeEqBWKSgyPTEqm+++YajR4/SokULmjZtysmTJ1m4cCGffPIJc+fOzZViHB0dKVasGID5f69evWreX6ZMGU6ePJkr1xIRkcLFYDAwsY0jHzUzEZ8E5TxsPzoMUNYdHAyQmib/+nraph4RsZbpQLxlyxaaNm3KhAkTzG1lypRh4sSJ/Pvvv7nySuVSpUpx5swZAFxcXChVqhR79uyhQ4cOABw6dCjHTxGKiEjh5l3UgHdRW1dxW3wSpLfAaWxi/tciIunL9O+S/v33X5o3b27R1rJlS0wmU47eHf1f9erVY+vWrebtdu3asWjRIj766CNGjBjB0qVLrWoQERFJqyAtsX8jBdKr5npyOo0iYhOZHiFOTEy0Gp318PAAICkpKVeKefrpp6lSpQoJCQkULVqU/v37c+LECVauXAlAkyZNGDRoUK5cS0RECp8vfk/ls52pxNyAp6sZmNzOAXcX206dqFzcQLOysP3M7bay7vCoX8GY0iEiubQOsSGX1lP08/PDz8/PvF2sWDEmTJiA0WjEwcEBV1fXXLmOiIgUPquOpTJ4w+3lzOYcMuHmnMrX7R1tWNVNS7o68t7WVDZHm6jlY2BUCweKOikQixQUWQrEc+fOZc2aNebtlJQUDAYDX3/9tdXoscFgYPz48blSpF5NLCIid7P0H+uJCUuOmvi6vQ2KSaOkq4Fpj9o+mItI+rIUiI8cOcKRI0es2vfv32/Vlp1R47Vr17J161ZGjhyZ7v4PP/yQRx55hHbt2mW5bxERKdzKuhlIO1u3rMZTRCQTMh2Id+3alZd1APDTTz9Rvnz5DPc7ODjw008/KRCLiIiV/nUMzNgPp/5/zV8nB/iome3XIRaRgq9A/U1x/PhxqlWrluH+atWqcezYsXysSERE7hWl3Ax82sIBP0/wKQb9axvo4K95uiJyd7keiC9dusTs2bPp0aNHls9NSEjAwSHjkgwGA9euXctJeSIiUkjtOGOi96pUomLh4nWYvMfEu1tT736iiNi9XFllIjU1lS1btrB06VK2b99OSkpKtlaEKFu2LHv27OGpp55Kd/+ePXsoXbp0TssVEZFCaNSOFKv1fr/+n4kxLW1SjojcQ3IUiKOioli2bBm//PILly9fxsPDg8cff5y2bdvSuHHjLPcXEBDA7NmzWbJkCV27drXYt3TpUtavX89zzz2Xk5IzJSoqirFjx7Jv3z7c3Nzo2LEjoaGhODs73/E8k8nE7NmzWbBgATExMVStWpUhQ4ZQq1Yt8zG7d+9mwIABVue2b9+ezz77LNfvRUTEXkTHWbcl6OUXIpIJWQ7E169fZ+3atSxbtoz9+/fj6OhInTp1uHz5Mu+++y5t2rTJdjF9+vRh06ZNfPrpp8yfP988n/ivv/7i+PHj+Pr60rdv32z3nxmxsbEMGDCAihUrEhYWxvnz55kwYQIJCQkMHz78jufOnj2badOmMWjQIKpUqcKCBQsYNGgQ8+bNs3pY8MMPP7RYc9nb2zsP7kZExH60rmBg7wXLMeLyHjYqRkTuKZkOxHv27GHZsmWsX7+ea9euUa1aNYYMGcJjjz1GXFwcwcHBOS7Gzc2NmTNnMmnSJNatW8fq1asB8PT0pFu3boSGhub5msQLFy4kPj6esLAw89rKKSkpjBkzhpCQEEqWLJnueTdu3GDWrFn06tWLnj17AlC3bl2Cg4OZO3cub731lsXxDzzwADVq1MjTexERsSdvN3bgpyMpnIm/ue1ogK/bFahnx0WkgMp0IH7xxRcpUaIETzzxBEFBQVSuXNm8z2g05lpB7u7uvPXWWwwfPpyYmBjg5uhpbr0N7262b99Oo0aNLF40cms6w44dO+jUqVO65+3bt4/4+HiLJeGcnZ1p3bo1GzZsyPO6RUTs3f1uBg6FODL/TxNXEqB7VQNVS2iVCRG5uyxNmbhx4wZGozFXA3BGDAYDxYsXz/PrpBUVFUXnzp0t2jw8PPDx8SEqKuqO5wEW0yAA/P39+f7770lISKBo0aLm9tdee42rV6/i4+NDhw4d6N+/v8V+ERHJOq8iBgY+rBAsIlmT6UC8YMEClixZwqpVq1i2bBlly5YlKCiIwMDAXCvmjz/+yNRx9erVy7VrphUbG4uHh/WkMw8PD2JjY+94nouLC0WKFLE6z2QyERcXR9GiRXF3d6d3797Uq1ePIkWKsGvXLubOncvx48f54osvMuw/MTGRxMRE83Z8fHzWb05ERERErGQ6EPv5+fH6668zaNAgNm/ezNKlSwkPDyc8PJzKlStjMBgwmazfI58V/fv3z9TUiMjIyBxdx5aqV69O9erVzdsNGzbEx8eHsWPHcuDAAWrWrJnuebNmzSI8PDy/yhQRERGxG1leZcLJyYk2bdrQpk0bLl68yLJly1i+fDkmk4kPPviAVatW0aZNG1q2bJnlB+A+/PBDq7bk5GROnz7N8uXLKVu2bK48vHcnnp6e6U4JiYuLw9PT847nJSYmcuPGDYtR4ri4OAwGQ7qjzre0b9+esWPHcvjw4QwDcd++fc0P68HNEeLcHJ0XERERsVc5WofYx8eHkJAQQkJC+P3331m6dCm//vormzZtwtnZme3bt2epv6CgoAz3Pffcc/Tq1Ssn5WaKn5+f1Vxho9HIxYsXreYHpz0P4MSJE1StWtXcHhUVRenSpXM8P9jFxQUXF5cc9SEiIiIi1nJtPZr69eszcuRIVq9ezfDhwy1WocgNnp6edOnShTlz5uRqv2k1a9aMyMhI4uJur/AeERGBg4MDTZo0yfC82rVr4+bmRkREhLktOTmZDRs20Lx58ztec82aNQBahk1ERETEBnLl1c3/5e7uTvfu3enevXtud42npyenT5/O9X7/q1u3bvz4448MHTqUkJAQzp8/z8SJEwkODrZYg3jgwIGcPXuWJUuWAFCkSBH69u3L9OnTKV68OJUrV2bBggVcvXrVYmT7/fffp3z58lSvXt38UN38+fMJCAhQIBYRERGxgUwH4qtXr2a58/+u5ZtTN27c4JdffuG+++7LtT7T4+npyZQpUwgLC2Po0KG4ubnRtWtXQkNDLY5LSUkhJSXFou3555/HZDIxd+5crly5QtWqVfnqq68s3lJXqVIlVq1axbx580hMTKRs2bL07ds3z9/AJyIiIiLpM5gyuTREw4YNs/RyDIPBwM6dO7NUzEcffZRue2xsLPv37+fKlSu89tpr+TKXuKAzGo0EBASwcePGPH97n4iIiOScYVyyxbbpjVz/Rb1kU6b/SwQGBloE4hs3brBu3TqaNGmCj49PrhSzYsWKdNs9PT3x9fU1vypaRERERCS3ZDoQjxgxwmI7JiaGdevW0bt3bxo2bJgrxezatStX+hERERERyaxcW2VCRERERORepEAsIiIiInbNprO5u3Tpkq3zli5dmsuViIiIiIi9smkgLlWqlNXKFefPnyc6Oho3NzfKlSsHwOnTp4mPj6d8+fLcf//9tihVRERERAqpHAfirCzFltb06dMttg8fPkxoaChDhw6lW7duODs7A5CUlMSCBQv45ptv+Oyzz3JUr4iIiIjIf2U6ED/99NMW26mpqRgMBj7++GOKFStmdbzBYOD777/PUjFffPEF7dq1s7qWs7Mzzz77LMePH2fixIlMmTIlS/2KiIiIiGQk04E4Pj7eajS4dOnSmEwmrl27livFHDx4kHbt2mW4v1q1aqxZsyZXriUiIiIiAlkIxMuXL8/LOgAoUqQIBw8epHv37unu379/Py4uLnleh4iIiIjYjwK17FpAQAArV64kPDzcYtT52rVrTJ8+nVWrVhEQEGC7AkVEBIB/403M/zOVrdEmW5ciIpJjubLKRHJyMgcPHuTChQv4+/vzwAMPZKuf1157jb/++ovp06fzzTffmF8JffHiRVJSUqhevTqvvfZabpQsIiLZtOKfVLovS+VGys3tTg8YWNzFAUeH7D9kLSJiS5kOxLt372bDhg288MILlChRwtx++vRp3njjDf755x9zW2BgIB9++GGWi/Hw8GDmzJksW7aMTZs2cfr0aQAaN25Mq1at6Ny5M05ONl0pTkTErplMJl799XYYBlj+j4nl/5joWkWBWETuTZlOlytWrGDfvn0MGzbMov2jjz7i6NGj1KlTh5o1a/Lbb7+xcuVK6tevT1BQUNYLcnIiODiY4ODgLJ8rIiJ5KyEZjl+1bj94CbpWyf96RERyQ6bnEB88eJAmTZpYtEVFRfG///2PunXrMmPGDF5//XVmz55NhQoVWLlyZY4KS0xM5Pz58yQlJeWoHxERyT3FnA00LG3dHlBBo8Micu/KdCC+dOkSFStWtGjbvXs3BoOBrl27mtuKFi3KY489xt9//52tgg4fPsyAAQNo2bIlQUFB7NmzB4DLly8zcOBAdu7cma1+RUQkd8x6zJFq/z9zrqgTfNTMgeblFIhF5N6V6UCcmJhIkSJFLNoOHToEQL169SzaS5UqhdFozHIxR44coV+/fkRHRxMYGGixr0SJEty4cYMVK1ZkuV8REck9D/kY+LOvI0dCHDk30JEPmhWoBYtERLIs03OIS5cuzbFjxyza9uzZQ/HixSld2vL3ZwkJCXh4eGS5mKlTp1KyZEnmzZtHYmIiy5Yts9jfsGFD1q1bl+V+RUQkdxkMBqqWuPtxIiL3gkz/WF+3bl1WrlzJ0aNHAdiwYQOnTp2iWbNmVscePXqUkiVLZrmYPXv20LVrV1xdXdPdX7p0aS5evJjlfkVEREREMpLpEeI+ffqwatUqnn32Wby8vLh69SrOzs706tXL4riUlBQ2b95MmzZtslxMYmIi7u7uGe6Pj4/Pcp8iIiIiIneS6RHicuXKMX36dJo3b46XlxfNmjVj2rRpVi/h2L17N15eXrRq1SrLxZQrV47Dhw9nuH/Xrl34+/tnuV8RERERkYxk6S0XNWrUYMKECXc8pnHjxvz444/ZKuaxxx7jm2++oV27dlSrVg24OU8NYO7cufz2228MHTo0W32LiIiIiKSnQL327bnnnmPnzp288sor+Pn5YTAYGD9+PFeuXOHSpUs0btyYHj162LpMEZH8czgaos5D8wfBo5itqxERKZQKVCB2dnZm8uTJ/Pjjj6xevRoXFxdOnDhBxYoV6dmzJ08//TQODlreR0TsgMkE/b6Gmetvbnu5wqLh0KaWbesSESmEClQghpuvbu7Zsyc9e/a0dSkiIrazbu/tMAxw9RoMmApHJoFBL8EQEclNGm4VESmI/jhm3fb3WTAm5H8tIiKFnAKxiEhB1KSqddtDFTSPWEQkDygQi4gURAE1YXAnuPXcRClvCA+1aUkiIoVVgZtDLCIi/298XxjSCU5ehAYPgIuzrSsSESmUFIhFRAqy8j43v0REJM9oyoSIiIiI2DUFYhERERGxawrE6YiKiiI0NJQWLVrQoUMHJk6cSFJS0l3PM5lMfPvttwQGBtK8eXP69u3L/v37rY67cOECw4YNo2XLlrRp04aPP/4Yo9GYF7ciIiIiInehQJxGbGwsAwYMIDk5mbCwMEJDQ1m8eDHjx4+/67mzZ89m2rRpPPvss0yYMAEfHx8GDRpEdHS0+Zjk5GQGDRrEyZMnGTVqFG+99RY7duzgvffey8vbEhEREZEM6KG6NBYuXEh8fDxhYWF4eXkBkJKSwpgxYwgJCaFkyZLpnnfjxg1mzZpFr169zG/Zq1u3LsHBwcydO5e33noLgIiICI4dO8aCBQvw8/MDwNPTk0GDBnHgwAFq1qyZ9zcpIveExBQTC/8yERULHf0N1Llfb6gTEckLGiFOY/v27TRq1MgchgHat29PamoqO3bsyPC8ffv2ER8fT7t27cxtzs7OtG7dmm3btln0X6VKFXMYBmjcuDFeXl4Wx4mIfUtMMdHqhxSeXZnKO1tSqTsnhRn7Um1dlohIoaRAnEZUVJRFWAXw8PDAx8eHqKioO54HWJ3r7+/PuXPnSEhIMB/n6+trcYzBYMDX1/eO/ScmJmI0Gs1f8fHxmb0lEbkHLf7bxI6zt7dNwLtbU0lJNdmsJhGRwkpTJtKIjY3Fw8PDqt3Dw4PY2Ng7nufi4kKRIkWszjOZTMTFxVG0aFHi4uLS7d/T0/OO/c+aNYvw8PAs3ImI3MtOpPPXwflrcD0Z3F3yvx4RkcJMgfge0bdvX/PcZID4+HgCAwNtWJGI5KWO/gbe2nxzZPiWRqXB3UXziEVEcpsCcRqenp7pLoEWFxeHp6fnHc9LTEzkxo0bFqPEcXFxGAwG86iwh4dHuv3HxsZSqlSpDPt3cXHBxUXDQiJ54deTqcz/04SXCwx82IHKxW0fOm+kWIZhgCRNIRYRyROaQ5yGn5+f1Vxeo9HIxYsXreYHpz0P4MSJExbtUVFRlC5dmqJFi2bYv8lk4sSJE3fsX0TyxrxDqbT9KZVv9psY/7uJBnNT+CfG9vN0I05Y1/C/8xCXaPvaREQKGwXiNJo1a0ZkZCRxcXHmtoiICBwcHGjSpEmG59WuXRs3NzciIiLMbcnJyWzYsIHmzZtb9P/3339z8uRJc1tkZCRXr161OE5E8sfYXZbDrldvwLS9th+KrVrCuq20K7g5538tIiKFnQJxGt26dcPV1ZWhQ4eyY8cOli1bxsSJEwkODrZYg3jgwIF07drVvF2kSBH69u3L3Llz+f7779m1axfvvPMOV69epVevXubj2rVrR6VKlXjzzTfZsmUL69atY+TIkbRo0UJrEIvYQFyidVtsOm35rbK3dVtJV3Aw2H46h4hIYaM5xGl4enoyZcoUwsLCGDp0KG5ubnTt2pXQ0FCL41JSUkhJSbFoe/755zGZTMydO5crV65QtWpVvvrqK8qXL28+xsnJia+++oqwsDDeffddHB0dad26NUOGDMmX+xMRS9VNRo7jfrvBZKK5xw3AzWY1Aaw+bt22/yLEJ5pw04N1IiK5ymAymTQh7R5kNBoJCAhg48aNuLu73/0EERtLSjGx4C8T+y+YaF7OQGAlA4YCMNr54AdnOOx5v0Xb68l/M+GtB21U0U3f/5nKsystp274FINzAx1xdLD95yYiWWcYl2yxbXpD45IFhaZMiEi+CPgxhZ4rUxkdaaLT4lSeX2X7eboAySnWbUmxN/K/kDSCqxho4JZg0TayQYrCsIid6tOnDwaDweorKCjI1qWZRUVFYTAY+Pnnn21dSpbpRxMRyXO/nkxlT1QCr++IoPbZk2z1r87slJaMaWmgjLttfy7vlxzFW9weIXZKSaZPTduHziJnL7H5rSH8WLU+UcXvp+PhP2j0exn4XtOrROxVpUqVmDdvnkVb8eLFbVRN4aJALCJ5bt2+a6yd/gnNTxwBoO/ujQT8c5DtgYPoVtO2gfjNt+tw6NODLPPwo2hyEu86H6fBsw1tWhMAC3+jWKyRPrs33W5bcBxmvgzFimR8nogUWsWKFbvjileSfZoyISJ5rvtvG81h+Jae/9tK86MHbFTRbZOPFWPO/Q8RU8yNcx7evOFWl/0XCsCjFe5FrduKuYCTY/7XIiL3hG+//ZbatWtTtGhRypUrx7vvvmuxAMC3336LwWBg9+7dPProo7i6ulKtWjUiIiJITU3lvffeo1SpUpQqVYq3336b1NTbU9sOHz7M008/TYUKFXB1daVGjRp8/vnnFsdkt66CQIFYRPJc/fLWIc7BZKJ0aVcbVGNp8h7Lv8xvpMCM/QVgfvOTzaGij2Xbq4HgrF/sidiz5ORki69bxo8fT79+/ejQoQPLly9n+PDhfPnll7z77rtWffTu3ZugoCAWL15M2bJlCQ4O5rXXXuPUqVPMmTOHl19+mdGjR/PDDz+Yzzl9+jTVqlXj66+/5pdffuGll15i5MiRfPzxx3esNyt12ZL+ZhWRvNenDWdHLGV0k47sL1OR5lFHGPbXFjybVLN1ZaSkMxicWgAGiPF0hR2jYdIqiDoPgfXh2Za2rkpEbOjgwYM4O1u+nWfLli3UqVOHDz/8kDfffJNPP/0UgPbt2+Pi4sKQIUMYNmwY9913n/mcV155hYEDBwJQrlw5atWqxe7du/ntt98A6NChA8uWLWPBggU8++yzALRt25a2bdsCN9+w26JFC65du8akSZP48MMP0603Li4uS3XZkgKxiOS55CIutBkxgcPxN/8i31C5JpG9OrPGxnUBvFTbgWGbbo8IOzlAXxvPazYrUwI+6WnrKkSkgHjggQcsRm0BqlevzrZt2zAajfTo0cNi1Lhdu3Zcv36dAwcO0KpVK3N7+/btzd9XrVoVwBx2/9v+119/mbcTEhL47LPPmDdvHidPniQpKcm8z2g0prsE7Pbt27NUly0pEItInvv1pMkchm9Ze86Zf2JMPOBt2xUd3mjogKcLzPszFe8iBoY0cKBeKduvMiEiklbRokVp0KCBVfvFixcBqFevXrrnnTp1ymLb29vb/L2Li4tV2632hITbSz8OHz6c8PBwPvzwQ+rXr4+3tzdLly5l1KhRJCQkpBuIs1qXLSkQCwDHY0ysjjLh5wkd/A16Pazkqoxe/1MgpiYAL9Vx4KU6BWRUWEQki0qUKAHAokWLqFChgtV+f3//HF9jwYIF9O/fn+HDh5vbVq5cafO6cosCsfDj4VR6rkw1z6VsW9HA6u4OOOkFAJJL2lQ0UNElkZOJLua2Zt6JVClu+4fqRETudU2bNsXV1ZXo6GieeOKJPLnG9evXzaPJACkpKVbTN2xRV25RILZzqSYTb2xKtXiwaP1JE0uPmuhWVYFYckdybAIJVxLA7fZfpvH/XABTRdBvI0REcsTb25uRI0fy5ptvEh0dTUBAAI6Ojhw7doylS5eycOFCXF1zNgDRvn17wsPDqVGjBj4+Pnz99dfcuHHnt3rmR125RYHYzl1Pgug46/a/r+R/LVJ4rf/lBOfdKlu07b2vHId2nKFG03I2qkpEpPAYOnQo5cqVY/z48Xz11Vc4OzvzwAMPEBQUZDGym11fffUVAwYM4JVXXsHV1ZU+ffrwxBNP8OKLL9q0rtxiMJkymt0nBZnRaCQgIICNGzemO5E9K5rNT+a3M5Ztu3s5Ur+0Ru4kd2xac4KA/ZbB15CaSlRwPBUr67WjImIfDOOSLbZNb2hcsqDQUyTC7McdqXv/ze89XeCL1g4Kw5KrWj5akSZXT1u0PX3tuMKwiIgUCPrRRKhS3MAfvZ04F2/CuwgUdVIYltxlMBhYO7wsU787xr6L0KKiIyGvV7F1WSIiIoACsfxHaTcFYck7HjeuM+zXpXDgJDxSA57ud/MtGCIiIjamQCwieS81FR4IhavXbm4fPg0b9sPfX9u2LhERETSHWETyw9Q1t8PwLUfPwa6/bVOPiIjIfygQi0jeO3M5/fbTGbSLiIjkIwViEcl7rwZav4CjiDN0bmibekRERP5DgVhE8t793vDdq+D1/28kKuUNaz8AB/0VJCIitqeH6kQkf/RsdfOrgDp40YSnC1Tw1GorIiL2RoFYROzaqVgTnRansPcCGIBnHzTw7eMOODkoGIuI2Av9vlJE7NrQjansvXDzexMw708Tsw/qjfYiIvZEgVhE7Nq2M9bhd2u0ArGIiD1RIBYRu1bbx3pqRJ37NV1CRMSeKBCLiF0b28qBUq63t5uXgxdrKRCLiNgTPVQnIvkrMQlcnG1dhVmtkgaOv+hIxEkT3kUMtCgHhrRrJouISKGmEWIRyR8Re6H6K1DkKWg4DPYet3VFZsWcDXR6wIFHyhsUhkVE7JACsYjkvZh4eGIMHDl9c3v3P/DEWEhNtW1dIiIiKBCLSH7YsB+MCZZtx/+F/SdsU4+IiMh/aA5xOjZv3syUKVM4ceIEpUuXpk+fPnTu3Pmu5xmNRsaPH8/GjRtJTk6mSZMmvPnmm/j4+JiPmTZtGuHh4VbnvvXWW3Tv3j1X70OkwKjgY93m5Ahliud/LSIiImkoEKexZ88ehg0bRpcuXRg6dCi7du3i448/xtXVlXbt2t3x3Lfffptjx47x9ttv4+Liwtdff82rr77KnDlzcHK6/VEXKVKEqVOnWpxbrly5PLkfkQKhQWUIbgKLdtxuG9wJ7ve2WUkiIiK3KBCnMWPGDB566CHeeecdABo0aEB0dDTTpk27YyDet28fv/32G5MmTaJJkyYA+Pr60qNHDzZs2ED79u3Nxzo4OFCrVq28vRGRgubdbvDPWTh6DupVgkGP27oiERERQHOILSQmJrJ7926r4Pvoo49y/Phxzpw5k+G527dvx8PDg8aNG5vb/Pz8qFq1Ktu2bcuzmkXuCdduwGMfw94TEH8DtvwJnT61dVUiIiKAArGF6OhokpOT8fPzs2j39/cHICoqKsNzo6Ki8PX1tVqyyd/f3+q8Gzdu0K5dOxo3bkyPHj1YvHhxbpQvUnBF7IULsZZt+07AwZO2qUdEROQ/NGXiP2Jjb/6D7eHhYdHu6elpsT+jc9Oed6uv/55XoUIFXnnlFapVq0ZiYiKrV6/mk08+wWg08txzz2XYf2JiIomJiebt+Pj4zN2USEFQ3N26zWAAL1frdluYtgbmbblZzxtdoNVDtq5IRETyUaEPxEajkYsXL971uPx6qK1jx44W2y1atCApKYlvvvmGZ555xuLhu/+aNWtWuqtTiNwTWjwIj9SALYdut/UOgPLprD6R38YtgWFzbm+v/h/sHA31HrBZSSIikr8KfSCOiIhg1KhRdz3u559/No8EG41Gi323Rnhv7U+Pp6cn//77r1V7XFzcHc8DaN++PevXr+fUqVPm6Rlp9e3bl549e5q34+PjCQwMvGO/IgWGwQD1/G8HYgcDNCwggXP6Osvt5BSY9asCsYiIHSn0gbhr16507do1U8cmJibi5OREVFQUTZs2NbffmgOcdm7xf/n5+REZGYnJZLKYRxwVFUXlypWzU7oFFxcXXFxcctyPiE0cOgUTV97eTjXBK99ASFsoVsR2dQE4pvMohYMerxARsSf6W/8/XFxcaNCgAevXr7doX7duHf7+/pQtWzbDc5s1a0ZsbCyRkZHmthMnTnDkyBGaN29+x+uuWbMGDw8PKlSokLMbECmoJiyzbjOZ4McCsALLy2mWfyviDP3uvOa4iIgULoV+hDir+vXrR//+/Rk9ejTt2rXj999/Z/Xq1Xz22WcWxzVu3JjAwEA++OADAGrXrk3Tpk0ZOXIkgwcPNr+Yo0qVKrRu3dp8Xq9evQgKCsLPz4+EhARWr17Nhg0bGDp0aIbzh0XueZ7F0m/3dsvfOtIzqCMUd7v5UJ23G7weBLV8bV2ViIjkIyWwNB5++GHGjh3LlClTWLp0KaVLl+a9996zWps4JSWF1NRUi7bPPvuM8ePH88knn5CSkkLjxo158803LYJuhQoVmD9/PpcuXQKgcuXKfPzxxzz+uF5SIIXY651h/ArLNkcH6PCwTcqx0rMV+x9tiacL+HoZ7n68iIgUKgaTyWSydRGSdUajkYCAADZu3Ii7ezpLWokUJAmJUCoEYq/dbqtcGv7+2nY1/b+TsSaCFqWw/yIYgKerG5jT0QEnBwVjEcldhnHJFtumNzQuWVBoDrGI5L2IfZZhGG6+wvnPaNvU8x9DN6ay//9XZjQB3x828e0BjROIiNgTBWIp+M5dgb8zfm223APci1q3GQzgZuMVJoDtZ6zD77bTCsQiIvZEgVgKrpQU6DcZyr0IVQdBw2Fw+pKtq5LsaFkDGqZZfrBHU6hY0jb1/EdtH+upEXXu13QJERF7okAsBdfczfDNerj18OLuf2DotzYtSbLJwQEiRsCnPeHpFjDpRfjuNVtXBUBYKwdK/2exi0fKw0u1FYhFROyJZnNLwbX5kHXbpoP5X4fkDk9XeLubrauwUrOkgeMvOvLrSRPeRQw0K6cwLCJibxSIpeB6KJ0XldSsmP91SKFX1MlAx0oKwiIi9kpTJqTgeqk9NK5ye/s+Dxjb23b1iIiISKGkEWK5ad0eWPE7+N0PIW3AqwC8Qcy9GGz/DDYcgJj4my9xcM/gjWciIiIi2aRALDBuCQybc3t7+lr4fRy42n5JLBwcoG1tW1chIiIihZimTNi7lBT4dJFl2+HTsGC7beoRERERyWcKxPYuMfnmdIS0LlzN/1pEREREbECB2N4VKwJB9S3bnJ2ga2Pb1CMiIiKSzxSIBWa9As8+Al6uUNsXFr0JlcvYuioRERGRfKGH6uTmcmbzBtu6ChERERGb0AixiIiIiNg1BWIRERERsWsKxCIiIiJi1xSIRURERMSuKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxawrEIiIiImLXFIhFRERExK4pEIuIiIiIXVMgFhERERG7pkAsIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrTrYuoCDavHkzU6ZM4cSJE5QuXZo+ffrQuXPnO56TlJTE119/zYEDB/jzzz9JSEggIiICb29vq2P37t3LF198wV9//UXx4sXp3r07zz//PAaDIY/uSEREREQyohHiNPbs2cOwYcOoVasWX375Je3bt+fjjz8mIiLijuclJCSwZMkSXFxcqFu3bobHnTp1ildeeQUfHx8mTJjAM888w7Rp05g7d25u34qIiIiIZIJGiNOYMWMGDz30EO+88w4ADRo0IDo6mmnTptGuXbsMz/Pw8ODXX3/FYDCwfPlyfvvtt3SPmzNnDl5eXnz66ac4OzvTqFEjYmJimDlzJk899RQuLi55cl8iIiIikj6NEP9HYmIiu3fvtgq+jz76KMePH+fMmTN3PD8zUx62b99OQEAAzs7OFv3HxcWxb9++7BUuIiIiItmmQPwf0dHRJCcn4+fnZ9Hu7+8PQFRUVI76v379Ov/++y++vr4W7X5+fhgMhjv2n5iYiNFoNH/Fx8fnqBYRERERuUlTJv4jNjYWuDn94b88PT0t9mdXXFxcuv07OztTtGjRO/Y/a9YswsPDc3T9O9pyCFb+Dr4l4blW4F4s764lIiIiUoAU+kBsNBq5ePHiXY8rV65cPlSTfX379qVnz57m7fj4eAIDA3On8y9Xwmvf3N6esgZ2joZiRXKnfxEREZECrNAH4oiICEaNGnXX437++WfzSLDRaLTYd2vk9tb+7Lo1Mpy2/6SkJBISEu7Yv4uLS948cJeSAiN/smzbfwJ+/g2eC8j964mIiIgUMIU+EHft2pWuXbtm6tjExEScnJyIioqiadOm5vZbc3vTzi3OqmLFilGqVCmrucInTpzAZDLluP9sSUyGy0br9rNX8r8WERERERvQQ3X/4eLiQoMGDVi/fr1F+7p16/D396ds2bI5vkazZs3YvHkzycnJ5ra1a9fi4eFBnTp1ctx/lhUrAh0etmxzdIAujfK/FhEREREbUCBOo1+/fuzfv5/Ro0eze/dupk2bxurVq+nfv7/FcY0bN2bkyJEWbdu2bSMiIoJDhw4BN994FxERwbFjx8zH9O7dm8uXL/POO++wa9cuvv/+e7777jtCQkIslmLLV7NfgScag4sTVCkDPw6FagV7TrWIiIhIbjGYTCaTrYsoaDZt2mT16uYuXbpYHNOgQQOCgoIYMWKEua1Tp06cPXvWqr8XX3zRIlDv3buXCRMmmF/d3KNHjyy/utloNBIQEMDGjRtxd3fP+k2KiIhIvjKMS7bYNr1R6Geu3jMUiO9RCsQiIiL3FgXigktTJkRERETErikQi4iIiIhdUyAWEREREbumQCwiIiIidk2BWERERETsmgKxiIiIiNg1BWIRERERsWsKxCIiIiJi1xSIRURERMSuKRCLiIiIiF1TIBYRERERu6ZALCIiIiJ2zcnWBUj2mEwmAOLj421ciYiIyL3Jzc0Ng8Fg6zKkADCYbiUruaf8+++/BAYG2roMERGRe9bGjRtxd3e32fUnT57M5MmTAXj55Zd5+eWXbVaLvVMgvkelpqZy4cIFXF1dC/1Pt/Hx8QQGBrJy5Urc3NxsXc49QZ9Z1ukzyzp9Zlmnzyxr8vrz0gix3KIpE/coBwcHSpUqZesy8pWbm5tNf5K/F+kzyzp9Zlmnzyzr9JlljT4vyWt6qE5ERERE7JoCsYiIiIjYNQViKfBcXFx48cUXcXFxsXUp9wx9Zlmnzyzr9JllnT6zrNHnJflFD9WJiIiIiF3TCLGIiIiI2DUFYhERERGxawrEIiIiImLXtA6xFGgrVqxg/vz5REVFUaxYMR566CHGjh1L0aJFbV1agbRp0yZmzpzJ8ePHKVasGHXr1mXQoEGUL1/e1qUVCKdOneK7777jwIED/PPPP/j6+vLTTz9ZHbdkyRLmzJnDuXPn8PX1JTQ0lEceecQGFdve3T4zo9HIvHnz2LZtGydPnsTFxYWHHnqIl19+mcqVK9uwctvJ7J+zWzZu3Mgbb7xBpUqV7nhcYZXZzysuLo6pU6eyfv16YmNjKVmyJD169KBXr142qFoKGwViKbC++eYb5syZQ9++falVqxYxMTHs2rWL1NRUW5dWIO3evZthw4YRGBhIaGgoV69eZerUqQwaNIgffvhBP0QA//zzD9u2beOhhx4iNTU13T9La9as4ZNPPiEkJISGDRuydu1a3njjDWbMmEGtWrVsULVt3e0zO3fuHIsWLaJLly6EhoZy48YN5s6dS58+ffjuu+/w9/e3UeW2k5k/Z7ckJCQwfvx47rvvvnyssGDJzOd1/fp1+vfvj6OjI0OGDKFEiRKcPHmS+Ph4G1QshZFWmZACKSoqiqeeeorx48fTvHlzW5dzT/j000/ZuXMnS5YsMb+KdPfu3QwYMIDw8HDq1q1r4wptLzU1FQeHmzPFRowYwaFDh6xGooKDg3nwwQf55JNPzG0hISG4u7vz5Zdf5mu9BcHdPrPr169jMBgsfuC6du0anTp1okOHDrz55pv5XrOtZebP2S1Tp07ljz/+oGzZsnc8rjDLzOc1ZcoUVq9ezQ8//ECxYsVsUaYUcppDLAXS8uXLKVeunMJwFiQnJ+Pq6moOw4D5Vaf6ufemW//oZiQ6OpqTJ0/Svn17i/ZHH32UXbt2kZiYmJflFUh3+8yKFStm9dsHV1dXypcvz4ULF/KytALrbp/ZLdHR0cydO5c33ngjjysq2DLzeS1ZsoTOnTsrDEueUSCWAmn//v088MADzJgxg/bt29OkSRNCQkI4cOCArUsrsDp16sSxY8dYsGABRqOR6OhoJk+eTLVq1ahTp46ty7snREVFAeDn52fR7ufnR1JSEmfOnMn/ou5BcXFx/PPPP3Y5XSIrxo0bR2BgIFWrVrV1KQXamTNnuHTpEt7e3gwePJimTZvSpk0bRo0axbVr12xdnhQSCsRSIF26dImdO3fyyy+/MHz4cMaNG4fBYODll1/m8uXLti6vQKpbty7jxo1j0qRJBAQE0LVrVy5dusSXX36Jo6Ojrcu7J8TFxQG3R9Zv8fT0BODq1av5XtO96Msvv8RgMNCtWzdbl1Jgbd68mX379jFw4EBbl1LgXbp0CYCJEyfi6enJxIkTCQ0NJSIiglGjRtm4OiksFIilQDKZTFy7do0xY8bQrl07WrRowfjx4wHsco5dZuzdu5cPPviArl27MnXqVEaPHo3JZOL1118nISHB1uWJnVi2bBmLFy9m+PDhlCpVytblFEg3btzg888/56WXXsLb29vW5RR4tx6yq1ixIh999BGNGjWie/fuvP7666xdu5bo6GgbVyiFgVaZkALJw8MDLy8vqlSpYm7z8vKiWrVq/PPPPzasrOAaN24cDRo0YPDgwea2WrVqERQUxC+//EJwcLANq7s3eHh4ADeXEvPx8TG3x8bGAjf/DErGtm3bxieffEK/fv0ICgqydTkF1vfff4+DgwOPPfaY+bcSSUlJmEwm4uLiKFq0KM7OzjausuC49RuaBg0aWLQ3bNgQgGPHjmlpSckxBWIpkCpVqpThT/32+GBTZhw7doxWrVpZtJUqVQpvb2+NoGTSrbnDUVFRFvOIo6KicHZ2ply5crYp7B6wf/9+hg8fTlBQEAMGDLB1OQVaVFQUp06dol27dlb7WrduzVtvvUX37t1tUFnBVL58eVxcXDLcr38TJDcoEEuB9Mgjj7B8+XKOHDlCtWrVAIiJieHw4cM8++yzNq6uYCpTpgyHDx+2aDt79iwxMTGULVvWRlXdW8qXL0/FihVZv349AQEB5vZ169bRsGFDjdpl4NixY7z++us0bNiQt99+29blFHh9+vSxGkGfPXs2J06c4IMPPsDX19dGlRVMzs7ONG7cmMjISIv2nTt3AlC9enVblCWFjAKxFEgBAQHUqFGD4cOHExoaSpEiRfj2229xdnbWyEkGunXrxueff864ceN45JFHuHr1Kt988w0lSpRIdyTKHiUkJLB161bg5g8L8fHxREREAFC/fn2KFy/OSy+9xPvvv0/58uWpX78+69at48CBA4SHh9uydJu522dmMpl45ZVXKFKkCM8++yx//vmn+Vw3NzcqVapkk7pt6W6fmZ+fn9VKJitWrODff/+1mhZgDzL7/8uQkBDee+89goKCOHnyJJMnT+bxxx/XdAnJFXoxhxRYMTExfP7552zZsoWkpCTq1q3LkCFD7PIf2MwwmUwsXLiQhQsXEh0djaurK7Vr1+bll1+2+sfXXp05c4bOnTunu2/q1KnmMLJkyRJmz55tfnXzyy+/bLevbr7bZwZkOEWiXr16TJ8+Pc9qK6gy++fsv+72Ao/CLLOfV2RkJJMmTeLo0aN4eHjw+OOPExoaesfpFCKZpUAsIiIiInZNy66JiIiIiF1TIBYRERERu6ZALCIiIiJ2TYFYREREROyaArGIiIiI2DUFYhERERGxawrEIiIiImLXFIhFpMB56aWX6NSpk63LyLTly5fToEEDdu/ebetSREQkGxSIRUQyYffu3UybNo24uDhblyIiIrlMgVhEJBN+//13wsPD0w3EHTt2ZNu2bdSrV88GlYmISE452boAEZF7naOjI46OjrYuQ0REskkjxCKSqxITE5k5cyZPPvkkzZo1IyAggMGDB3P48GGrY2NjYxk1ahRt27alRYsWvPTSS/z555/p9tugQQNGjBhh1Z7R/F2j0cjkyZPp3r07zZo1o23btrzwwgusWbPGfExUVBSjR4/mySefpGXLljRv3pxevXqxZMkSi75GjBhBeHg4AJ07d6ZBgwY0aNCAadOm3bGGmJgYxowZQ2BgIE2aNCEwMJAxY8YQExOT7j3s2rWL7777ji5dutC0aVOCg4NZsWJFup9HeqZNm0aDBg2Iiopi8uTJdOzYkaZNm/LMM8+wdetWi2N3795NgwYNWL58uVU/I0aMoEGDBhZtt+Z1nzlzhjfeeIOAgABat27NiBEjuHbtGqmpqcycOZPOnTvTrFkzevbsyZ49ezJdu4iILWmEWERyTXJyMq+88gr79u2jY8eOPPnkkxiNRhYvXswLL7xAeHg4NWrUMB87aNAgDh06RMeOHalVqxZ//fUXoaGheHl55aiOuLg4XnjhBY4dO0bbtm3p3r07KSkpHDlyhK1bt9KhQwfgZij8448/aNGiBWXLliUhIYGIiAhGjRrFlStX6Nu3LwDBwcHEx8ezYcMGhgwZgre3NwBVqlTJsAaj0UhISAinTp2ic+fOVK9enSNHjvDzzz+za9cuZs+ejZubm8U5kydP5saNGwQHB+Pi4sLPP//MiBEjKF++PA8//HCm73/EiBE4OTnRq1cvkpKS+P7773njjTdYtGgRZcuWzdqH+R/Xr19n4MCB1KtXz/zfbtmyZdy4cQNvb28OHDjAk08+SXJyMnPnzmXIkCEsX77c6j5FRAoaBWIRyTU//vgjv//+O1999RVNmzY1t3fv3p2nnnqKL774gunTpwOwbNkyDh06xIsvvkj//v3Nx/r7+zN+/HjKlCmT7TomT57MsWPHeOeddwgODrbYl5qaav4+MDCQ7t27W+x/9tlnGTBgAN9++y3PPfccTk5O1K5dm8qVK7NhwwYCAgIyFSpnz57NyZMnGT58OD169DC3V61albFjxzJnzhwGDhxocU5iYiJz5szB2dkZgLZt29KlSxd++umnLAVib29vJkyYgMFgAG6Orj///PMsWrSIQYMGZbqftGJiYujduze9e/c2t8XFxREREUH16tWZNWsWTk43/1nx9/dn6NChrF69mm7dumX7miIi+UFTJkQk16xatQo/Pz8efPBBYmJizF/Jyck0btyYvXv3kpCQAMDGjRtxdHSkZ8+eFn107949RyOKqamprF27Fn9/f6swDODgcPuvvWLFipm/v3HjBjExMcTGxtKkSRPi4+OJiorKdh0bN26kePHiPPHEExbtwcHBFC9enA0bNlid06NHD3MYBrj//vupWLEip06dytK1n376aXMYBnjooYdwdXXl5MmTWbwLS46Ojjz11FMWbQ8//DAmk4lu3bqZwzBA3bp1AbJcu4iILWiEWERyzfHjx7lx4wbt2rXL8JiYmBhKly7N6dOn8fHxwd3d3WK/i4sL5cqVy/byZrdC7X9HqDNy7do1pk+fzrp16/j333+t9sfGxmarBoAzZ87w4IMPWoREACcnJypWrJjunOpy5cpZtXl5eXHu3Dnz9tWrV0lKSrI4xsfHx2K7fPny6fZz9erVLN1DWj4+PhQpUsSizdPTE8Bq1PxWe06vKSKSHxSIRSRXVa5cmcGDB2e4v3jx4rl6vZSUlGyf++6777J161aeeOIJ6tWrh5eXFw4ODmzbto358+dbTK/ID/8dvf4vk8lk/n7YsGH88ccfFvvTPsyXmX7+O4KcVkafaUb9ZvaaIiIFlQKxiOSaChUqcOXKFRo2bHjH8AQ3R0N37tyJ0Wi0GCVOTEzk9OnT5hHGWzIa4Tx9+rTFtre3N56envz99993vH5cXBxbt26lY8eOvPPOOxb7IiMjrY6/U4BMT7ly5Thx4gTJyckWo8TJycmcPHky3dHgzBg8eHCORq5vufXgYmY+UxGRwk5ziEUk1wQGBnLp0iXmzZuX7v5Lly6Zv2/VqhUpKSlWx/7888/Ex8dbnVuxYkX2799vnoMMN6c0LFu2zOI4BwcHOnTowLFjx6yWT4PbI5a3AnvaEcyLFy+me56rq6v5mpnRqlUrrly5YtXXkiVLuHLlCq1bt85UP2k9+OCDNG7c2OIrO8qWLYujo6NV+N+7dy/79+/PVp8iIvcqjRCLSK555pln2LlzJxMnTmTXrl00bNgQNzc3zp07x65du3BxcTGv3du5c2cWL15MeHg4p0+fpnbt2hw5coSIiAjKly9v9Wv7J598kvfff58BAwbQsWNH4uLiWLJkCWXKlLEI2gADBw5k165djBo1ip07d1KnTh0Ajhw5QnJyMh9//DFubm40adKEVatWUaRIER566CHOnj3LokWLKFeunNXIac2aNQH48ssvefzxx3FxceGBBx6gcuXK6X4Wzz//POvXr2fs2LEcOXKEatWqceTIEZYuXYqvr6/FSg224OrqSqdOnViyZAnvvPMO9evX59SpUyxfvpwqVarw119/2bQ+EZH8pEAsIrnGycmJL774gp9//plffvnFHH5LlizJQw89RFBQkPlYZ2dnJk+ezMSJE9m0aRO//vorNWrUYPLkyXzxxRecPXvWou/HH3+cCxcu8NNPPzFhwgTKlStHv379cHBw4MCBAxbHenp6MmvWLGbOnMmGDRvYsGEDbm5u+Pv7W6yS8PHHH/PVV1+xZcsWVq5cSYUKFQgNDcXJyYmPPvrIos+HH36YV155hUWLFjFq1ChSUlJ48cUXMwzE7u7ufPPNN0ybNo3NmzezbNky7rvvPrp160b//v0LxNq8Q4YMwWQysXHjRjZt2sSDDz7I+PHjWbx4sQKxiNgVg0lPPIiIiIiIHdMcYhERERGxawrEIiIiImLXFIhFRERExK4pEIuIiIiIXVMgFhERERG7pkAsIiIiInZNgVhERERE7JoCsYiIiIjYNQViEREREbFrCsQiIiIiYtcUiEVERETErikQi4iIiIhdUyAWEREREbv2f9TGuZsO4oYkAAAAAElFTkSuQmCC\"\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 29\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Scoring Overall Feature Importance via Permutation Shuffling\\n\",\n \"\\n\",\n \"Note that if you'd like to understand how much each feature contributes to AutoGluon's general predictive accuracy (rather than explaining individual predictions), AutoGluon offers a built-in method for this based on [permutation-shuffling](https://explained.ai/rf-importance/):\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:48.393043Z\",\n \"start_time\": \"2026-01-07T01:36:48.275280Z\"\n }\n },\n \"source\": [\n \"predictor.feature_importance(test_data)\"\n ],\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Computing feature importance via permutation shuffling for 14 features using 50 rows with 5 shuffle sets...\\n\",\n \"\\t0.67s\\t= Expected runtime (0.13s per shuffle set)\\n\",\n \"\\t0.11s\\t= Actual runtime (Completed 5 of 5 shuffle sets)\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \" importance stddev p_value n p99_high p99_low\\n\",\n \"marital-status 0.104 0.029665 0.000715 5 0.165080 0.042920\\n\",\n \"education-num 0.084 0.021909 0.000508 5 0.129111 0.038889\\n\",\n \"age 0.064 0.035777 0.008065 5 0.137666 -0.009666\\n\",\n \"capital-gain 0.064 0.029665 0.004249 5 0.125080 0.002920\\n\",\n \"hours-per-week 0.008 0.017889 0.186950 5 0.044833 -0.028833\\n\",\n \"workclass 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"capital-loss 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"education 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"relationship 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"race 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"native-country 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"sex 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"fnlwgt -0.008 0.010954 0.911096 5 0.014555 -0.030555\\n\",\n \"occupation -0.012 0.010954 0.964758 5 0.010555 -0.034555\"\n ],\n \"text/html\": [\n \"\"\n ],\n \"text/html\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 33\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Again we can compute the Shapely values for an individual prediction. Note that some SHAP functions like `force_plot`, `dependence_plot` will not work in this multiclass setting, as they can only operate with a single set of Shapely values. For these plots, we will thus only focus on the Shapely values corresponding to one target class of interest. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:08.493822Z\",\n \"start_time\": \"2026-01-07T01:37:08.488717Z\"\n }\n },\n \"source\": [\n \"shap.initjs()\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"\"\n ],\n \"text/html\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 34\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:08.655971Z\",\n \"start_time\": \"2026-01-07T01:37:08.544Z\"\n }\n },\n \"source\": [\n \"\\n\",\n \"ROW_INDEX = 0 # index of an example datapoint\\n\",\n \"class_of_interest = ' Not-in-family' # can be any value in set(y_train)\\n\",\n \"class_index = predictor_multi.class_labels.index(class_of_interest)\\n\",\n \"\\n\",\n \"single_datapoint = X_train.iloc[[ROW_INDEX]]\\n\",\n \"single_prediction = ag_wrapper.predict_proba(single_datapoint)\\n\",\n \"\\n\",\n \"shap_values_single = explainer.shap_values(single_datapoint, nsamples=NSHAP_SAMPLES)\\n\",\n \"print(f\\\"Force_plot for class: {class_of_interest}\\\")\\n\",\n \"\\n\",\n \"shap.force_plot(\\n\",\n \" explainer.expected_value[class_index], # The base value for the selected class\\n\",\n \" shap_values_single[ROW_INDEX, :, class_index], # SHAP values for the selected sample and class\\n\",\n \" X_test.iloc[ROW_INDEX, :] # Original feature values of the selected sample\\n\",\n \")\\n\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" 0%| | 0/1 [00:00\"\n ],\n \"text/html\": [\n \"\\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ]\n },\n \"execution_count\": 30,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 30\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Features with near zero or negative importance score above hardly contribute at all to AutoGluon's overall accuracy on the test data, whereas features near the top of this list contain the most predictive signal.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Explaining predictions for multiple classes\\n\",\n \"\\n\",\n \"Let's now repeat this same analysis for a multiclass classification problem with more than 2 classes. We'll use the same dataset and predict the `relationship` variable based on the other columns' values. For interpreting predictions in multiclass classification, SHAP will quantify how much each feature contributes to the predicted probability for each class separately.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:36:48.407160Z\",\n \"start_time\": \"2026-01-07T01:36:48.398518Z\"\n }\n },\n \"source\": [\n \"label = 'relationship'\\n\",\n \"\\n\",\n \"y_train = train_data[label]\\n\",\n \"y_test = test_data[label]\\n\",\n \"X_train = pd.DataFrame(train_data.drop(columns=[label]))\\n\",\n \"X_test = pd.DataFrame(test_data.drop(columns=[label]))\\n\",\n \"\\n\",\n \"display(train_data.head())\\n\",\n \"print(\\\"Possible classes: \\\\n\\\", train_data[label].value_counts())\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" age workclass fnlwgt education education-num \\\\\\n\",\n \"6118 51 Private 39264 Some-college 10 \\n\",\n \"23204 58 Private 51662 10th 6 \\n\",\n \"29590 40 Private 326310 Some-college 10 \\n\",\n \"18116 37 Private 222450 HS-grad 9 \\n\",\n \"33964 62 Private 109190 Bachelors 13 \\n\",\n \"\\n\",\n \" marital-status occupation relationship race sex \\\\\\n\",\n \"6118 Married-civ-spouse Exec-managerial Wife White Female \\n\",\n \"23204 Married-civ-spouse Other-service Wife White Female \\n\",\n \"29590 Married-civ-spouse Craft-repair Husband White Male \\n\",\n \"18116 Never-married Sales Not-in-family White Male \\n\",\n \"33964 Married-civ-spouse Exec-managerial Husband White Male \\n\",\n \"\\n\",\n \" capital-gain capital-loss hours-per-week native-country class \\n\",\n \"6118 0 0 40 United-States >50K \\n\",\n \"23204 0 0 8 United-States <=50K \\n\",\n \"29590 0 0 44 United-States <=50K \\n\",\n \"18116 0 2339 40 El-Salvador <=50K \\n\",\n \"33964 15024 0 40 United-States >50K \"\n ],\n \"text/html\": [\n \"| \\n\",\n \" | importance | \\n\",\n \"stddev | \\n\",\n \"p_value | \\n\",\n \"n | \\n\",\n \"p99_high | \\n\",\n \"p99_low | \\n\",\n \"
|---|---|---|---|---|---|---|
| marital-status | \\n\",\n \"0.104 | \\n\",\n \"0.029665 | \\n\",\n \"0.000715 | \\n\",\n \"5 | \\n\",\n \"0.165080 | \\n\",\n \"0.042920 | \\n\",\n \"
| education-num | \\n\",\n \"0.084 | \\n\",\n \"0.021909 | \\n\",\n \"0.000508 | \\n\",\n \"5 | \\n\",\n \"0.129111 | \\n\",\n \"0.038889 | \\n\",\n \"
| age | \\n\",\n \"0.064 | \\n\",\n \"0.035777 | \\n\",\n \"0.008065 | \\n\",\n \"5 | \\n\",\n \"0.137666 | \\n\",\n \"-0.009666 | \\n\",\n \"
| capital-gain | \\n\",\n \"0.064 | \\n\",\n \"0.029665 | \\n\",\n \"0.004249 | \\n\",\n \"5 | \\n\",\n \"0.125080 | \\n\",\n \"0.002920 | \\n\",\n \"
| hours-per-week | \\n\",\n \"0.008 | \\n\",\n \"0.017889 | \\n\",\n \"0.186950 | \\n\",\n \"5 | \\n\",\n \"0.044833 | \\n\",\n \"-0.028833 | \\n\",\n \"
| workclass | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| capital-loss | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| education | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| relationship | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| race | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| native-country | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| sex | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| fnlwgt | \\n\",\n \"-0.008 | \\n\",\n \"0.010954 | \\n\",\n \"0.911096 | \\n\",\n \"5 | \\n\",\n \"0.014555 | \\n\",\n \"-0.030555 | \\n\",\n \"
| occupation | \\n\",\n \"-0.012 | \\n\",\n \"0.010954 | \\n\",\n \"0.964758 | \\n\",\n \"5 | \\n\",\n \"0.010555 | \\n\",\n \"-0.034555 | \\n\",\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Possible classes: \\n\",\n \" relationship\\n\",\n \"Husband 219\\n\",\n \"Not-in-family 134\\n\",\n \"Own-child 66\\n\",\n \"Unmarried 43\\n\",\n \"Wife 25\\n\",\n \"Other-relative 13\\n\",\n \"Name: count, dtype: int64\\n\"\n ]\n }\n ],\n \"execution_count\": 31\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Fit an AutoGluon classifier:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:08.358229Z\",\n \"start_time\": \"2026-01-07T01:36:48.454124Z\"\n }\n },\n \"source\": [\n \"predictor_multi = TabularPredictor(label=label, problem_type='multiclass').fit(train_data, time_limit=20)\"\n ],\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"No path specified. Models will be saved in: \\\"AutogluonModels/ag-20260107_013648\\\"\\n\",\n \"Verbosity: 2 (Standard Logging)\\n\",\n \"=================== System Info ===================\\n\",\n \"AutoGluon Version: 1.4.1b20251106\\n\",\n \"Python Version: 3.12.6\\n\",\n \"Operating System: Linux\\n\",\n \"Platform Machine: x86_64\\n\",\n \"Platform Version: #26~22.04.1-Ubuntu SMP Wed Feb 19 06:54:57 UTC 2025\\n\",\n \"CPU Count: 96\\n\",\n \"Pytorch Version: 2.7.1+cu126\\n\",\n \"CUDA Version: 12.6\\n\",\n \"GPU Memory: GPU 0: 22.05/22.05 GB | GPU 1: 22.05/22.05 GB | GPU 2: 22.05/22.05 GB | GPU 3: 22.05/22.05 GB\\n\",\n \"Total GPU Memory: Free: 88.18 GB, Allocated: 0.00 GB, Total: 88.18 GB\\n\",\n \"GPU Count: 4\\n\",\n \"Memory Avail: 318.79 GB / 363.85 GB (87.6%)\\n\",\n \"Disk Space Avail: 176.62 GB / 1938.16 GB (9.1%)\\n\",\n \"===================================================\\n\",\n \"No presets specified! To achieve strong results with AutoGluon, it is recommended to use the available presets. Defaulting to `'medium'`...\\n\",\n \"\\tRecommended Presets (For more details refer to https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets):\\n\",\n \"\\tpresets='extreme' : New in v1.5: The state-of-the-art for tabular data. Massively better than 'best' on datasets <100000 samples by using new Tabular Foundation Models (TFMs) meta-learned on https://tabarena.ai: TabPFNv2, TabICL, Mitra, TabDPT, and TabM. Requires a GPU and `pip install autogluon.tabular[tabarena]` to install TabPFN, TabICL, and TabDPT.\\n\",\n \"\\tpresets='best' : Maximize accuracy. Recommended for most users. Use in competitions and benchmarks.\\n\",\n \"\\tpresets='best_v150': New in v1.5: Better quality than 'best' and 5x+ faster to train. Give it a try!\\n\",\n \"\\tpresets='high' : Strong accuracy with fast inference speed.\\n\",\n \"\\tpresets='high_v150': New in v1.5: Better quality than 'high' and 5x+ faster to train. Give it a try!\\n\",\n \"\\tpresets='good' : Good accuracy with very fast inference speed.\\n\",\n \"\\tpresets='medium' : Fast training time, ideal for initial prototyping.\\n\",\n \"Using hyperparameters preset: hyperparameters='default'\\n\",\n \"Beginning AutoGluon training ... Time limit = 20s\\n\",\n \"AutoGluon will save models to \\\"/home/ubuntu/workspace/code/autogluon/examples/tabular/interpret/AutogluonModels/ag-20260107_013648\\\"\\n\",\n \"Train Data Rows: 500\\n\",\n \"Train Data Columns: 14\\n\",\n \"Label Column: relationship\\n\",\n \"Problem Type: multiclass\\n\",\n \"Preprocessing data ...\\n\",\n \"Train Data Class Count: 6\\n\",\n \"Using Feature Generators to preprocess the data ...\\n\",\n \"Fitting AutoMLPipelineFeatureGenerator...\\n\",\n \"\\tAvailable Memory: 326441.43 MB\\n\",\n \"\\tTrain Data (Original) Memory Usage: 0.25 MB (0.0% of available memory)\\n\",\n \"\\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\\n\",\n \"\\tStage 1 Generators:\\n\",\n \"\\t\\tFitting AsTypeFeatureGenerator...\\n\",\n \"\\t\\t\\tNote: Converting 2 features to boolean dtype as they only contain 2 unique values.\\n\",\n \"\\tStage 2 Generators:\\n\",\n \"\\t\\tFitting FillNaFeatureGenerator...\\n\",\n \"\\tStage 3 Generators:\\n\",\n \"\\t\\tFitting IdentityFeatureGenerator...\\n\",\n \"\\t\\tFitting CategoryFeatureGenerator...\\n\",\n \"\\t\\t\\tFitting CategoryMemoryMinimizeFeatureGenerator...\\n\",\n \"\\tStage 4 Generators:\\n\",\n \"\\t\\tFitting DropUniqueFeatureGenerator...\\n\",\n \"\\tStage 5 Generators:\\n\",\n \"\\t\\tFitting DropDuplicatesFeatureGenerator...\\n\",\n \"\\tTypes of features in original data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('int', []) : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\\n\",\n \"\\t\\t('object', []) : 8 | ['workclass', 'education', 'marital-status', 'occupation', 'race', ...]\\n\",\n \"\\tTypes of features in processed data (raw dtype, special dtypes):\\n\",\n \"\\t\\t('category', []) : 6 | ['workclass', 'education', 'marital-status', 'occupation', 'race', ...]\\n\",\n \"\\t\\t('int', []) : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\\n\",\n \"\\t\\t('int', ['bool']) : 2 | ['sex', 'class']\\n\",\n \"\\t0.0s = Fit runtime\\n\",\n \"\\t14 features in original data used to generate 14 features in processed data.\\n\",\n \"\\tTrain Data (Processed) Memory Usage: 0.03 MB (0.0% of available memory)\\n\",\n \"Data preprocessing and feature engineering runtime = 0.06s ...\\n\",\n \"AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\\n\",\n \"\\tTo change this, specify the eval_metric parameter of Predictor()\\n\",\n \"Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 400, Val Rows: 100\\n\",\n \"User-specified model hyperparameters to be fit:\\n\",\n \"{\\n\",\n \"\\t'NN_TORCH': [{}],\\n\",\n \"\\t'GBM': [{'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}}, {}, {'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'ag_args': {'name_suffix': 'Large', 'priority': 0, 'hyperparameter_tune_kwargs': None}}],\\n\",\n \"\\t'CAT': [{}],\\n\",\n \"\\t'XGB': [{}],\\n\",\n \"\\t'FASTAI': [{}],\\n\",\n \"\\t'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\\n\",\n \"\\t'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],\\n\",\n \"}\\n\",\n \"Fitting 11 L1 models, fit_strategy=\\\"sequential\\\" ...\\n\",\n \"Fitting model: NeuralNetFastAI ... Training model for up to 19.94s of the 19.94s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.72\\t = Validation score (accuracy)\\n\",\n \"\\t0.37s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMXT ... Training model for up to 19.56s of the 19.56s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.1/318.8 GB\\n\",\n \"\\t0.82\\t = Validation score (accuracy)\\n\",\n \"\\t1.18s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBM ... Training model for up to 18.37s of the 18.37s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.1/318.8 GB\\n\",\n \"\\t0.81\\t = Validation score (accuracy)\\n\",\n \"\\t1.95s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestGini ... Training model for up to 16.42s of the 16.41s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.79\\t = Validation score (accuracy)\\n\",\n \"\\t0.43s\\t = Training runtime\\n\",\n \"\\t0.06s\\t = Validation runtime\\n\",\n \"Fitting model: RandomForestEntr ... Training model for up to 15.91s of the 15.91s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0, mem=0.0/318.8 GB\\n\",\n \"\\t0.79\\t = Validation score (accuracy)\\n\",\n \"\\t0.42s\\t = Training runtime\\n\",\n \"\\t0.07s\\t = Validation runtime\\n\",\n \"Fitting model: CatBoost ... Training model for up to 15.40s of the 15.40s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0\\n\",\n \"\\t0.81\\t = Validation score (accuracy)\\n\",\n \"\\t4.58s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesGini ... Training model for up to 10.81s of the 10.81s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0, mem=0.0/318.7 GB\\n\",\n \"\\t0.78\\t = Validation score (accuracy)\\n\",\n \"\\t0.43s\\t = Training runtime\\n\",\n \"\\t0.06s\\t = Validation runtime\\n\",\n \"Fitting model: ExtraTreesEntr ... Training model for up to 10.29s of the 10.29s of remaining time.\\n\",\n \"\\tFitting with cpus=96, gpus=0, mem=0.0/318.7 GB\\n\",\n \"\\t0.78\\t = Validation score (accuracy)\\n\",\n \"\\t0.42s\\t = Training runtime\\n\",\n \"\\t0.07s\\t = Validation runtime\\n\",\n \"Fitting model: XGBoost ... Training model for up to 9.78s of the 9.78s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0\\n\",\n \"\\t0.82\\t = Validation score (accuracy)\\n\",\n \"\\t0.79s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: NeuralNetTorch ... Training model for up to 8.97s of the 8.97s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.0/318.7 GB\\n\",\n \"\\t0.78\\t = Validation score (accuracy)\\n\",\n \"\\t1.91s\\t = Training runtime\\n\",\n \"\\t0.01s\\t = Validation runtime\\n\",\n \"Fitting model: LightGBMLarge ... Training model for up to 7.05s of the 7.05s of remaining time.\\n\",\n \"\\tFitting with cpus=48, gpus=0, mem=0.4/318.7 GB\\n\",\n \"\\t0.77\\t = Validation score (accuracy)\\n\",\n \"\\t6.39s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"Fitting model: WeightedEnsemble_L2 ... Training model for up to 19.94s of the 0.64s of remaining time.\\n\",\n \"\\tFitting 1 model on all data | Fitting with cpus=96, gpus=0, mem=0.0/318.7 GB\\n\",\n \"\\tEnsemble Weights: {'LightGBMXT': 1.0}\\n\",\n \"\\t0.82\\t = Validation score (accuracy)\\n\",\n \"\\t0.05s\\t = Training runtime\\n\",\n \"\\t0.0s\\t = Validation runtime\\n\",\n \"AutoGluon training complete, total runtime = 19.43s ... Best model: WeightedEnsemble_L2 | Estimated inference throughput: 32229.2 rows/s (100 batch size)\\n\",\n \"TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\\\"/home/ubuntu/workspace/code/autogluon/examples/tabular/interpret/AutogluonModels/ag-20260107_013648\\\")\\n\"\n ]\n }\n ],\n \"execution_count\": 32\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Now we will use a baseline value that is a vector of predicted probabilities (for each class), here simply taken over a random subsample of the training data. For a particular prediction, we will produce a separate set of Shapely values for each class, which quantify how much each feature contributes the particular prediction deviating from the baseline probability value for that class.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"scrolled\": true,\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:08.437596Z\",\n \"start_time\": \"2026-01-07T01:37:08.406351Z\"\n }\n },\n \"source\": [\n \"baseline = X_train.sample(100, random_state=0)\\n\",\n \"\\n\",\n \"ag_wrapper = AutogluonWrapper(predictor_multi, X_train.columns)\\n\",\n \"explainer = shap.KernelExplainer(ag_wrapper.predict_proba, baseline)\\n\",\n \"print(\\\"Baseline prediction: \\\\n\\\", ag_wrapper.predict_proba(baseline).mean()) # this is the same as explainer.expected_value\\n\",\n \"\\n\",\n \"NSHAP_SAMPLES = 100 # how many samples to use to approximate each Shapely value, larger values will be slower\\n\",\n \"shap.initjs()\"\n ],\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Since target_class not specified, SHAP will explain predictions for each class\\n\",\n \"Baseline prediction: \\n\",\n \" Husband 0.427679\\n\",\n \"Not-in-family 0.274159\\n\",\n \"Other-relative 0.021522\\n\",\n \"Own-child 0.136484\\n\",\n \"Unmarried 0.081629\\n\",\n \"Wife 0.058526\\n\",\n \"dtype: float32\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"| \\n\",\n \" | age | \\n\",\n \"workclass | \\n\",\n \"fnlwgt | \\n\",\n \"education | \\n\",\n \"education-num | \\n\",\n \"marital-status | \\n\",\n \"occupation | \\n\",\n \"relationship | \\n\",\n \"race | \\n\",\n \"sex | \\n\",\n \"capital-gain | \\n\",\n \"capital-loss | \\n\",\n \"hours-per-week | \\n\",\n \"native-country | \\n\",\n \"class | \\n\",\n \"
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 6118 | \\n\",\n \"51 | \\n\",\n \"Private | \\n\",\n \"39264 | \\n\",\n \"Some-college | \\n\",\n \"10 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Exec-managerial | \\n\",\n \"Wife | \\n\",\n \"White | \\n\",\n \"Female | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \">50K | \\n\",\n \"
| 23204 | \\n\",\n \"58 | \\n\",\n \"Private | \\n\",\n \"51662 | \\n\",\n \"10th | \\n\",\n \"6 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Other-service | \\n\",\n \"Wife | \\n\",\n \"White | \\n\",\n \"Female | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"8 | \\n\",\n \"United-States | \\n\",\n \"<=50K | \\n\",\n \"
| 29590 | \\n\",\n \"40 | \\n\",\n \"Private | \\n\",\n \"326310 | \\n\",\n \"Some-college | \\n\",\n \"10 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Craft-repair | \\n\",\n \"Husband | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"0 | \\n\",\n \"44 | \\n\",\n \"United-States | \\n\",\n \"<=50K | \\n\",\n \"
| 18116 | \\n\",\n \"37 | \\n\",\n \"Private | \\n\",\n \"222450 | \\n\",\n \"HS-grad | \\n\",\n \"9 | \\n\",\n \"Never-married | \\n\",\n \"Sales | \\n\",\n \"Not-in-family | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"0 | \\n\",\n \"2339 | \\n\",\n \"40 | \\n\",\n \"El-Salvador | \\n\",\n \"<=50K | \\n\",\n \"
| 33964 | \\n\",\n \"62 | \\n\",\n \"Private | \\n\",\n \"109190 | \\n\",\n \"Bachelors | \\n\",\n \"13 | \\n\",\n \"Married-civ-spouse | \\n\",\n \"Exec-managerial | \\n\",\n \"Husband | \\n\",\n \"White | \\n\",\n \"Male | \\n\",\n \"15024 | \\n\",\n \"0 | \\n\",\n \"40 | \\n\",\n \"United-States | \\n\",\n \">50K | \\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ]\n },\n \"execution_count\": 35,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 35\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Next we again plot aggregated Shapely values for predictions for every datapoint in the test data (for the class of interest), as well as a summary_plot, and dependence_plot (for the class of interest, as well as a particular feature of interest).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:12.922116Z\",\n \"start_time\": \"2026-01-07T01:37:08.662751Z\"\n }\n },\n \"source\": [\n \"shap_values = explainer.shap_values(X_test, nsamples=NSHAP_SAMPLES)\\n\",\n \"\\n\",\n \"shap.force_plot(\\n\",\n \" explainer.expected_value[class_index], # The base value for the selected class\\n\",\n \" shap_values[:, :, class_index], # SHAP values for all samples with respect to the selected class\\n\",\n \" X_test # Original test data (feature values)\\n\",\n \")\\n\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \" 0%| | 0/50 [00:00\"\n ],\n \"text/html\": [\n \"\\n\",\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \"
\\n\",\n \" \"\n ]\n },\n \"execution_count\": 36,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 36\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:13.607236Z\",\n \"start_time\": \"2026-01-07T01:37:12.970222Z\"\n }\n },\n \"source\": [\n \"shap.summary_plot(shap_values, X_test)\\n\",\n \"print({\\\"Class \\\"+str(i) : predictor_multi.class_labels[i] for i in range(len(predictor_multi.class_labels))})\"\n ],\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"\\n\",\n \" Visualization omitted, Javascript library not loaded!
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\\n\",\n \" Have you run `initjs()` in this notebook? If this notebook was from another\\n\",\n \" user you must also trust this notebook (File -> Trust notebook). If you are viewing\\n\",\n \" this notebook on github the Javascript has been stripped for security. If you are using\\n\",\n \" JupyterLab this error is because a JupyterLab extension has not yet been written.\\n\",\n \"
\"\n ],\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAABAkAAAKoCAYAAAAVl6bwAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3Xl4U1X6B/Dvzdo06b63FCiUtYAIVER2UEGBggjKuCEqirszPx1HZxxxxHHU0XFH3ACX0VFQVlFBUcCllh0EylZaWui+Jmm2m/v7oyQkTdIlFNK038/z+EhOb+49SU7uPXnvOe8RJEmSQERERERERESdnizQFSAiIiIiIiKi9oFBAiIiIiIiIiICwCABEREREREREZ3BIAERERERERERAWCQgIiIiIiIiIjOYJCAiIiIiIiIiAAwSEBEREREREREZzBIQEREREREREQAGCQgIiIiIiIiojMYJCAi6qC6d++OqVOnttm+br311jbZF51/P/zwAwRBwA8//BDoqvi0bNkyCIKAEydOBLoqnR7bC7UHJ06cgCAIWLZsWZvul9ev9isnJweXXXYZtFotBEHA7t27W/xcQRCwcOHC81a3zo5BAiIiIjpv/vnPf2LVqlWBrgYFCbYXcvXVV1/xh2AHZbVaMXv2bFRWVuI///kPPvzwQ3Tr1i3Q1fLpwIEDWLhwYacJVDJIQEREROeNrx99N998M+rr69t1p5AuPLaXzqtbt26or6/HzTff7Cz76quv8NRTTwWwVnS+HDt2DPn5+Xj44Ydx55134qabbkJUVFSgq+XTgQMH8NRTT3WaIIEi0BUgIqK2ZTQaERoaGuhqEDVJLpdDLpcHuhoUJNheOi6bzQa73Q6VSoWQkJBAV4cukNLSUgBAZGRkYCtCXnEkAVE7k5+fj3vuuQd9+vSBRqNBTEwMZs+e7TVyuXfvXowdOxYajQZdunTBokWLsHTpUq/zNjds2IDRo0dDq9UiLCwMU6ZMwe+//35hXlQntnfvXgiCgDVr1jjLduzYAUEQMGTIELdtr7rqKgwfPtz5+M0330RGRgbUajWSk5Nx7733orq62u0548aNw4ABA7Bjxw6MGTMGoaGhePzxx33WZ/ny5VAoFHjkkUecZXa7Ha+88goGDhyIkJAQxMXFYfLkydi+fbvP/VRWVuLhhx/GwIEDodPpEB4ejquuugp79uzx2Pa1115DRkYGQkNDERUVhWHDhuG///2v8+91dXV46KGH0L17d6jVasTHx+OKK67Azp07fR6/sygqKsJtt92GhIQEqNVqZGRk4P3333fbprCwEDNmzIBWq0V8fDz++Mc/wmw2e+zL17zccePGYdy4cW5lJpMJCxcuRO/evRESEoKkpCTMnDkTx44dc27z73//G5dddhliYmKg0WgwdOhQrFixwm0/giDAYDBg+fLlEAQBgiA46+Brjnlr2v2BAwcwfvx4hIaGIiUlBc8//3zTb2gHx/bivb1QyyxcuBCCIODw4cO46aabEBERgbi4ODzxxBOQJAknT57E9OnTER4ejsTERLz44ovO51osFvz973/H0KFDERERAa1Wi9GjR2Pz5s1ux3DkHfj3v/+Nl19+GT179oRarcaBAwc8chLceuuteOONNwDA2R4EQXDuqyVtitqnW2+9FWPHjgUAzJ49G4IgYNy4cbj11luh0+lQVFSEGTNmQKfTIS4uDg8//DBEUfS5v3Ppa9ntdixcuBDJyckIDQ3F+PHjceDAAbdz4LJlyzB79mwAwPjx451tsT3ncTlXHElA1M7k5OTg559/xpw5c9ClSxecOHECixcvxrhx43DgwAHnHeKioiLnieqxxx6DVqvFu+++C7Va7bHPDz/8EHPnzsWkSZPw3HPPwWg0YvHixRg1ahR27dqF7t27X+BX2XkMGDAAkZGR2LJlC7KysgAAW7duhUwmw549e1BbW4vw8HDY7Xb8/PPPuPPOOwE0dNaeeuopXH755bj77ruRm5uLxYsXIycnBz/99BOUSqXzGBUVFbjqqqswZ84c3HTTTUhISPBal7fffhsLFizA448/jkWLFjnLb7/9dixbtgxXXXUV7rjjDthsNmzduhW//vorhg0b5nVfx48fx6pVqzB79mykpaWhpKQES5YswdixY3HgwAEkJycDAN555x088MADmDVrFh588EGYTCbs3bsX2dnZuOGGGwAACxYswIoVK3Dfffehf//+qKiowLZt23Dw4EGPi3tnUlJSgksvvRSCIOC+++5DXFwcNmzYgNtvvx21tbV46KGHUF9fj4kTJ6KgoAAPPPAAkpOT8eGHH+L777/3+7iiKGLq1Kn47rvvMGfOHDz44IOoq6vDxo0bsX//fvTs2RMA8MorryArKws33ngjLBYLPv30U8yePRvr1q3DlClTADSce+644w5ccsklzrbteL43rWn3VVVVmDx5MmbOnInrrrsOK1aswKOPPoqBAwfiqquu8vv1Byu2l6bbC7Xc9ddfj379+uFf//oX1q9fj0WLFiE6OhpLlizBhAkT8Nxzz+Hjjz/Gww8/jMzMTIwZMwa1tbV499138Yc//AHz589HXV0d3nvvPUyaNAm//fYbBg8e7HaMpUuXwmQy4c4774RarUZ0dDTsdrvbNnfddRdOnTqFjRs34sMPP/SoZ0vaFLVPd911F1JSUvDPf/4TDzzwADIzM5GQkICPP/4Yoihi0qRJGD58OP79739j06ZNePHFF9GzZ0/cfffdXvfnb18LAB577DE8//zzmDZtGiZNmoQ9e/Zg0qRJMJlMzm3GjBmDBx54AK+++ioef/xx9OvXDwCc/++QJCJqV4xGo0fZL7/8IgGQPvjgA2fZ/fffLwmCIO3atctZVlFRIUVHR0sApLy8PEmSJKmurk6KjIyU5s+f77bP4uJiKSIiwqOc2t6UKVOkSy65xPl45syZ0syZMyW5XC5t2LBBkiRJ2rlzpwRAWr16tVRaWiqpVCrpyiuvlERRdD7v9ddflwBI77//vrNs7NixEgDprbfe8jhut27dpClTpkiSJEmvvPKKJAiC9PTTT7tt8/3330sApAceeMDj+Xa73W1fc+fOdT42mUxudZMkScrLy5PUarX0j3/8w1k2ffp0KSMjo8n3JyIiQrr33nub3KYzuv3226WkpCSpvLzcrXzOnDlSRESEZDQapZdfflkCIH322WfOvxsMBik9PV0CIG3evNlZ3vgzdBg7dqw0duxY5+P3339fAiC99NJLHtu6tonG5yqLxSINGDBAmjBhglu5Vqv1etylS5e6nav8afeu50Sz2SwlJiZK1157rcexOgO2lwbe2gu1zJNPPikBkO68805nmc1mk7p06SIJgiD961//cpZXVVVJGo3G+VnZbDbJbDa77a+qqkpKSEiQbrvtNmdZXl6eBEAKDw+XSktL3bZ3/G3p0qXOsnvvvVfy9XOlpW3KV1umwNq8ebMEQPr888+dZXPnzpUAuPUjJEmSLr74Ymno0KFuZQCkJ5980vm4tX0tSWroCysUCmnGjBlu+164cKEEwK3dfP755x7nyY6M0w2I2hmNRuP8t9VqRUVFBdLT0xEZGek2/Prrr7/GiBEj3KLz0dHRuPHGG932t3HjRlRXV+MPf/gDysvLnf/J5XIMHz7cYyggtb3Ro0dj586dMBgMAIBt27bh6quvxuDBg7F161YADRFvQRAwatQobNq0CRaLBQ899BBksrOn6fnz5yM8PBzr1693279arca8efN8Hv/555/Hgw8+iOeeew5/+9vf3P62cuVKCIKAJ5980uN5rsM6G1Or1c66iaKIiooK6HQ69OnTx62dRkZGorCwEDk5OT73FRkZiezsbJw6dcrnNp2NJElYuXIlpk2bBkmS3L67kyZNQk1NDXbu3ImvvvoKSUlJmDVrlvO5oaGhbndJWmvlypWIjY3F/fff7/E31zbheq6qqqpCTU2Ns637o7XtXqfT4aabbnI+VqlUuOSSS3D8+HG/jh/M2F6aby/UcnfccYfz33K5HMOGDYMkSbj99tud5ZGRkejTp4/z+yaXy6FSqQA0DN+urKyEzWbDsGHDvH7G1157LeLi4s6pnm3dpqj9WLBggdvj0aNHN3tub21fCwC+++472Gw23HPPPW778nY+62w43YConamvr8ezzz6LpUuXoqioCJIkOf9WU1Pj/Hd+fj5GjBjh8fz09HS3x0eOHAEATJgwwevxwsPD26La1ITRo0fDZrPhl19+QWpqKkpLSzF69Gj8/vvvbheu/v37Izo6Gvn5+QCAPn36uO1HpVKhR48ezr87pKSkODtnjf34449Yv349Hn30Ubc8BA7Hjh1DcnIyoqOjW/WaHHkM3nzzTeTl5bnNFYyJiXH++9FHH8WmTZtwySWXID09HVdeeSVuuOEGjBw50rnN888/j7lz5yI1NRVDhw7F1VdfjVtuuQU9evRoVZ06krKyMlRXV+Ptt9/G22+/7XWb0tJS5OfnIz093SOg07jttMaxY8fQp08fKBRNdxHWrVuHRYsWYffu3W5z2psKLjWlte2+S5cuHseKiorC3r17/Tp+MGN7OctXe6GW69q1q9vjiIgIhISEIDY21qO8oqLC+Xj58uV48cUXcejQIVitVmd5WlqaxzG8lbWWP21KFEWUlZW5lUVHR/u8htKF58iN5CoqKgpVVVVNPq+1fS3g7Hmkcd85Ojq6Xa+0cCEwSEDUztx///1YunQpHnroIYwYMQIREREQBAFz5szxmK/XEo7nfPjhh0hMTPT4e3MdOzp3w4YNQ0hICLZs2YKuXbsiPj4evXv3xujRo/Hmm2/CbDZj69atuOaaa/zav+vdlMYyMjJQXV2NDz/8EHfddVebdMyAhmXKnnjiCdx22214+umnER0dDZlMhoceesitnfbr1w+5ublYt24dvv76a6xcuRJvvvkm/v73vzuXtbruuuswevRofPnll/j222/xwgsv4LnnnsMXX3zRKeeWA2e/tzfddBPmzp3rdZtBgwa1ap++Os6iKLY6a/zWrVuRlZWFMWPG4M0330RSUhKUSiWWLl3qlpTyfPJVZ9fAamfB9kJtydvn29z37aOPPsKtt96KGTNm4JFHHkF8fDzkcjmeffZZtwSWDk1dt1rC3zZ18uRJj+vg5s2bPZJxUuD4u4rJ+e5rdTb8dUDUzqxYsQJz5851yxpsMpk8sjV369YNR48e9Xh+4zJH0qf4+HhcfvnlbV9hapZjGPTWrVvRtWtXjB49GkBD1NtsNuPjjz9GSUkJxowZAwDOdcBzc3Pd7qZbLBbk5eW16nOMjY3FihUrMGrUKEycOBHbtm1zJhUEGtrHN998g8rKylaNJlixYgXGjx+P9957z628urra426TVqvF9ddfj+uvvx4WiwUzZ87EM888g8cee8y53FVSUhLuuece3HPPPSgtLcWQIUPwzDPPdNogQVxcHMLCwiCKYpOfd7du3bB//35IkuT2oy43N9dj26ioKK9Z3/Pz893aWc+ePZGdnQ2r1eoz8dvKlSsREhKCb775xi1Z6tKlSz22bemd4rZs950N2wvbS6CtWLECPXr0wBdffOH2GXqbytYavtpDa9qUq8TERGzcuNGt7KKLLjqnOlL70Nq+FnD2PHL06FG34FFFRYXHyAV/Rz0FK+YkIGpn5HK5x52w1157zWPpl0mTJuGXX37B7t27nWWVlZX4+OOPPbYLDw/HP//5T7fhfw6Nh93R+TF69GhkZ2dj8+bNzgtXbGws+vXrh+eee865DQBcfvnlUKlUePXVV93awnvvvYeamppWZ23u0qULNm3ahPr6elxxxRVuw0OvvfZaSJLkvKvvqqk7st7a6eeff46ioiK3MtdjAQ0X8f79+0OSJFitVoii6DaNBmgIaCUnJ3tdlq2zkMvluPbaa7Fy5Urs37/f4++O7+3VV1+NU6dOuS37ZTQavQ4579mzJ3799VdYLBZn2bp163Dy5Em37a699lqUl5fj9ddf99iH4zOXy+UQBMHtvHTixAmsWrXK4zlarbZFS9K1dbvvTNhemm4vBQUFOHTokNvzy8vLcejQIRiNRmeZ0WjEoUOHUF5e3uzxyZ3j7q/rZ5GdnY1ffvnlnPar1WoBwKNNtKZNuQoJCcHll1/u9p9jWLmvz//QoUMoKChwK/PWpijwWtPXAoCJEydCoVBg8eLFbvvxdj7z1RY7Ko4kIGpnpk6dig8//BARERHo378/fvnlF2zatMltnjcA/PnPf8ZHH32EK664Avfff79zCcSuXbuisrLSGfEMDw/H4sWLcfPNN2PIkCGYM2cO4uLiUFBQgPXr12PkyJFeT4bUtkaPHo1nnnkGJ0+edLtAjRkzBkuWLEH37t3RpUsXAA13BR977DE89dRTmDx5MrKyspCbm4s333wTmZmZbsnaWio9PR3ffvstxo0bh0mTJuH7779HeHg4xo8fj5tvvhmvvvoqjhw5gsmTJ8Nut2Pr1q0YP3487rvvPq/7mzp1Kv7xj39g3rx5uOyyy7Bv3z58/PHHHnkErrzySiQmJmLkyJFISEjAwYMH8frrr2PKlCkICwtDdXU1unTpglmzZuGiiy6CTqfDpk2bkJOT4zaapjP617/+hc2bN2P48OGYP38++vfvj8rKSuzcuRObNm1CZWUl5s+fj9dffx233HILduzYgaSkJHz44YfOpVJd3XHHHVixYgUmT56M6667DseOHcNHH33kscTcLbfcgg8++AB/+tOf8Ntvv2H06NEwGAzYtGkT7rnnHkyfPh1TpkzBSy+9hMmTJ+OGG25AaWkp3njjDaSnp3vkBBg6dCg2bdqEl156CcnJyUhLS3Nbo9rhfLT7zoTtxXd7ueWWW/Djjz+6/YB9/fXX8dRTT7kNNf/tt98wfvx4PPnkk1i4cOE5fBqdz9SpU/HFF1/gmmuuwZQpU5CXl4e33noL/fv3h16v93u/Q4cOBQA88MADmDRpEuRyOebMmdOqNtVSvj7/fv36YezYsfjhhx+cZd7aFAVea/paAJCQkIAHH3wQL774IrKysjB58mTs2bMHGzZsQGxsrNvogcGDB0Mul+O5555DTU0N1Go1JkyYgPj4+Av6Gi+YC7iSAhG1QFVVlTRv3jwpNjZW0ul00qRJk6RDhw55XcJn165d0ujRoyW1Wi116dJFevbZZ6VXX31VAiAVFxe7bbt582Zp0qRJUkREhBQSEiL17NlTuvXWW6Xt27dfwFfXedXW1kpyuVwKCwuTbDabs/yjjz6SAEg333yzx3Nef/11qW/fvpJSqZQSEhKku+++W6qqqnLbZuzYsT6XGHRdAtEhOztbCgsLk8aMGeNcPspms0kvvPCC1LdvX0mlUklxcXHSVVddJe3YscNtX42XQPy///s/KSkpSdJoNNLIkSOlX375xWN5tCVLlkhjxoyRYmJiJLVaLfXs2VN65JFHpJqaGkmSGpate+SRR6SLLrpICgsLk7RarXTRRRdJb775Zove146upKREuvfee6XU1FRJqVRKiYmJ0sSJE6W3337buU1+fr6UlZUlhYaGSrGxsdKDDz4off31116XanrxxRellJQUSa1WSyNHjpS2b9/u8ZlJUsPSYn/961+ltLQ053FnzZolHTt2zLnNe++9J/Xq1UtSq9VS3759paVLlzqXUHN16NAhacyYMZJGo3FbUqrxknYO59Lu586dK3Xr1q1F721HxPbiu700Po7j2K6v2bEkm+uyap2J4z0pKytzK587d66k1Wo9tnf9Htrtdumf//yn1K1bN0mtVksXX3yxtG7dOo/vpGOZwxdeeMFjf96WQLTZbNL9998vxcXFSYIguH2OLW1TLV0C0dfnD8CjzXtrU9Q6vpZA9NbWvH2u3j4rf/paNptNeuKJJ6TExERJo9FIEyZMkA4ePCjFxMRICxYscNv2nXfekXr06CHJ5fIOvxyiIEkMgRF1JA899BCWLFkCvV7vd/IXIiIiIqLOqLq6GlFRUVi0aBH++te/Bro6AcGcBERBrL6+3u1xRUUFPvzwQ4waNYoBAiIiIiKiJjTuSwPAyy+/DACdetUL5iQgCmIjRozAuHHj0K9fP5SUlOC9995DbW0tnnjiiUBXjYiIiIioXfvf//6HZcuW4eqrr4ZOp8O2bdvwySef4Morr8TIkSMDXb2AYZCAKIhdffXVWLFiBd5++20IgoAhQ4bgvffec1vehYiIiIiIPA0aNAgKhQLPP/88amtrnckMFy1aFOiqBRRzEhARERERERERAOYkICIiIiIiIqIzGCQgIiIiIiIiIgAMEhARERERERHRGUxcSBTkrFYrli5dCgCYN28elEplgGtEwYDthvzBdkP+YLshf7DdkD/YbtoGRxIQEREREREREQAGCYiIiIiIiIjoDAYJiIiIiIiIiAgAgwREREREREREdAaDBEREREREREQEgEECIiIiIiIiIjqDQQIiIiIiIiIiAsAgARERERERERGdwSABEREREREREQFgkICIiIiIiIiIzmCQgIiIiIiIiIgAMEhARERERERERGcwSEBEREREREREABgkICIiIiIiIqIzGCQgIiIiIiIiIgAMEhARERERERHRGQwSEBEREREREREABgmIiIiIiIiI6AwGCYiIiIiIiIgIAIMERERERERERHQGgwREREREREREBIBBAiIiIiIiIiI6g0ECIiIiIiIiIgLAIAERERERERERncEgAREREREREREBYJCAiIiIiIiIiM5gkICIiIiIiIiIADBIQERERERERERnMEhARERERERERAAYJCAiIiIiIiKiMxgkICIiIiIiIiIADBIQERERERER0RkMEhARERERERERAAYJiIiIiIiIiOgMBgmIiIiIiIiICACDBERERERERER0BoMERERERERERASAQQIiIiIiIiIiOoNBAiIiIiIiIiICwCABEREREREREZ3BIAERERERERERAWCQgIiIiIiIiIjOYJCAiIiIiIiIiAAwSEBEREREREREZzBIQEREREREREQAGCQgIiIiIiIiojMYJCAiIiIiIiIiAAwSEBEREREREdEZDBIQEREREREREQAGCYiIiIiIiIjoDAYJiIiIiIiIiAgAgwREHULkSRO67KyDVGEIdFUoiJTVh+NITTIq9fZAV4WCiN0kh608BDYL2w21nLrCjqgDNojV5kBXhYg6OFu9CvWl4TDW2gJdlaDFIAFREJMkCZbb/ocrninAkPcrYE77J2xr9ge6WhQEnvzUiE8OjcOPhy7ClU/V4tvd7LhT87I/KkTtl+nQf9Md78zZgfwdVYGuEgWByid+xqV/NeCiV4w42f091H15JNBVIqIOasvnpSjaNAilv/bBS/NzseeHykBXKSgpAl0BIvKf+P0R7P+2HDsvngCLQolIYx0mPLQBXab2hyBjDJC8yzlqQf7/TuChg8ehsdpQpgvFG+YBmPB6VyjkQqCrR+1U2TEDfn03D5p6M2SiHaJBga8WHcJdKy6FjO2GfDDvLYPhuV8QDhPksMNmlKPs9g3QTekBQSUPdPWIqAOpOGXG9/8tBdBwTbJZJKx+vQD9Lo2AKoTnm9ZgkIAoiJVtLcKv3TOcj6tDw/CDlIYbK40QYnUBrBm1Z4dzanBjznb0qymAzmpCaUgkVN9bUanvgvgIXkTJuxM5ldBV6xFfZUCI2YZanRpldjvqysyISAwJdPWonar/Ph9aGOEIIykgQlNVCWtBLVTpUQGtGxF1LIV7azzKzCYJ5afMSO4RGoAaBS8GCYiCWKFdB6DaraxSGwGzUgVNQGpEwSDtWBH6lR2EQmqYU97VWIZJBRbo6q8EIngRJe/EciP655VCW28FACRU6qE1WiCTmJuAfBPqzgYIHOSwQ2YXA1IfIuq4ogvLIEgSJOHsWUdtsSKiRg+A/ZvW4HhkoiAmt3omZLEKgFJk54t861lZ7AwQOMSbaqBip52aEFVtdAYIHBIq9JDbpQDViIJBpdozZG2VyWBWKgNQGyLqyOyihKEHj0AmNvRxFDYbRuw7CCun4LYaRxIQBbGc5ATsj1Oia40BdWoVYoxGrOqThruVCn65ySejRu0RT7fJZLAp5Ww35JNN6dk65JIEGYME1ISfunVFt6hY9Kkqd5at6DcA16lU0AawXkTU8dghQ0ZeIXoUlaBGp0VMTR0UogiP4UzULPYHiYJYeVgY1vSJQpTJCkEQYAdwSquGVWDElHyrC1FBFhqGaGOds2x/Ug8M4289akJpghaREGHH2bwVMmU9ajUhYEYC8kUQBMiqQnAaMVDBinqo0T3fBKOCIwmIqG1FWEwoA6CxWKGprAYAiIKAEAuXQmwtBgmIglhCXT2izRrgzNwrGYDUunqobWoATEBH3tWGSlg/bDyGFRxFRL0eJ6Pi8W3fPrhMtIBz9siXjOLD0GAnitALJmgRjgqkWnMhq68BEBPo6lE71e9UJeohQO9ybonV1yO5zgAkRASwZkTU0ZjVSlgVchztkoharQYxNXr0KCqBJONQgtZikIAoiNWFet6/k2QymBRKJi4kn75Pz8DisTJckRuLGIMJR+Mi8G2frlgYpkJSoCtH7Za6zo4wVKMvcpxlImSQBHYlyLdDCZHo1qisSqNGsS4UaQGpERF1VNtTk1A4qC/qdA1ByeKYKJRGhkGKiUL/ANct2HBMMlEQS6yu8yiLqDchxGb1sjVRg1qoUatWokytRIVShmKNGna7CMnOLPXkW6UsDnVwv/NbhDRYJY5aIt8Ox0Xh48w+cJxdzAo5Xrr8Ytjk7IISUduSQXAGCBzKoiOhZmLmVmP4nyiIpZ+qwOBCA/akJEISBIRYrbji4DHIpfhAV43asUHHivHwVyeQWGcEAIw4dgo/dE9G+B39gXD+4CPvjLoQHMRQJKEESthgRAhOIwnxTFxITehZVomnxl2E1Rf1RLfKWuxPjkGdRo0Qqw2cFkdEbal7eQ1+91IebjADHGPbKgwSEAWxsFoTHvp2O6wKGUrDdEgvKUdliA4CLg101agd61dZA8OZAIHDmBOnIJf6BahGFAx2dY/DOFRCBQGAEjqI0KnqUBapRWqgK0ft1kXl5YiwJuJUWixOpccBVjuGH8hDvCEBgDrQ1SOiDiTCYILaZIE5ROUsi6wxQGPmCNvWYpCAKIhpYIOsvh4AEF9naPi/sQ7gqHFqQois3qNMBgAC7wiTbz3Ly6CCe4boOFstwkwmIIIJL8m77LQU2GrUgOrMqAG5DLXx4TgZFYH0wFaNiDqYikgNBu4vQGFqLPTaEETWGBBeUYd6jRK6QFcuyDBIQBTELKGeS0gpJDtk4I898k2vlcGkkCPEdnaO3vHYcKhFRtrJty61Ro8yhV2C2salpci30+HhMDTKsXuwSwLsArONn2/51RK+P2FHr2gBo7oyBwR1fHuT4pCksWJgbgHskEMOG3b3jsOpMC3iAl25IMMzBlEQK0iM9QgHVITrUK/k+tPk2089e+GLoemQBBvkdhvqQgS8P+oilISFB7pq1I7lJSfCqFK5lR2PT4BR6bnKCpFDcq3Bo0xutyPUykRi59OHe0X0fM2C29bYMHqZFdetsEKSeAOBOrY+p0vRr6IEkUI1oK1HDKow8fBhxBo9g9zUNI4kIApiBrUKh1ISkH66DEq7HXUhauzvmoxB7AhQE+Jq6vHOp58iXn+28x5nqkTog9cDOlUTz6TOzKhU4bo7bsI9P/yE9LJybE1Pw+ohQ7CBI5eoCf1PVyClWkJR5NnBvsPzS6C1RANgQPt8sIoS/u9bG0SXr+bnB+xYMFTChDSO4KCOK95Qj83p6Xh9zCjUaDRIrK3Fo5u+w2X1lkBXLegwSEAUxEJMZhyLjsTpyHAoRTssSgUkAHJ22qkJ444cQZjeABFy2CGDHDZM2/c7NDYLAAYJyLvcqHDkJCkwf9Y1iKyvR7lOBygAs4pdCfJNbbNhxo7D+HxgT9SEqpFSpcdVu45CZhsW6Kp1WOVGoMzLjdOD5RImpF34+hBdKAdSU/Dc5TGwyRtyoBSHh+OpyZPwYVQ4OFaydXhlJwpip7QNy7nIJAlKyQaLJIddJoNNxjsF5FuEoR4GhMLqDAhICIERci5lR00o06gxZ1cOnvnmK8QZDNifkIg7rr0eVhnXNiDfKiJ0WN47CnooAKOI42oNPu6ThjvVHEVwviSFCegXK+Bgufs5fXx39g2oY6tTKmGTu+dXqg4NhVngDPvW4jtGFMR2dklAl6oyTD20C5Nz9+LKw/twIjIUZhnXnibfrJLCJUAAAAJMCAVjBNSUnlWVeHPVSsQZGqapDCgpxrLPP4Ek8YcH+XYoPAx6tQqQCw0rHAjAobgo1Cl4n+p8sdeZsTLkCK6pLgQAhKmA1yYr0D+O3X7q2OKqaz3KVDYrwo2mANQmuPEMTRTE0murMLwwz/k4zGLClNz9kIRxAasTtX8KmbeEYQIEZhunJlxxJBdyux2FYTGoVYciUV+FvmWlCK2uAiKiA109aqe09WZAGwpozowckCQItWao7SLYDW171uyTqL7qI8RU1eMtAG+M7YHwdTdCq+PNA+r4BhTn4brfT+OzjDHOsgd/XYVo8wwAYQGrVzDi2ZkoiI08dsKjLKOkDNH19UAo1y0n70JQi2KlEnmxMTCq1Yip06N7RSnkkhngSsLkQ3iNCT90G4Si8Fhn2cWnj2AYRy5RE8Ik29kAAQAIAmRaJdQ2rm5wPtT98WtIVfXOx/Ifj0P22T7gtiEBrBXRhaERLHjqh49xzcGfcTimC4aePoKeVcWw2K4OdNWCDoMEREHscEwMGs8GLg7TQRWiRmQgKkRBwSxp8GuPNJjOLGdXFhYGQ4gSXQUl+HOPfCnWRaEo3OxWtjehBy6Wy9mZIJ+K4iKhPWxD73I9wiw2lIeqcDhGh2ptCNctPw/EvSUeZbY9xQGoCdGFV6hNRppchcEleRhc0jDStkIdBbsmiuebVuLkJKIgtjm9B9b16+N8bJXJ8OSkyyExcSE14WREvDNA4FAYFQOrnD/1yLeieM8pBaJMDgsTQlETYuqtuKygEl3qTIgw29CzyojMoipozRxJcD4oR3f1UtYtADUhuvAkO2ASomGHEhIE2KFCvRAF2O2BrlrQYY+QKIil1Fvwp2umY8mIYnSvqsLWtO6IsgvgCojUFLPgucyhBEACg0vkW15cOATJDsklKKCQ6lGl1XCmJ/mkNtsQIrp30GPrreDppu1Vmez4YME09Kz7Fpf89DsgF6CZ3B1SrwgU/FSKCJMBWsEG+ag0CFy6tFlmvRXF+2sQmRqKiBRO4QwGyYYKhNpE2BAGASIkKBBvqoNkMQKICnT1ggrPEERBbGDpKVxZpMKx8HDsioxGb4MJXYxmqMQIcKAQ+VIYpoNBqYDWanOW7U+IBURGl8i3EXn7oTj5I35JugQGpRZxxnKML9oCnf4KIDwy0NWjdiqhugZoNJFJsEtQW60AuAxiW1m+T8S81TZI9lDgyhlIHz8Bv77+OIq3GbD5xiiIMgUEScLA0jxcpKiE5us7IR+YHOhqt1vHt5Ti67/vh7VeBARgyA3dMPqB3oGuFjXDolBABwPkMEJAww0QEWEwy+VQB7pyQYa/IoiC2PATB6G2iehXbcAlZTXoYjSje+lpaM31zT+ZOq0DXeOxvm9XKG0mhFpMMMtE/O+iXqjWaQJdNWrHzAoFetXk4aZDn+G23z/CrGNrEGmuhU3OTBbkW7eySsTU1rmVDc4rQGi92cczqLVsdgn3fGWD5DJg46gyHC8OuRxbUkZAlDXcE5QEAXvj01BdboP5j6sDVNv2T7TZ8d2/DjYECABAAnZ+nI+Sg57L61H7YgjVOAMEQMOAJUEwol7BgGRrcSQBURBT1Uu4/YdvYdJZEG7WozQ0Hv3yKmAXLmIEkHxKr6rGdSu/RLj5bCc9ua4S2sduAMIYayfvjsclYTAAGSSo7RYAgFGpRk1IKLQBrRm1Z5XR4bhzw0b8lNELpRHh6Hm6FBkFRTCFDgXDkm2j2AAYLZ7lOSnpSC077V4oCKjS6BCxo/DCVC4IGcrMMFZ4vqGlh2qR0C88ADWilkqpK0a9TIPfw/ujWhmJOHMpMuoOItpcDSAm0NULKgwSEAWxX7t2x1VFn0EBq7OsSh6HuhA1IgJYL2rfLjlxAqFm97t4Y4/nIcRqBTggj3y4tOAYGgYgSmf+ExBmMSOsuhKIYN5o8q5OrcKiayfg/o2/YvKOfcju2QVPXjsRn9k5vamtJGkBrRowmNzLR548BLVNDbMi5GyhJCGmvg7yCUxm6IsuPgS6ODX0Ze7XycQB7Fm1d3abEpviJ6BO2RDMKQ2JR6UqBmPsSt48ayUGCYiCWKStyi1AAAAaoRqSKIJfb/JFcumbG1UqVGt1iK6rBeQyVFbaUFFhQ1qaGgoFM4tRg1KDhE+kZPxfw+BNAECNIgwVIWFQq3RICWz1qB07GBuJ7zKS8F1GD2eZIEnIy6tBTFcmg2sLcpmAd6fKceMXojOJe0/osWDvrzghxGFV35FI0psRYrPhopLjCO8SipCXZ8D2ewlgE6G4iLkJXMnkAi7/Wwa+emIvKi1yqGDHqBtTEdeLKVrbs32HzThaEQ29KgI1uhCUaTXoWlGD00iCURECjgFpHf6K6CAMBgOWL1+O7OxsFBYWwmg0IiEhARMnTsT8+fMREnI2ilxdXY1XXnkFW7ZsgcViQUZGBh566CG89NJLOH36NNauXeu27wMHDuD999/Hrl27YDQakZSUhClTpmDu3LlQKNiEAknUmzzK1DYr9D+dBK5KD0CNKBjUqLSQVGocTknFL736waZUQGGzIeGtEvx6vGGloIgIOR58IB69e4U0v0Pq0J79VcTfttlhN6cjbNAY3LbvJ2yNGYFTmoYfFrax32DUWyPQc0JCgGtK7VGkQQ94mVjw2Qv52LP4CP6wJBOhEZwvfC52l0r481bArpIBJhEKsw0LPvsVxyoHAQD6HanEwjljcN3u3xFtisPR5N6YfMsKyH4rAADIL0lF2Lq5kMXpAvky2pWQtHAcG5yO4pKGBL8hFi0ybRKD5+1QXZ2IP/+jGHUlFkQaBKSkJUAuSVDJBCy7eCiGHTqJaVxOpdX4C6+DKCsrw+rVqzFhwgRMnjwZcrkcO3fuxAcffIDc3Fy8/vrrAACLxYJ77rkHhw8fxrRp05CRkYEjR47g3nvvRXi4Z4xt27ZteOSRR5CamoqbbroJ4eHh2LdvH5YsWYLDhw/jueeeu9AvlVw8HjUUG5UfQ2s9OyROjyiUPLQFYQwSkA82uYDf4tMRU2zClXm7UR4Thn2DuqMgpwb2yHBAEFBTI+Ldd8vx/HNdAl1dCqBDFRIe39YwtQBqORaOn4NDqSOhEGQIrzWgy8kSKE1WbPvzDvTIuQqCwI4YuRt3JB9DC0VcVlCKBL0RR2MicEqtQrjJilMWGba8k4fJDzNr/Lm4e6OIk3UAbBJgkzD7t1yMOHrK+ff00mrc8MN+vHnFEHStMmJgbh5kpSedfxd/O4n6f3wH7WvTA1D79umj/1Y6AwQAkP2bAYMGajB2LEcTtDcff16FuhILZHY7UiqqID8zXFJpl3BVbgFeHTkA/xC8JO2gJjFI0EGkpKRg/fr1bnf2r7vuOixevBjvvfce9u/fjwEDBmD16tU4fPgw7r77btx+++3ObdPT0/Hcc88hKSnJWWY2m/H0009jwIABWLx4sXPf1157LXr16oX//Oc/2L59O4YNG3bhXii5KaxX4+XMWXjo53VQwgQjIlGOVNgPV0GsNkEeybvA5OlAfCx6FO6EcCayHltRh4t3HsPPo/pDKdphVTRkqz912gqj0Y7QUM7k66yyT5+dmxJiEzGzsBRQqWADUBkTAYtSgT6HC4BKE0x1NmjCeUeY3FkFNa7fcwySvOE80q+sGsP09TBpQmAMVaNwT3VgKxjkJEnCr47chGLDXIOMwjKP7TKKylEdGoIqTQhijHUef7f9etKjrDM7etRzpObRY2YGCdqhA4cbbpSFWKzOAIGDQpIQZbbiaHwMLg5E5YIYe34dhFKpdP6It9lsqK2tRXV1NS655BIAwP79+wEAW7duhVwuxx/+8Ae358+YMQM6nfsws+zsbFRUVGDatGnQ6/Worq52/jdy5EjnNu1FZWUlzC7J2PR6Perqzl4ILRYLKioq3J5z+vTpJh8XFxdDcjnhtLdjTCorQV1INIwIhxHhMCAcdiiAVC1kEeo2OcaFeB1tdQx/dNT3oqlj9Dpd6QwQOETWGKGwWGGTn70sxMQIcJmp1O5eR1sdwx8d9b1ofIyB0WfvpKVX1EHVKNmcPlwLq1IBKUKNEJ2i3b4OtpvAHaMsMswZIHAw6DQwqRvaS1K/sKB4HW11DH80VU9BEDA47szx5A3n9cOJ0R77OJIYhXCTGZEmM6o0ntMKxP7R7fL9TkxMPO/H8PY6unfzTOLbrZuqTY/Rmtfhj476HWp8jB7dGs4lZqUS9kaj2UQBOB2mQXGpex3a4+toL+3GQZCkRiEXClqff/45Vq5ciePHj8Nut7v97a677sL8+fNx7bXXwmQyYf369R7Pv+GGG1BXV+fMSbB8+XK89tprTR4zKysLf//739vuRVCrfPKPg5jwwn+QoC93lpUKXaH44n5Ez+gZwJpRe1Z2/xocfr3ErUwOG3b/aQy2lTUkEgsJEfDQAwkYMICLlHV2f/lRxHM5dszYfQxpRpvb32SiHYP2H0Xmy8PRbxrTF5Kn3a8cwrdflLoXShLMahVidHbc/G4mwmJUgalcB/HLKQlTvxBRWS8BRhEaowUv/vd7DDozoqAoSoe/zhmHyUfy0K+kEroQ4Oq6I5AdargOyPrGIXzTHZCltK/s/VarFUuXLgUAzJs3D0rlhRupVFhowb/+eQrV+oafSQMHaPDHh+KhUvH+antTVWXDw0+ehrnahug6PZIrqyADYAfwTZ9U/NCrC3JukmNYEj+71uB0gw7io48+wssvv4xLL70Uc+bMQWxsLJRKJcrKyrBw4UKPoEFLOOJHDz74IHr39j5fMC6Oy14F0iWG424BAgCIEk5BGs955OTbsSQtwlGJWpy925QiHMPAx25BlkWJsjIbevcOgUbDCyoB/xorx50XyXDoz2XYdUqOisiz+WsG5BXgmjVjEZbB9afJO7nFDl1NPfQRZwOOUWV6XPbXvuh/Qw8IMuaxOFcjkgWcvEuOLSclWF7ej6IvCqBXqWAVALvWjppL4nHvieOwWERYkiMx7P96IebyKbD9nA/Y7FCM6g5BzvO9g+14FZTT/4dHDpQjLzEB0dN6YuAj45lzpZ2KilLgnZe7YNsuE1buNGHVnmhMOH4aMgi4qKwO+bF6aKxaAAxGtgaDBB3EV199heTkZLz66quQyc6e6H/++We37ZKTk/Hbb7/BaDQiNPTs0kM2mw2nTp1CWNjZuVZdu3YFAGg0GgwfPvw8vwLyR7fqQo8yhd0GQTSD692TL3uT+2KG5mMk1IfChFBEohx7UrtDrQpBcqwcycm8kJK7HpECopMldF2ajYPdU1EVpkOX0gr0KiqCRjcq0NWjdqwmTodBB3Mhj66HIVSDyOo6nEJXJIxLYYCgDYUqBVz8Wx5+f203+pwps0MBpcmKWr0MolwEAFjq7fjmxaPoeVkM1CO7B6y+7Vn1fRtg218KBYBep04DS07DdEUSNNf2C3TVyAeZTMCYoRqE5pUj9EQ5BJkcEgCtVcQ1vxcgTt8dDBK0DsOGHYRcLocgCG7zUmw2G5YtW+a23ejRoyGKIj755BO38i+//BJ6vd6tbMSIEYiOjsayZctQU1PjcUyTyQSDwdB2L4JazSJ4nvAEAJDY8SLf5JIcQ+f/A59cMhD707X4+5VTMfvau6GxWANdNWrHzJAh3GbAxUePY9yu/ehTVAirWoIksCtBvoUba9FTuRubMlLw/iUDsSs9AoPE36C08nzT1qq3lHiUVerCIMrlbmXWehElR/Qe21IDy9YCjzLzlvwA1IRay6xWe/y4DRHtsKqYVLe1OJKgg5g4cSJef/11PPDAAxg/fjwMBgO++eYbt9UOgIYEhV988QUWL16MwsJC5xKImzZtQmpqKkRRdG6r0Wjw1FNP4eGHH8a1116LrKwspKamoq6uDidOnMDmzZvxwgsvcHWDADqp6wpFZByeu+RqHI+Mw6S8/bh1zy8IE1TgugbkS2G4FnUhdnzd70r8ZrbgeJgWZaFa6NVqMG8z+fJdehp2jTXhvq2/QGGXUB2ixoMzp+LzUDWiAl05ardSagsxbv5fsT+pYXTiJxePwh2/fYfFVgPAM06b0g70/CaGG40QJAmSy1B5mUJAdGqox7bUQDEgDtZfi9zKlAPjA1Qbao2CEM/er8xuQ4VCAWbNaR0GCTqIm2++GZIkYfXq1XjxxRcRExODK664AllZWZg9e7ZzO5VKhcWLF+OVV17Bjz/+iI0bN2LAgAF48803sWjRIphM7ku+jBgxAsuXL8fy5cuxYcMGVFVVITw8HF26dMGNN96IXr16XeiXSi6ORiVh7o1PoiJUCwD4rlsGNnXJxCoZ7+yRb1EmK245XIQwa0MSuu51RiQaTQixdgPAaDt5Zw6R4d8TRuOjYYPRrbIae5ITYVIpYeL5hprwc/ce2C/r6la2dOg4PKuzIjZAdeqokm5NR8knx1GztWFEgV0ORMrrMaC6FPuiEwAJgACMvr07dEwW6VPES1ei4qr/QqppyBKvGt8doTcNCnCtqCX6lxzHdlsNQhQNSTjtALR1JxFv7A6AedRag0GCDkIul2PevHmYN2+ex9+2b9/u9jgqKgoLFy50KxNFEYWFhRgwYIDH89PT0/H000+3aX2pbexKikFFjXvU9Pse3WFRyMCc9ORLqsGIEqt7lvq+1XWQ+9ieCAAmH81FRP1FKA4PQ3F4wx3gCccOIbGuCxDNxIXk3a7kNKDR6G1RLkdliIpBgjYm1ygw5MerUL2lBD9tqcW2fRZYbEBiFxWm35wAe60ZiX3DEJXCHkJT1CNSkVjwEMwbj0MWHwr16G6BrhK1kCADXhoyFLF2O+LrzSgM1cBk64p7RVPzTyY3DBJ0QiaTCSGNhuOsXLkSdXV1TFAYZOKMNUDjiQWSBJlkB/iTj3woD/U+GUXkDWFqgsIow4YP3sDfJ07F4dgETDiei398ux6Wp59gmlTyKcZoQnqxEVccPIGEGj0OJ0ZjV2oCBHtioKvWIQmCAEOXCHy/6+zKR8WFFnz1WTn++AKXRm4pWbiaiQqD0K9dewNHZCgXhLN9HbsSx6IjMCSwVQs6DBJ0Qs888wzMZjMGDRoElUqFffv24euvv0ZqaiquueaaQFePWmHsyXzEGHXO6QYAMOfgLoRahoPDxsmXQ7HhEJVyaK1nc5AcjItAvVzOueXkk0yU0P9UFdZ9uBgCADtkMCACnG1ATck4XY4HvzsA+ZnEyvFHi3DpsVOI/dNYIFYX4Np1TIf3eSYlPHmsHiajiJBQ3kCgjktUKABBci+UCW45OahlGCTohIYPH47PP/8c7733HoxGI2JiYjBjxgwsWLAAWq22+R1Qu3EgpRs2/uMt/Hv4OByPjMGVeYcxtvA4TIqx4CdJvoSINrw1KA2X5Zch2mRBXqQWP3eLw2JeRKkJteEaaKGFBRrIIEKEAoAAO9sNNUFXb3YGCBwUkuTRj6e2E5/sObYnIloBVQgjetSxpVQbALgn5RQkCdFGs0c5NY1Bgk5o6tSpmDp1aqCrQW3gcFgkLq6S462vvoQKFhihxVdpg2FSyhkkIJ/SSqtRKY/Buh5nc/0qzHaE15sBHdfFIO+K4yKQqFZgV/feqNFqkVJZgf5Fh1Gn1nC6AfkUW1/pWShJUNstF74ynUT/oWHoP1CDIzvrYFUqIJMD0+clQSZjQI86NpMgAHYJcGnrkl2CTc4AWWsxSEAUxMYcyUOxPQF7Y3vDFKJCVJUB/fOqEWY2Az7mnRMVh3pG020yGQwqBYNL5FN6WR7+m3kpajXhAIDCmDgYdHbMMFaBWaPJF5nMBLlkhiicDSXFm05DZrc18Sw6F5VPbMPYV3ZiTL0NlktTkfThVYjpyeUmqeMzaNQYcbAQV+8+hqowDaJrjfhgzEAYFAmBrlrQYZCAKIgllRvwzaDusIcICDMZsa9bKrofD4HnGhVEZ5VEhKJhLSx3Iu8yURNKQ+NQq7G6le1P7IdJqlAO4iSfvk8fjEemyjAq7xQSaw04HB+NwsjBGBoeje6BrlwHVPdZLiqf/c35WPXLSVj/9QvwzpUBrBXRhdG7qg5/X/kTVKL9bNnpKsT+JQv82ds6fLeIglhZTCT6VfyGjKITkAEwKZT4ofdFsDf7TOrMBpQVIcoQharQECjtEqxyGRJr66C2RwDgkDzyrkwbC+C0R7lNxq4E+XYqTIdajYSv+rtn1rcw4+V5YdiQ51Fm/NqzjKgjSiquRL3djtKkCBh0aoRXGxFdpkdYrQngWMlW4ZWdKIipBTPSi044H4fYrBhx/ABEWVbgKkXt3rS9vyPyf/U42KcL5IIAWGwYvP84ov40FYiIDnT1qJ2SQwIkCXBJVCiznykj8sEqyACIbmUKUYQg4wo854OyZ6RnWQ/PMqKOKM5UhvWDU1F9ZuWUU4hBckEFxtj1AGICW7kgwzAuURCLNdZ6lEWYjAixWr1sTdTAKIvCsZ5J0BlN0NYZoBZtON4zBTaB6efINwGAwmpFnUIOvUIOo0yAwsZ55dS0oXmnMfpgAQBAITa0l/nf7YaunokLz4fIBYOgTI90PhZCFYhZNDJwFSK6gKrDtc4AgczeMK72dJcomNWqQFYrKHEkAVEQKw8NQ2qjstqQUIQolZwjTD4VxcYgtP6kcwkyucUOq0IBk1IB3tsjXySriA8yeuFQTKSzbOrRfNwTuCpREOhZXoM3P12NFOURRFlrUayMwylrX4QbMwAwwW5bk8eGotueW6BfeQRijRm6a9KhTGHSQuocSsITkFqxHeNzdyLaUIfTEdHY1D8Ttaow6AJduSDDIAFREDPb1TgSlYj0qmIIACwyObYn9UCWnVkJyDebXPBYo1xhs4FpC6kpR6PCcSjGvZu1sXsKLHK2HPLNHKFABvZDb41DGeIRatUjXdgLkzaEM4TPE1moEuE39w90NYguuFCbGdP2bINKbJjilFRTiWm7t0FhHx7gmgUfBgmIgpjCYsGu6K44HpaAUKsZVSFaiDJ5oKtF7ZzM6n2IuOBlxQMih2ptCBJr9Zh6KA8JeiMOxUVhfd8eMCiUnOlJPqVVnkAFekJEw3BfC3QIkaoRqa8EkBTYyhFRh5JYUwGl6J4DJbLeALXRACAyIHUKVgwSEAWxOp0GAGBUqmFUNswnlwDYmW6EmlCm0wB2CXBZ8lBuEyFyLAE1oX9JBf64LRdhloacJym1eqRW1yGqfigQqQlw7ai9Uhol6OE+H9iECEBgF5SI2taBuFhc1KjMIpejIiKcS662En9JEAWx8qgwSI1+11lDlJC43j01waSWQW0yQWa1QRBFyC1WhBvqYFAzIwH5FmEwOwMEDr0qqiE0nrtC5EL0kRBVYlCSiNpYSUgk1mZkuJW9PmY0zFyqt9X4jhEFsb2JMbB0iccPiXEo14Sgb2U10sxmGBQK8L4e+TKk6DS+1WmhNFsgt4oQlQrY1WrE6w1AGBNckXdF4Z4zyCUAFgW7EuSbQiZCgA2SS5dTAQvA3DlE1MZ6FVdi/qgrcDCpD3QWEeVaFXbGxeH+GiPAnnGr8MpOFMSKdKFYPjQDNlnDoKCjsZEIkex4UiYBvEtDPhiVobCprLCpzo4cMEGDeoWC2X/Jp0qd55opFrkcBrkC0QGoDwUHmUxEHA5Dj3jYEAI19AhFFWRMeElEbexg1zjMem8nJLkMgIA4gxVXVxYgP747r1OtxOkGREFMbrE5AwQOZnhmridypZd7rhcsSRIEBpaoCf0qKjzKQkQRkfWmANSGgoXcboQSFkShEHE4inAUQwEzBNHa/JOJiFohzmg6EyA4S1QrEGPgdaq1GCQgCmKp1XUeZTJJgoLDOKkJB6LCUa5xX588OzEOVjkvCeRbakV5wz8kCYJoByQJVpmAEP7YoyaUaD3XvrDJ5DCqOfSXiNpWtdrLTRBIqFcx51JrsUdIFMQGl1YizGxxKxteVAqljUEC8q0qQofDSQk4EheLwsgI7EtOhD48HCY5l88k33Z1SURhmBbKeiuUZhuU9VZsSe8Co5dOGZHD5p4DsCWtn1vZf0ZPQa3ac/oKEdG5iKrXI7nSfdRb79NFkNu9L/1MvjEnAVEQi4pV4U+r92BjWirKQkOQUV6FsYXFkNAj0FWjdqyHVkCOTIaSiLNJCkMkCSpOU6EmVISFo0dJlfOxAGD4kSLUqQYiInDVonauSheCO+b/FTfu2oZ+JYXYnD4AX/Ubghkq3qciorY1uKgYfX/fjbz4RFRpdUioqUZqRRmiqscB8Ey+S74xSEAUxHqOicevS0/gD4eOQSbaYVMqENonCqGRHFZFvl12UQj+92WNW1l8nBxREey0k28Dq2uwr1FZtMGECIMJiORdYfIuo64OVnk03r9kgrMsrs6IWIMCiGG7IaK2Y1NpAAlILzntLDMLCligBMe8tQ57hERBLLFPGIbf2QPGyFDoo8Kh7BGJKX/vH+hqUTuXlqrCzdfoIBNEAEBMlAwPz4+BIDBxIfnWY1iUR5kqVg1tfIiXrYkaZPTV4KbsgwivNwMAkqr1uHPfIYQlqANcMyLqaLQjk1GCGFjRMH3SDAXKQuIR0ovj3VqLIwmIgtzwG7tgf913kOoVuO2BG6AOYayUmjdjkg6lBV+i3hKC++6ehRDOK6dmxA6IQv95PfD70mMQIEARKseYRRdDpuT9BvItoV8EbslUYPCXP6NepUCYKOLyv/WHQs0cKETUtkL7RSHsT5k48dI+yGCHXa5Az5dHQhHGPk5rMUhA1AHI1HZAbeG609QqSoUNSoUechnbDbXMsD9nYJ9yO1Apx5w/Xw9tNIeLU/PG/F9vHMFvUFercMOD0xGZpAt0lYiog+r6r2HYFrYP6iIZpj42C7oekYGuUlBikICIiIhaLtIORNqhCmPuE2o5WbQVsmgrtLGcZkBE55c5ueE/dSqTFfqLYwSJiIiIiIiICACDBERERERERER0BoMERERERERERASAQQIiIiIiIiIiOoNBAiIiIiIiIiICwCABEREREREREZ3BIAERERERERERAWCQgIiIiIiIiIjOYJCAiIiIiIiIiAAwSEBEREREREREZzBIQEREREREREQAGCQgIiIiIiIiojMYJCAiIiIiIiIiAAwSEBEREREREdEZDBIQEREREREREQAGCYiIiIiIiIjoDAYJiIiIiIiIiAgAgwREREREREREdAaDBEREREREREQEgEECIqJOSxRF1NfXw2AwBLoqRERERNROMEhARNQJ5eTkID8/H6dPn8YLL7yA7OzsQFeJiDowBiWJiIIHgwRERJ2MXq/H+vXrnY9FUcTatWuh1+sDWCsKFvyxR63FoCQRUXBhkICIqJMpLi6GKIpuZTabDcXFxQGqEQUL/tij1mJQkogo+DBIQETUyYSFhXktDw8Pv8A1oWDCH3vkDwYliYiCD4MERESdTGlpqdfykpKSC1wTCib8sUf+SExMhCAIbmUymQxJSUkBqhERETWHQQKiDoBzhKktNO7IE7lKTEyEXC53K1MoFPyxR83ydm6RJCkANSEiopZgkIAoyHGOMLVWfHx8q8qJAECn0yEjI8OtbMCAAdBqtQGqEQWD4uJi2O12tzK73c4RKERE7RiDBERBjHOEyR+cbkD+0Ov1+P33393K9u/fz/MNNSkxMREymXt3k9MNiIjaNwYJiIIY5whTW+J0A2oKzzfkL29TCzjdgIjagl6vx9GjRz0C1qIo4tixYwxk+0kR6AoQkf8cc4RdO+6cI0zN4XQD8gfPN+SP4uJij4CAY7pBenp6gGpFRB1BdnY21qxZA1EUIZfLkZWVhSFDhqC2thbl5eVYtmyZs3z48OGBrm5Q4UgCoiCm0+kwZcoU5x1ghUKBadOmcY4wNamurs5reW1t7QWuCQUT5iQgf3DJVSI6H/R6vTNAAJydcltWVoby8nLndpyK6x8GCYiCXGZmJrp27YqkpCQ8/PDDjJRSs9hpJ38wJwH5g0FJIjoffE2By83N9diWU+Naj0ECog5ALpdDo9Hwjh61CBMXkj+Yk4D8kZiY6JHvhIkLiehc+VqWt0+fPh7bcmpc6zFIQNQBiKKI+vp6GAyGQFeFghgTF1JTfHXI2PGi5ng7tzBxIRGdC51Oh6ysLCgUDSn2HFNu4+LiEBsby6m454iJC4mCXE5ODvLz8wEAL7zwApOzULOYuJD84chJsHfvXmcZcxJQc4qLi2G3293KmLiQiNrC8OHDkZGRgeLiYiQmJkKn08FqtSI8PBxarRYTJkxASkoKdDpdoKsadDiSgCiI6fV6rF+/3vmYyVmoJTjdgPzBnATkD043IKLzSafTIT093SMQIJfL0aNHDwYI/MQgAVEQ4xxhakucbkBN4fmG/MXpBkREwYVBAqIgxjnC5A9ONyB/JCYmQiZz7zbwjjA1p6npBkRE1D4xJwFRENPpdJgyZQrWrl0LSZKYnIVapKklyRgooKZ4u/vLO8LUFEdwyTVQwOBS6+j1erc5182VU/vT0s+Qn2nzXN8jg8GA7du3w2KxoEuXLujatSuqqqpgsVggiiLKyspQUVEBAEhLS2v2PeX7fxaDBERBLjMzE3v27IHFYsHcuXMRGRkZ6CpROxcWFua1PDw8/ALXhIJJcXGxR0CACeioJRhcOjfPPPMMJEmCXC53JifOzs7GmjVrIIqiWzm1P74+q8blAwcOxL59+7x+ptnZ2fx84f5eCoLgdh7Jzs722P7VV191/lsmk2H69Ok+30d+p9xxugFRO6HX63H06FHo9Xq3fzf3nN9//x0GgwGiKKK8vBw//vgjSkpKoNfrsW/fPuzbt89jP479l5SUtOg41LEwcSH5g9ObyB9NBZeoZRzvnyM5cUlJifPHjGs5r+Xtj16v9/pZefsMd+/e7fUzdeyjs2v8XrY20Gi327FmzRqv3xNfn1Nn/k5xJAFRO+AavZTJZJAkyeOugbfnrFq1yu0k+e677wIANmzY4LatIAiYMWOGR+TagRFTApi4kJrGJRDJHxy51LZsNhsOHTrkM4koR/W0L74Svubm5nqUN+aaGLa5bTsDb+9la4mi6PV70lRi3s76neJIAqIAaxy9tNvtHncNvI0EWL16dYujqJIkYc2aNR6RawdGTDsXJi4kf3AJRPJHUzlQqPUUCgX69evHUT1BwtcIrL59+3qUN+b4TL3tozNqi/dBLpd7/Z5wpJwnBgmIAqy5yKi3Jca8ZYtujiiKTUauuZRZ58HpBuQPLoFI/uCqGOfO8f45khPHx8cjKysLCoXCrZyjetofnU7n9bPy9hkOHjzY62fq2Edn1/i9bO3oR5lMhqysLK/fE1+fU2f+TnG6wQV06tQpZGVlYf78+bjrrrvaZJ933nknTp8+jbVr17bJ/ujCc0Qvff149xbJ9JYtujlyuRx9+/bFN9984/VYnT1iSpxuQE0LCwvzet6xWq3MCE1NYuLCc/P44497fL+GDx+OjIwMfu+CgK/Pylu5r3Mpp4M2aPyeeVvdoLq6Ghs3boQoipg9e3aLVzfgd8odgwREAeaIXq5duxY2m815x8But/uMZOp0OkyfPh2rV69uUaDAET11RK4dx3JgxLRz4XQDai1HLhNv55vly5c7/838JuTg+LFTVVXlM3FhYmIiO+QtoNPpvM6L9lVO7U9LP0N+ps1zfY8cS4G7io6OhkqlAgDExcUhOTnZr313dgwSELUDjaOXAJrtODmec+zYMWzcuNEZCDh16hT69OkDrVaLvLw8AO7RU9djhYWFoa6ujh20TqapOcIMFFBjjfOmNMWR3yQjI4PnlE7MW4JcVzKZDKdOncLSpUu53BgRUTvEIMEFYDKZnHNciHxpHL1sSSRTp9Ohf//+zrVhu3Xr5va8gQMHNnushISEc6k2BSFf2ca3bt3KCDp5aG1G6c6eEbqza0lQSZIkt6lvDC4REbUvneaX6+nTpzFt2jSPfAD33Xcffv31V/zxj3/EjTfe6CyfO3cuDAYDVqxYAQA4cuQIlixZgl27dqG+vh4pKSmYOnUqbrrpJrdsmAsXLsS6deuwceNGvPrqq/jpp59QVVWF1atX+6zbL7/8gkcffRS9e/fGSy+95FwW6OTJk3j//feRnZ2NyspKREZGon///pg/fz769evnc3/79+/HihUrsHfvXpSUlEAulyM9PR0333wzxo8f77ZtcXExlixZgpycHFRUVECn0yE1NRUzZ87E1KlTATQMC/z000+xZs0anDp1CoIgICYmBoMHD8bjjz8etAGQ8zGH1t996vV6513/+Ph45919AG6jAQB4jACora1FZWUlFAoF8vPzUVRUhL59+wIADh06hK5du0Kv18NkMjVZh5CQkGbna7Xm9blu6+t1cARDYPhKXJibm4uSkhIGjsjt++srqOSLr+zR1PHp9Xrs2LGj2aCSJElek2A6rhNA8/OHO5OSkhIcOnQIffv2dTs/N77O+tv/4PX4/Gr8Hjseh4WFOa/Hjr6RtxGg7V1btiFv7018fDxKS0t99mNNJhPKy8thtVqhVCoRHR2N6upq2Gw2bN68GYIgID09HeXl5SgtLUVERAQiIiLc3mPX43KEbYPg/HXnh6SkJKSkpCAnJ8cZJLBardi9ezdkMhm2b9/uDBLo9XocOnQIM2fOBAAcOHAAd955JxQKBWbPno2YmBhs3boVr732Go4cOYJFixZ5HO/ee+9FTEwMbr/9dtTX1yM0NBRGo9Fju3Xr1uHpp5/GmDFjsGjRIqjVaucx7777bthsNkyfPh09e/ZEbW0tdu7ciT179jQZJPjhhx9w4sQJXH755UhKSkJNTQ3WrVuHRx55BIsWLcLkyZMBNFyQ7733XpSVlWHWrFnOH5NHjx7Frl27nEGC999/H2+99RZGjx6Na6+91jlMcMuWLbBYLEEZJHAdCtlWwxz93Wd2djZWrVrlMWfTkUTOtbxx0jBBENz+/u677wIANmzY4NdrEAQBM2bM8Frv1rw+12291VkQBNjtdg4xDZBt27b5/Ftubi6DBJ2c6/e38TmmJURRxP79+/m97mSam2LQEv/973+d7a2p61Fn85///AdAw7V98ODBmDNnjsd1VpIkSJLU6v5HW/eFyF3j93jgwIHYt2+f1++J6/lWJpNh+vTpzs8jOzu7XX42bdmG2uIc0tj3338PAPjuu+88/uY4xwDwOC6/D50oSAAAmZmZWLduHUwmE0JCQrBv3z6YTCZcddVV2LJlC2w2GxQKBXbu3AlRFDFs2DAAwL///W9YrVYsXboUvXr1AgBcf/31eOyxx/D1118jKysLl1xyiduxevbsiaefftqtrHGQYOnSpXjjjTcwa9Ys/PnPf3YmrJMkCQsXLoTVasXy5cudxwSAefPmNZuo7vbbb8d9993nVjZnzhzccMMNeO+995xBgry8POTn5+P+++/H3Llzfe5v8+bNSEtLc16kHO6///4m69FeNR4K2RbDHP3dp16vx+rVq1uc+bnxZ9/W2aElScKaNWs86t2a19d4W291dtSbQ0wvvJKSEuTn5/v8uyPZD3VOjb+//p5jvvzyS36vO5HW5K1oimt783U96ux2796N4cOH+7zOtqb/0dZ9IXLn7T3evXu3z+1d27/dbne2f6DhR2x7+8Halm2orc4hrSFJElavXg1BEDyOy+8DIGt+k45j2LBhsNls2LVrFwAgJycH0dHR+MMf/gCDwYADBw4AALZv3w5BEDBs2DBUVlZi7969GDNmjNuPdUEQcNtttwFo+BHd2E033eSzHpIk4bnnnsMbb7yBBQsW4C9/+YvbGsK5ubk4fvw4pk2b5nZMh8brDTem0Wic/zaZTKiurobJZEJmZiby8vKg1+sBwNnod+zYgcrKSp/70+l0KC0tbfLE1h5UVlbCbDY7H+v1ercEbRaLBRUVFU2u9X369Gm38uLiYreTtq9jtGafro+Li4tbtYzhhSCKIgoLC51LxgAtWx/d8V75O38ZQJPvlesxHBp/Hv5oabtpql7+tptAHOPQoUNoiuvf2/PraMtj+KOjvhd5eXlt1klzfK876nvlj476XrRlu3EliqKzHXWU96ot/Pbbb02+3zabzdmn9VXPAwcONHldby/vt2Mqxfk8xvl6Ha3tDzXmaP/+7ud8vxe++oau04Zaeoxzfa/8ZbfbfR7X9fvQGc83nW4kAdAQHBgxYgS2b9+OoUOHom/fvggPD0dOTg4GDRqE7du3o1evXoiIiMD+/fsBAD169PDYX1paGmQyGYqKijz+1q1bN5/1+OSTT2AwGHDPPfc4Aw2uTp48CQDo06ePX6+zsrISixcvxo8//uj1x79er4dOp0NSUhJuu+02LFu2DJMnT0bv3r2RmZmJyy+/3Bm5BBqmTjz88MO44447EBcXh6FDh2LUqFGYOHEilEqlX3U8H6Kjo90eN478qVQqxMTEQK1WQy6Xu50UFAoFkpKSPJYAbHxx8nUMQRBavE/XubqJiYle1x0PJLlcjtTUVLd6JyYm+nx9rtv42rYprvtpPI+58ePmPg9/tLTdNFWv1tYzkMfo27dvk9NRLr300nM+RlP1bo/vlT866nuRlpbWqu9vUxzH7qjvlT866nvRlu3GlWt+i47yXrWFSy+9FHv37vX5fisUCrd+HOBZz/79+2P16tU+r+vt5f12vRlxvo5xvl6Ht/5mazjav2MaSWud7/fCV9/Q9TdTS4+h1+vPyzmkOTKZzOtIAsD9+9AZzzedaiRBTEwMevToge3bt8NkMmH//v3IzMyETCbDkCFDkJOTg+rqahw5csQZUPBXSEiIz79dcskliIyMxJdffonCwsJzOk5jkiThvvvuw7p16zBlyhQ8++yzeO211/DGG284pxm4/iC955578MUXX+BPf/oTunTpgtWrV2Pu3Ll49dVXndsMGjQIq1atwnPPPYdx48bh8OHD+Nvf/oYbbrgBNTU1bVr/C0Gn0yErK8uZS0GhUGDatGkeP+YvxD51Oh2mT5/udXSI48TlIAiCx3auf28LjmUUG9e7Na+v8bZNvY62eO+pdRISEpyJLRsLCwvz+TfqHLx9f/1xzTXX8HvdiTRuN/5yvVb4uh51doMHD0a3bt08vqetva6ej74QufP2Hg8ePNjr98SRr8nBtf079tPetGUbaqtzSGs48j54Oy6/D51sJAHQMOVgxYoV2LJlC6xWqzOXQGZmJl555RX8/PPPkCTJGSRITk4GABw/ftxjXydOnIDdbkdKSkqr6pCeno4FCxbg7rvvxl133YXFixeja9euzr87/n348OFWv74jR47g8OHDHqs4AMCqVau8PqdLly6YM2cO5syZA7PZjPvvvx8ffPABbrrpJmdEKjQ0FBMnTsTEiRMBAJ9//jmee+45rF69Grfcckur6xlow4cPR0ZGRptm9PV3n47nncvqBh988AEUCgVmzJiBU6dOOUeh5ObmIjU1tU1WN2jN62u8ra/XweyxgTFq1Civ0w6uv/76ANSG2htv39+DBw/i8OHD2Ldvn8/npaSkoH///hg+fDi/151Q43ZTVlaGn3/+GQaDwWsfatasWQgNDcW+ffuQkpKCwYMHAwjO7O7n2x//+Efk5uaiT58+zsSy3r6n/vY/eD0+f7y9x/6sbtDe8hE4tGUbct1Xa1Y3MJvNKCsrc+aWi46ORnZ2Nmw2m/P3XHOrG7gel6sbNOh0QYLMzEx89tlneOedd5CYmIguXbo4yy0WC5YtWwa5XI6LL74YQMOwjUGDBmHLli04evSoc91nSZKwdOlSAPBYVrAlevbsibfffhsLFizAnXfeibfeegvdu3cHAPTu3Rs9evTAmjVrMHv2bPTs2dPtuZIk+byD7Jr80NXRo0fxww8/uJXp9XqEhIS4Rc/UajW6d++OnTt3ora21rmMSGRkpNtzHXcba2trW/vS2w2dTtfm63j7u0+dToeBAwc6H7tml3ctB+Dcv2Mbm80GlUoFu90OvV6PIUOGAGjoLAwZMqTNljR01LOlr6/xtr5eB114vpa0cyy/StT4+5uZmYmoqKgmgwQXXXQRxowZcyGqR+2Ua7vR6XRIS0uDXq/HP//5T7dRjDKZDH379oVOp0P//v3d9tH4WkEN13tvq840/p762//g9fj8avweuz5u/LkGY/tvyzbk671pzapLVqsVubm5AIBx48Y5p0Y7AjGtOW5n1umCBEOHDoVMJkNeXh6mTZvmLO/RowdiYmJw/PhxDBw40G14ycMPP4w777wT8+fPdy6BuG3bNvzyyy+YPHmyx8oGLdW9e3e8/fbbzhEFb775Jnr27AlBEPDkk0/innvuwdy5c51LINbV1WHnzp0YMWIE5syZ43WfaWlp6NGjBz744AOYTCZ069YNBQUF+OKLL5Ceno6DBw86t92+fTueeeYZTJgwAd26dUNoaCgOHjyI1atXY8CAAc6gxaxZszBw4EBkZGQgLi4O5eXl+PLLL6FUKnHllVf69dqpbTRePvHTTz91WzqxqSVcuPRR5+WIzjdWUlKC+Pj4C1wbChbN5VBpamle6ry8Jei12+0oLi7mj1Mionaq0wUJwsPD0bt3bxw6dMi5xKFDZmYmvv76a4/y/v374/3338eSJUuwYsUK1NfXIyUlBffff3+Tqxi0RNeuXbFkyRLcfffdWLBgAd5880306tULGRkZWL58Od577z1s2rQJK1euRGRkJDIyMpxD8ryRy+V45ZVX8PLLL2PdunWor69Hz549sXDhQhw+fNgtSNCrVy+MHz8eO3bswNdffw1RFJGYmIh58+a5va6bbroJP/30E/73v/9Br9cjOjoaAwYMwLx589C7d+9zev3kP1/LJ7o+9rWEC5c+Im/aOscFdTy+lkQcPHgw4uLiLnBtKBgkJia6rf8ONIwkaJyAi4iI2g9BauuF1onogjh69CjefffdFm17xx13uN2x8fXcxttRx1RSUoL//Oc/HuV/+tOfOJKAfPJ13rj22mvPOdkvdVy+phs8/vjjDEpTk6xWq3Nq77x589rVilrUfjSeOmu1WvHuu+/CYrFg9uzZMBqNzDHgh043koCoo2jp8omNlyp0PLe5JQ2p4+J0A/KHr/NG4znlRK443YCIzhdvU2dFUUR+fj4AOFdr47Ta1utUSyASdSS+lk9syRKDXPqIvOF0A2qKTqfzWHt9wIABPG9QkxwBbVecbkBE58rb1Nk1a9Zg3bp1Hts6ptXq9foLXc2gxZEEREFs+PDh6NOnD95//33Y7XZcccUVztUwmlu1gEsfdV6+RgtwFAE1Ra/X4/fff3cr279/P/R6Pc8f1CRvM1s525WIzkVxcbHbyDYAHo9d2Ww2jmBqBY4kIApyWq0WOp0O4eHh6N+/P3Q6nXMpl+Y67i3djjqWpqYbEPnirUPm6HQR+VJcXOwREHBMNyAi8pdjCpwruVzuMXLJgdNqW4dBAqIOQhRFHDt2jEOpyG+cbkBN8dYhY6eLmsPpBkR0PnibOpuVlYWpU6d69Gc4rbb1ON2AqAOora1FeXk5li1bxuQs1CxONyB/OHIS7N2711nGnATUEpxuQETng7eps1arFXv27OHqBueIIwmIgpzBYEB5ebnzMZOzUHPq6uq8ltfW1l7gmlAwaSonAZEvnG5AROeTt6mzcrkcGo0GcXFxnFbrJwYJiIKct44W5wlTU8LCwryWh4eHX+CaUDBhTgLyB883RETBh0ECoiCXmJjoUcZ5wtQUJi4kfzAnAfmDI5eIiIIPgwREQU6r1SI2NtaZpIXJWchfTFxITXHkJHDFnATUHCYuJKLzSa/X4+jRo25T30RRRH19PQwGQwBrFtyYuJCoAwgPD4dWq8WECROQkpLCuVfUJCYuJH80lZOA5xxqChMXEtH5kJ2djTVr1kAURWfiblEUkZ+fDwB44YUXmMzbTxxJQNRByOVy9OjRg511ahanG5A/mJOA/MHEhUR0Puj1emeAAGgYPbBmzRqsW7fOuQ2TefuPQQIiIgLA6QbUtMTERI82wmHj1BxONyCi88Fb4FoURdjtdrcyBrP9wyABEVEnw+kG5C9vgSQOG6fmcLoBEbU1b8l05XK5R1CSCXb9wyABEVEnw2zj5I/i4mKPOzQcNk7N4XQDIjofdDodsrKyoFA0pNhTKBTIysrC1KlTmcy7DTBxIRFRJ8N1y8kfjrs2rsM7eYeGmuOYbuAaYOJ0AyJqC8OHD0dGRgaKi4uRmJgInU4Hq9WKPXv2wGKxYO7cuYiMjAx0NYMSRxIQEXUyTFxI/uASiOQvTjcgovNFp9MhPT3dLXG3XC6HRqPh9ekcMEhAREQAmLiQmtbUEohEvnC6ARFR8GGQgIiok2HiQvIHl0Akf3B6ExFR8GGQgIiok2HiQvKHt0zSzElAzeH5hogo+DBIQETUyfDHHvlDp9NhypQpzBpNrcLzDRFR8GGQgIiok+GPPfJXZmYmunbtiqSkJDz88MMYPnx4oKtE7RzPN0REwYdLIBIRdUKZmZlcIoj8wqzR1Fo83xARBReOJCAi6qT4Y4+ILhSeb4iIggeDBEREREREREQEgEECIiIiIiIiIjqDQQIiIiIiIiIiAsAgARERERERERGdwSABEREREREREQFgkICIiIiIiIiIzmCQgIiIiIiIiIgAMEhARERERERERGcwSEBEREREREREABgkICIiIiIiIqIzGCQgIiIiIiIiIgAMEhARERERERHRGQwSEBEREREREREABgmIiIiIiIiI6AwGCYiIiIiIiIgIAIMERERERERERHQGgwREREREREREBIBBAiIiIiIiIiI6g0ECIiIiIiIiIgLAIAERERERERERncEgARFRJyWKIurr62EwGAJdFSIiIiJqJxgkICLqhHJycpCfn4/Tp0/jhRdeQHZ2dqCrRERERETtAIMERESdjF6vx/r1652PRVHE2rVrodfrA1grChYcgUJERNSxMUhARNTJFBcXQxRFtzKbzYbi4uIA1YiCBUegEBERdXwMEhARdTJhYWFey8PDwy9wTSiYcAQKERFR58AgARFRJ1NXV+e1vLa29gLXhIIJR6AQERF1DgwSEAUpvV6Po0ePcl4wtRpHEpA/EhMTIQiCW5lMJkNSUlKAakREROSJuXPOnSLQFSCi1svOzsaaNWsgiiLkcjmioqL4A49arLS01Gt5SUkJ4uPjL3BtKJgIggBJktzKGj8mIiIKFEfuHAB44YUXkJWVheHDhwe4VsGHIwmIgoxer3cGCICGaGlFRYXHMGCi1mp8l5jIVXFxMex2u1uZ3W7ndAMiImoXmDun7TBIQBRkvM0LliQJFoslQDWiYONrtABHEVBTEhMTIZO5dxs43YCa4jotjsN/Oy9HO+APNTrfmDun7XC6AVGQSUxMhFwudzsJCoIAlUoVwFpRMGkqcSEDBdQUb1MLON2AvHGdFuc6TYXDfzuXxtMj+dnT+eStj6xQKBjM9gNHEhAFGZ1Oh6ysLCgUDTE+hUKBmJgYyOXyANeMggUTF5I/iouLPQICnG5A3jSeFufabjj8t/PwNj2Snz2dTzqdDlOmTHFOn1QoFJg2bRq0Wm2AaxZ8OJKAKAgNHz4cGRkZKC4uRmxsLD777LNAV4mCCBMXkj8cqxu4/uDjdAPyxtuQX1eO4b/p6ekXsFZ0oTU19JufPZ0vmZmZ2LNnDywWC+bOnYvIyMhAVykocSQBUZDS6XRIT09ndJTaDBMXUnO8tRFON6DGHEN+feHw387BWzvgZ08Xglwuh0ajYR/5HDBIQBQEvCX9cU0IRdQaTFxI/uDqBtRSjafFuSa85PDfzsPb9Eh+9nQhMFHqueN0A6J2zlvSHwBuZVFRUZxPTi3GxIXkD043oNZoPC3uk08+4fDfTsi1HSQmJkKn0wW6StTB5eTkID8/HwATpZ4LBgmI2jFvSX/WrFnj/Lfj/xUVFYzMU4sxcSH5q3GQAOB0A/LNMS3OarVy+G8n5mgHROebXq/H+vXrnY8dyTIzMjIYoGolTjcgase8Jf0RRdGjTJIkWCyWC1k1CmJNJS4k8oXTDYiIqD1rKlkmtQ6DBETtmLekP3K53KNMEASoVKoLWTXqgJi4kJrimG7gitMNiIiovWCyzLbD6QZE7Zgj6c/atWths9mcSX8AuJVFRkY2mUmayBUTF5K/ON2AWkqv12PXrl3Iz89HZGQkysvLoVAo8PPPP0Ov1yM+Ph5du3ZFXV0d56oHKb1e75ZrwPE4LCwMdXV1CAsLQ2lpKUwmk/M5ISEhSEtLc/u8fT3PMXKyPfRvGr9Wap90Oh2mTJmCtWvXQpIkJss8BwwSELVzvpL+uCaE+uyzzwJcSwomTFxI/mhqugHnG5Or7OxsfPnll17/tmHDBo8yR1JeJhcLHo2TKg8cOBD79u3zGOrtjSAImDFjBoYPH+62H19iY2Pbsuqt5i2BNNtq+5WZmYk9e/YwUeo5YpCAqB1zRK7lcjmKiopgsVhQVFQEjUYDSZJQW1uL8vJyWCwWiKIIg8Hg9WTICDi5YuJC8kdiYiJkMplboIDTDagxvV6P1atXt+o5TC4WXLwlVd69e3eLny9JEtasWYPu3bs3GyAAgPLycp/9m/PN22tlW22/SkpKsHnzZpw+fRqhoaHOcvaDW49BAqJ2qiXR9ca8LfXCCDg1xpEE5C9vUws43YBceRtx0hKO5GIcldL+eUsO11qiKOLQoUMt3k9JSUlAggRNJcJjW21fPv30U7dglcViwb/+9S8MHjzYOcqF/eCWY+JConaoceS6pRwRbr1e73U/jf9OnRNHEpA/iouLPQICXN2AGnOMOGktJhcLHt6Sw7WWXC5Hv379WryfhISEczqev5gILziUlJT4HM2ye/du9oP9wCABUTt0LlF616VeuBQMecMlEMkf3n78cboBNabT6TB9+vRWPYfJxYKLI6myQtEwIFmhUGDw4MHOx82RyWTIyspCfHy82358iY2NDVjb8PZa2Vbbn0OHDrV4W/aDW4bTDVrpzjvvxOnTp7F27dpAV6VF1q5di6eeegpvvfUWhg0bFujqUAs5Itf+BApcI9ze9sMIOPnCJRCpOZxuQC3hSLi7e/dunDhxApGRkdi7dy8UCgUuvfRS1NXVcXWDIOctqbI/qxu47sfb6ga//PJLwFc38JVAmtqPvn37ek2K6g37wS3DIEEHsH37duzYsQM33HCDz2HEFFwaL33YUo0j3L6WUGQEvHPjEojkj6amG3BuLjWm0+kwatQojBo1ClarFadOnQIAjBgxAkql0rldoIaR07nT6XRu333Xx47PtSWfr6/nWa1W/Pbbb21dbb80fq3UviQkJGDw4MFepxwMHjwY+/fvZz+4lRgk6AB27NiBd955B9OmTfMIElx99dW48sor3S7IFBxcI9dyuRwnT55EbGysc3UDAKipqUFsbCxycnIgiqLXpV4YAafGmLiQ/JGYmOi1/NSpU+w8ExFRQM2ZMwfjx4/HDz/8gN9//x2hoaFYsGABIiMjubqBHxgk6ODkcnnAh2mR/1wj12lpaQCA/v37u21jtVqxZ88eAPAZGWUEnFwxcSH546effvJa/tVXX2HIkCHseBERUUAlJCRg5syZqKmpAQC3kbXsB7dOpwoSWCwWfPTRR/j6669RWFgIlUqFiy++GHfddRf69u3rtm1tbS1effVVbN68GWazGf3798cf//hHr/sdNmwYpk6dioULF7qV+8oHoNfrsXz5cmzevBmnTp2CRqNB9+7dcd1112HSpEkAgBMnTuDTTz/Fzp07ncnn0tLSMGvWLMyYMcO5r4ULF2LdunUAgKysLGf5/Pnzcdddd/msQ3V1NZYsWYItW7agoqICMTExGDNmDO666y63O9GO5y9evBiHDh3CihUrUFpaiqSkJNx2222YOnVqqz4D8uQtuqnX63Hw4EEUFRVBkiTExcVBrVbDbDajvLwcFosFKpUKYWFhsFqtqK+vBwAYDAYoFArk5eUBaBg+7jq/D4DbXMDWRFbPJQrLCG77UlBQ4LV83759mDhx4gWuDQWL7Oxsn3/jlIPOofGcc9e56Hl5eTCZTAgJCUF8fDxKS0tRWFiII0eOQBRFVFRUQKPRYOXKlTAYDIiJiUHv3r1RWFgIjUaDyMhIt+sTBYavPom3ssafuaO/UVBQgKqqKvTq1Qt6vd5nTgLHfuVyOQoKCtC1a1eIoujssxgMBuj1eufoSX9fBwBnv8hxw8XXa2xJf6nx8xu/Py3t87Bv1LYcfefc3FycOHECdrsdr7zyCmQyGeRyOWw2GyIiInDZZZdBFEVn29XpdCgoKEDfvn05/clFpwkS2Gw23H///di7dy+uvvpqXHfdddDr9fjyyy9x++2345133nHeobXZbLjvvvtw4MABXH311Rg4cCAOHz6Me+65BxEREedUj7q6Otx+++04fvw4Jk6ciFmzZkEUReTm5mLbtm3OIMH27duxc+dOjBo1CsnJyTCZTNi0aRMWLVqEqqoqzJs3DwAwc+ZMGAwGbN68GX/605+cP/B79erlsw56vR633XYbTp48iaysLPTt2xe5ublYsWIFcnJysHz5co870m+88QbMZjNmzpwJlUqFFStWYOHChejSpQsGDx58Tu9JZ5adne1cotCxdisAfPnll37t7/nnn292fWqZTObMPN342L7WjfVWz5auMXsuz6XzY9u2bV7LN27ciLKyMsyZM+cC14iCgVarhdFo9Po3Tjno+FzP5Q5yuRwDBw7E3r17m732AA19IMec4SNHjuDXX391+7sgCJgxYwavEQHiq0/irWzVqlXNJi39/vvvvZbLZDIMGjTIuXZ9S+Tk5OCyyy5r9euQyWSQJMmtrjKZDHa73etrbFzPxv0lQRAgCILz+QMHDnS+Dm+PffV52DdqW9nZ2V77zuXl5W6Py8rKcPToUa/72LBhAwYPHsw+0BmdJkjwv//9Dzt27MBrr72GESNGOMtnzZqF66+/Hi+//DLefvttAA0nggMHDjjvxjukpaXhpZdeOqeMmG+88QaOHz+Oxx9/HDNnznT7m+sFdsqUKZg1a5bb32+44QYsWLAAy5Ytw8033wyFQoFBgwYhPT0dmzdvxrhx45CcnNxsHZYvX46CggI8+uijmD17trO8d+/eeP755/HBBx/g7rvvdnuOxWLBBx984MxtMHHiREyfPh2fffYZgwR+0uv1bhclURS9XqRaoyWdNLvdjtWrV0MQBI91YzMyMjyi2d7q6Wvbxs7luXR+lJSUNLnU4e7duzF+/HhG08lNSUkJysrKfP59w4YNnHLQgTU+lzuIouhzbXJ/SJKENWvW8BoRAL76JI5/u5bZ7fZzWtXEbre3ut2sX78egwYNanW/w1u/yFHm7TU23m7NmjWQJMn5HNeAQ+P27+2xtz4P+0ZtS6/XY9WqVW2yL/aBzpI1v0nHsGHDBnTv3h39+vVDdXW18z+bzYbhw4djz549zuFQP/zwA+RyOW688Ua3fcyaNeucsmHa7XZ8++23SEtL8wgQAHBbf9p1aJXZbEZ1dTVqa2tx6aWXwmAw4MSJE37X44cffkBUVBSuueYat/KZM2ciKioKmzdv9njO7Nmz3ZIfOpYuOnnypN/1aGuVlZUwm83Ox3q93i1Bm8ViQUVFhdtzTp8+3eTjxtm82/IYjmkkrs4lQNAadrvd41iu68a6vg5v9bTZbDhw4IBbmbf36uTJkz6Pcz4+D38EW7s512O0ZC3h3Nzcdv862vIY/uio74W/7UaSJOf5oz2/jrY8hj+C9b04cODABbs+iaKI4uLioH2vgrXdOKaFuBJF0WtZIJY9dbQLf15Hc/ttantRFFt0A8YXR5/Htd6++lWOcygQPO2mPRzj5MmTbdomc3NzO8x7dS46zUiCvLw8mM1mXH755T63qa6uRmJiIoqKihAbG+sRzVOpVEhJSfGZGbw5jh/6riMZfDEajXj77bexceNGr3f9amtr/aoD0DAstF+/flAo3D9+hUKBrl27eu0MpqSkeJRFRES4ndACLTo62u2xt88vJibGrazxqJDGjxtn827LYyQmJkIul3sM3bwQHTGZTOY2kgBwXzfW9XV4q6dCoUBGRobbPr29VyqVyutzk5KSzsvn4Y9gazfneoyWrCXcr1+/czpGU/Vuj++VPzrqe+FvuxEEwe3c1l5fR1sewx/B+l70798fq1evviDXJ7lcjqSkJI+bMsHyXgVru+nVq5fXPgkAj7JzHUngD1/toiWvo7n9Ar5v0sjlcreRBK3l6PO41ttXv8r1MwqWdtMejpGamgpBENqsTfbr16/DvFfnotMECQAgPT3dZ/JBAIiKimrT453LxfSvf/0rtm3bhmuuuQZDhgxBREQEZDIZfvrpJ/z3v/89p6imP1xHObgKRDS5o9DpdMjKysLatWvd1m4FWjbXzxtvc++8beOYY9f42N5GyviqZ0tG1ZzLc+n8SEhIQPfu3X2ORho8eDDi4uIubKWo3UtISECfPn2co0wamzFjBr/XHVjjc7mDQqHAgAEDWpyToDkymQxZWVlsSwHQVJ/EW9nq1av9/swdOQkca9e3xJQpU/zqdzTuF7nmFPD2GhvX05GzwHV/AJzPHzBggPN1eHvsrc/DvlHb0ul0mDFjht99Z1fsA53VaYIEqampqKqqQmZmps8fvA4pKSnIzs6GXq93i/JYLBYUFRV5LBMWERHhXGrDVVFRkdvjyMhIhIeH48iRI00ev66uDtu2bcPVV1+Nxx9/3O1vv/32m8f2giA0ub/GUlJSkJ+f7zwxOdhsNhQUFHgdNUDnx/Dhw5GRkeGR3TYjI6PFqxvYbDbn6I+5c+e2anUDb8duTT3P5TVS4IwcOdJrkGDatGkYOXLkha8QBYUePXp4DRJMnDiRCbc6AddzeePVDaZOnepzdYOjR49CFEWUl5dDo9EgPT0der3eubpBUVERNBoNIiIiuLpBgDXVJ/FW1tTqBtXV1c7PuiWrG5w8eRKpqakeqxv8/PPP0Gg0yMzM9Pt1AM2vbuDatpvrLzV+vj+rG7Bv1LYc76djdYMDBw7AbrcjNjbWubqBKIqIiIjAiBEjPFY3OHnyJPr06cNcBC46TZBgypQpeOWVV/Dxxx/j5ptv9vi7YxlAABg7dix+/vlnfPzxx26JC1esWAGDweARJOjatSv27dvnbGxAw3QARzIUB5lMhkmTJuHzzz/HqlWr3JYyBBruyguC4AxiNI6GlZeXe03MERoa6jxmSxIXjh07FkuXLsWqVavckiOuWrUKVVVVXvMl0Pnjbe1WnU6HzMzMFl0UrVYr8vPzATRkH1cqlRg4cKDz744TnrcTX2vWjT2XNWa5Pm37Eh8f77W8qVVRiHxNOWDy2s7D9Vzuek3R6XRu1x3H3x1lVqsVS5cuBdCQ/8g1x5FjZSlqH3z1SbyVefvMXf/fmmM5fry77stqtWLv3r2tqr+vOjeua1OvsSX9Jdd/N/5bS/s87Bu1LUffefDgwc7zzbx589zON740bn/UiYIEf/jDH5CdnY1XXnkFOTk5yMzMhFarRXFxMXJycqBSqbBkyRIAQFZWFr788ku88847KCoqwqBBg5Cbm4tNmzahS5cuHtMIrrvuOjzxxBNYsGABrr76atTV1WHVqlVISkrySEBx9913IycnB4sWLUJ2djYuuugiAA1JMmw2G55++mlotVpceuml2LBhA9RqNTIyMnD69Gl88cUXSElJ8Ri1MGDAAADAq6++iquuugoqlQo9e/b0eeKZO3cuvvvuOzz//PPIzc11DiFdvXo1unXrhltuuaVN3nNqO43XEu7bty8AYM+ePYiIiHAuodOSfTBiTb7yqtTW1voMIBBptVqPeZ+CIPi1hjkREVFb0+v1KCwshMVigSiKMBgMiIyMZB/YD50mSKBQKPDyyy9jxYoV+Oqrr5wBgbi4OGRkZGDq1KnObZVKJd544w288sor+PHHH/H999+jf//+eOONN/Dyyy97ZJa86qqrUFZWhs8++wz/+c9/kJKSgjvuuAMymQz79+932zY8PBxLly7F+++/j82bN2Pz5s3QarVIS0vD9ddf79zu6aefxmuvvYatW7di/fr1SE1NxT333AOFQoGnnnrKbZ+DBw/G/fffjy+++AKLFi2CKIqYP3++zyCBTqfDe++9hyVLlmDLli1Ys2YNYmJicO211+Kuu+7inKh2xtva1N7u5sXGxrZoH1yPl8LCwryWNx4lReSqcRZl4OyqBrwbRkREgeStv/zCCy9g4MCB2LdvH/vArSRIzDxH1G7p9Xo8++yzLU6C+Ze//AWRkZHN7kOhUOAvf/kLo6md1L59+/Dxxx97lN94440eQzKJHPR6PZ555hm3QIFMJsNf//pXBpfJg16vd84DT01NxSeffAKLxYLZs2fDaDTyjl4H5GtufuMcFi3lOk2lpcPGqXNqTX+ZfeCW6TQjCYiCkbe1dJtSUlLiESRoaj1e3v0jV61Ngkqdj7dlpnivgRrLzs52yzTu2m5effVVAOAdvQ6m8YhF17u3DvzM6XxpTX+ZfeCWaTrNPxEFlGMt3ZbylmzH2z4ar8dLnYuvvAPMR0BNKS4u9ljyzG63o7i4OEA1ovZIr9dj9erVbsEjb4EkURSxdu1a6PX6C1k9Og/0er3bMG9RFLF7926PH238zOl8aU1/mX3glmGQgKgdc6yl67pUpS+xsbFeh/w23gfX46WmEhcS+ZKYmOixhLBMJmNni9x4Cyb54rijR8HNn7u4RG3JV39ZoVBg8ODB7AP7gdMNiNo517V0HWsJ9+nTBwCwd+9ehIeHY+fOnU1GULkeL7li4kLyl7c7wpxuQK4cwaSWBAp4R69jcNzFbel8cH7mdD44+rpFRUXYuHEjRFHE3LlzubqBnziSgCgIONbSTUtLw5gxY5CQkOBcFaNPnz4tGmLl2AdPjlRaWuq1vKSk5ALXhIKJt9UNON2AGtPpdJg+fbrbqBNv+U54R6/j8DZi0fXurQM/czrfdDodevTo4dEvZh+49TiSgCgINU4QFBUVxbvAdM6YuJCa4u0OMacbkDeOO3pc3aDz8DZi8VxXNyDyR05ODvLz8wE0LIHIZJn+YZCAKMh4SxBUUVHByDy1GBMXkr843YBaSqfTOZdUtVqtkMvl0Gg0iIuL41J2HZTjbq23x94SKxO1Nb1ej/Xr1zsfO5JlZmRkMEDVSpxuQBRkvCUIkiQJFoslQDWiYMPEheQPTjcgIqL2rKllv6l1GCQgCjLelnkRBAEqlSpANaJgw8SF5A+ubkBERO0Zl/1uOwwSEAUZbwmCYmJiWrw+LBETF5K/ON2A/CWKIurr62EwGAJdFSLqoHQ6HaZMmeLMscRkmf5jTgKiIOSaICg2NhafffZZoKtEHQATF1JTmppu4DoPmagxJhIjogslMzMTe/bsgcVicS6BSK3HkQREQcqREIjRUWotJi4kf3C6AfnDVyIxvV4fwFoRUUfmSJTKPrL/GCQgIupkmLiQ/MXpBtRaTCRGRBcapzedOwYJiIg6GSYuJH9wdQPyBxOJEdGF5JjedPr0abzwwgvIzs4OdJWCEoMERESdDEcSkD8SExM98lZwugE1h4nEiOhC4fSmtsPEhUREnQxHEpC/BEHwGE3A6QbUHCYSI6ILoanpTUyw2zocSUBE1MlwCUTyR3FxMex2u1sZpxtQSzGRGBGdb5ze1HYYJCAiIgBcApGaxukGRETUnnF6U9vhdAMiok6GSyCSvzjdgIiI2jNOb2obHElARNTJMHEh+YPTDYiIKBhwetO5Y5CAiKiT4Zw98gfbDRERUefAIAERUSfDOXvkD7YbIiKizoE5CYiIOiHO2SN/sN0QERF1fBxJQETUSXHOHvmD7YaIiKhjY5CAiIiIiIiIiAAwSEBEREREREREZzBIQEREREREREQAGCQgIiIiIiIiojMYJCAiIiIiIiIiAAwSEBEREREREdEZDBIQEREREREREQAGCYiIiIiIiIjoDAYJiIiIiIiIiAgAgwREREREREREdAaDBEREREREREQEgEECIiIiIiIiIjqDQQIiIiIiIiIiAsAgARERERERERGdwSABEREREREREQFgkICIiIiIiIiIzmCQgIiIiIiIiIgAMEhARERERERERGcwSEBEREREREREABgkICIiIiIiIqIzGCQgIuqERKsdhpNRqDmUhJMHDYGuDhERERG1EwwSEBF1MnZRwod/z0Pljh6oPZSMpX85hm0riwNdLSIiIiJqBxgkICLqZA5vr0HBAffRA99/fApWiz1ANaJgcfKgAaVbe6NowyB88WIB9FXWQFeJiIiI2pgi0BUgIqIL6/QxI5RWG3oUFENnrEdJbBQKE2NgNohQqhg7Ju8MNTZ8vDAPFlMYAGD/lmroq2y4/bk+Aa4ZERERtSUGCYiIOhubiPG/7kWEvh4A0KOwFIfSUiBJFwW4YtSe5f5WDYvJfbTJiX161FVaERatDFCtiIiIqK0xSEAU5E4eNKB0Wy/YDGqsqy/E5NtTodHxq02+JRZVwnomQODQK/801DJONyDfNDoFToVpsLlnEipC1Uir1OPKE6ehDOHoEyIioo6EvySIglhdpRUfLcyD1RQOANj5TSWMNSJu/Ht6gGtG7Vmo1QIjTIhDAdSoRx2iUW5PgWAVA101asdiB4bjkyEa1MvlAIA9ydEQUrV4PlQe4JpRMLBUhcJaF4LqUgviUjjyhIjanmSxwbIuF0k7jSgdEBLo6gQ1BgmIgtjBX6phbTT8Nze7BiajiBB23MkHfbgKvbALKpgBAGGogqAwwQIZ2HUnX746AWeAwGG3qEaxQUKiVghMpSgorH3tJEp+7AcAeO2uQ5h+fzcMnRQb4FoRUUciluhRNeY9iIcrcAmA+kgZxImVUPZLCHTVghLHCBIFMZXG8yusUAmQK9hhJ9+SawudAQKHBPEUQmS2ANWIgkG42rNMKQM0vN1ATSjMNWDXpirnY8kOfP1eIaxmTm8iorZj/PdPEA9XOB9rqu2of/rHANYouPHSThTE+l8Whc1Jp1B52uIsGz4tnhnqqUkVYSGIalQmAQBjS9SEkcmATJBgl842lCQtEKFmwyHfSgvqPcpMehG1FRbEJHM4cGf069pS7NpYAblSwIjp8Rg4JjrQVaIOwPZ7qUeZeLA8ADXpGPhLgiiIqUJkuO35dIT3PYXQ1ArM/L9UXDkvJdDVonbup579YW90+j8VFo9SlTZANaJg8M0JuAUIAKCgDigxSIGpEAWF7gPCIDTqbUbGqxCV6GVoCnV42etKsX7xSZw6asTJgwZ89q88HM6pCXS1qANQjU/zKFOM7RaAmnQMDBIQBbn6YiN0x02IPGKAcXcpbCYO4aSmjTp6DLVIRT0iYIUGesQivFaFeLMh0FWjdizUS8IKhUyCmulPqAnRSWpcdWcKBEVDYtTwWCVmPZIGmYwjUDqjXZsqPMu+8ywjaq3QBy6F+roBzlGRJf3V0PxtTGArFcQ43YAoiNVXmrH+zhygTgMA2Lv8BIwlFlzx/JAA14zas+TyGpggQAEz5LBAASVMiILA+QbUhClxZvSqqMaRmDhn2a1FhxAZMjCAtaJgMOyqGOwpXAfRpMQd9/wB6hBVoKtEAaJQet6f5BRJaguCWoHI/10H87+vwP/++wlMUQr0i+CUJn8xSEAUxPK+L4alzj3Z3LGNpzHWYINKy683eScpFdDJCvF7Yl9UaKORXn4cSVVFkCwchUK+KdfvxfevrcCbI8fgcHwCxh/JxbycXyE9+iyE+PBAV4/aOZnCDpnODJmcwcjO7LJr4lFwQA/pzCwluULA8KlxTT+JqBVkiTqYotgHPld8B4mCmMxLRF4mFyBwGCc1QQk9lmXegILoVADAD+mjceWh7zBawSABNUGtQLShHg9/uxV2yKGAFTIZACXnG1DTak8aYd8UCalKgd/qDuOSu/tAGcouaGfU/7IozH2mF3ZurIBSKSBzShxSejEfDvlvRa4db+2RgHoLpm3ZjqTdJagN74n6KBW+lU5h4k1dEKLldaq1eIYmCmI9Lk/Cb6/nwlBicpb1m5kKpYYnQ/LtWGQXFES7n/5/TB+FSxUqeJl2TtRgXB/o5VGQxIYgpAgl7KlxCIliB598s9aLWDf/N0hloQCAvcvzUHPCiKtfywxwzShQeg4OR8/BHH1E5+7LI3bMXuu4waHE5j7D8ejpvZBDAEzAr6vLUXrCjHnP9g5oPYMRJwERBTGVVoGspcMhXKQH0upx2V/6YeSjAwJdLWrnqsJjPcrMCjXMAuPG5Jtl43FngMDBWmCAvYwJL8m3gq0lMJaZ3cryfyyBoczk4xlERC3zzl731XXSKvUNAQIXx/fUofK0+zmImsceIVGQ09ZWYfzxnxBWbkTCcQkysSsg5/1g8i21ogKQZIBw9kIaaqqHxmQCOJaAfBC8zSUXBIBzzKkJXqe/CT7KiYhaofHlR/JxWmm8DCs1j28ZUTCrNkAx/kn0+bkIyYerIH/iU+DutwNdK2rnykK0bgECADCrVLBpmHGcfFNN6OHRI5OlhkMWHRqgGlEw6Do6HmEpGreynlcmITRGHaAaEVFHcc9g93WZTkTqIEruowt6Z0YgKoHnm9biSAKiYLbyFwilNe5lH/4IvHo7oOWyL+RdZWQkgGK3MlEmhw0y8DJKvli+Ow6I7p0ve0EN7BVGyGIYKCDvFGo5pr57CT59dD2kKgWGZQ3ExbemB7padI5Em4RfVpXg6M5aRCepkXmJDuXLD8N4Qo+4K1PQ/b6+kClkqFlzHHnP7kJNpQ0FvZMRJavDRfmHAEHAvvQMFHbtCl2dHoMO5CKhqBhyhQ07LrkYhUldEJeuhWhVQK62NV8h6pSu6iHD17OAJXskwGTF9B+yEVZRihO6CBhiVRiRlY7Rs5IDXc2gxCABUUfUKIpK5Eoh2SET7bDLzw4m0+hNsIsBrBS1e5IkQR8Sgp1pvVCl1SG1ogwXFRzn+YaapY0PgWxMLQBg8LweUHBFjKC39vV87Pi2AgBwIqca0t9+htLc8GO+4vtiGI7WouuEWORf+xVkAKIAdDl8FF1Q6NzHhN2H8fnFl+PSA4egNZthB2AHIDedxPGBMTh+oB6KsN5InHDggr8+Ch5Xdpfhyu4AIAeuHwur1YqlS5dCC2DM9SOh9LISGDWPQQKiYDbzUkiPfQShrPZs2U1jAJ3G93Oo04sorkWXE8UoTomBTaFAiMmMtMOnoQSXQCTf5JN6Y+XIMahTNZxfTsbFo2pAV9wQy9UNqGmSJEF/MhLak2YU/r4RkZN7IyJGAaFPAoQIXq+CjblexO7vK52PY8pqnQECh6IPjiH0wCm3sihUuT0WAIw8tgfaRjnl+hYVYGu/AbApFLDVaWAuD2vT+lPHYTKIKC80ITQEOLWrEqFWC6oEJWK+N0OjrYd0vRWIcM+1JEkS6vZVQWazQS1aoRiUAEHNn8SN8R0hCmZROth++AeOz38BYWVGJNwyCfI/XxPoWlE7J7PbYVMLiC0/28mrjVJD4A1hakLuPqMzQOAss4fBWGdDaBi7E+Rd+SkT3rj7d9isPVEFIGf378h6/SNY7TpAq4Hi1eshv+2yQFeTWkGSAMnucsHwcu2QJEASmw88yyQJgGe2OQFN759o+9fl+GrJSVjN9oYGJwjO/0cZ++GmHZ/B9MEDkD55EOpZAwEA9QUG5FzzPfS/VwMAosQ6pEXWIfqz2VCN7xHAV9P+cPwF0Xm2r0zC/G9EzFotYuVh/+/UHv3mFL68dzveunUXlv3lEHZ91zDMTwwJxW7tJViVPAuLf0rAE9N34akrfsVzE7fhu3/9DtHCMeTkzh7WcCFN0pejT0U+wswGiEo5bHb2xMg3u5fTl93e6McCUSP/feoYbNazj/clZ2B/cl8ooAcMFtju/gRSaa3vHVC7ExIqx4Cx0c7HlfFhsDWaQpI8Jw0xCwa6lVUj0mNfP/cYhHqle9LcI0kpsCoa7v4qtCaoY+vaqObUUdRVWrHuzYKGAAFwNhnzmf9XhUZhbcZkhNsqYb7lXUimhpPQwcd2OAMEAFAlD0N5pRw1c79sUVCrM2Hon+g8OlwpYcR/RRjOdJBWHpGw+HJgweDWxef2/TcPPz5/EGUJcbDL7UCpAcf2GlBVYETdmznYH9ff7dtsU8qhhxK/fVUBQ/4OZC2+pA1fFQW7lJoiTCzYgW61JQAa5oDuSOyNUGEMACa8JO/SLw6DIAMkl35URLwK2ggum0m+eVuf/GRkCoYU7mt4YLHBvqMA8qsGXOCa0bmY8WA3xCSpnYkLBz+WhsrluQ2JCyeloMf/ZUCubggc5D27C3U1Inb3GohooRYX5x+EJAjYk56BqrQe2DqgKy4+eBDxp0ogV9hguKQbuiSqEd9bi9P4gcvXkYeiIwaItqYD1AWRXQAA8vo62HIroLwoEdW/lnlsp5dpEHeyBPbCWsi7RZ6P6galgAQJ7rzzTpw+fRpr164NxOFbbe3atXjqqafw1ltvYdiwYYGuDgWRd/banQECh1d22lsdJNj7UR7qQzVuieYA4JfVpRCjUnw+z6gNxcmfS6AvrocukfM+qUGMoQbhZwIEQMOQsqElhz2WRSRylbdP7xYgAIDacgvq62zQcLoB+RAWo0R1icWtLKmuBJKjCyqXQTbQ93WM2ielSoYJ/9/efcdHVeX/H39NS6+kExISCL0I0myguCqKiijY1oLYRXf3t7qrrrtf17Zr2RXbWhAVxK6IgLoq6oIFlQ7Sew1JSO/JtPv7I8nAMEmAITBJeD8fDx+7c+beuedOPtyZ+dxzPufajpx97f7K8elnJ/psF3t1D2Kv7nFQ6yUAdPJqG+b5f6Pr/7euAN1BX6JEgOSMUJ+k9cFSGm6E2EKxdI0FILJ/LDXZVV7bhRm1mBPDMadEHLP+tkXKzdVbunQpU6ZMobxcQ5qk5dQ0MtK/xo+VfFx2F0YjP+BcLgP3Yfywczk0hEr2cwX5LldnNgC74kSatnlhPrGVxUTUlNOhogiLywFG3XVIpClX/aULpgMKnmQW7uCkPWtwEQFmE5Z/jMHUKTaAPRSRtiYmKZjfXNdx/yiTg1bZCbVXc+H6r6g0R2B99lrMEXULPPf858kEd9x/0yzCXU2CtZLIly7CFKRk94H0btRbtmwZU6dO5eKLLyYy0ruK6ujRoznvvPOw2TSkUo7MhD5mXl7p8lpa/MZ+R56b63lpOqWvbaM8KsLrbu/AkbFUvLOadUlpje4XUl1DfM8ootNUfVz2c7htmPEuFeXCgmG1NlI+SgSWfVvAiu9KMcL3/5gzuV10spUTEaPPRmlaavdw/vJhP55/YB7xW8o5PaqW0gdvJ6F/FOaBaZg6xwW6iyLSBp15ZQr9z+pA7rZqbG4nuYvysVTbqTAs2L/9kR3DEkmc+gjBafvrZ0T2iuGs9ZdS9F0upuJKIkLdBJ2ejjnO9+bJiU5JgsNgsViwWLSmrxy5wckm/jvOzFOLDYprDX7b08wfBx/5z7DBt3fHFmph1X/zKHAHYY0Opv858Zx5ZQpVp9pw3fsL+ywJuM0WyqzBmIBgu51uPUK4+KmTWv7EpE1zOczUEkEIlZgxcGGhkihCzEoRSOO+e34zhsm7uJhhtmDKLYGSSohRIlKaZg0yE9snB1cf6DJxom66iEiLiE0KJjapbpRA1unxQP00lcSfyCaUU5J9l8+0BFtIOE9TnA7liJMEdrudt99+my+//JI9e/YQFBTEwIEDue222+jZs6fXtmVlZTz//PPMnz+f2tpaevfuzR//+MdGX3fw4MFcdNFFPPTQQ17tTdUDqKio4M0332T+/Pns3buX0NBQMjIyuOKKKxg1ahQAO3bs4P3332f58uXk5ubicrnIzMxk/PjxjB071vNaDz30EJ999hkAY8aM8bTfcsst3HbbbU32oaSkhClTpvD9999TWFhIXFwcI0aM4LbbbiMmJsbnHF5++WU2bNjAzJkz2bdvHykpKdx4441cdNFFh/XeT5kyhalTpzJz5kw+//xzPv/8c4qLi8nIyODOO+/kjDPO8Gy7dOlSbr/9dv7+979z8cUXe71Ow/kuXbrU09ZQJ2LKlClMnjyZpUuXYjKZOPPMM7n33nsJCQlh+vTpzJ49m4KCAjIzM/nzn//MgAEDDqvvJ5LaX/PY9X8/UrmvBteYvgwcl8kju/Zgr3DQrWcqZlN8o/sVL8pnz1tbsQRbSLsxi8g+++/YmS0mEk5J4qSvl5H4v9WEFNdQ9ZGFDycPYEmnbqzN6EtCSSmXrt/C3oRENicnYA+z0nFlLj/c9gP9/jKQ0loL6xYWEx5jY9hFCUQneH/hX/tjMRsXlxKdEMTQCxOI7KAvce1VaWQ45eHxdKgMoW5tKTOr0zI4s8apOWjipabSxYKXtlNT6aJz+S7O2fgLwXYnBUEJlNsi2RcZxbbB04g+P5NtSSlUOCx0OTeZzsOTAt11CTCX2+DfPzmZMq+CK37aQOfcjpgNN9P+8xVhTjs9d++ig6WG4IdG0fEPAzCpJkqrUbCxjPWzdmG4DHpemkZkxzDWfriD0l1VpJ+eQNYFHXFtLKB6ylKMSjsh1w8g6IzO/LTHzQcfFxKys4zeeXkkbM8j2GEnq3wfUWVFWK21fHfGEGb37ceV8z5neOFOSocMZc2GSGr3VZFp2kd8ZC1rU7qwOTUDc0ww+QVdCYqpoqrMSXRcy30vqSx1svjzfWzY62RxchymSAtXrVvEWetXszRtIHvDEonfVUy03U6H0xJJm9AVc1DTNwwNw2Dz59ns/rmAqAgTmXuzCap1EHxNf9y7Sil/dy3lRQb2wZmk/a4vET2jAcids4u8z/YQmhpG+m3dCUnZf0e7ptLF4s/z2bermvF/ymyxc2/Laj/dQM0n67FE2wizlVC7fA97qjuwOCSTUocJcOOyWEjNDadTcSE7X36N+Kt7ssGUQMnPe9kUFsaPGZ1J3l1BYn45VquJ2PQwBpyfwKBR8Vhtdd+CqsqcLPosn+y1ZVBchd0FoR0j6H9OPH1Ob/npUW6Hmz1vbqH4p3wi+8eSfnM3rBGB/R5+REkCp9PJ7373O3799VdGjx7NFVdcQUVFBZ988gk33XQTU6dOpXfv3p5t77rrLtatW8fo0aPp168fmzZtYtKkSURHRx9Vp8vLy7npppvYtm0bv/nNbxg/fjwul4uNGzfy448/epIES5cuZfny5Zxxxhl07NiRmpoavvnmGx577DGKi4uZOHEiAJdddhmVlZXMnz+fu+++2/MDv1u3bk32oaKightvvJHdu3czZswYevbsycaNG5k5cyZLlizhzTffJDzc+87Kiy++SG1tLZdddhlBQUHMnDmThx56iE6dOh3Rj+2HHnoIq9XKtddei8Ph4L333uNPf/oTs2bNomPHjod+gSZUV1dzxx13cPLJJ3v+dnPnzqW2tpaYmBjWrFnDFVdcgdPp5O233+buu+/m008/9TnPE5ljVS6FQ6YQ5XARBbh/2ch/PxxMlavuorP+k92c++TJZJ3v/XfKn5fN0kvnY9TPS9j1+mZOXXA+0QPqhkhtXVFG8djnyNpVQC3ROAkjyAGd9hRxzdi+GCYT8VU1pBrBBLkN4srtxOcXUG3AlnXVbL1uIQXxcdSG1GVbl39dwJ3/6e1JBCx4L4dv39rr6c+Kbwq566XehIRrBE17tPK9LQyqqqJuwoEJl8nE+rR0iv6xiSteUnFWqeNyGUy5Yw0FBU4yyvK4dslXuLFSTiyJ1VUkUkXXgjy2xCUS+uJi4m02VnXry4bZuxn+QF/6XpUR6FOQALruvy7mLKnl3h/WkVhQBGYT7vo0ZF5UNJtPO43f/LKKzn+cy4btNfR67tQA91gA8n4tZvbEn3HX1zJa9/EuwuKDqcyrAWDTp3vYtzCXbs99gVFZV5Cy+rXlbHnpCh5dHsGQPQWEVlVTVVTCTmyAje2WNM4uKCPc4eLM976ga8K39Mrfy15zJ1ZudgBFAOw2wqiOquR/SakYe1ywpwqIoSY3hjfu3cKdL/YhKOToU9n2Gjev3rOB7YVuXhvanepcK+TCNNvpTKwNJXWFhd7Lf6WqpIoqIOe97RTOz+Hkd89s8jV/+tc6fn17u+fxxppqzt6yhuqpyzxtEUD1kh389O4OTvlhNPv+u4dND670PL/nra0MX3oRtthgXC6D1+/bSO62aqwOh5IEQOWzP1Hxxy8BN3FswYydUKAbUJIygLkDLvRsW5oaRr8duwgqcPBxbTJ2ayUANuz0yQZnaCi1WKh1QuW2Gva8tJutK8v57d+64qiti4/C7INWZtldxrpFZZwzoSNnXpnSoue26qaF5Hy4o+7Bu5A3exenzB8V0OTpESUJPvjgA5YtW8YLL7zAqafuv5iPHz+eK6+8kmeffZZXX30VgLlz57Ju3TrP3fgGmZmZTJ48mZQU/9/cF198kW3btvHAAw9w2WWXeT3nPmAh5wsvvJDx48d7Pf/b3/6W22+/nenTp3PddddhtVrp378/WVlZzJ8/n7POOuuwfmi/+eab7Nq1i/vuu4/LL7/c0969e3eeeuopZsyYwR133OG1j91uZ8aMGZ5hdr/5zW+45JJL+PDDD48oSRATE8MzzzzjCZzBgwczYcIEZs2axV133XXYr3OwkpISrr/+eq6//npPW3l5Od988w09e/Zk2rRpWK11IZOZmck999zDl19+ybhx4/w+ZntT8tRPWBz7qxWagb67dvBjZg+C7HVFvlZM2+qTJNj27DpPggDAXeNix4sbOGnqaQAsfXsbl+5aRzHeiavu+/I4KTuHlZ06MiSniKD69cojKirrCtHVMzARWVbuSRJUljhZ8U0hI65Ixu0yWDgrz+t1S/PtrP6uiCGjE476PZHWxbE+l5Q9RZgPKPJjMQz679jK2vJUSndXqoaFALBlWRkFBXWVVkdu/gWw4CAYA+/kYXpxIbkRUXSsKKVTSRHb4xJZMW2rkgQnsNxKg/fWuBm9ex9xRaU+K6eEVlVTFh3Jps6dSF+dS8Grq3H+czDWcI1gC7Rf397uSRAAGC7DkyBosPbzHDKrHPuvBIZBzXO/MLDLQAAiyiu8tndYrGyPS6Bv7h6qiadX/gYAtlm8v9M4TVYWdhmA0ciah0U5dtb/XMxJI4++hsW6n4op2lvLyswkqm3eP4Xm9O7NvfNWEF3iXQE/d9YuKreWE97Vd/i6vdLJ2g93erWVh4SSGxlDalmxV3uoYSeiopQdL20g75NdXs/VZFeR/cEOMm7vwZalpeRuqwYgvMK7LyeqqqcWAhBMGVa8V0wZkLeGL5zn4rDWjZKtDQpibVoG8cXF2K3e15W44jLyQn1X/Fr/UwmF2TVkb67yTRAcYOHHeYy4IrnFfsBX76ok56MdXm3Fv+RTvHAfHc4I3Ki8I0rHffHFF2RkZNCrVy9KSko8/zmdToYNG8aqVauoqam7kCxYsACLxcI111zj9Rrjx48/qjvPbrebefPmkZmZ6ZMgADCb959S6AEBUFtbS0lJCWVlZZxyyilUVlayY8cOv/uxYMECYmNjufTSS73aL7vsMmJjY5k/f77PPpdffrnXPLzExETS09PZvXv3ER37qquu8grMPn36EBYWxq5du5rZ69AsFgtXXnmlV9uAAQMwDINx48Z5EgQAAwfWfRAcad+PpaKiImpr9/+jrqio8Fqtwm63U1hY6LVPTk5Os49zc3MxDvgxdahjOIp9L+RBTgfuAz7wqku9P2xzc3Nxlvku8eMsd3iOYS+pwtRESbmI+v4Eu/YnJ0yGb7Vxs9u7cn1led1+breBvZFlGIoKvFf6aOn36lj8PfzRGuLmeB7DVVKNqZE1gyqsdV/37BX7l99ozefRksfwR3t9Lw48Rm3V/utCkLPuemQ0ch2yGm6c9XV7rO66fewV+69pgT6PY3UMf7TX9+LgYxTXr/0b6nT5VB2H+jFMhoHDasGNCZPTTe7u1nceJ2Lc2CsPvQSTy23gOuiHvK3agbX+9Uxu37+5w1x3jTAw0/Cs0+R7r9JhbTpRVFPpapG/aVF+GQC1jdQbs1stWJyNr/TT8L3s4GOUFZXhamR1IEcT9cwsuCnfV4azwve9dpbV/fgtrO8j0Ohn9uFob/+GjLK645jxfT8sbqfn88dzbKvVE3cHauw7coOaKpfXZ19j7DVu3O6We6+cFXU3EQ/mLHcE5HrT4IhGEmzfvp3a2lrOOeecJrcpKSkhOTmZ7Oxs4uPjiYjwXnMyKCiI1NRUv5cabPihf+BIhqZUVVXx6quv8vXXX5OXl+fzfFlZWSN7HZ69e/fSq1cvrx/OAFarlfT0dDZs2OCzT2qqb5GM6OhocnNzPY9LS0txOLx/MMbHe89h79TJe2XZhtcpLS09onM4WHx8PMHBwV5tUVFRAD6jKxraj/aYLalDhw5ejxuLvbg47wz0wSNaDn6cnJx8RMfocNsgSr/Y5NW2KSWV4AP+wfa6pLPPMWquzqR0qfdFNPXqTM8xelzWjexZCUSU1OAixLNNQXg4v2SkA7A6IYa+BXV/j+qwUEKrvZMRlQck58xmGDCybpSA1Wam7/AO/LqgyPO81WZi6EFFXVr6vToWfw9/tIa4OZ7HMIbFsiQjldRVJV7Pv3rKIM5yVBHfM6pNnEdLHsMf7fW9OPAY3YdEE2QFuxOWpffl/HVLsFFLDeEcuDbG3qgYkstLcZlMZEfXzdXsfuH+60egz+NYHcMf7fW9OPgYvRKDOCnRwU8l8QzatpeYMu/vfPYgGy6rlc578wnCTvCITnTq6b1KT2s4jxMxbrqNTmXXD/u8njPbTLgd+3+IpPWKIGi19w+p6ot7sa0ogi7FFVSHhRJ54GgCwyCtpO47TgilVASHEFlbQ0fXbrZYe3lt12fvVn6MHsjBbCFmep8W61MvyZ+/6eBzUvnx/RL65BWzJC3ea4npQXtyKYsNpzbYSnDt/h/xEX1iiDopttFjxKd1IO20BHb/lO9ps7pcpJSV+JyHCzMl5ggG3tSHnKidZL+7zfOcOchMyri674gnj0xlwYwiqitcVIf53vU+HO3t31DINf2pfnUpNUQRQa5XsmB7bAbVQfvfJ5Nh0D1nD+HVNWxM7Oj1N65q4v1MSAuhY1YYUXFBfPnaHuw1jSdn+g6PxWIxtdx71RuiToqlbNX+USfByaHEjUzBEuKd5Dge15sGR1y4MCsrq8nigwCxsS1bzMHlaj6b05y//vWv/Pjjj1x66aWcfPLJREdHYzabWbhwIe+++67X1ITj4cBRDgc6MAP05z//meXLl3s9f2CBwcN9neaGwDT1njb1uod7TIGQS3pR+dxFlDy+EFeVg31Du5M+YQDuT3djr3DQ/cJUBt3qW+ui86SeuGvd7J6+BXOQmYxJPUm+JN3z/NALE1g8+S5qHn6H+N3luN1B2G0mlnbrwtk7cliVEIultpaQ0kJqIqLYFx3BnvAQemTn0cHkpPcNWeSHRbJuYQkRMTbOujqFlC77i+OM+V06oREWNiwqJSYxiLOv7UhscrBPP6XtM5nNuO4azt8/TuGapStwmc28fspgFnTrypQJZhUPE4+QcAs3/rsncx/eyEpXfzoX7qF33m7CKaaWcFxYsWIn1BmOtWssa9M6Y7FFMOC8FIbe1SPQ3ZcA+/JKK5d+EMYn5VlctWgdoTU1uM1mHEE2XME2hq7eQO+8LZSd1Y8BM88NdHelXvcLU7FXOFjz/k4Mt0Hv8ekk9Yth0QsbKdtdSdrpiZz6x164L4qlavJPGJUOQiYM4Nz7z2DF9y7WfLiHpDwrZoubuMJSQpwOeuTvIbG6BJulkm/6d2fKKcN5+IsPGJCzA1dCHNsrknE73XR25ZCWm0dVdBjrUrMwgq3YXXaCoqu5+k8DW6ygclRcEBMe68a3b2Xj2pXNzxmJOINMXLV+MXd++wUL+pzNrjO7kLEjn/DyGuJOT6TnPwc1+/l4zhMD+fmZ9ez5OZ/IYIPeu3cSlhlF8Pg+uHLKqZ69kWqnlcL0NPrcP4TECzrRYUQS1hgb+z7bQ0incLr9tT/hXesS9SHhFm74Z3e+np5N/m5NwwGIfPYCTBFB1MxaR0VEDOHmfNw78tljTmJBx2FElpVRG1p3I63fzu1EVldTExbE2clFrA7tROmeKjbGR/Ndzy4M2lNETI0Ds2FgtZroNjSG0bd2wmQyEdnBxoTHuvHNjGxyN1dirrbjNpkxh9vod3Yc597Q8isjDPp4JBv+spzin/YR2T+Wnv842SdBcLwdUZIgLS2N4uJihgwZ0uwPSqi7a75o0SIqKiq8skp2u53s7GzPnegGTd0Jz87O9nocExNDVFQUmzdvbvb45eXl/Pjjj4wePZoHHnjA67nFixf7bH+kX4xTU1PZuXMnTqfTazSB0+lk165djY4aOBx//OMfj2qEQ4OG4pCH855Ky4r7/VDifj8UgO71bUOuzWh2H5PJRJe7+9Dl7j5NbjN0YjeY+JBX2wTg6qJy1l/6JIlbK4nu2pn1sd341egAPTtwztM9SeuxfwTBBTd736lpEBxq4aJJ6Vw06VBnJ+3ByY4qPkjL4KXYJDrU2CmMDuf8bTlEJagwknhL7R7OHe+cDIBrugvnxBnYcGFj/+dUr81/w5QSTcuWcZK2LjncxM83BsONqTgciUybNg0LcKuWQGz1+l6ZQd8rM7zaLnn9oBG81w0g9LoBXk33j7TAyC7Nvva4+v+gbt9e9f8d6NL6/xwOB9OmTQMgvdcZtKTOfSK48Yke3OjVeiZwJtc0vkuzQmKCGPlw00tOx9T/74HvjjXcRp/JQ+kzeWij+3TMCmPCY00XUT/RmEJtRD59PpFPn+9pswCZwE0HbFcXN6v5lngm1l9vuh/8YviOyj5Qeu+6+DheQjuFM/Ct4cfteIfjiGoSXHjhhRQWFvLOO+80+vyBc07OPPNMXC6Xz7YzZ86ksrLSZ9/09HRWr17tqWkAddMB5s6d691hs5lRo0axbds2Zs+e7fM6DXe2G5IYB9/pLigoaHS/sLAwzzEPx5lnnklxcbHPa82ePZvi4mJGjhx5WK9zsF69ejFs2DCv//zRsWNHLBaLT0Jk1apVrF692q/XlNbHqHVg9H6Unt8X0iG7Bsv3GylYnsO28iC2/VrBq3/cwNaVR590kvYlvqycEbv3kVVSQYcaO4PyiumfW4it2rc2hojH+X1wHTS/syY9BVPK0a1YJCcGwwC3w4xR5cCwH3reu4jI4TIMA3dZ3W/IymI7RpWL4HL7IfaS5hzRSIKrr76aRYsW8dxzz7FkyRKGDBlCeHg4ubm5LFmyhKCgIKZMmQLAmDFj+OSTT5g6dSrZ2dn079+fjRs38s0339CpUyefIe9XXHEF//d//8ftt9/O6NGjKS8vZ/bs2aSkpPgUvLjjjjtYsmQJjz32GIsWLeKkk+oydxs3bsTpdPLoo48SHh7OKaecwhdffEFwcDB9+vQhJyeHWbNmkZqa6nOHvW/fvgA8//zzXHDBBQQFBdG1a1eysrIafS8mTJjAt99+y1NPPcXGjRvp0aMHGzduZM6cOXTu3NlrhYBACAsL4+KLL2b27Nk88MADDBo0iN27d/Ppp5/SrVs3Nm3adOgXkVbP9dFyyPOe63nqtl/5ObM/rvqCOV+9vodJL/QORPekldoXGw0m74rLNcFB2MOC0P09aUrB8jK2WHrR1dhFuFFFkTmGLcXpnFnuwBqpyJGmrf+5lILPe3HmkrUU/u5JzGE2wv/fKUQ81nSNKxGRw1H75WbK7/yMkl2VbIhPIz23lG4RoWzqEoHliZvhy/+DQY3/npOmHVGSwGq18uyzzzJz5kz++9//ehICCQkJ9OnTh4suusizrc1m48UXX+S5557ju+++43//+x+9e/fmxRdf5Nlnn/WpznjBBReQn5/Phx9+yDPPPENqaio333wzZrOZNWvWeG0bFRXFtGnTeOONN5g/fz7z588nPDyczMxMr+r8jz76KC+88AI//PADn3/+OWlpaUyaNAmr1crDDz/s9ZoDBgzgd7/7HbNmzeKxxx7D5XJxyy23NJkkiIiI4PXXX2fKlCl8//33zJ07l7i4OMaNG8dtt912VCs4tJS7774bwzBYsGAB3333Hb169WLy5Ml88sknShK0Fzm+00mCXQ6sbqcnSVBd7n9dD2mfKoMaqTehWgRyCPb8GkrNUSwP6ntAY10FZiUJpCllBXY+/vcuzli1hW65e+saK+1U/uN7LH0SCb26f2A7KCJtlru4mtLxH2BU2tkW25n03LrvxfEV1cSuruGD3idz9ahHYd+0uqrdcthMhirPibRZxq4i7Jl/gwOWG9oWl8q7Qy+o38BgxFUpnDuh5YusSNu1YNpuvv0wzycxcO/b/VusMJS0PzU5Vfyv68ccuPpUSGoYZ28dF7hOSau34ptCZk3ewXULvia62nuZ4JBr+hP99vgA9UyOp327qlnzfTHBYRYGnB1LyA8bMH7ahumkVDinN7XvrsadX0nwZb0p7p7MW6tdlFS7sK6fTYq11DO3XORANbPXU3rpe7gxkU+cz/PzBqXyp2XvwK/PQL/OjbyCNOWIVzcQkdbDlN4BZkyg9pa3CKp24bTZWJOchc3lxLCY6XlmHOdc3/HQLyQnlNiCci5a+gu5sR0oDQsnrTCfCmswQc7eoAkH0oSyVcUcvDy1vbAWZ4UDa4TiRhoXkxQEQHlomE+SwJLZsitiSeu0ZXkZbz+0BZez7oaG9f6PGLB5LQBuTFSEJeGuX5u+4p/fc++1Y3mrW900SQsXc1fMt4HpuLR6lowYAEwYVITYiKjxrq0URDWYTdCx5ZYGPFEoSSDSxpnG9GfhN10Jy3cy5L7xXD5cFeqleUklhZgK9pFRsH8tbJfJjEWfCNKM2txqTIZBjLucEMNBuTmMippQnKVKEkjTMvtF0n1IJIuKepK09Gds9TWpzJ1jCL3Lv+LM0rYseC/HkyCIqKmk/+Z1nufshHgSBAAmt8Et//2et/5QlyRwYeHzypP41/HtsrQRtgEphFzTn5p3fsURauCuBXP94Npf0xO4bNM38LvREBcZ2I62QfpKKNKGGQ4XZefO4ORFdUXoyj+fBs9eQNgfTj3EnnIiC7c4qDqozWK4MTtVv0KalnBeR7q59hLpqo8eVxGFndMISQ0LbMek1bvygQxeNq9g7oDeXBwbT3y3aIIv74M5spH6KNLulBXurzIfXluNmf1TJI1GFlpLLq/welzqDj12nZM2L+qtcYRcdxKDV+aSGxfLysWlFG/cxJDKRSS/fAeMHhzoLrZJShKItGG1n27EtSjbq63iofmE3jEEU5D+eUvjTGaDunHj+7+cmbFjuFWiRppmWp29P0FQLy4/F6PSjik8KEC9krbAZDYRmlIKKZA8cazmlp9gep8Wy8JZeQDkRcVRHBpJbHXdykw27NTinWj8orf3+vQDgncBMcejq9IGmUwmgkd1I3hUN7oC6RMcTJu2lkX0pve5JwW6e22WyjyKtGHuPWU+bUZJDUal1ruXpgXbi6kyu6jBhgGUm0IwUYzZphUOpGmNXW+ocuAurj7+nZE2xe0yqNobTdnGZPZsrAx0d+Q4+831HRl8QTy2YBPhsTay/3kdpjO6gtmEbWAyEfeeirlTFARZCLmmP8aT55MYDqFWGB6ykUvCVwT6FKQNqLAbvLKwlj9PKSTnx464f44kd0XxoXeURml1A5E2zLm1iMKez4NzfzUx2/DOdPj+pgD2Slq7kv/7nJ/+VeTVFmZUcGb2NZjiowLUK2ntnHtKKUifjOnArw2dYkjafXfgOiWtnmEYzHhwM1uWlXvazpnQkTOvTAlgr6QtcDgcTJs2DUCrG0iziqsNhr5ay5byupsdFpeb2xeupE9uIYNu7cbQu3oc4hXkYBpJINKGWbt2IOKtS6mMt2CYwHpuF6Lf0XJS0ryCUN8v51WmCGodmqIiTateX0qJEYmz/quDHRuFRTbcVRq5JE3bsbrCK0EA8N37udRWqQaKiLSM11a6PAkCAJfFzNy+WQCsmLaV6mJ7U7tKE/SNUKSNs43tyY8LU7Hmm7jgrxdhSYsOdJeklSsM963y67CZcYfpLo00zb6zDDtBFNIBMAATVBk4C2sIUuxIE0r2+X45d9S6qSx1EhxmCUCPRKS92VnqOzC+KDwEALfDTVV+DaGxqp1zJDSSQKQNM1xull/6HfFTg4mZHcTPQ79g97TNge6WtHI1oVYIc5Flz6Zv7Q7SnPsojQtCs8+kORHnpYPnRk3d/7F2DCcoTUtLSdO6DozEYvWud5KQHkKHFK1sICIt46JuvgnHvjkFAESmhtEhS59TR0pJApE2bN+X2RTNz9vfYMDG/1uB2+Fueic54cXYq+lfuoNoo4pgHCS6SulftAsUN9IMx5ZSOCiP5Cq14652BqZD0iZExQUx/t50LGG1AHTsFspVD3QJcK9EpD05v6uZJ0ZaiMCFyTA4aU8el6/YSFyPSM5/djAmswozHylNNxBpw6q2V/i02QtqcZY7COqguzTSuOSiQnB7JwSSy0sJMWmOsDStdmspBpAfG0VFeCgJxWVEVlbjLKjWaAJpVo9h0aScuwbDZeKmW25QAToRaXH3nW7lT6daqKiw897bX2LqZ+LSW3W98ZeSBCJtWMI5HVlvwuvuXvTgOCUIpFnVtmBCD2pzmc24LWY0Q1iaEnluOj+f3JPspLi6BsNgaHEu/ZUgkMNgMoHJqilNInLsWMwmwkLNmK3gM/RNjoimG4i0YRE9o+n94lBcEXUXwqjBHRgw/YwA90pau+zERErCwr3a1qZlUGMobyxNyy437U8QAJhM/JqaisOuaSoiIhJYhtOF88FPcfd4mIsfWU+XX4oOvZM0Sd8IRdq41Ou6sGzNPMJznAz+xwWEdNE699K8iOJqZp5yBn137SCstpZ9MbHsik7gTKcT0LA8aVxhdo1PW021m8oSJzGJqhotIiKB4/rHF7ge/S8AHYDh03ZijF8Po/sHtmNtlJIEIm2Y4XZTPv5DzpibD0DpzP/AW+MIuapfgHsmrVlwSTWRhTUszeoBJhMml5vua/dgsqqwjzSty0lReJY+rBcTaVaCQEREAs797hKfNuOD5UoS+EnTDUTaMPu8rTjmbtzf4HRT/scvMFwa/itNq4kJIT8lum6SMGBYzOzukojh1Pw9acbWErqu24vVXreaQWhlLV2W7MBVq4KXIiISYDFhjbQdXIFJDpdGEoi0Yc4N+T5t7twKjJIaTHGNXCxFgPIO4WCq9mqrigjBbrWikpfSlIoNpSTmlpKek4fN7KLGHYzDZMWeV0NoevihX0BEROQYsdx3Hs7xU8Gou+FhDzETfKvqdPlLSQKRNixopO9a09b+SZiVIJBmJJbs4+CBZCH2WkLMDlSTQJoSd1YynZ15xLvKgLqJB3tTMwhJ0/VGREQCy3LZQEzf343jrV9Ys2Mzm4bHM757YqC71WZpuoFIG2Y7KZmwf5+HM7hu2Li5exxRb40LcK+ktetclE/Gvtz9DYbBaRvXYnFq2Lg0zbavxJMggLrKBKkVueBQ3IiISOCZz8jC/J8rWTYulfJEjY08GhpJINLGhfx+GB8GrSG43M0Vf7oZW5CKiEnzXBHhXLjsW3bFJ1AaFkFaYT6xlRUYjkD3TFoz55p9vo2lNbhzKrB0jjnu/REREZFjQ0kCkXbAFWymKtiMyaTq9HJotjMyMAGdC/KBuroW5pQIzAkaNi5NCxrR2afN0iUWc5qWXRUREWlPNN1AROQEE3RGZ4JvOXl/Q6iVyJcvxmS1BK5T0upZeyYQ9vQB05s6RRH19jhMZn2VEBERaU80kkBE5AQU/uKFfNExm/B8J6MenEhwku4Gy6GF/G4YH1nWEFrsYtz9NxMUojmfIiIi7Y2SBCIiJ6jyjjbKO9owd9A6wnL4nKFmykPNmCwaQSAiItIe6RNeRERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEBERERERERFASQIRERERERERqackgYiIiIiIiIgAShKIiIiIiIiISD0lCUREREREREQEUJJAREREREREROopSSAiIiIiIiIigJIEIiIiIiIiIlJPSQIRERERERERAZQkEBEREREREZF6ShKIiIiIiIiICKAkgYiIiIiIiIjUU5JARERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEGkHXC4X1dXVVFZWBror0oYobsQfihvxh+JGRI4XXW+OnpIEIm3ckiVL2LlzJzk5OfzrX/9i0aJFge6StAGKG/GH4kb8obgRkeNF15uWoSSBSBtWUVHBZ5995nnscrmYM2cOFRUVAeyVtHYVFRV8/vnnnscul4tPP/1UcSPNUtyIPxQ3InK86HrTcpQkEGnDtm/fjtvt9mpzu91s3749QD2StiA3NxeXy+XV5nQ6yc3NDVCPpC1Q3Ig/FDcicrzoetNylCQQaYdMJlOguyCtWGRkZKPtUVFRx7kn0pYobsQfihsROV50vWk5ShKItGGJiYlH1C4CsG/fvkbb8/LyjnNPpC1R3Ig/FDdyvFVUVLBlyxYNMT8B6XrTcqyB7oCI+K+5i6ESBdKUmpqaRttra2uPc0+kLVHciD+aipum2kWOxqJFi5g7dy4ulwuLxcKYMWMYNmxYoLslx4muNy1HIwlE2jBdDEVEpC3StDhpaRUVFZ4EAahoneyn682RU5JApB3SxVCaExISckTtIqC4Ef8obuR4UdE60fWm5ShJICJyglEtC/GH4kb8obiR4yU5ORmLxeLVZrVaSUlJCVCP5HjT9ablKEkg0g4ZhhHoLkgrpsI+4g/FjfhDcSPHS0REBGPGjMFqrSu5ZrVaufjiiwkPDw9wz+R40fWm5ahwoUgb1tTwqdDQ0OPcE2lLVMtC/KHCheIPXW/al4qKCnJzc0lOTiYiIiLQ3fGRkZHBKaecgslkYvDgwSQlJQW6S3Ic6XOq5ShJINKGaViVtCTVshCR40XXm7anta8c8P7777Ny5UrP4x9//JGxY8e2qj6KtBWabiDShmlYlfhDhX3EH4ob8Yfipn1o7SsH5OXleSUIoG7q5dy5c1tNH+XY0/Wm5ShJINKGaViV+EMjUMQfihvxh+KmfWjtKwds2LCh0XaXy9Vq+ijHnq43LUdJAhGRE4xGoIg/FDfiD8VN+9DaVw7o2bNno+0Wi6XV9FGOPV1vWo6SBCLtkFY3kOZoBIr4Q3Ej/lDhwvahta8ckJSUxIABA7zaTCYTY8aMaTV9lGNP15uWo8KFIu2QCkKJyPGipKT4Q59TbUtFRQVxcXHcdNNN7N69mx49ehyzlQMaVlCIj48/ov0uuugiOnXqxJYtW4iJieHUU0/V6gYC6HrjDyUJRNowFWgRfyhuxB9aclX8oetN23fgqgYNvvrqq2OyusHBKyjExsYSFRV1WPvNnj3bK2m5aNEirW5wgtH1puVouoFIG6YCLeIPxY34Q3Ej/lDctG0Hr2rQ4FisbtDYCgqFhYU+x25svzlz5viMatLqBiceXW9ajpIEIm2YCrSIPxQ34g/FjfhDcdO2NbaqQYOWXt2gsWMZhoHdbj/kfm63u9HntLrBiUXXm5ajJIFIG6ZCYuIPxY34Q3Ej/lAhsbatsVUNGrT06gaNHctkMhEUFHTI/czmxn/SaHWDE4uuNy1HSQIREQFUgE5Ejh8VEmsbDl7VoMGxWN2gsRUU4uLimkxSHLjfJZdc4pMoMJvNWt1AAF1v/KHChSIiJxgVoBN/qCCU+ENx0/YNGzaMjIwMNm7cSHx8PHv27CE0NJSwsDAqKiqIiIho0WMlJiayYsUK+vfvz//+97/D3q9Pnz6sXLmSrVu3EhUVxWmnnabVDU4wut60HCUJRNoh3RGW5qiwj/hDcSP+UNy0fY2tbtDAZDK16AoC77//PitXrgRg8eLFhIeHH/YP/c8++8yzb8P+Wt3gxKLrTcs5IaYb7N27l8GDBzNlypRj8voXX3wxt9566zF5bRF/aFiVNEeFfcQfihvxh+KmbWtqdYMGLbmCQF5entePfIDKyspDFi5sal/DMJgzZ45WNziB6HrTck6IJEFjysvLmTJlCkuXLg10Vxo1ZcoUFixYcNSv8+677/Lpp58efYekVdKwKvGHCtCJPxQ34g/FTdvW3OoGDVpqBYENGzY02l5VVeX3vm63W6sbnEB0vWk5J0SSICUlhYULF3LTTTd52srLy5k6dSrLli0LYM+aNnXq1BZJErz33ntKErRjGlYl/lBySfyhuBF/KG7atuZWN2jQUisI9OzZs9H2sLAwv/c1m81a3eAEoutNy2nXSYLKykqgbuh1cHCwT2VWkbZOw6rEH0ouiT8UN+IPxU3b1tTqBg1acgWBpKQkBgwY4NUWHh5+yCUQm9rXZDJxySWXaHWDE4iuNy3nmP1q/vTTT3n44Yd56aWXWLVqFXPmzKG4uJisrCz+9Kc/0a9fP5YtW8ZLL73Exo0bCQ8P5/LLL+fmm2/2vMYvv/zCnDlzWLduHQUFBdhsNvr06cONN97IoEGDvI536623kpOTw8svv8zzzz/P0qVLKSsrY+nSpezdu5cxY8Zwyy23cNttt7F06VJuv/12oO6O/dSpU4G6EQcNd90/+ugjFixYwLZt2yguLiY6OpqhQ4dyxx130LFjR7/fl9raWqZPn85XX31FXl4eNpuNpKQkTjvtNP7whz94+gp1BVg+++wzz74NUyPmzZvHF198waZNmygqKiIsLIwBAwZw++23061bN8/2gwcPBiAnJ8fz/wHmzp1Lx44dGTx4MBdddBEPPfRQo3+7V155xbNfaWkpr732Gt9//z35+fmEhoaSkpLCeeedx/XXX+/3+yFHR+vBij+aSi6tWrWKU089tUUrVUvbV1FRQW5uLsXFxY0+v3PnTn0BEx8NcZOTk9Po84qbtqNhdYNVq1ZhNpspKiqirKyMsLAwoqOjycjIaLFjXXXVVQwbNozFixeTnJzMunXrDnvfiy66iE6dOh3z1Q0qKirYvn07NTU1hISEkJmZqc/NADvU51ReXh5hYWHk5uaSnJysv9dhOOa31v/zn//gcrm46qqrcDqdvP3229x11108/PDDPProo1x66aVccMEFfP3117zyyit07NiR0aNHA3U/VktLSxk9ejRJSUns27ePOXPmMGnSJF555RUGDhzodayqqipuu+02+vfvz6RJkygqKmq0T5mZmdx9991MnjyZkSNHMnLkSMB7ONPbb79N3759ufLKK4mOjmbr1q3Mnj2bJUuW8P777xMTE+PX+/Hkk08yd+5cLrzwQq655hpcLhe7d+9myZIlAMTGxvLII4/w4IMPMnDgQC699FKf1/jwww+Jjo7m0ksv9SxF88knn3DTTTfx9ttvk56eDsAjjzzC5MmTiYmJ4cYbb/TsHxsbe8T9vv/++1m+fDnjxo2jW7du1NbWsn37dpYtW6YkQSukwoXSnKa+dH377bcsWLCAMWPGqBq0AM1XNW/w8ccf43a7FTPicThxM2vWLMVNG7Fo0SJmz57d5MpJP/zwAwMGDOCqq65qkWPNmTMHt9vtaYuPj/erj8didYPG+tfSKzzIkTnwetPU999169bx/vvv43K5sFgs+p5zGI55ksDlcjF9+nRsNhtQ9wP9nnvu4b777mPatGn07t0bgEsuuYSLLrqIjz76yJMk+Nvf/uazbve4ceO44oormDZtmk+SoLS0lHHjxjFp0qRm+xQXF8dZZ53F5MmTycrK8hzvQO+//77PsUeMGMGkSZOYM2cOEyZMOLI3ot6CBQs47bTTePjhhxt9PjQ0lNGjR/Pggw+SmpraaN9eeOEFn75deOGF/Pa3v+Xdd9/l/vvvB2D06NG8/PLLdOjQodHXOVwVFRUsWbKE8ePHc++99/r9OiISeBUVFaxatarJ510uF59++il9+vRRpv0Ed6iq5gdSzEiDw40bwzAUN21ARUUFc+bMOeTSyitXrmTkyJFHdee+IXYO/AEOUFBQQGVlZZM36JrqY8PKCy0VY031r6WPI4fv4OtNU3G6atUqz99N33MOzzGvSTB+/HhPggDw/LDv27evJ0EAeKYS7Nq1y9N24A/hqqoqSkpKsFgs9O3bl7Vr1zZ6vOuuu65F+t1wbLfbTUVFBSUlJXTv3p2IiAjWrFnj9+tGRESwbds2tmzZctR9MwzD07fY2Fg6d+58VH1rSnBwMEFBQaxZs4a9e/e2+Ou3lKKiIq/qpRUVFZSXl3se2+12CgsLvfY5eBjkwY9zc3O9Ljit9RgHKykpaZPncbTH8Ed7fS+aOkZubq7PF5yDOZ1OcnNzW/V5tOQx/NFe34sDj3E4Vc0bNMRMazyPY3UMf7TX90Jxc/jH8Ecg34vD+cxo0DAq9kiP0WD37t1Nxk5DraXGziMnJ6fJPrpcLrZv3+55fDTvVXOxfeAKD4qb1ne9OTg+Drz2tIbzOFbHOBrHfCRBamqq1+OoqCiARuf1R0VFUVpa6nm8Z88eXnzxRX755RevNxEaH04dGxtLZGRkS3SbJUuWMHXqVNauXevzZh/cl4NVVFT4zAmPjY3FYrFw99138/e//52rrrqK1NRUBg8ezPDhwxkxYgRm8+HlbDZs2MArr7zCsmXLqK6u9nru4Pe7JdhsNu6++26efvppxowZQ5cuXRg8eDBnnXUWQ4cObfHj+atDhw5ejw/ODgYFBREXF+fVdnDF24MfJycnt+pjNFWt9eDXbO3n0VLH8Ed7fS+aOkZwcDAWi6XZD1Wr1UpKSopPsafWdB4teQx/tNf34sBjNFQ1P5wvYA0xc6THgLb7Xvmjvb4XipvDP4Y/AvlemEwmzGbzYSUKDh6+faTvd1paWpOx0zBCobHziIiIaLKPFouFLl26eB4fzXvVXGwfuMKD4ub4HeNwrzcHx8eB157WcB7H6hhH45gnCZr64Xuo5VSqqqq45ZZbqK6u5uqrryYrK4vw8HBMJhPTp0/3yVZCyy1vsXbtWu666y46derEXXfdRceOHQkODsZkMvHAAw8c8kL573//26vgIOwvFnjWWWcxd+5cFi5cyPLly1m8eDFz5sxh4MCBvPTSS16jLhqTm5vLrbfeSnh4ODfddBMZGRmEhIRgMpl4+umnfZIG/mjsH9r48eM566yz+PHHH1m2bBnffvstH374Ieeeey6PP/74UR9T/KMqrnKkIiIiGDRoEIsXL270eavVysUXX6xq0OKpav7pp5/idDoxmUyNDuU0m82KGfE4OG6aorhpGyIiIrjkkkt85uEfbMCAASQkJBz1scaMGdNoTYLm4qSpPrbkygvN9a+ljyOH7+DrTVPJosGDB7N8+XKcTqe+5xymVrsm4OLFi8nPz+fBBx/0VPtv8PLLLx/16zdX2O3LL7/E5XLx/PPPe92Zr66uPuQoAoDrr7+eCy64wKvtwMxQdHQ0o0ePZvTo0RiGwQsvvMCMGTP47rvvOOecc5p97fnz51NVVcXkyZO9ViyAupoMBy8T09x5RkdHe43caJCdnd3o9vHx8YwdO5axY8ficrl48MEH+eqrr7j22mvp06dPs/2WY6O5JRCVKJCmpKWlNZokGDFiBCNGjNAcPfEYNmwYffr08Uw/OTgBDnU1cVQASg50YNzs2LGDb775xmeb0aNHK27aiANXN7BYLJSXl2MYBlFRUbjdbvr3799iqwg0HGvp0qW43W527tx5WEsgHrhfZWUlYWFhDBkypMVXN2iIba1u0HoczudUWloa5513nlY3OAKtNknQMNLg4LsWv/zyS4vMu2+Y119WVnbYx37jjTcOa7hVly5dvIY2NXC5XFRVVXlNiTCZTPTo0QPA6wd7WFhYoz/gG0ZmHNy3Tz75hMLCQp9hKKGhoY2eI0B6ejqrV6/2XOSg7v2YO3eu13YNUycOHKlhsVjo1q0bX331VZOvL8eelkCUlpSYmKgPTvERERFBVlZWk0tLBQcHH+ceSVuguGk/Glut4lhViG9slQJ/VzdYuHDhMVl1ICIign79+rXoa8rROdT1xmQyebaRw9NqkwQDBgwgLi6OZ599lpycHBITE9m0aRP//e9/ycrKOqrCfwAxMTGkpaUxb948OnXqRIcOHQgNDWXEiBGcddZZvPvuu/zhD3/g0ksvxWazsWjRIrZs2eL30odQN4Xi/PPPZ8SIEfTo0YPY2Fj27t3LzJkziYqKYsSIEZ5t+/bty+LFi5k+fTrJycmYTCZGjRrF6aefzgsvvMCDDz7IFVdcQWRkJKtWreKnn36iU6dOPlMF+vXrx5w5c3j55ZfJzMzEZDIxYsQIQkNDueKKK/i///s/br/9dkaPHk15eTmzZ88mJSXFq3DGzp07ufXWWxk5ciRdu3YlMjKSHTt2MHPmTFJTU31WmZDA0xKI0pympma11JQtaZ8UN+KPpuLj4FWapHVqarWKY1EhvqlVClrL6gbS+ulzquW02iRBZGQk//nPf3j++ef54IMPcLlc9OzZk+eee445c+YcdZIA4NFHH2Xy5Mm8+OKL1NTUkJKSwogRIxgwYABPPfUUr732Gq+88grBwcEMHTqUV199lVtuucXv44WEhHD11VezePFiFi9eTFVVFfHx8YwYMYKJEyd6zeW6//77efLJJ5k2bRqVlZUAjBo1ik6dOvH888/z4osvMm3aNMxmMyeddBJTpkzhqaee8ql6OWnSJEpLS/noo488w8Pmzp1LaGgoF1xwAfn5+Xz44Yc888wzpKamcvPNN2M2m71GayQlJTFmzBiWLVvGggULcDgcJCQkcOmllzJhwgT9wxNpY1TLQvyhuBF/KG7atuaqxzdUiG+pu7PNraSQl5fXZJKguf0aVh3QHeQTg643LcdkHGrhUxFptZYsWcLHH3/s0z5u3DiGDBkSgB5JW7B69Wreeecdn/ZrrrlGQyilSYob8Yfipm2rqKjg8ccfbzRRYLVa+ctf/tJiBeAqKir45z//2egP/vvvv7/ZkQRN7WexWHjggQdUpO4EoetNyzm8NfdEpFXSME7xR1M1K1pqbV1pnxQ34g/FTdvWUD3eavUefHwsKsQ3rFJw8Mpoh7u6wcH7adWBE4+uNy2n1U43EJFD07AqERFpizSQte04sHq8xWJh9+7d9OjRo8VXDmg4VkZGBr/++itRUVEsX778kMumH7hfw+oG4eHhDB48+Jj0UeREoCSBSBumJRDFHyrsI/5Q3Ig/NOKtfYiIiKCwsNBTxPCrr746ZqsbNBzDYrEQGxtLVFTUYe138OoGP/744zFZ3UBaL31OtRxNNxBpwzSsSvyhESjiD8WN+ENx0z4cvMpBw+oGFRUVx/QYhYWFTRZOPHC/5lY3aMk+Suum603LUZJAROQE09wIFJGmKG7EH4qb9qGxVQ4aVjc4lscwDAO73X7I/Q61uoGcGHS9aTlKEoiInGA0AkX8obgRfyhu2ofk5GSf2gBWq5WUlJRjegyTyURQUNAh9zu4aGEDi8XSon2U1q2p601T7dI0JQlE2iEVhBJ/KG7EH4obkfbv4FUOjtXqBgcfIy4u7pCFC7W6gRyKyWQKdBfaHBUuFGnDVBBK/KG4EX8obsQfKiTWfhy4ykFycjIRERHH9Bjx8fF8+OGHR7Tf9u3bqampISQkhMzMzGPSR2m9dL1pOUoSiLRhKtAi/lDciD8UN+IPxU37EhERQVZW1nE5hsPhOOL9MjMzj2kSQ1o3XW9ajpIEIm2YlkAUfyhuxB+KG/GH4kaOl4OXTzwWSzRK66brTctRTQKRNkwFocQfihvxhwpCiT90vZHj4Xgs0Sitn643LUdJAhGRE4zm7Ik/VJNA/KHrjRwPx2OJRmn9dL1pOUoSiLRDqjYuzdGcPfGH4kb8obiR4+F4LNEorZ+uNy1HSQKRdkhLvUhzmpuzJ9IUxY34Q3Ejx8PxWKJRWj9db1qOCheKtGEaViX+0Nxy8YfiRvyhOcJyvAwbNoyMjAw2bNhAz549SUpKCnSX5DjT9ablKEkg0oZpWJW0JI1AEX8obkSkNThwdYN58+ZpdQPx0DTcI6fpBiJtmIZViT80AkX8obgRfyhu5HjQ6gYCKrDbkpQkEGnDNKxK/KERKOIPxY34Q3Ejx4NWNxDQ9aYlKUkgInKC0QgU8YfiRvyhuJHjQasbCOh605KUJBAROcGoAJ34QyOXxB+KGzketLqBgK43LUmFC0XaMM31FJHjRdcbaUkqJCYtbdiwYfTp04fc3FySk5OJiIgIdJdE2iwlCUTasMzMTMxmM26329NmNpvJzMwMYK+ktVNhH/FHZmYmJpPJ68edrjdyKLreyPEUERFBVlZWoLshAaJkdsvRdAORNiwiIoKLLrrI89hisXDJJZdoeJ00qyG5dCD92JNDiYiI4OKLL/Y81vVGDkdDculAut6IyLGg603LUZJApI0bMmQInTt3JiUlhT//+c9aE1gOSckl8ZeuN3KklFwSkeNF15uWo+kGIu2AxWIhNDRUF0E5bEOGDGHVqlXY7XYmTJhATExMoLskbYSuN3KkdL0RkeNF15uWoZEEIiInKP3YE5HjRdcbETledL05ekoSiIiIiIiIiAigJIGIiIiIiIiI1FOSQEREREREREQAJQlEREREREREpJ6SBCIiIiIiIiICKEkgIiIiIiIiIvWUJBARERERERERQEkCEREREREREamnJIGIiIiIiIiIAEoSiIiIiIiIiEg9JQlEREREREREBFCSQERERERERETqKUkgIiIiIiIiIoCSBCIiIiIiIiJST0kCEREREREREQGUJBARERERERGRekoSiIiIiIiIiAigJIGIiIiIiIiI1FOSQEREREREREQAsAa6AyIicvzZt5TQ7c1aQve5Kdq7hIT7h2IO0UeCHB6XYQp0F0REROQY0TdCEZETjKusluyRs0jOcwJQ9PAinFtKSX37ggD3TFq7eWscPLfxPIodEXz1YgXPXBNFZoIl0N2SVs7ugsWOLux1xZCxG87rEugeiYhIczTdQETkBFP+yRZceVVebWXvb8RVUhOgHklbsLfYxaQZVRQ7IgBYvM3FbdPKAtwrae0Mw2DMHHi9+iy+sA9g1CwTf1/oCnS3RESkGUoSiIicYKrL60YQFIeFsC45HofZjOEGTBpCLk1bsMGB2+7mrE07uG7xr/TZu481e1zsLdYPPmna/3YZ/G+397XlqSUGpbVGgHokIiKHoukGIiInmD1J8UwfOYSpZwzEYbUQX17Jff9bTLphJiLQnZNWq2OUicfnfsugPbkA3LBoFdOHDyQm7MwA90xas+wK37YaJxRVQ3Tw8e+PiIgcmkYSiIicYHZFhvPSmYM4aec+xi7eRIjdxeOjTiMoRB8J0rRBu/Z6EgQNrl2ymhCnM0A9krbgvM5gNnmPGkgOg8wYjVwSEWmtNJJApA37etoulr+5DXN1FzZ1jqLXSBfDu9kC3S1p5fbGhPPP975l+MY9ALhN8NTFp1Do6ENKSIA7J62Ws5FbwtYaJ67iGszhuu5I49YXgtsJmOsTBQYUV0FZrUFUsBIFIiKtkW4bibRRO37KZ8N/1hNWXkuI00X/rcW88tBWqhya5ynNG7Al15MggLrv7pO+WUGcTXPLpWkRJ8cCbq+24DAHtk6RgemQtAm7ywC3ganKgbmsFlOti1onFFQHumciItIUJQlE2qjFn+X5tPXYmc/8nUoSSPO6FJT6tEVV1UKJPQC9kbbCtnIjnViPjbpfd2GU0qlqJWQXBrZj0qqdm2kiqLQaW3E11vJabIWVdDRq6aLpBtKEqgoXP39TzKL/leJyaIlVOXKmChO2JUHs+GgnjgpHoLvTJmm6gUgbFZ0YzMFpgvKQIJIj9MVLmuc0mTCAAyPFZTZhjtCQcWlGYhSRFBFJEQYmTBhgs0BUWKB7Jq3YljxnXaXCA1QW1VJRG06EphvIQfbtreW5v+2goqxuZJvFNojMob8GuFfSlpSsKSHihUhMtSaWfb6M9ZPXc86nvyEsRZ9VR0IjCUTaqNN/m4YjIsjz2GUyUTAyg0HJ+tIlzXPYrNRarDSMOXFhosZsxV3rbnY/OcEF708imRqix2wCq75KSNN2FLrAMIh1ukizOwl3ual2QF65rjfi6+tPCjwJAgCXI4iC7WkB7JG0NeueXY+pdv934aq9VWx8dVMAe9Q2aSSBSBsVkRjCrbNO5cPX97BybQ6uzBpevCch0N2SNmBvSixVQTYcDjMmwDCZyEuKxhFsJeiQe8sJa0e+b1utEwrKIU1r2Unjzu5u4yS7g0TH/h9+RXFBdIlTckl8Feb5Dg23V6uirhy+yl2VPm0VOxtZi1WapSu0SBsWERfCdX/sTN/T13NSx+3YNHVPDkOwYWfhGT3ZlR7P7rQ4tnZJZMXgTCwWjUKRptkH9cTAO0aqw2IhLT5APZK2oDTf6ZUgAEgud1BrV/0c8dVrQIRPW0RccQB6Im1V0plJPm0pZ6cEoCdtm5IEIm3Y7jI3E14t5OP1A8nPDQ10d6SN6Je9FWeYhbUDM1hzciab+qaRlb+b4NLyQHdNWrHCX2tYxwAKSCKXThSQyK9VJ1GbUxXorkkrlrPP6dNmtxsUl2q6gfgaeXEcw86OwWIFiwViUnOJy8gOdLekDen9x144+tgxTAbmYDPdb+pG1992CXS32hxNNxBpoz7e4OLyT90YpnhIjedbRz+CH17Knx47NdBdk1YuJzIep9XkuSdsAvJj4nFHh6LBKNIUa2wQBXSkmLo7MgbgtlqwhOurhDStX0/fqSgRkWaSE3S1EV9Wm4nfTurIuBuTcTgcvPvej4HukrQx1jAr1ZdXQ201199wPSGRmq7iD40kEGmjbv/cjmHaP/TXZbPyekUyxaVaxk6atzvedyheaUgEFTWabiBNs0basLB/iLgJsFrAHKSvEtK03WUG24NsNMw0rzaZWGeyUa1VyaQZwSFmgkN0bZGjEAyWECUj/aX0v0gbVei2ctD0YPZGx7JnezmxA+IC0ylpE6rMViosZlZGhFNqs9Cpxk6v6mqCw/RhKk2r3VpGOBWks5NQqikhml21Gdj31RCS7juPWARgyz4XxVYLxRYzVuqWYMUJeaVuMjWaQESkVVKSQKSN6hHuZEOV97r2/XJ20aNfvwD1SNqKapuFjxPjqLbU3aXZHRJMdmgw1TUGISpSL02IHhhNB1Zjpa4IXTI1hIW5CE4LD3DPpDU7Lav+c8pkoqE6QWyEic5a3UBEpNVSkkCkjfrvNcEMeqWaYkswGAadCgt48IIwglShXg5hn9XiSRA02BNswxKs2JGmhazbhoF3lfqoqgLYVwZJ0QHqlbR2OVVQEmQlyu7EDDhNkGO2UuuCUOUJ5CCGYVDywXK+n7Od704aQFVZN4LsJqYvsnN+fytpUc1/TlVuKKFiVRHRpyQS0lkjnE5E735bybwVZ9B3dw5LP/iA9PuHE1XjxBRsJfw3aZgsuvAcDiUJRNqozGgzRfeFs3hLBZ+9+ykdM8oZedHEQHdL2gCr0+XbaIBLxcalORG+w0wMqwVTsL5KSNPW5bmptloIczqJr3WwOyyEKqeZ7FKDrHglJmU/o9pOYb9/ssCcyJ/OvQz7XisQQ7TLxS8fVPK7hVaeO9/GbSc1/iNv21+XsfOfv9Y9MJvo9txQOt3V+/idgASUw2FwwQP5dN1VTKoLikNSeC84mbFXf0NKYRkAwX3jSJ9/OdZ4rQh2KEqliLRxAzsH0ym5HLP+NcthGpSdT3RNtVfbqbv2EFqtopfStD1dulAQHuPVtqpzb9zh+rIlTRueaeHsvEJu2JnDRbkF3LptD2cUFZIRYxx6ZzmhuF5bSND2Qh4eMRq7ZX/ysdRiweJw0bGkmnvmuymr9Y2dqi1l7Hz81/0NboOt9y7FUVx7PLourcBbX5aTkFtBiGv/jRCTycTPfXt6HteuKaTomeWB6F6bo/S/SBuUu62Kb2fsYdmvxQzb/BOn5e1gU0Ian4Rnc8U1GYHunrRyHSvz+e6jGfzx7N+ypUMyF25ewV9++Qpr6MuB7pq0Yvv2Ovl2yKUM2bWa+MoStselsiK1F12KHMQkqZiFNC64pJaBJeWexxbgjJxCnJ+VYx2rGjqyn7FsFwVh4RSG+U4TqDWZCK91UemE7aVwUqL385Vri+Gg3IG72kX1ljJsQxKOYa+ltViyxeGVIGhQHOUdT7W/5h+vLrVpShKItDGVJQ6m3buBbIeZWxe9R/+8XQD0zd/Npj9s57v+/+LMfmEB7qW0ZonOAv549ni+6jcYgJeSkxlWsIPflpZBWIcA905aq+SuYdTYgvmx62BPW1CImeiEoAD2Slq7vZsqfNoc1iBKlueSrCSBHMB8Xm+S31xMemkRu6K9P4vCDDc7Q20khELPRj6mooYlYLKZMRz7581ZY4MI7xt7rLstrcSFA4N4c6WVKLv3+qopBcVej8NGdDqe3WqzNEBZpI1Zt7CYmgonTnOlJ0HQoHthDj+9uDIwHZM247v0Hrzf7zTPY6fFyh9HT8ARGxnAXklrV7SnBhNQarOyMzyMWrMZZ62bqlLnIfeVE1dm/yiMg27xut1u4s/uHKAeSWtlvmoQVRf04d/zZpJcUVrXZhgkOJ2UhAdTEx/MW6PNBFt9a1kEJ4fR49XTsETWraZhiw+m14wRWEJ1P/REceGZEVi7R1AaZPNccQy3m2Hr13u2ibw0i9jfDQhI/9oa/csRaWMs9R+OLlMTOT4VEZNDWJ2SAau9h+TtC4uiwGElJSQwfZLWz2w18VNiHKtiY8Bkwup2c25OHmatqCLNyCl180N6IqfuKcDmNqi0WfhvVho3VFbTP9Cdk1bFZDYT/99JnJ5dzCufbWVnr1g2LvoaS42JcyZcwkmpQc2u4JRyQzcSxmdQs62csB7RmIMtx7H3Emgmk4n3/prAll0RPD51Gf32ZDOmaxIp027Gkl+JKdiCLU03Qw6XRhIEQHZ2Nvfccw/nnHMOgwcP5qGHHjqi/R966CEGDx586A2lXep9RiyR8UFEuEL5Mb2H13MrUrpw/v8bEJiOSZuRWFnj0xbtcNDBpkJi0rSa1EhWdYgFU92XdKfZzMK0JKzh+iIuTVu4qpolafFM75fOV0mxfNWxA7tiw/l5WVWguyatlCk1lqG3DWbsqZEkRpcSl1TCgBTzYS3xbI2wEdG/gxIEJ7DOKVaGpa0j7NRS0u47ndBoG0FZMUoQHCHdcgyAhx9+mM2bN3PjjTcSFxdHp05tZ27M0qVLWbZsGb/97W+JjNQ/tkAICbdy8+ReLJyZwxfhV/Brygp65O9ie1wKWf+6lLMyNT9YmhdndzBkXwUr4qNxms1E2J0MzynE7gglWCMJpAnr89zYzAbdq2uIsjvYFxTE9qBg8krddOqgL+TSuN8MCyfzmQJ2mG2sCQkDw6DjlnwGsR7oGujuiYhII5QkOM7sdjsrVqzgiiuu4Lrrrgt0d47YsmXLmDp1KhdffLGSBAEUkxjMhZMyuHBSBg5HX6ZNmwbA6aepQI8cmtXpwuRwcP2vmwl3u3EaUBYagkaNS3PK91UzMr+YCEfdVJWUylpiwh3EmaOoq1kv4qvYDXtMVoz6ESiYTOwNDeHprYm8t6cAOsUHtoMiIuJDSYLjrKioCMMwiIqKCnRXpI2q3VtJzbZyasJtzP26FPPKbZSWRlMVEc6c3b+SnhlFcGI4YekRpOwtwLpuB7YhnbAM777/NfJrqNxURnlhLQ4ndLmkk+YVn0BmbzMYvmcf6cW7CLVXsjemC1F2B44aF0Tox5407osFFaQ7vGtZpFXWsOzFrZzx194B6pW0dnPnFuEw+y6RuTYugfI5G3Cf2ouogR0wmfQZdCIyVu+mvMjOyl8chIRAt/5RlG3JZsmGEmYn9Ma0ticRtQa7XQVcfHI4tbnV5JQ66d3JSp+zEindU0VNqYOo3ELMOYVYTk6n/PtcyiLCqYmPpMvZCQSF6HOtPXO5DZbsrSuPuinPwYwvyynJGcVvtmxg9bMvUjx4IInxVpKTDCwdowgf1Y3gxNBAd7vVMxmGoUmox8lDDz3EZ5995tP+97//nYcffphbbrmF3r17M3XqVLZs2UJkZCSjR4/mzjvvxGq1+rzO0qVLAfjss8946KGHeOWVVzy1CpxOJyNHjqS6upq3336bnj17AlBZWcnZZ5/N2LFj+ctf/uJ5zZkzZ/Luu++Sk5NDcnIyV111FWFhYTz88MOe122q/7fccgu33XZbi75X0rid/7eUXY+voiAmlKqwugq+9nALW7ukUxMUjGEY/NwxHjsOJixfza2L3iOYagCc3TOwrnic7VO3svGvKzDsbjAMbC43NXGhDP72fJL7Rwfy9OQ4GX7XNv4553WG76mr+Ls3PIaP+15IfmJn7nqhD4md9eEp3sqXFXDDsyWk1HovLWUAF25azahvL8ISp6VXZb/qciev3/4rqZ9v4HfjTiE7Msbr+cc/+4mTdtUtTRZ5UiyD555NSIpi6ERhlFVjjHmW+XsiWBnfhyCHm56rswlyuCgLsTF5zMmsS0qiwGrB7a77qWK2mYmxGhSF18XJ+Ru2csXSjfTelUtUdV2tHQtOwqjGbrXy7UkDyElI4PIHutJruJb3bY82FLgZ/b6D7eUmwitqGFJQSlxlLUl2ByagxmTislXziNvloMiRACYTJpNB97/3p8sDAwLd/VZNhQuPo8suu4y7774bgJEjR/LII4/wyCOPkJmZCcDChQt55JFHOO2007j77rvp3r07b731FjNmzGj2dRsSA0uWLPG0rVmzhurqasxmsyeZALBixQpcLhdDhgzxtE2fPp0nnniCsLAw7rzzTi688ELeeustPvzwQ5/+jxw5EoC7777b0/+zzz77KN4VOVwVywvY/dhKKkKsngSBAWzqkkFNUN1dGpPJxGk5heyMiSWqcpcnQQBg3bSDintmsuG+5XUJgrodcFjMhBVUsfjmn4/3KUmAXLH4G0+CAKBjZQlnbvsBc62L/76yq5k95US1fNIiolxu3Ae12w03n6dlUfDwjwHpl7ReC97dS+yPu+hRsYXpX79JTE1doUKz280dKxdy7rYdnm3LVxWz+eFVAeqpBILx7y8oXJLL0sT+1IaHkLUhl6D6kUofD+3BxqQkSi1mT4IAwO1wU2SzeR5/2bMrcSVlngQBgAsrdoIIdjo5c81q3G6DT/69Hafj4KuXtAe//8rJ9lLABJk1tYRVO0muTxAAhBgGH590LiWuGE/RXcMwsfGh1VRuKQtUt9sETTc4jvr37098fDyTJ08mKyuL0aNHA7B3714Atm3bxocffkjHjh0BGDduHFdeeSUffPABN954Y5Ovm5ycTFpaGkuWLOGOO+4A6hIGMTEx9O7dm8WLF3Pttdd62k0mkyexUFpaytSpU8nKyuL1118nOLjux+bYsWMZN26cT/+zsrKYP38+Z511lqefcnyU/7IPAHvQ/n+29iAbjiCbz7YpldWs7JTKhDXe7aXf7gZ3qnejyVS3nuyW0hbusbRWmUX7fNq6F+VgNgz2rK8MQI+kNXM73eza44BuZmpsJmwuN2bDwGU2EVxtZ11sNHkrNpEU6I5Kq7JjdTmdSqqJNJdyzu49ZL9+P0uSMuhSWkB8VRWbONlr+5JFBQHqqQTEL1vZHZuM1e0Gh4uwKrvnqS3JdfWVHI3tZ3fDAfWZI6trfTZx1ddICa+tJaK6mjJzOCV5duI7qTJve/NLtgFmE9hdxNqdWPAdIB9sQGlEFJHl3is7lSwuIDxL07+bopEErcjBP7wbfswXFhZSVdX8UkGDBw9m3bp1nu2WLFnC4MGDGTp0KCtXrsTpdAJ1qxNkZWURExMDwKJFi6itrWX8+PGeBAFAfHw8F1xwQQuf4bFVVFREbe3+D4uKigrKy8s9j+12O4WFhV775OTkNPs4NzeXA2fkBPIYEYPqijsF2ffPCQ6yO7A6nBwsLzyEvntzfNqjTu8IB0/7NAxMgLtzxHE5j2N5DH+097hp7Bi7OiRwsK2xSRgmEynd9g/3be3n0VLH8Ed7fS8aO0ZxaTGpyVYwDAyTCbvVQo3NitNspsRspntpBYl9Y1r9eShuju8xEjIsVESFUG7UfQkPczo4M3szaRXFlFijcR5U7DKoV2irPA/FzTE6xqAMUkv24TSbcFnNVIfuv+GRua/upoXvLRDA5v3TpTzEd0UnM3Xfk6qDgqgIDSHYCtGJQcfmPFDcBPIYg1JM4DbAZqYkyIqrkdomtSYTkZUVPu1RA/dPQQn0eRyrYxwN1SQ4zvbu3cuYMWO85vE3tN10002ekQANpkyZwtSpU/n0009JSUkBfGsSAMybN48HHniA5557jkGDBnH22Wdz991307dvX6699lpee+01MjMzOffcc7nyyiu55557gLqpBv/5z394+eWXvaYgALz33ns8/fTTXrUOGvozd+5cjSQIgG1//IXs59awLy6MmvoP1OrIILZ2ScNhtWEYBkuTO1BiMbhz8VImLv0QG3UXC2daCta1/2bLC5vZ/Oiv4DI8NQnsUcH0/+o8Og2LC+TpyXHy1/fyGf3npzk9exMAhSERvN/vQooTu3DTc31I6ap5weKtZGEef/x3PoQF1w3ZNAxCq2v4NSyIP637lYu/uABLUsShX0hOGJUlDl67aRURi3cxsuQHulTUjZqssoSy2dKHvNBkXPX3P8KyIhn65bmEpocHsMdyPBlFFRjn/ZuvylNY26E7oTVOeq7OxupyUxwWzJNjh7AtrgP5VjNGwwxJm5mIYBPlIXUjAkZs2cmNP6+hz84cwmvrRiKYcRFOFU6zmfn9T2JnUjKX/SmT/udoFY32aFWem/Pfc5BbYyKyvJqh+0rpUFVLYv0NNLvJxKh1C0jbUkGhM7F+yoFB13t60f3xIc2/+AlO0w1aEbO56YEdh8rlDB48GJPJxNKlS7FardjtdoYMGUJ6ejrR0dEsWbKEoqIi3G63TzJA2o4uz5xCyp29qNlaTonVwpx5pYSv2kxS5SYKo2PpkhrDBZnFRCRGEn71mVTu7E7Q2q0EDeuE9ZK6oZ1ZD/Qn9dqulK8roWRvDQ5MdL+mM7ZgVf89UcR37cAZdz3KtUt+oMe+PNbE92BVRio/PRpPrFY3kEbEnJ5E8mJYu8ZOh9Jiagw3u6JjibRY6HtXXyUIxEd4jI3fzxzEvS+kcXPJaC75dSXjV6zjp7Qsppw1mDs723mkjwN3rZvY0xIwWTS49URi6hABS/7O+b9sZeg+O0sW2rGOTqTXwEhK1+9g0tb1vB91MtGbdhPkgJBTMhh3cgiOvdVklxbTJ93Gqfd2pWB7MtVFtcTuysW6txDzoM6ULsihIiySgakxXHlRCmGR+rnTXp2UZGbHXUH8uNug1hXBqj0hfPJNGbsKqjh9x1Zu2LCYwu59MfcPZUC8HWuXDkSM7UNoZ31mHYr+1bQTHTp0oEuXLixevBiLxUJSUhKdO3cGYNCgQSxZsoTi4mIsFgsnn7x/HmDD6ISdO3f6JA927tzpcxwtURR4oVnRhGZFEwv8v9+k4HB0Zdq0aYRRxrUTz8Z2QFEfzugA9PF9jfRwQtPDSTxuvZbWJMhVd1vm7SHDPW0mw8DQl3Rpxs2D7Ny1Fgpi6oZohgLuqiq6jMsMbMek1TKZTIw9J5p/z4Q5/Qcwp/8Az3Nn9w4iZmhMwPomgWcymeDULOKA8y/Z355IF7oBVzkcTJv2BQATJw7x/n5TL6FH/ZzyU/dPo0sckanvNyeQYKuJ32TW/T4ZnWXhT6fHMG3aJ3AynDTxnkbjRg5N3wjbkSFDhrB582bmz5/vmR7Q0L569Wp++uknevbsSUTE/uzZsGHDCAoKYubMmV5zWAoKCvjiiy98jhEWVjcMuaxMFUFbgx/vXczEsSuZseBMXlx5DuvzVb1XDq1nYSlWl3espJdUEOJwNbGHCHTdsJ675r+Lqboau8tFVP4+pn38JOYSfR5I004PKqZ/rnex1Izick6N9a2nIyIirYNGErQjgwcP5v3332fnzp1MnDjR0z5kyBAcDgd79uzhnHPO8donJiaGW265hRdffJGbbrqJCy64gJqaGj755BM6d+7MunXrvEYP9O3bF4Dnn3+eCy64gKCgILp27UpWVtbxOUnxWHHd19xDF3Z3jfa0nf9EKWueSKBDmEZ8SNOqrBacboMwRy1gwmk2UWmxYLEpbqRprp5pRO7awqTdv3ja8oOySEqJDWCvpLWr/Ho9V24M4dS9RWyIi6ZrcRldSyvZtaADSd1VWVxEpDVSkqAdGTRoEBaLBZfL5TWSICMjg4SEBPLz873aG0ycOJHw8HDef/99/vOf/5CcnMx1112HYRisW7fOa9WDAQMG8Lvf/Y5Zs2bx2GOP4XK5uOWWW5QkCICFyyrYfU60V5vJbeKRb2t59mIt8yNN2x4TwaNzv+bKpWsBqLFauOfy8yl29SI5wH2T1iu/MJxKUyRmw0UwtVQTyh5XJ3pUGdh8C4yLABB6chqWD7Mx22LpXuXAZAuhNNRFwkkqJCeNq6g1WLzNQddIjTaRI+dwQb47kviSCkpWF9OhTxwW1d06YlrdQJr01FNP8eGHH/Lll18SH68P89bmmb5z+NfZp/u0n3NGODOuCA1Aj6St+P6DXaRc9YFXW35UOIP23UawPkilCdlvbaX4hvfo4V6PDSeVhLHScjKDs28mOEnXHGlcQZ6dx29fQ3mId6Gwv/0znfRMJbTF2x8/q2HlnH30LK3EAlRGuRgycDG33Hqd5pbLIf13m5sbv3CRV20itryaP878hUG5+fR67QwyL00PdPfaFNUkkEbX0ywoKODzzz+na9euShC0UoNi3UTUeP/tzC4XfxupW3rSvH55+3zaEsoqsRRVB6A30lYk9rLSx70aG3V398KpYlDIaoIT9UNPmrZmRYVPggBg+eLyRraWE9nCnS7mfVlKn/oEAUB4mYWNG3sFtF/SNlTYDa7+zE1edd3UyeLIUP752zNwVbpZddtCagpqAtzDtkXTDYRly5bx3HPPcfbZZ5OYmMjevXuZPXs21dXV/O53vwt096QJZ3x9EX8ZM58PIjuSHR2J2wy/m5hA9wTdCZbmhboqfNqsJgfmaN2lkabZlm/2aQupLIXdhZCuZLI0Lj0zBAyjfn3y/TKzNPpEvM3b7KJTle8PuZpi1a6QQ1uWB2V277aaYBvr0+Ppsyef/KWFpJ2fGpjOtUFKEghpaWl06tSJTz75hNLSUoKCgujduzc33HADw4YNC3T3pAnmEBsT553HtQ4H06ZNA2Di0ImH2EsELDipIIhw7JgAF2YchhlTRS2EBR9yfzkxObt39PnSUBseRnBSdKPbiwBYkkLJCwsiqcruSRSUh9noOzAswD2T1qZvkpkvgnyT1ZYw3xGvIgfLigGLCVwHTKQ3u92k5ZfhtJmI6hoZsL61RUoSCGlpaTz99NOB7ob4qbTKRW5VNEmhpYHuirQRBcSzIL0r75/em9yYcPruymfigl/pHhyiDwVp0pqIdGrT+zFs12oA3CYTH/YbxXiXGd0TlqbM3+lmbpeOdCkpo3d+MWsT49geHcGjJSayOgS6d9KaXNrbwlM9oihaVkUHuwMAu8VEz24bgH6B7Zy0eqmRJv56iolHft6fJbhywTo6VFQR+4c+RHfTiJQjoe+DIm3YqBcr+DbfjMs0hk57q4n9oYorztZdPWleXkQEz144BIe1bmrKii7JVIXYuER1bKUZVZUu/jZ6IjgKySjZx4qOXelaDhfXGoTqprA0oXsHE1f+uoSnv/iQlIoyVid2ZNLVN5IS0TXQXZNWxmoxsfj/hfP+qs7M/7GcPrFubCVzsNpcge6atBEPn27hki4Onnv/R3qtKub8nh3p9MhFxA/WlLgjpSSBSBv1zPe1zCu0esqP7okI48HPKrhsuBurTTVJpWkrMpNxrPae97mxYxylFhv6GJWmRHYNZ0F6KIYpEagrJJbjdhATq68S0rRTKWbwrDewutwA9Nu3l08/mUrYM08EuGfSGplMJq4eYOPqAR1wOBxMm6YEgRyZfvFwevhmOA36TLxYq2L4Sb8kRNqo95c7fNr2hIWwZI2qt0rzOlT6rmIQ4nQSpoUxpBm/FJgxDio+l2e2satUI1CkGf9bj9XlxgBqrHVf1mN25cG2/MD2S0REmqT0v0gb1S3ezOIy77Zou4Muab5LTYkcaPCuHLrnBbMpKc7TNn7pemxFkZASHsCeSWuWGeWbDAgzGSRoqoE0p0sCyzv14L3B51EUHk1G4V5uXPUVnZI0P1hEpLVSkkCkjXruslA+fbKSMkvdP2Or2805HRwkxWtYlTQvJzqSx2d9yfweGeyNieDkXblkFZZixI4IdNekFUsuqiKzxMn2mAhsThcOq4VT9hRgciaBTUuvSuNK+nTllRHjcJnqBq/uiOvIy6Ov4Z8RIQHumbRmLreBy6VRSiKBoiSBSBsVF2Yi569h3PNBBSvW5TMkYiuT7xkZ6G5JG7AjNYGCrp24YO1WABwWM++OHEKfWojT93ZpQkWpk0nf/crJa7YSW1HFnoRYFg3sQU1VAiGhShJI49auqfIkCBrk1ASxb5+DxEQltcWbYRi8PbuMud9W4HAadIzpR7+MtYHulsgJR0kCkTYsLMjM81eFMm3at4HuirQhhTYrH545iB/7diW+tJJtKfHsiwwlJFRlaqRp3ZNNWBevweKuu7vXKb+Y6FXriIkbHOCeSWsWn+CbCLAFmYiMVGJJfP3vpyo++Lzc83hXQRpBNnsAeyRyYlKSQETkBFNlMbMxOgynyUROXAwVVgtro8OocUK4ihdKE9w/Z4MBGzunUBwVTlpuIal5xdj3VBDUSbVQpHEx6SFkR4eSWrq/YOr29GhClZSURiz61bf4cl5JYgB6InJiU5JAROQEY7KY2RsRQm54MDa3Qa3FDBgcVLhexIs1LYI5Zw9mT3JdwcvF/bM4bd0W+scGB7hn0prN22HwTUYincqqiamxkxMRSkF4MNtKDLrE6KIj3hJifUeYhAb5rsgjIseW0rgiIieYuIpKANxmE7X1BedsLjc2fSJIM3YnxXsSBA2W9u2K06Jh49K0TpFgmEzsjg5jdVIMBeHBhFggPjTQPZPW6JJzI4iN3v9hZDE76d5xSwB7JHJi0kgCEZETzKiNm4l09aA8IgRMJjAMRqzeSlhxZ0jWsHFpXHGJi73BQSyJjqDUZiWlxs6pJWVUVroJDlGGSRrXJxbMTjdu6/4YiTG5iArWV1DxlRhn5cWHk/h+cTXVNU6yt84lLNh3CoKIHFu6QouInGDWJyVSVRHMybvySCivZmNyB9Z3TsGI1tIG0rTELqF8Gw+u+nkp2aHB/BwaS4c4fZWQps3b4sJd7gCbGSwmcLrJdRpsL7KR2UHJJfEVFWHhorMjcDgcTNujBIFIIOiTXUTkBPNr545Men0Z3faVAHDh6m18MrAb+a5YUgLbNWnFfsrbnyBokI2V7FI3qdH6sSeNS4msjw2HGxx1/zfYCh3CVI9ARKS10qe6iMgJpsfeIk+CoMHotduIs7oC0yFpE5IjfX/UhdogOkQ/9qRpZ3cxc2aGd4zcc7pNcSMi0oopSSAicoJJsfsO3wy2u3BWKkkgTRvV3cKp6d4/7P48IoiIYP3Yk6aZzSY+u9bKDXE/MSpqLZ9fa+Uf52mtVRGR1kzTDURETjDpp8VjtphwuwxPW2KfKMLitJSdNM1iNvHVDUHc9dL35Dqi+MPY/ozupZiRQwu2mjg1YjsA52YNDHBvRETkUDSSQETkBBOdGsZvHu0DEU4AUgbGcP4/Twpwr6QtCLGZOD1qG+PiVnJulpY+FBERaY80kkBE5ATU9exEvt+ZjeGCS24+G5vNFuguiYiIiEgroJEEIiInMJNuBouIiIjIAZQkEBERERERERFASQIRERERERERqackgYiIiIiIiIgAShKIiIiIiIiISD0lCUREREREREQEUJJAREREREREROopSSAiIiIiIiIigJIEIiIiIiIiIlJPSQIRERERERERAZQkEBEREREREZF6ShKIiIiIiIiICKAkgYiIiIiIiIjUU5JARERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEBERERERERFASQIRERERERERqackgYiIiIiIiIgAShKIiIiIiIiISD0lCUREREREREQEUJJAREREREREROopSSAiIiIiIiIigJIEIiIiIiIiIlJPSQIRERERERERAZQkEBEREREREZF6ShKIiIiIiIiICKAkgYiIiIiIiIjUU5JARERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEBERERERERFASQIRERERERERqackgYiIiIiIiIgAShKIiIiIiIiISD0lCUREREREREQEUJJAREREREREROopSSAiIiIiIiIigJIEIiIiIiIiIlJPSQIRERERERERAZQkEBEREREREZF6ShKIiIiIiIiICKAkgYiIiIiIiIjUU5JARERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEBERERERERFASQIRERERERERqackgYiIiIiIiIgAShKIiIiIiIiISD1roDsg0hIMw6C8vDzQ3QgIh8NBdXU1AGVlZdhstgD3KLAiIyMxmUyHta3iRnHTQHFzeBQ33hQ3h0dx401xc3gUN94UN4dHcePtSOLmQCbDMIxj0B+R46qsrIzo6OhAd0NagdLSUqKiog5rW8WNNFDciD8UN+IPxY34Q3Ej/jiSuDmQkgTSLjSVMa2oqODCCy/k888/JyIiIgA9Oz50nvu1RKZd72f7orhpWTrP/RQ3h0/nuZ/i5vDpPPdT3Bw+ned+/o4k0HQDaRdMJlOjWTKz2YzFYiEqKqpdXyR0nv5R3Og8/aG40Xn6Q3Gj8/SH4kbn6Q/Fjc7zqF+7RV9NRERERERERNosJQlEREREREREBFCSQNq5oKAgbrnlFoKCggLdlWNK59k2jxNoOs+2eZxA03m2zeMEms6zbR4n0HSebfM4gabzPHoqXCgiIiIiIiIigEYSiIiIiIiIiEg9JQlEREREREREBNASiNIOff/997z88svs3LmT5ORkbrjhBsaMGdPsPnv37m10m759+zJ9+vRj1NPDs2PHDp566il+/fVXwsPDGT16NJMmTcJmszW7n2EYvPnmm3z00UeUlJTQvXt37r77bvr163ecen5k/D3Piy++mJycHJ/2hQsXEhwcfNjHV9zUUdwobhQ3TWupuIH2FTuKG8WNPxQ3iht/KG6OT9woSSDtysqVK/nzn//MJZdcwj333MOSJUt49NFHCQsL45xzzjnk/nfeeSeDBw/2PA4LCzuW3T2ksrIybr/9dtLT0/nXv/7Fvn37eOaZZ6ipqeG+++5rdt8333yTKVOmcNddd9GtWzc++ugj7rrrLt555x06dep0nM7g8BzNeQL85je/4dprr/VqO5IiLoqb/RQ3ihvFTfOONm6gfcWO4kZx4w/FjeLGH4qb4xc3GCLtyJ133mlMnDjRq+2BBx4wxo8f3+x+2dnZxqBBg4yvv/76WHbviL3xxhvGGWecYZSUlHjaPv74Y2Po0KHGvn37mtyvpqbGGDFihPGf//zH02a3242LLrrIePzxx49pn/3h73kahmFcdNFFxhNPPHFUx1fc1FHcHBnFTR3FzZFrT7GjuFHc+ENxo7jxh+Lm+MWNahJIu2G321m6dKlPVvS8885j+/bt7N27N0A9899PP/3E0KFDiY6O9rSde+65uN1ufvnllyb3+/XXX6msrPR6L2w2GyNHjmThwoXHtM/+8Pc8W4LiZj/FzeFT3OynuDky7S12FDeKG38obhQ3/lDcHJ+4ARUulHZkz549OJ1OMjIyvNozMzOBurk9h/LEE08wdOhQzj33XB577DFKS0uPQU8P344dO3zOJzIykvj4+GbPp+G5xt6L3NxcampqWrajR8nf82zw5ZdfcuqppzJ8+HB+//vfs2XLlsM+tuLGez9Q3BwOxY33fqC4OVztLXYUN4obfyhuFDf+UNwcn7gB1SSQdqSsrAyo+0d0oKioKK/nGxMUFMT48eM55ZRTiIyMZM2aNbzxxhusW7eOGTNmYLUG5p9KWVmZz/lA3Tk2dz5lZWUEBQX5FCiJjIzEMAzKy8sJCQlp8f76y9/zBBgxYgR9+/YlOTmZ7Oxs3njjDW666abDnmOmuPHeT3GjuDmY4ma/o42bhuM3HO9AbTV2FDeKG38obhQ3/lDcHJ+4ASUJpJWrqKigoKDgkNulpqYe1XHi4+O5//77PY8HDRpE165d+X//7/8xf/58zj333KN6fTl2/vznP3v+/8CBAznllFO47LLLeOmll7j11lub3Vdxc+JS3Ig/GoubcePGMW3aNK677rpD7q/YOTEpbsQfihvxR1Nx8/bbb3v9PQ9FSQJp1b755hsee+yxQ243c+ZMT1a0oqLC67mGjFvD84fr9NNPJzQ0lPXr1wfsQhgVFeVzPgDl5eXNnk9UVBR2u53a2lqvrGl5eTkmk6nR7GQg+XuejYmPjyclJYV58+Yxb968ZrdV3Pjup7hR3BxMcdO0+Ph4BgwYwJIlS5gzZ84ht2+PsaO4Udz4Q3GjuPGH4sb/uFm/fv0R7ackgbRqY8eOZezYsYe1rd1ux2q1smPHDk499VRPe1PzkNqCjIwMn7lHDaMrmjufhud27txJ9+7dPe07duwgOTm5VQ2pAv/PsykN5/jmm28eclvFjfd+oLhR3OynuDk8sbGxzJ0797C2bW+xo7hR3PhDcaO48Yfixv+4OVIqXCjtRlBQEIMHD+bbb7/1av/666/JzMykY8eOR/R6P/zwA9XV1fTu3bslu3lETjvtNBYvXkx5ebmn7ZtvvsFsNnPKKac0uV///v0JDw/nm2++8bQ5nU7mz5/P6aeffkz77A9/z7Mx+fn5rFy58rD/boqb/RQ3ihvFzbGNG2h/saO4Udz4Q3GjuPGH4ub4xA1oJIG0MzfffDO33XYbTzzxBOeccw7Lli3jyy+/5PHHH/fabtiwYVx44YU8+OCDADzzzDOYzWb69u1LZGQka9euZfr06fTu3ZuzzjorAGdSZ9y4cXzwwQfcc8893Hjjjezbt4/nnnuOyy67jISEBM92d9xxBzk5OcyePRuA4OBgJk6cyKuvvkpsbCxZWVl89NFHlJaWcu211wbobJrm73l++eWX/Pjjj5x++ukkJCSwZ88epk+fjsViOaLzVNzMBhQ3ihvFzfGIG2hfsaO4Udz4Q3GjuPGH4ub4xY2SBNKuDBgwgKeeeoqXX36ZOXPmkJyczN/+9jef9WFdLhdut9vzODMzk5kzZzJr1ixqampITExkzJgx3HbbbQGrNA51c5Jefvll/vWvf3HPPfcQHh7O2LFjmTRpktd2LpcLl8vl1TZhwgQMw+Dtt9+muLiY7t2788ILLxxRZdPjxd/zTE1NJT8/n6effpry8nIiIyMZMmQIt9122xEV7FHc7Ke4Udwobuocq7iB9hU7ihvFjT8UN4obfyhujl/cmAzDMFrkbERERERERESkTVNNAhEREREREREBlCQQERERERERkXpKEoiIiIiIiIgIoCSBiIiIiIiIiNRTkkBEREREREREACUJRERERERERKSekgQiIiIiIiIiAihJICIiIiIiIiL1lCQQERERaQOmT5+OyWRiwYIFge5Kq7JgwQJMJhPTp08PdFdaXHs+NxFpvZQkEBERkXZn27Zt3HrrrfTs2ZOwsDBiY2Pp1asXEyZMYP78+V7bZmRk0Ldv3yZf64YbbsBkMlFQUNDo8+vXr8dkMmEymfjhhx+afJ2GbRr+CwkJoVu3btx9990UFRX5d6JH6KGHHmL27NnH5VgtaeXKlTz00EPs2LEj0F0REWn3rIHugIiIiEhLWrp0KWeeeSY2m43rr7+ePn36UF1dzebNm5k3bx6RkZGMHDmyxY73+uuvExkZSWhoKG+88QbDhw9vctsBAwZwzz33AFBUVMR///tfnnnmGb7++muWLVtGUFBQk/ted911XHXVVc1ucygPP/wwEyZMYOzYsX6/RiCsXLmShx9+mLPOOouMjAyv50aMGEF1dTU2my0wnRMRaWeUJBAREZF25eGHH6aqqoqVK1dy0kkn+Tyfm5vbYsdyOBy89dZbXH755URHR/Pqq6/y/PPPExkZ2ej2qampXHvttZ7Hv//977n44ov57LPPmDNnDpdffnmTx7JYLFgslhbre0srLy9v8ryPJbPZTEhIyHE/rohIe6XpBiIiItKubN68mbi4uEYTBADJycktdqxPP/2Uffv2MWHCBG644QYqKyv54IMPjug1Ro0aBcCWLVua3a6xmgQNbf/73//497//TdeuXQkODqZ79+68+eabnu127NiByWQC4M033/Sa9nCgb775hvPOO4+YmBhCQkLo378/r7zyik9fMjIyOOuss1ixYgWjRo0iOjqa/v37A3XJgr/97W8MGzaM+Ph4goODycrK4v7776eqqsrntQzDYOrUqQwbNoyIiAgiIiLo168fDz74IFA3RWLixIkAjBw50tPvG264AWh63n5lZSV/+ctfPO9JcnIy119/PTt37vTa7sD9p02bRp8+fQgODqZz58489dRTzf5NAEpKSggJCeGyyy5r9Pm//OUvmEwmVq5cCcDevXu55557GDBgALGxsYSEhNC7d2+efPJJXC7XIY/XXG2KxkZaQN3omksvvdTz9+jRowf/+Mc/cDqdhzyeiJx4NJJARERE2pWuXbuyceNGZs2a1eQPt4O5XK4maw7U1tY2ud/rr79OZmYmw4cPx2QyMXDgQN544w1uvvnmw+7v5s2bAYiPjz/sfQ72wAMPUF1dzW233UZwcDAvv/wyN9xwA1lZWZx++ukkJCTw1ltvcd111zF8+HBuvfVWn9d49dVXuf322znllFP461//Snh4OF9//TV33HEHW7du5V//+pfX9rt27eLss8/m8ssvZ9y4cVRUVACQnZ3Na6+9xrhx4/jtb3+L1Wrlu+++46mnnmLFihV89dVXXq9z3XXX8c477zBs2DD++te/EhMTw4YNG5g5cyaPPPIIl112GTk5Obz66qs88MAD9OrVC6j7OzfF4XAwatQoFi5cyPjx47nnnnvYvHkzL7/8MvPmzWPp0qV06tTJa59XXnmFvLw8brrpJmJiYnj77be577776NSpE7/97W+bPFZMTAxjxoxhzpw5FBUV0aFDB89zbrebd955h/79+zNgwAAAfv31V2bNmsWll15K165dcTgcfPnll9x///1s27aNKVOmNHksf3z++edcdtllZGVlcc8999ChQwd+/vlnHnzwQVauXMlHH33UoscTkXbAEBEREWlHfvrpJ8NmsxmA0a1bNxUZxlkAAAnLSURBVGPixInGSy+9ZKxbt67R7Tt37mwAh/wvPz/fa7/s7GzDYrEYf//73z1tzz77rAE0eizAOO+884z8/HwjPz/f2LRpkzF58mTDZrMZ0dHRRl5eXrPnNW3aNAMw5s+f79M2YMAAo7a21tO+Z88eIygoyLjqqqt8+jBhwgSf1967d68RHBxsXH311T7P/f73vzfMZrOxdetWn/ds6tSpPtvX1tYadrvdp/1vf/ubARiLFi3ytH3wwQcGYFx77bWGy+Xy2v7Ax42de4P58+cbgDFt2jRP26uvvmoAxp///GevbT/77DPP8Q7ePyUlxSgpKfG0V1ZWGvHx8cYpp5zic8yDNbzuiy++6NX+zTffGIDx9NNPe9qqqqoMt9vt8xrXXnutYTabjb179zZ7bs29F2eeeabRuXNnz+Pq6mojKSnJGD58uOFwOLy2nTx5cpOvIyInNk03EBERkXbl1FNPZdmyZUyYMIHS0lKmTZvGpEmT6N27NyNGjGDbtm0++2RkZPD11183+t95553X6HGmT5+O2+3m+uuv97Rdc8012Gw23njjjUb3mTdvHgkJCSQkJNC9e3fuvvtuevfuzbx580hMTPT7nCdNmuRV0DA1NZXu3bt7RikcysyZM6mtreWmm26ioKDA67+LL74Yt9vNN99847VPhw4dPNMADhQUFOQpIuh0OikuLqagoIBzzjkHgEWLFnm2feeddwD497//jdns/bX04MdH4pNPPsFsNvOXv/zFq/3CCy9kwIABzJkzB7fb7fXcxIkTiY6O9jwOCwvjlFNOOaz3cNSoUSQlJTFjxgyv9hkzZmC1Wrnmmms8baGhoZ5pHna7naKiIgoKChg1ahRut5ulS5ce8fk25euvvyYvL4+JEydSUlLi9XcdPXo0UBeTIiIH0nQDERERaXf69evnmaO+c+dOvvvuO1577TV++OEHLrnkEp+VBMLDwz0/Yg/29ttv+7QZhsEbb7xB//79cbvdXvUETj/9dN566y0ef/xxrFbvr1rDhg3jscceA/DMe09PTz/a06VLly4+bXFxcT7z75uyfv16gCbfA4C8vDyvx127dm2ykOJLL73EK6+8wtq1a31+jBcXF3v+/+bNm0lJSSEpKemw+nm4tm/fTseOHYmNjfV5rk+fPqxcuZKCggKvxExT72FhYeEhj9eQCJg8eTKbNm2ie/fuVFZWMmvWLM477zyv83M6nTzxxBPMmDGDLVu2YBiG12sd+P4crYa/64033tjkNgf/XUVElCQQERGRdq1z585cf/31nvn4CxcuZPHixZxxxhl+v+Z3333H1q1bAejWrVuj23z22Wc+Sw3Gx8c3+0PcX039WD/4B2hTGrabMWMGKSkpjW5z8I/osLCwRrebPHky99xzD+eddx6///3v6dixI0FBQWRnZ3PDDTf4JA1ai6NdOeL6669n8uTJzJgxg8cee4xZs2ZRUVHBhAkTvLa7++67eeGFF7jyyiv561//SmJiIjabjeXLl3Pfffcd8v05uNjkgQ4uRNjwd/3Xv/7lqYlwsI4dOx7G2YnIiURJAhERETkhmEwmhg0bxsKFC8nOzj6q13rjjTcIDg5mxowZjQ6Lv+2223j99dd9kgStVUOioyWSGG+99RYZGRl88cUXXu/Nl19+6bNt9+7dmTNnDnl5ec2OJmjuh3FjunTpwpdffklJSQkxMTFez61bt46oqKijKhTZmJNOOomTTjqJt99+m0cffZQZM2Z4ihoe6K233mLEiBG8//77Xu2HWt2iQUNhxKKiIp/ntm/f7pnqAfv/rs2NlBEROZhqEoiIiEi78vXXXze6tFt1dbVn/nXv3r39fv3S0lJmzpzJeeedxxVXXMH48eN9/hszZgxffPEFOTk5fh/nWIiIiGj0x+UVV1xBcHAwf//736murvZ5vrS0tNlVHg5ksVgwmUxeoxgahtgfrGGu/r333utzB/3A/SMiIoDGfxg3ZuzYsbjdbp9jfvHFF6xYsYIxY8YcVc2DpkyYMIGdO3fy7rvv8r///Y8rr7ySkJAQr20sFovPCI/KykqeeeaZwzpG9+7dAXxqRLz33nvs3bvXq23UqFEkJibyxBNPNPreVVdXU15efljHFZETh0YSiIiISLvyxz/+kcLCQsaMGUO/fv0ICwtj9+7dvPvuu2zatInrr7+efv36+f367733HtXV1YwbN67JbcaNG8f06dN58803uf/++/0+Vks75ZRT+Oabb3jyySdJT0/HZDJx1VVX0alTJ15++WVuvvlmevXqxXXXXUfnzp3Jz89n9erVzJ49m3Xr1pGRkXHIY4wfP56//OUvXHDBBVx22WWUlZXx7rvvet3hbnD55Zdz5ZVXMmPGDDZv3syYMWOIjY1l06ZNfPXVV6xZswaAIUOGYDab+cc//kFxcTHh4eFkZmYybNiwRvtwww038Oabb/Lkk0+yY8cORowYwZYtW3jppZdISkrin//851G9j0255ppruPfee5k0aRJut9tnqgHUvT9Tpkzhyiuv5JxzziEvL4833niDuLi4wzpGjx49OOecc5gyZQqGYTBgwABWrlzJJ598QlZWFg6Hw7NteHg4M2bMYOzYsfTo0YMbb7yRrKwsSkpK2LBhA7NmzeKTTz7hrLPOaqm3QETaASUJREREpF2ZPHkyc+bM4ccff+Tjjz+mpKSE6Oho+vfvz3333ccNN9xwVK//+uuvY7VafYaRH+jcc88lMjKSadOmtaokwUsvvcSdd97JP/7xD88d5Kuuugqoq+7fvXt3/v3vfzNlyhRKSkqIj4+nR48ePProoyQnJx/WMf785z9jGAavv/46f/jDH0hOTubKK69k4sSJjY7gePfddxk+fDivv/46jzzyCBaLhczMTC6//HLPNunp6bzxxhs8+eST3HHHHTgcDiZMmNBkksBms/HVV1/x2GOP8cEHHzBr1ixiYmK4/PLLeeyxx0hLSzvSt+6wJCYmcv755/PZZ5/RrVs3Tj31VJ9tJk+eTGRkJB9++CFz5swhLS2NW2+9lSFDhhz2lIC33nqL3/3ud7zzzju89dZbDB8+nPnz53PHHXewY8cOr21HjRrFkiVLeOKJJ3j77bfJz88nNjaWrl27cvfdd9O/f/+WOHURaUdMxuFWtBERERERERGRdk01CUREREREREQEUJJAREREREREROopSSAiIiIiIiIigJIEIiIiIiIiIlJPSQIRERERERERAZQkEBEREREREZF6ShKIiIiIiIiICKAkgYiIiIiIiIjUU5JARERERERERAAlCURERERERESknpIEIiIiIiIiIgIoSSAiIiIiIiIi9ZQkEBEREREREREA/j9Qhk/10GFi/gAAAABJRU5ErkJggg==\"\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"{'Class 0': ' Husband', 'Class 1': ' Not-in-family', 'Class 2': ' Other-relative', 'Class 3': ' Own-child', 'Class 4': ' Unmarried', 'Class 5': ' Wife'}\\n\"\n ]\n }\n ],\n \"execution_count\": 37\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:13.720551Z\",\n \"start_time\": \"2026-01-07T01:37:13.613694Z\"\n }\n },\n \"source\": [\n \"dependence_feature = \\\"marital-status\\\"\\n\",\n \"print(f\\\"Dependence_plot for class: {class_of_interest} and for feature: {dependence_feature} \\\\n\\\")\\n\",\n \"shap.dependence_plot(\\n\",\n \" dependence_feature, \\n\",\n \" shap_values[:, :, class_index], \\n\",\n \" X_test\\n\",\n \")\"\n ],\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Dependence_plot for class: Not-in-family and for feature: marital-status \\n\",\n \"\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"
\"\n ],\n \"image/png\": \"iVBORw0KGgoAAAANSUhEUgAAAroAAAJMCAYAAAAVNgUdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAg5hJREFUeJzs3XdUFNffBvBn6L2JgmIBe0VUQCxBrBhBxZbYYo8FTYwSY4qxRBONGP2pUVRUjL1FBdEokoi9K/ZoLBhRFBEpC8LC7r5/+LJxXeoKDCzP5xzPYe7M3H3IBv1y9869gkKhUICIiIiISMvoiB2AiIiIiKgksNAlIiIiIq3EQpeIiIiItBILXSIiIiLSSix0iYiIiEgrsdAlIiIiIq3EQpeIiIiItBILXSIiIiLSSix0yxiFQgGJRALu40FERET0fljoljFpaWnw8vJCWlqa2FGIiIioMIR+an9WrFiBxo0bo3HjxlixYoXYCSssgVsAly0SiQReXl6IioqCmZmZ2HGIiIioIEI/9TbF76Wfg9RwRJeIiIiItBILXSIiIiLSSix0iYiIiEgrsdAlIiIiIq3EQpeIiIiItBILXSIiIiLSSix0iYiIiEgrsdAlIiIiIq3EQpeIiIiItBILXSIiIiLSSix0iYiIiEgrsdAlIiIiIq2kJ3YAIqLyYv99OU7GKtCssoCPGwjQ1xXEjkRERPlgoUtEVAhf/CXD0suK/z9SYMstAX/01xU1ExER5Y9TF4iICvA8TYFfryhU2g7FKHD6iSKPO4iIqCxgoUtEVICE14Asl5o2Lo2FLhFRWcZCNxcxMTHw9/dH+/bt4e3tjaVLlyIrK6tIfWzduhWurq744osvSiYkEZWaxpWABjaqbRYGQJdanKNLRFSWsdB9R0pKCsaPH4/s7GwEBgbC398fe/fuxeLFiwvdR0JCAoKDg2FjY1PwxURU5gmCgFBfoKNOIvTkMrSQJyG8mxSWhix0iYjKMj6M9o7ff/8daWlpCAwMhKWlJQBAJpPh559/xqhRo1C5cuUC+1i2bBk8PT0RFxdX0nGJqJQ0mLwMf20/+V/DzjrA+Z8BHY4XEBGVVfwb+h2nT5+Gu7u7ssgFgK5du0Iul+Ps2bMF3h8dHY1jx47hs88+K8mYRFSaYhOAHadU2y7dB6JuipOHiIgKhYXuO2JiYuDo6KjSZm5uDltbW8TExOR7r0wmw8KFCzFy5EjY2toW6vWkUikkEonyT1pamobJiajEpGUCilwePJO8Lv0sRERUaJy68I6UlBSYm5urtZubmyMlJSXfe3ft2oXXr19jyJAhhX69kJAQBAcHFzknEZWiBg6Aez3g/D//tVW1Bro2Fy8TEREViIVuMUlMTMTq1asxZ84c6OvrF/q+kSNHqhTGaWlp8PHxKYmIRPQ+wr4Bvt4EnLgNNKsJ/DQUMDYUOxUREeWDhe47LCwsIJFI1NpTU1NhYWGR532rVq1CvXr10KJFC6SmpgJ4M5VBJpMhNTUVxsbG0NNT/89tYGAAAwOD4vsGiKhk2FkBIZx7T0RUnrDQfYejo6PaXFyJRIKEhAS1ubtvi4mJweXLl9GxY0e1cx07dsSyZcvQtm3bYk5LRERERHlhofuOtm3bIiQkBKmpqcq5upGRkdDR0YGHh0ee9wUEBChHcnMsXrwYhoaGmDhxIurVq1eiuYmIiIhIFQvdd/Tr1w87duxAQEAARo0ahfj4eCxduhR9+/ZVWUN3woQJiIuLw759+wAADRo0UOvLzMwMJiYmcHV1La34ubt4D3iWBHRsCpgaiZuFiIiIqJSw0H2HhYUFgoKCEBgYiICAAJiamsLPzw/+/v4q1+XMvy3TsrKBfguB/RffHNuYAQe+AzzUi3IiIiIibSMoFLktDklikUgk8PLyQlRUFMzMzN6vs83HgE+Wqra51wPO/fx+/RJVVCdvv/nTrCbwYUvuikZEbwj91NsUv5d+DlLDEV1tdjVGvS36YanHINIK328F5u3+73hAW2Dnl+LlISKiAnE4Qpvp66q3cY4uUdG9TAUW7lNt23X6zfx3IiIqs1joajNptnobtywlKrqnibn/PMXEl34WIiIqNBa62qxJDfW2pjVLPwdRedekBuBYRbXN2ADo2EycPEREVCgsdLXZYE/Aq+l/x6ZGwOKR4uUhKq90dIA9XwHNHd8c17YDdk8DKpmLGouIiPLHh9G0maE+8NccIOrGm3V0vV0AG/7DTKSRFrWB6MVvpv+YGgGCIHYiIiIqAAtdbScI/HiVqBg8lSjwZZQcJ5/oo5mtHAs8ddCsMotdIqKyjIUuEVEh9Norw6Xnb75+nKrAxecyPPxUFyb6LHaJiMoqztElIirAnUSFssjNEZ8OHHnE/XaIiMoyFrpERAUw0wdyG7e1MCj1KEREVAQsdImICuBgLmBIY9VS190e8KrBaQtlzY6/5ai7NhtGS7LxUZgML19z1J2oIuMcXSKiQgjprgPP6gqciFWgma2A8S4CBK68UKZcf6HA4ANyyP+/tt11V4EsuRx7/XLZJZKIKgQWukREhaCnI+BTZwGfOoudhPISek+hLHJzhN1XIFuugJ4Ofykhqog4dYGIiLSCval6W2VjsMglqsBY6BIRkVYY2FBAIxvVtjnt+M8cUUXGqQtERKQVzAwEnBuqi003FXicqkCvujpoU42juUQVGQtdIiLSGuYGAvxbsLglojf4mQ4RERERaSUWulou5LoctVZnw3BJNgaHy5CUwTUliYiIqGLg1AUtdvapAqMOy5XH2/5WQEeQY7MP15QkIiIi7ccRXS22755crW3PPxzRJSIiooqBha4Wq2qq/kBGNTMRghARERGJgIWuFhveREAdq/+OBQBz2vItJyIiooqBc3S1mJWRgItDdbHplgLP0xXwq6sDV3suu0NEROKKTVVg1x0FDHTfbPRRyZj/NlHJYKGr5ayMBHzWkn+BEBFR2XDxmQIdd8ggyXpzPPcMcGawLpys+G8VFT9+jk1EVAj3XsnhvCEbxkuy4bQmG1H/qj/sSUQFm3tGrixyAeB5OrD4En+eqGSw0CUiKoRWm+S4ngBkyICYFKDLLjlSMvmPM1FR/ZuqvvrPvykiBKEKgYUuEVEB/oyRI0Wq2iZTAIsvcrk+oqLyqa0+RcG3DqctUMngHF0iogIY6efebsi9V4iKbIaHDuIkcmy+rYC+DuDvImBMMxa6VDJY6BIRFaCdgw7sTeR4lv5fm6EuMNWV/zgTFZWRnoB13XUR1FUBHQHQ0+HPEZUcTl3QchnZCmy9Lcfii3Lce8WPWYk0dXOkDrwdgcrGQGt7IHqYDgz1+FcokaYMdAUWuVTiOKKbi5iYGCxcuBDXrl2DqakpevToAX9/f+jr5/H55f/7/vvvcePGDbx48QL6+vqoW7cuRo8eDQ8Pj1JKrkoiVaDdNhmuvXhzPP04sKunDvzq8R9noqKSKwQ4WQp4IlGgthX/gSYiKg9Y6L4jJSUF48ePR82aNREYGIj4+HgsWbIEGRkZmD59er73ZmVlYciQIahRowakUilCQ0MxefJkrFq1Ci1atCil7+A/G28qlEUuAGTLgW9OyFnoEmnAL1SGU0/efH0jQYETT2S4N1oXhnoseImIyioWuu/4/fffkZaWhsDAQFhaWgIAZDIZfv75Z4waNQqVK1fO894FCxaoHLdt2xa9evXCwYMHRSl0Y1LUpyo8TC71GETl3j+vFMoiN0dsKhDxSIGefFqciKjM4tDeO06fPg13d3dlkQsAXbt2hVwux9mzZ4vUl66uLszNzZGVlVXwxSWgh5P6P8C5LetCRPnLa3UFI666QERUprHQfUdMTAwcHR1V2szNzWFra4uYmJgC71coFMjOzkZSUhI2bdqEx48fo2/fviUTtgBeNXUwpZUA/f9/lxvaAL92ZqFLVFQ1LQT0eXVPpa3Zqzh0qs4HPImIyjJOXXhHSkoKzM3N1drNzc2RklLw1i2hoaGYN28eAMDExAQ//fQTnJ2d87xeKpVCKv1vJfq0tDQNUufuQZICa64pkPX/mzf9nQisiFZgXvtiewmiiuFxArYGzsavbb1xwqkhmsX9iyknDkC305dA57x/vomISFwsdIuZl5cX6tevj6SkJERGRuKbb75BYGAg2rVrl+v1ISEhCA4OLpEsm24pkPbOrIlVV1noEhXZaymMsqT48th+fHls/3/taRniZSIiogKx0H2HhYUFJBKJWntqaiosLCwKvN/KygpWVlYA3jyMlpKSgqVLl+ZZ6I4cORJDhgxRHqelpcHHx0ez8O/IbZICJy4QaaB+NcCjPnD27n9tVa2Bbi6iRSIiooKx0H2Ho6Oj2lxciUSChIQEtbm7hdGwYUOcPn06z/MGBgYwMDAocr+F8UljAYEXAMlbo7oTmrPUJdJEyq6v8eOv93Ai2xLOSMGMIVVR3ahkfnaJiKh4sNB9R9u2bRESEoLU1FTlXN3IyEjo6OhotPHD1atX4eDgUNwxC8XJSsCJQboIvCDHszSgXz0BE1xY6BJpwveEGU7YugAAzgA4cA544KyAvi5/poiIyioWuu/o168fduzYgYCAAIwaNQrx8fFYunQp+vbtq7KG7oQJExAXF4d9+/YBAE6ePIkDBw6gffv2sLOzQ0pKCg4dOoQzZ87gxx9/FOm7AVyqCNjiwzWQiN7HgyQFTry7jq4ECH8gR596/PkiIiqrWOi+w8LCAkFBQQgMDERAQABMTU3h5+cHf39/letkMhlkMpnyuHr16pBKpfj111+RlJQEKysr1KtXD6tXr0arVq1K+9sgomL0MCn3ZcSi44E+9Uo5DBERFZqgUCi4EGQZIpFI4OXlhaioKJiZmYkdh4gAxKYqUGO1TK09or8OujpyOXKiCk/op96m+L30c5Aa/g1NRFSA6uYCPmqgOhfXpTLQuRbn5xIRlWWcukBEVAibe+igU00FTsQq4Fz5zYOdOgILXSKisoyFLhFRIejrChjXXMC45mInISKiwuLUBSIiIiLSSix0iYiIiEgrsdAlIiIiIq3EQpeIiIiItBILXSIiIiLSSix0iYhIq1x+rkDYPTlSMrkfElFFx+XFiIhIK8jkCgwKl2PX3TcFroUBEOqnA6+aHNMhqqj4009ERFoh7L5CWeQCQIoUmHg4W8RERCQ2FrpERIWULVfgn1cKvM7iR+Jl0dXrSWptt5J1kCXj+0VUUbHQJSIqhKh/5XBcI0P9dTI4rJZh00252JHoHe1uX1Nrc3t8D/oKvldEFRULXSKiAmTJFBgYLscTyZvjVxnA6MNyxEk4UliWdLXPxpTj4dCVyQAAtRLjEXx0K6CnK3IyIhILH0YjIirA7UTgebpqW5YcOBGrwEcNBXFCkbqB7bF4yXRMiwpDnIU1mj+Nge7q8WKnIiIRcUSXiKgAtSwAk1yGBRpXYpFbppgaAWcXoOrCj9FyeEvonpkPfNpV7FREBRoxYgQEQYCHh4faOYVCgRo1akAQBMyePbtI/UZFRUEQBFy8eLGYkpY/HNElIiqApaGABZ46mPyXHDmTFSa1ENC0MgvdMsfMGBjvLXYKoiIzMzPDuXPn8PDhQzg5OSnbT5w4gefPn8PQ0FDEdOUXR3S13Pk4BfqFytBuazYWX5RDJuecQiJNfFYvE3dfhWPD5T24FLsPyxsmiR2JiLRIrVq10Lx5c2zfvl2lfdu2bfD29oaRkZFIyco3Frpa7N4rBbx2yLDnHwVOPwUCouSYcZJPHxNppN9C1P1pI4Zv246WS7cCXjOBLK7RSkTFZ9CgQdi2bZvyODs7G7t378bgwYPVrj1z5gx69eqFatWqwdTUFC4uLti0aVOBr6FQKLBo0SLUr18fhoaGqF27NpYsWVKs30dZwkJXi22+Jcfrd/4dXnONI7pERfbgGXDkqmrbvTggUn05KyIiTQ0cOBA3btzArVu3AAARERF4/fo1evXqpXbto0eP0K5dO6xduxb79+9Hv379MHr0aPz222/5vsbkyZMxc+ZMDB8+HAcOHMCIESMwffp0rFq1qkS+J7Fxjq4W09URAKgWtrqcUkhUdEIePzj8eSKiYlSrVi20adMG27Ztw9y5c7Ft2zb06tULpqamatcOHDhQ+bVCoYCnpydiY2OxevVqDB8+PNf+79+/j19//RWrVq3C2LFjAQBdunRBeno65syZg7Fjx0JHR7vGQLXruyEVwxoLMDdQbZvYgm85UZGZGuZe1JpyzhwRFa9BgwZh+/bteP36NUJDQzFo0KBcr3v16hU+//xz1KpVC/r6+tDX18eaNWtw9+7dPPuOjIwEAPTr1w/Z2dnKP126dMGzZ8/w+PHjEvmexMSqR4vVshRwepAuetcF3OyAlV0EzGzDISiiInuR8u6HI2/EJ5d6FCLSbgMGDMDDhw8xc+ZM6Ovro3v37rleN2LECGzbtg1ffvklIiIicOHCBYwaNQoZGRl59p2QkACFQgFbW1tlcayvr4+uXd8sw6eNhS6nLmgxmVyBH8/JEXrvzfFjiQIeVYEWduLmIip3GtcA6lUF/on7r83cGOjsLF4mItJKdnZ26NSpExYvXozRo0dDX19f7ZqMjAyEh4dj8eLF+Oyzz5Ttcnn+D5zb2NhAEAScPHkSBgYGaucbNGjw/t9AGcNCV4vtvqvA9r//G4Z6lgZ89pcMJwfxbScqEkEAQr8Bxq0CTt4GmtYElo0GrNTnzRERva/PP/8cJiYm+PTTT3M9n5mZCblcrlKspqamIiwsLN9+O3fuDAB4+fIlevbsWXyByzBWPFrswjP1z1ovPBMhCJE2aFQdOD4PUCjyfjiNiKgY+Pr6wtfXN8/zlpaWcHNzw4IFC1C5cmXo6elhwYIFsLS0RHx8fJ731a9fHxMnTsQnn3yCadOmoXXr1sjKysLdu3dx9OhR7Nu3rwS+G3Gx0NVi7vbqqy6424uThUhrsMglojJg69atGDduHIYPH45KlSrh888/h0QiwaJFi/K9b9myZWjQoAFWr16NH374AWZmZmjQoAEGDBhQSslLl6BQKLiwahkikUjg5eWFqKgomJmZvVdfMrkCw/+QY8vtN2+xgxlwoK8umlfhP9RERETFRuin3qb4vfRzkBqO6GoxXR0Bm310MbONAs/TAY+qgD4X0iUiIqIKgoWulkuTKnDqiQLP0oFKRgIa24qdiIiIiKh0sNDVYimZCrhtluHuqzfHM04CO3vqoF99Lp9MRERE2o8VjxZbGS1XFrkAIFcAEyPzX2OPiIiISFtwRDcXMTExWLhwIa5duwZTU1P06NED/v7+uS7anCMhIQFbtmzBuXPnEBsbCzMzM7Ro0QKTJk1C1apVSzH9fw7HqD9nGJ8uQhAiIiIiEbDQfUdKSgrGjx+PmjVrIjAwEPHx8ViyZAkyMjIwffr0PO+7ffs2jh49il69eqFZs2ZISkrC2rVrMXz4cOzYsQPW1tal+F284ZDLog06fBaNiIiIKggWuu/4/fffkZaWhsDAQFhaWgIAZDIZfv75Z4waNQqVK1fO9T4XFxfs3r0benr//Sd1dnaGr68vDhw4gKFDh5ZK/rdNddXF1tsylZV0BzYs9RhEREREouAc3XecPn0a7u7uyiIXALp27Qq5XI6zZ8/meZ+5ublKkQu82a/a2toaL168KLG8+WlpJ2BXLx00rQRUNgbGNxewuquuKFmIiIiIShtHdN8RExODXr16qbSZm5vD1tYWMTExRerr0aNHSExMhJOTU57XSKVSSKVS5XFaWlqRXqMg/epzlQUiIiKqmFjoviMlJQXm5uZq7ebm5khJSSl0PwqFAosWLULlypXh7e2d53UhISEIDg7WKCsRERER5Y2FbglZs2YNzp8/j+XLl8PY2DjP60aOHIkhQ4Yoj9PS0uDj41MaEYmIiIi0Ggvdd1hYWEAikai1p6amwsLColB97N27F8HBwfj+++/h7u6e77UGBgYwMDDQKCsRERER5Y2TN9/h6OioNhdXIpEgISEBjo6OBd5/9OhRLFiwAOPHj0fv3r1LJiQRERERFYiF7jvatm2L8+fPIzU1VdkWGRkJHR0deHh45HvvxYsX8d1338HPzw9jxowp6ahERERElA8Wuu/o168fTExMEBAQgLNnzyIsLAxLly5F3759VdbQnTBhAvz8/JTHDx8+xJdffokaNWqgR48euH79uvJPbGysCN8JERERUcXGObrvsLCwQFBQEAIDAxEQEABTU1P4+fnB399f5TqZTAaZTKY8vnHjBiQSCSQSCUaPHq1yra+vL2bPnl0a8YmIiIjo/wkKhUJR8GVUWiQSCby8vBAVFQUzs1z28CUiIqKyRein3qb4vfRzkBpOXSAiIiIircRCl4iIiIi0EgtdIiIiItJKLHSJiIiISCux0CUiIiIircRCl4iIiIi0UpEL3fT0dMyZMweRkZElkYeIiIiIqFgUudA1MTFBREQEJBJJSeQhIiIiIioWGk1dqF27NuLi4oo7CxERERFRsdGo0B02bBh2796NR48eFXceIiIiIqJioafJTTExMbCzs8PAgQPRvn171KxZE0ZGRirXCIKAMWPGFEtIIiIiIqKi0qjQXbNmjfLrqKioXK9hoUtEREREYtKo0A0LCyvuHERERERExUqjQrdq1arFnSNft2/fRnJyMlq0aAFDQ8NSfW0iIiIiKp80KnTflpSUhKdPnwIAqlWrBisrK4372rRpEy5fvowlS5Yo27777jscOXIEAODg4IC1a9eiUqVK75WZiIiIiLSfxoXu3bt3sWjRIkRHR6u0u7i4YNq0aahXr16R+4yIiEDTpk2VxxcuXEBERAS8vb1Rt25drFu3Dhs3bsSUKVM0jU1EREREFYRGhe69e/cwZswYZGZmokOHDqhduzYA4MGDBzh+/DjGjBmD9evXo06dOkXq9+nTp/D19VUeR0VFwdbWFnPnzoUgCEhKSsLx48dZ6BIRERFRgTQqdFevXg09PT2sW7dObeT23r17GDt2LFatWoXAwMAi9ZuRkaEyB/fChQtwd3eHIAgAACcnJ+zevVuTyERERERUwWi0YcTly5cxYMCAXKcn1K1bF/3798fly5eL3G/lypVx//59AEBcXBwePnyIli1bKs+npqbCwMBAk8hEREREVMFoNKKbkZGR7wNhtra2yMjIKHK/H3zwAXbv3o3s7GzcvHkTBgYGaN++vfL8/fv3S33FByIiIiIqnzQa0XVwcMDJkyfzPH/y5Ek4ODgUud9PP/0ULi4u2L17N+7fv4+pU6cqC+qMjAwcPXoUrq6umkQmIiIiogpGoxHdHj16YMWKFfjuu+8watQoODo6AgAePnyIDRs24OzZs5g0aVKR+7WwsEBQUBAkEgmMjIygp6cab82aNbCzs9MkMhERERFVMBoVup988gnu3LmDiIgIHDlyRPmwmEKhgEKhQJcuXTB06FCNQ5mZmam1GRkZoX79+hr3SUREREQVi0aFrq6uLubPn4/evXvj2LFjePLkCYA3Uxq8vLzQunVrjcIU9gG2tx9QIyIiIiLKTaEK3Tlz5qBfv37KzRwuX74MJycneHh4wMPDo9jCjBs3Tjk6nJ/z588X22sSERERkXYqVKEbHh6O1q1bKwvd8ePH44cffkD37t2LNcysWbPU2rKzs/HkyRPs378f1apVQ9++fYv1NYmIiIhIOxWq0LWyssLLly+VxwqFokTCvL0r2rs++eST95r3S0REREQVS6EKXWdnZ6xfvx7Pnj2DhYUFAOCvv/7C48eP87xHEASMGTOmeFLizYoMvXv3xsaNG/MtiImIiIiIgEIWugEBAZg9ezZ27NgBhUIBQRBw9OhRHD16NM97irvQBd4UuzkPvhERERER5adQhW61atWwZs0aZGVl4eXLl+jZsycCAgLQoUOHks6nlJmZiYMHD+a7IxsRERERUY4iLS+mr68Pe3t7+Pr6omnTpsW+He+cOXNybU9JScH169fx6tUrTJ48uVhfk4iIiIi0k0br6Oa2OkJxCA8Pz7XdwsICtWrVwtSpU4t9pYfcxMTEYOHChbh27RpMTU3Ro0cP+Pv7Q19fP9/7du3ahVOnTuHGjRtISkrCggUL0KVLlxLPS0RERETqNCp0S8qFCxfEjoCUlBSMHz8eNWvWRGBgIOLj47FkyRJkZGRg+vTp+d574MABAEC7du2UXxMRERGROMpUofvs2TNYWVnByMgo1/MZGRlISkqCvb19iWX4/fffkZaWhsDAQFhaWgIAZDIZfv75Z4waNQqVK1fO897169dDR0cHT58+ZaFLREREJDIdsQO8rVevXoiKisrz/PHjx9GrV68SzXD69Gm4u7sri1wA6Nq1K+RyOc6ePZvvvTo6Zeo/JxEREVGFVqYqs4I2opDL5YXaIvh9xMTEwNHRUaXN3Nwctra2iImJKdHXJiIiIqLiU6amLhQkJiYGZmZmJfoaKSkpMDc3V2s3NzdHSkpKsb+eVCqFVCpVHqelpRX7axARERFVRO9d6D5+/BgvX75E3bp1NSpCw8PDVVZbWL9+Pfbt26d2XUpKCu7fvw8vL6/3SFv2hISEIDg4WOwYRERERFpH40L3xIkTWLRoEeLi4gAAK1asgJubGxITEzFq1ChMmjSpUEtrpaam4unTpwDe7Kb26tUrZGRkqFwjCAKMjY3Rq1cv+Pv7axq5UCwsLCCRSHLNmbP9cXEaOXIkhgwZojxOS0uDj49Psb8OERERUUWjUaF78eJFfPnll2jQoAF8fX2xZs0a5TkbGxtUr14dERERhSp0Bw0ahEGDBgEA3NzcEBAQUCpr5ebF0dFRbS6uRCJBQkKC2tzd4mBgYAADA4Ni75eIiIiootPoYbS1a9eifv362LBhAwYMGKB2vlmzZrhz506R+71w4YKoRS4AtG3bFufPn0dqaqqyLTIyEjo6OvDw8BAxGREREREVhUYjurdu3cK4cePyXE7Lzs4OCQkJ7xVMLP369cOOHTsQEBCAUaNGIT4+HkuXLkXfvn1V1tCdMGEC4uLiVOYT37p1C0+fPkVSUhIA4MaNGwAAa2trtGrVqjS/DSIiIqIKT6NCVy6X5/txe1JSUoHb5eYlNjYWW7Zswc2bN5GSkpLrkmOhoaEa9V0YFhYWCAoKQmBgIAICAmBqago/Pz+1ucEymQwymUylbefOnSoP1m3evBkA0LJlS5XpHURERERU8jQqdJ2cnHDlypVcpy0Abx5Uq1+/fpH7vXfvHkaPHo2srCzUqlULT548Qe3atZGcnIyXL1+ievXqqFKliiaRi8TJyQkrV67M95rcCtfZs2dj9uzZJZSKiIiIiIpCozm6vXv3xp9//ol9+/YpR1wFQUBGRgYCAwNx/fp19OnTp8j9rlq1Cvr6+ti6dSuCgoIAAF9++SUOHTqEb7/9FqmpqZg+fbomkYmIiIiogtFoRLd///64evUqfvzxR/zvf/+DIAj47rvvkJSUBLlcjp49e+LDDz8scr/R0dHo06cPHB0dlfNccwrpPn364MqVK1i+fDmWLFmiSWwiIiIiqkA0Xkd37ty56NSpEw4ePIhHjx5BoVCgSZMm8PHxQefOnTXqMz09HdWrVwcA5Rzf169fK883b94cK1as0DQyEREREVUg77UzWseOHdGxY8fiygIbGxu8fPkSAGBqagpjY2P8+++/yvOpqalqD4AREREREeXmvbcALk7169fH7du3lcctW7bE9u3b0aRJE8jlcuzcuVOjh9yIiIiIqOLRqNANDg4u8BpBEDBmzJgi9du9e3fs2rULGRkZMDIywvjx4zFu3DiMHz8eAGBoaIiJEydqEpmIiIiIKhhBkdtCtQVwc3PLu0NBgEKhgCAIOH/+/HuFA4Bnz54hKioKOjo6aNu2rXIOr7aSSCTw8vJCVFQUzMzMxI5DREREBRH6qbcpfi/9HKRGoxHdsLAwtTaZTIbY2Fhs3boVEomk2NaTtbe3x8CBA4ulLyIiIiKqODRaR7dq1apqf6pXrw4PDw8sXboUOjo62L9/f5H7dXd3x6FDh/I8HxERAXd3d00iExEREVEFo1Ghmx9BENC5c2ccOHCgyPcWNItCg1kWRERERFRBFXuhCwBZWVlITk4u9n6fPXsGExOTYu+XiIiIiLRPsS8vduvWLWzfvh2Ojo6Fuj4qKgrHjh1THu/Zswfnzp1Tuy4lJQXnz5+Hi4tLMSUlIiIiIm2mUaHbu3fvXNuTk5ORnp4OXV1dzJgxo1B93b17F+Hh4QDeTHu4cuUKrly5onadiYkJnJ2d8dVXX2kSmYiIiIgqGI0KXTs7OwiCoNImCAIaNGiAWrVqoU+fPqhWrVqh+ho7dizGjh0L4M2yZXPnzkX37t01iUVEREREpKRRobtmzZrizgHgzbJl1tbWJdI3EREREVUsZWoL4KpVq6q1ZWdn49ixY0hJScEHH3wAW1tbEZIRERERkViys7Ohp1f0srVEVl3Q1NKlSzFs2DDlsUKhgL+/P7755hv89NNPGDhwIGJjY0VMSERERETFacSIEUhPT8/z/L1799C2bVuN+i5Uaezm5qY2J7cggiDkunpCfs6cOaOyIcTx48dx5coVDBs2DPXr10dgYCA2bNhQ6AfdiIiIiKhs27RpE86dO4ft27ejefPmaucmTZpU5Do0R6EKXR8fH41foCieP3+OmjVrKo9PnDiBatWq4bPPPgMAPHjwIN+d04iIiIiofDl48CCGDRsGDw8P/Pzzz/j888+RlpaGCRMmYMuWLXB1dcW2bds06rtQhe7s2bM16ryosrKyoKurqzy+ePGiygivg4MDEhISSiULEREREZU8b29vXLt2DUOHDsWUKVPwxx9/4N69e3jw4AECAgLw008/aTQ/Fyhjc3Tt7Oxw7do1AMD9+/fx5MkTtGrVSnk+MTERxsbGYsUjIiIiohJgZ2eHiIgItG3bFocPH8aDBw+wbNkyLFy4UOMiFyhjqy5069YN69atw6tXr/DgwQOYmpqiXbt2yvN37txB9erVRUxIRERERMUtMTERI0eOxOnTp/HBBx/g2rVr+Oabb2BhYYFPPvlE4341LnSjo6OxYcMG3LhxA6mpqVAoFCrnNXkYbeTIkXj+/DmOHTsGMzMzzJkzB+bm5gAAiUSC48ePY/DgwZpGJiIiIqIy5vjx4xg6dCieP3+OBQsWYNq0aXj48CEGDRqEESNG4MiRIwgKCoKpqWmR+xYU71aohXD58mX4+/vDzMwMTZs2xalTp+Dm5ob09HTcvHkTdevWRcOGDTFr1qwiB8qLXC5Heno6jIyM3msIu6yTSCTw8vJCVFQUzMzMxI5DREREBRH6qbcpfi/9HOWUnp4eatWqhW3btqk8myWTyfDdd98hMDAQdevWxZ07d4rct0ZzdNevXw9bW1vs2rVL+aDayJEjsWHDBixfvhxPnz5F7969NelahVwux7Nnz5CVlQUdHR2YmZlpdZFLREREVNH0798fV65cUSlyAUBXVxcLFizAoUOHkJqaqlHfGhW6N2/ehJ+fH6ytrZXLjsnlcgCAh4cHevTogVWrVmkU6G2vXr1Cr169EB0d/d59EREREVHZs337dlhYWOR5vmvXrrh69apGfWs0PCqVSlG5cmUAgIGBAQCo7GhRv359HDx4UKNA79JgZgURERERlTMxMTGIjIzE8+fPMWTIEDg6OkIqleLZs2ewt7fXqE+NRnRtbW0RHx8PADA2Noa5uTnu37+vPB8fH88pBkRERERUKNOnT0e9evUwduxYzJw5Ew8ePAAAZGRkoHHjxli5cqVG/WpUjTZu3FhlCLl169bYunUr7O3toVAosHPnTjRp0kSjQERERETliRC4Q63t1xUrsGLFCgDAxIkTMXHixNKOVW6sXr0agYGB+Pzzz+Hr64tu3bopz1lYWKBXr17Yv38/vvjiiyL3rVGh27t3b4SHhyMjIwNGRkaYOHEioqOjMWfOHABApUqV8Pnnn2vStQpDQ0P4+voqp0kQERERlQcsbgtv5cqV6NOnD/73v//h5cuXauednZ3x66+/atS3RoWuh4cHPDw8lMfVq1fHnj17cP78eejq6sLFxaVYlsYyMzMr1iXKiIiIiKhsuXv3LiZMmJDn+cqVKyMhIUGjvottIq2xsTE6dOhQXN0RERERUQVgZGSEtLS0PM8/evQIVlZWGvWtUaE7ZMgQ9OzZE927d9f4hQFg/PjxRb5HEAQEBQVp/JqFERMTg4ULF+LatWswNTVFjx494O/vD319/XzvUygU+O2337Br1y4kJSWhfv36mDp1Kpo1a1aieYmIiIjKK3d3d+zduxcBAQFq5zIyMrBp0ya0a9dOo741KnRfvXqFX375BUuXLkXbtm3h4+MDT0/PIq+08OTJE+U6vGVFSkoKxo8fj5o1ayIwMBDx8fFYsmQJMjIyMH369Hzv/e2337B69WpMmjQJ9erVw65duzBp0iRs2bIF1atXL6XvgIiIiKj8mDZtGry9vfHJJ59g1KhRAIBnz57h8OHDmDVrFmJjY7F161aN+tZoC2CFQoFz587hwIEDOHbsGDIyMmBubo5u3brB19e3XK+4EBISgvXr1yM8PByWlpYAgD179uDnn39GeHh4ng/GZWZmolu3bvjoo4+Uk8+zsrLQt29ftGvXDl9//XWhXp9bABMREZUvwqJstTbFl1xmtSjWrFmDyZMnQyqVQqFQKAdCDQwMEBQUhBEjRmjUr0bvgiAIygfSXr9+jT///BMHDhzAnj178Pvvv6NmzZrw9fXVOJSYTp8+DXd3d2WRC7zZkWP+/Pk4e/Ysevbsmet9165dQ1paGrp06aJs09fXR8eOHXH06NESz01ERERUXo0dOxa9evXCrl278Pfff0OhUKBevXr46KOP4ODgoHG/Gm0Y8TZjY2P4+voiKCgIYWFhmDBhAhISEkp8Hm1JiYmJgaOjo0qbubk5bG1tERMTk+99ANTudXJywrNnz5CRkZHrfVKpFBKJRPknv8nYRERERNrK3t4en332GVasWIGlS5eiZs2aOHjwIJ49e6Zxn8U2rh4bG4sDBw7gjz/+QFpamsY7o6WkpCA0NBQ3btxAamoq5HK5yvmSfhgtJSUF5ubmau3m5uZISUnJ9z4DAwMYGhqq3adQKJCamgojIyO1+0JCQhAcHPz+wYmIiIjKoa+++gpHjx7FhQsXALyZItu1a1ecOHECCoUC3377Lc6ePYs6deoUue/3KnQlEgkiIiJw4MABXL9+XTnM/MUXX+DDDz8scn9xcXEYPXo0Xrx4ATMzM6SlpcHCwkJZ8FpZWcHY2Ph9Ipc5I0eOxJAhQ5THaWlp8PHxETERERERUek5dOiQytTP/fv34/jx4/jqq6/g4uKCzz77DAsWLNBoYFCjQvfEiRM4cOAATpw4AalUChsbGwwcOBC+vr6oX7++Jl0CeLMzRmpqKoKCglC3bl3l3NhmzZph3bp1iIiIwJo1azTuvzAsLCwgkUjU2lNTU2FhYZHvfVKpFJmZmSqjuqmpqRAEIddRYuDNJGsDA4P3D05ERERUDj1+/Bj16tVTHu/fvx9OTk5YsGABAODmzZvYsmWLRn1rVOhOnToVBgYG+OCDD+Dr64s2bdpAV1dXowBvu3DhAvz8/ODq6oqkpCQAb4avc7YZfvDgAZYvX4558+a992vlxdHRUW0urkQiQUJCgtr823fvA94savx2sR8TEwN7e/tcpy0QERERVXRSqVRlyuvRo0dVRnhr166NuLg4jfrW6GG06dOn49ChQ1iwYAHat29fLEUuACQnJyvnX+R8w5mZmcrzrVu3xrlz54rltfLStm1bnD9/Hqmpqcq2yMhI6OjoqGx7/C5nZ2eYmpoiMjJS2ZadnY2jR49qvMgxERERkbarUaMGzpw5A+DN6O2DBw9UdtuNj4/XeMlVjUZ0+/fvr9GLFcTa2lr5wJepqSkMDAzw9OlT5fmsrCyVwrck9OvXDzt27EBAQABGjRqF+Ph4LF26FH379lVZQ3fChAmIi4vDvn37AACGhoYYOXIk1qxZA2tra9StWxe7du1CcnIyhg4dWqKZiYiIiMqrgQMHYu7cuYiPj8fNmzdhYWGBHj16KM9fuXJFowfRgGJcdaE41K5dG//88w+AN6srNGnSBLt370aHDh0gl8uxd+/efKcPFAcLCwsEBQUhMDAQAQEBMDU1hZ+fH/z9/VWuk8lkkMlkKm3Dhw+HQqHA5s2b8erVK9SvXx/Lly/nrmhEREREefjmm2/w+PFj7Nu3D5aWlti4cSOsrKwAvPm0PywsDFOmTNGob412Rispu3btwubNm7Fjxw4YGRnhwoULmDRpEnIiCoKAhQsXqgxnaxvujEZERFS+cGe0kiOXy5GamgoTExPo6+sX+f4yVejm5tatWzh06BB0dXXh5eWF5s2bix2pRLHQJSIiKl9Y6JZdZf5daNy4MRo3bix2DCIiIiIqZ957C+Di1Lt3bxw7dizP8ydOnEDv3r1LMRERERERlVdFGtHNzs7GsWPH8PjxY1hZWcHLy0s5Wbg4PH36FK9fv87z/OvXrzVeR42IiIiIKpZCF7opKSkYN24c7t+/D4VCAUEQsHz5cvz6669o1KhRSWZUSkxM5MYLRERERFQohS50161bh3v37qF9+/Zo06YN/v33X/z+++/48ccfsXnzZo0DXL58GZcuXVIe//XXX3j8+LHadSkpKYiIiHivLYaJiIiIqOIodKF74sQJtGnTBkuWLFG2Va1aFUuXLsXz589hZ2enUYCLFy8iODgYwJvlw44ePYqjR4/mem2NGjUwdepUjV6HiIiIiCqWQhe6z58/x8cff6zS5unpif/973+Ii4vTuNAdPHgwevbsCYVCgd69eyMgIEBtnVxBEGBsbAxLS0uNXoOIiIiIKp5CF7pSqVSt0DQ3NwfwZmteTZmZmSnXi121ahWcnJxgY2OjcX9EREREREAxraMrCEJxdINWrVoVSz9EREREREUqdDdv3ozDhw8rj2UyGQRBwMqVK9VGewVBwOLFi/PtLzg4GIIgYNSoUdDR0VHO1c2PIAgYM2ZMUWITERERUQVUpEL3zp07uHPnjlr79evX1doKM8q7Zs0aCIKA4cOHQ0dHB2vWrCnwHha6RERERFQYhS50L1y4UOwvHhYWBgDQ19dXOSYiIiIiel/FMkdXU1WrVs33mIiIiIhIUzrF3eHLly/x22+/YcCAAUW6Lz09He7u7li7dm1xRyIiIiKiCqhYRnTlcjlOnDiB0NBQnD59GjKZDCYmJkXqw8TEBObm5rC2ti6OSERERERUwb1XoRsTE4OwsDAcPHgQiYmJMDc3x4cffojOnTujdevWRe6vVatWuHz5Mvr16/c+sYiIiIiIil7ovn79GhEREQgLC8P169ehq6uL5s2bIzExEd999x06deqkcZjJkydj7NixWL16NYYMGaLcSIKIiIiIqKgKXehGR0cjLCwMf/75J9LT09GgQQNMnToV3bt3R2pqKvr27fveYfz9/SGVSrFu3TqsW7cO1tbWMDIyUrsuNDT0vV+LiIiIiLRboQvdTz/9FDY2NujTpw98fX1Rt25d5TmJRFIsYezs7IptlzUiIiIiqtiKNHUhMzMTEomk2ArbdxVmwwgiIiIiosIodKG7a9cu7Nu3D3/88QfCwsJQrVo1+Pr6wsfHpyTzERERERFppNCFrqOjI7744gtMmjQJx48fR2hoKIKDgxEcHIy6detCEAQoFIqSzEpEREREVGiC4j2q04SEBISFhWH//v2IjY2FgYEB2rRpg06dOsHT01OjVROio6OxYcMG3LhxA6mpqWrFsyAIOHfunKaRyzyJRAIvLy9ERUVx1QkiIqJyQFiUrdam+FLUzWfp/73Xu2Bra4tRo0Zh1KhRuHTpEkJDQ/HXX3/h2LFj0NfXx+nTp4vU3+XLl+Hv7w8zMzM0bdoUp06dgpubG9LT03Hz5k3UrVsXDRs2fJ/IRERERFRBFNuvG61atUKrVq3w1Vdf4dChQwgLCytyH+vXr4etrS02bdoEQRDQtWtXjBw5Em5ubjh79iymT5+O6dOnF1dkIiIiItJiOsXdoZmZGfr374+NGzcW+d6bN2/Cz88P1tbWymXG5HI5AMDDwwM9evTAqlWrijUvEREREWmnQo/oJicnF7lzS0vLIl0vlUpRuXJlAICBgQEAID09XXm+fv36OHjwYJFzEBEREVHFU+hCt0uXLkXazEGTh8ZsbW0RHx8PADA2Noa5uTnu37+Pjh07AgDi4+Ohp8fJ3URERERUsEJXjT4+PiqFbmZmJo4cOQIPDw/Y2toWS5jGjRvj6tWryuPWrVtj69atsLe3h0KhwM6dO9GkSZNieS0iIiIi0m6FLnRnz56tcpyUlIQjR45g2LBhcHNzK5YwvXv3Rnh4ODIyMmBkZISJEyciOjoac+bMAQBUqlQJn3/+ebG8FhERERFptzI1D8DDwwMeHh7K4+rVq2PPnj04d+4c9PT04OLiUipryx4/fhxBQUF49OgR7O3tMWLECPTq1Svfe7KysrBy5UrcuHEDt2/fRkZGBiIjI2FlZVXieYmIiIhIXbGvulDcjI2N4eXlhfbt25dKkRsdHY1p06ahWbNmWLZsGbp27Yq5c+ciMjIy3/syMjKwb98+GBgYoEWLFiWek4iIiIjyV6ZGdAHg0KFD2LlzJx4/fpzrSg8lvTPa2rVr0aRJE3z77bcAAFdXV8TGxmL16tXo0qVLnveZm5vjr7/+giAI2L9/P86cOVNiGYmIiIioYGWq0F27di3WrFkDGxsbODs7w8LColRfXyqV4uLFi2rzgLt164bDhw/j6dOnqFatWp73F2VVCiIiIiIqWe9d6BZncbd79260atUKy5cvF2UZsdjYWGRnZ8PR0VGl3cnJCQAQExOTb6FLRERERGVHoavJgQMHqhzL5XIIgoC5c+fC2NhY7XpBELBt27YihUlLS0OXLl1EWys3JSUFwJtpCG/LGVnOOV+cpFIppFKp8jgtLa3YX4OIiIioIip0RZmWlqY2epuzvu3bu5e9jwYNGuD58+fF0lcOiUSChISEAq9zcHAo1tctrJCQEAQHB4vy2kRERETarNCF7v79+0syBwBgwoQJ+Oqrr9CpUyc0bNiwWPqMjIzEvHnzCrxu9+7dypFbiUSici5nJLck5gyPHDkSQ4YMUR6npaXBx8en2F+HiIiIqKIpUw+jtWrVCt9//z1GjhyJZs2aoWrVqtDRUV0BTRAEzJw5s9B9+vn5wc/Pr1DXSqVS6OnpISYmBm3atFG2x8TEAIDa3N3iYGBgAAMDg2Lvl4iIiKiiK5ZCNzs7Gzdv3sSLFy/g5OSEOnXqaNTPjRs3MHv2bGRnZ+PKlSu4cuWK2jVFLXSLwsDAAK6urvjzzz8xaNAgZfuRI0fg5OTEB9GIiIiIypFCF7oXL17E0aNHMXr0aNjY2Cjbnzx5gi+//BL3799Xtvn4+GDWrFlFDrNo0SLo6+vjl19+QYsWLdQeCisNY8aMwbhx47BgwQJ06dIFly5dwqFDhzB//nyV61q3bg0fHx+VovvUqVN4/fo1bt26BeDNDmsmJiaoXbs2ateuXarfBxEREVFFV+hCNzw8HNeuXcO0adNU2ufMmYN79+6hefPmaNq0Kc6cOYMDBw6gVatW8PX1LVKYf/75B2PHjoWnp2eR7itOLi4uWLhwIYKCghAaGgp7e3vMmDFDbbMImUwGuVyu0rZgwQLExcUpj3/44QcAwKeffopx48aVfHgiIiIiUip0oXvz5k14eHiotMXExODKlSto0aIF1qxZAwAYP348hgwZggMHDhS50LWxsYG+vn6R7ikJHTp0QIcOHfK95uLFi2ptpfHAHhEREREVjk7Bl7zx8uVL1KxZU6Xt4sWLEARB5WEvIyMjdO/eHf/880+Rw/Tq1Qt//PEHsrOzi3wvEREREdHbCj2iK5VKYWhoqNKWMxe1ZcuWKu12dnZqS3QVhouLC06cOIGRI0eif//+cHBwUFt1IbfXIyIiIiJ6V6ELXXt7ezx48EClLTo6GtbW1rC3t1dpz8jI0OhBMn9/f+XX8+bNU9ugQqFQQBAEnD9/vsh9ExEREVHFUuhCt0WLFjhw4AB69+6NunXr4ujRo3j8+HGu83Dv3buHypUrFzmMJis1EBERERHlptCF7ogRI/DHH39g8ODBsLS0RHJyMvT19TF06FCV62QyGY4fP45OnToVOUxRH14jIiIiIspLoR9Gc3BwwJo1a9CuXTtYWlqibdu2WL16tdrmEBcvXoSlpWWBqxYQEREREZUkQaFQKMQOQf+RSCTw8vJCVFQUzMzMxI5DREREBRAWqa8WpfiyWDafpfdU6BFdIiIiIqLyhIUuEREREWklFrpEREREpJVY6BIRERGRVmKhS0RERERaiYUuEREREWklFrpEREREpJVY6BIRERGRVmKhS0RERERaiYUuEREREWklFrpEREREpJVY6BIRERGRVmKhS0RERERaiYUuEREREWklFrpEREREpJX0xA5AREB6lgIRMQqYGQCdagrQEQSxIxEREZV7LHSJRHYrQYFOO2V4nv7muKUd8NdHurA0ZLFLRET0Pjh1gUhk356UK4tcALj8HPj1ikK8QERERFqChS6RyK6/UC9qbySw0CUiInpfLHSJROZZXX2KwgcOnLZARET0vljoEolsgacOPKq++VoAMLiRgE+dWegSERG9Lz6MRiQyO1MBZ4bo4Z9XCpjoAQ7mLHKJiIiKAwtdojKinjULXCIiouLEqQtEREREIhoxYgQEQVD74+vrK3Y0pZiYGAiCgN27d4sdpUg4oktEREQkstq1a2PLli0qbdbW1iKl0R4sdHNx/PhxBAUF4dGjR7C3t8eIESPQq1evfO+5efMmdu/ejStXruDFixeoUqUKOnfujNGjR8PY2LiUkhMREVF5ZGxsDA8PD7FjaB1OXXhHdHQ0pk2bhmbNmmHZsmXo2rUr5s6di8jIyHzvO3LkCB4/foxhw4Zh6dKlGDRoEPbu3YspU6aUUnIiIiLSVhs2bICzszOMjIzg4OCA7777DjKZTOW8IAi4ePEiunXrBhMTEzRo0ACRkZGQy+WYMWMG7OzsYGdnh2+++QZyuVx5799//42BAweiRo0aMDExQePGjfHLL7+oXKNpLrFxRPcda9euRZMmTfDtt98CAFxdXREbG4vVq1ejS5cued43fPhwlY8YXF1dYWFhgRkzZuD27dto1KhRiWcnIiKi8is7O1vlWE/vTZm2ePFifPXVV5gyZQp++eUX3L59W1lQLliwQOWeYcOGYfz48QgICMCCBQvQt29fDB8+HCkpKdi4cSPOnTuHWbNmoVmzZhg8eDAA4MmTJ2jQoAGGDBkCc3NzREdHY9asWZBIJJg1a1aeeYuSSzQKUsrMzFS0bt1asWXLFpX2qKgoRatWrRRPnjwpUn8PHjxQtGrVSnHs2LFC35Oamqpo1aqVIjU1tUivRUREROJAYJban6IYPny4AoDanxMnTihSUlIUZmZmim+++UblnqCgIIWxsbEiISFBoVAoFCEhIQoAipUrVyqvuX79ugKAwsPDQ+XeVq1aKfz8/HLNIpfLFVlZWYoff/xRUbVqVWX7w4cPFQAUu3btUigUikLnEhtHdN8SGxuL7OxsODo6qrQ7OTkBePPEYbVq1QrdX3R0NACo9fc2qVQKqVSqPE5LSyt0/0RERKQd6tSpg+3bt6u0NWzYEKdOnYJEIsGAAQNURny7dOmC169f48aNG+jQoYOyvWvXrsqv69evDwDo3LmzSr/169fH3bt3lccZGRmYP38+tmzZgn///RdZWVnKcxKJBGZmZmp5T58+XaRcYmGh+5aUlBQAgLm5uUq7hYWFyvnCSEpKwpo1a9ChQwfUrFkzz+tCQkIQHBysQVoiIiLSFkZGRnB1dVVrT0hIAAC0bNky1/seP36scmxlZaX82sDAQK0tpz0jI0N5PH36dAQHB2PWrFlo1aoVrKysEBoainnz5iEjIyPXQreoucSi9YWuRCJRvhn5cXBwKLbXzM7OVs7x/eabb/K9duTIkRgyZIjyOC0tDT4+PsWWhYiIiMovGxsbAMCePXtQo0YNtfM5nzq/j127dmHcuHGYPn26su3AgQOi5yoOWl/oRkZGYt68eQVet3v3buXIrUQiUTmXM5Kbcz4/CoUCc+bMwc2bNxEcHAxbW9t8rzcwMFD+xkVERET0tjZt2sDExASxsbHo06dPibzG69evVWoRmUymNo1CjFzFQesLXT8/P/j5+RXqWqlUCj09PcTExKBNmzbK9piYGAD5z7XN8b///Q+RkZFYunSpcm4MERERkSasrKzwww8/4KuvvkJsbCy8vLygq6uLBw8eIDQ0FL///jtMTEze6zW6du2K4OBgNG7cGLa2tli5ciUyMzNFz1UctL7QLQoDAwO4urrizz//xKBBg5TtR44cgZOTU4EPom3YsAFbt27F3Llz4e7uXtJxiYiIqAIICAiAg4MDFi9ejOXLl0NfXx916tSBr69vsXwqvHz5cowfPx6fffYZTExMMGLECPTp0weffvqpqLmKg6BQKBRihyhLoqOjMW7cOPTp0wddunTBpUuXsHbtWsyfP19lHd3WrVvDx8cHM2fOBAAcOnQIM2bMwIcffoiPPvpIpc/q1asXehs/iUQCLy8vREVF5Tr5m4iIiMoWYVG2WpviS44llgV8F97h4uKChQsXIigoCKGhobC3t8eMGTPUNouQyWQqO4acPXsWAPDHH3/gjz/+ULl21qxZ6NmzZ8mHJyIiIiIljuiWMRzRJSIiKl84olt26YgdgIiIiIioJLDQJSIiIiKtxEKXiIiIiLQSC10iIiIi0kosdInKituxwL8vxE5BRESkNfhIIJHYnr0Cev4EXLz/5vijtsCmyYCBvri5iIiIyjmO6BKJ7evN/xW5ALDzNBAcKV4eIiIiLcFCl0hsJ26pt528Xfo5iIiItAwLXSKxOdcqXBsREREVCQtdIrHNHwpUs/nv2K0uMPFD8fIQERFpCT6MRiSyTEd7fD36Mzx5mAK5rg7atbLGFAsTsWMRERGVeyx0iUQ2Z/Hf+J9FE6D5m+PfMwHHbf+gz6B64gYjIiIq5zh1gUhkv6fbqLXtuSEVIQkREZF2YaFLJDL77DS1Nju9bBGSEBERaRcWukQim9FCBv3s/wrbymkpmNTXXsRERERE2oFzdIlE1vXjhrgSFYNtfyXCTB8Y/lENVG1kJ3YsIiKico+FLlEZ0MTLEfO8HMWOQUREpFU4dYGIiIiItBILXSIiIiLSSix0iYiIiEgrsdAlIiIiIq3EQpeoLJDLgdN/A9dixE5CRESkNbjqApHYYuKBrrOBe8/eHHdxBkK/AUwMRY1FRERU3nFEl0hsI5f/V+QCQOQ14Oe94uUhIiLSEix0icR24rZ6W3BE6ecgIiLSMix0icQml6u3vc4q/RxERERahoUukdiqV1Jv69Ck9HMQERFpGRa6RGLb+zVg8NZzoVUsgU2fi5eHiIhIS7DQJRJb3CsgS/bfsTQbeJ4sXh4iIiItwUKXSGzzfwcUiv+Ok9KAFX+Il4eIiEhLsNAlEtvbS4vluPW49HMQERFpGW4YkYvjx48jKCgIjx49gr29PUaMGIFevXrle09sbCwWLVqEu3fvIikpCRYWFmjevDn8/f1Rq1atUkpO5dLbo7k55Lm0ERERUZFwRPcd0dHRmDZtGpo1a4Zly5aha9eumDt3LiIjI/O97/Xr16hUqRImTpyIZcuW4YsvvsCjR48wfvx4JCUllU54Kp9q2qq31a9W+jmIiIi0DEd037F27Vo0adIE3377LQDA1dUVsbGxWL16Nbp06ZLnffXq1cP333+v0ta4cWP07dsXZ8+eRffu3Us0N5VjU3sBQ/7337GRATDeW7Q4RERE2oIjum+RSqW4ePGiWkHbrVs3PHz4EE+fPi1Sf5aWlgCArCwu/k/5GOwJ7AgAWtcHOjcDjs8FmnG6CxER0ftiofuW2NhYZGdnw9HRUaXdyckJABATE1NgH3K5HNnZ2Xj69CkWLlwIOzs7dOzYsQTSkta48gAYtwo4dxf48zrwyTIgIUXsVEREROUepy68JSXlTXFhbm6u0m5hYaFyPj+zZs3CH3+8WRqqevXqWLlyJczMzPK8XiqVQiqVKo/T0tKKnJvKue+2vllSLMedJ8CvB4HZA8XLREREpAW0vtCVSCRISEgo8DoHB4dieb3x48dj4MCBePbsGbZt2wZ/f3+sW7cO9vb2uV4fEhKC4ODgYnltKqfu5jIl5k7RpskQERGROq0vdCMjIzFv3rwCr9u9e7dy5FYikaicyxnJzTmfHwcHBzg4OKBJkyZo164d+vTpg99++w3Tp0/P9fqRI0diyJAhyuO0tDT4+PgU+DqkRTo3A+6/s5ZuF2dxshAREWkRrS90/fz84OfnV6hrpVIp9PT0EBMTgzZt2ijbc+bmvjt3tyBGRkZwcnJCbGxsntcYGBjAwMCgSP2SllnwCfDoBXA4GtDXA8Z0BkZ2EjsVERFRuceH0d5iYGAAV1dX/PnnnyrtR44cgZOTE6pVK9raphKJBP/880+xTYsgLWVtBgxsD7Rr+GZ0t39bQIc/mkRERO9L60d0i2rMmDEYN24cFixYgC5duuDSpUs4dOgQ5s+fr3Jd69at4ePjg5kzZwIAVq9eDYlEgubNm8Pa2hpxcXHYvn07pFIpBg0aJMa3QuVF0CHAf81/x0euAqd+erPcGBEREWmMhe47XFxcsHDhQgQFBSE0NBT29vaYMWOG2tq6MpkMcrlcedywYUNs2bIFBw8exOvXr1G5cmW0bNkSCxYsQPXq1Uv726DyJOiw6rFMDqyNZKFLRET0nljo5qJDhw7o0KFDvtdcvHixyPcQ5erfF+ptN/8t/RxERERahhMBicSWlqne9vRV6ecgIiLSMix0icRmoKveZmZU+jmIiIi0DAtdIrH1b6veFtCr9HMQERFpGc7RJRJb8ARATxfYfhIw1AOm9wFGdhY7FRERUbnHQpdIbAb6wLqJb/4QERFRseHUBSIiIiLSSix0icqCo9cB3x+BwYuBu0/FTkNERKQVOHWBSGwr/wAmBv93vPM0cO5noFUd8TIRERFpAY7oEont+22qxzI5MHmdOFmIiIi0CAtdIpEpUl+rNz5JLP0gREREWoaFLpHIrtaup9Z2w7W5CEmIiIi0CwtdIpFlZMjU2mKeSUVIQkREpF1Y6BKJrN6LOLU2x4cxpR+EiIhIy7DQJRKZaWaGWpvjqxciJCEiItIuLHSJRGYkz1ZrMzPRFSEJERGRdmGhSyQ2eyv1tvYNSz0GERGRtmGhSyQ2I4Nc2gxLPwcREZGWYaFLJDbPJupt7RuVfg4iIiItw0KXSGwLhgLOtf477tMaGNtVvDxERERagoUukdhi4oH7z/87jo4BEiWixSEiItIWLHSJxDZrB5D21hJjD58Dyw+Kl4eIiEhLsNAlEtv9Z+ptD56rtxEREVGRsNAlEtuHLdTburuUegwiIiJtw0KXSGxdXQBB+O/Y2ADwbCxaHCIiIm3BQpdIbL+EAgrFf8evpcCKQ+LlISIi0hIsdInE9ixJvS3uVanHICIi0jYsdInE1qe1eltfj9LPQUREpGX0xA5AVOHNGQikvgY2RgFmRsCXvVnoEhERFQMWukRiM9QHVox984eIiIiKDacuEBEREZFWYqFLRERERFqJhS4RERERaSUWurk4fvw4Bg0ahLZt26Jv374ICwsrch8BAQFwdXXFpk2bSiAhERERERWEhe47oqOjMW3aNDRr1gzLli1D165dMXfuXERGRha6j1OnTuHGjRslmJKIiIiICsJC9x1r165FkyZN8O2338LV1RUTJkxAt27dsHr16kLdL5VKsWjRIkycOLGEkxIRERFRfljovkUqleLixYvo0qWLSnu3bt3w8OFDPH36tMA+Nm3aBHNzc/Ts2bOkYhIRERFRIbDQfUtsbCyys7Ph6Oio0u7k5AQAiImJyff+Z8+eYcOGDZg2bRoEQSihlERERERUGNww4i0pKSkAAHNzc5V2CwsLlfN5+eWXX9CxY0c0a9as0K8plUohlUqVx2lpaYW+l4iIiIjypvWFrkQiQUJCQoHXOTg4vNfrnD17FufOncPvv/9epPtCQkIQHBz8Xq9NREREROq0vtCNjIzEvHnzCrxu9+7dypFbiUSici5nJDfnfG4CAwPx8ccfw8jICKmpqcr2zMxMpKamqo0S5xg5ciSGDBmiPE5LS4OPj0+BeYmIiIgof1pf6Pr5+cHPz69Q10qlUujp6SEmJgZt2rRRtufMzX137u7bHj16hJCQEISEhKi0r1q1CqtWrcKpU6dgaGiodp+BgQEMDAwKlY+IiIiICk/rC92iMDAwgKurK/78808MGjRI2X7kyBE4OTmhWrVqed67atUqtbbx48ejX79+6Nq1K/T19UskMxERERHljoXuO8aMGYNx48ZhwYIF6NKlCy5duoRDhw5h/vz5Kte1bt0aPj4+mDlzJgDA1dU11/6qV6+e57ncKBQKAHwojYiISFOmpqZc/YgAsNBV4+LigoULFyIoKAihoaGwt7fHjBkz1NbWlclkkMvlxf766enpAMB5ukRERBqKioqCmZlZqb2e4kv1cmrFihVYsWIFAGDixIncSEokgiJnCJHKBLlcjhcvXsDExKTYfhvNecDtwIEDMDU1LZY+qfjxfSof+D6VfXyPyoeSfJ84oks5OKJbxujo6MDOzq5E+jY1NS3V33BJM3yfyge+T2Uf36Pyge8TlSTujEZEREREWomFLhERERFpJRa6FYCBgQE+/fRTrtdbxvF9Kh/4PpV9fI/KB75PVBr4MBoRERERaSWO6BIRERGRVmKhS0RERERaiYUuEREREWklFrpEREREpJW4YQSRyBQKBRISEmBtbQ09Pf5IEpF2CA8PL9L1vr6+JZSEKjKuuqAl5syZU6TrZ82aVUJJqLDOnDmD1atX486dO5DL5fjtt9/QsGFD/Pjjj2jZsiU+/PBDsSNWSG5ubkXaOvT8+fMlmIby0qtXryK9T6GhoSWYhnLj5uamcpzzfr1ddrz9HvJniUoCh4+0xJ07d1SOX7x4gaSkJFhYWMDGxgaJiYlISUmBlZUVqlSpIlJKynHo0CHMnDkTXbp0gZ+fH3788UflOQcHB+zfv5+FrkimTJmi/MdXJpNh27Zt0NPTg5eXF2xsbPDy5UscO3YM2dnZGDRokMhpK64OHTqoHP/111+QSCRwd3dHpUqV8PLlS5w/fx5mZmbo3LmzSCkrtqNHjyq/fvz4Mb7++mv06NEDnTt3Vv67FBkZiT/++APz588XMSlpMxa6WmLr1q3Kr0+fPo0FCxZg/vz5cHV1VbZfuHABc+fOhb+/vxgR6S3r1q3DoEGDMGXKFMhkMpVCt06dOti2bZuI6Sq2wYMHK79etmwZ6tevj19++QU6Ov890vDFF18gICAAL1++FCMiAQgICFB+vXHjRtjZ2WHHjh0wMzNTtkskEnz++eewsbERI2KF9/Z7sXz5cvTp0wcjRoxQttnY2KBu3bowNDTE8uXLERQUJEJK0nZ8GE0LLVu2DOPGjVMpcoE3HyONHTsWS5cuFSkZ5Xjy5AnatWuX6zljY2NIJJJSTkS5CQ8Px4ABA1SKXADQ0dFB//79izwHkUrGjh07MGLECJXCCnhTaA0fPhw7d+4UKRnluHbtGho2bJjruUaNGuH69eulnIgqCha6Wujx48ewtLTM9ZyFhQWePHlSyonoXZUqVUJMTEyu5/755x/Y29uXbiDKVWZmJp4+fZrrubi4OEil0lJORLlJTk7O85dDiUSClJSUUk5E77KxscGRI0dyPXf48GFYW1uXciKqKDh1QQs5OTlhw4YNaNmyJUxMTJTtaWlp2LBhA5ycnERMRwDQvXt3rFmzBo6OjmjVqpWy/d69e9i4cSP69esnYjrK0aFDByxfvhxGRkbw8vKCmZkZJBIJjh49il9//VVtniiJw83NDcuXL4ednZ3Kz9PFixfx66+/qj0URaVv5MiR+OmnnxAbGwsvLy9YW1vj1atXOHr0KK5cuYJvv/1W7Iikpbjqgha6evUqPv/8cwiCAFdXV+VfKBcvXoRcLseyZcvg4uIidswKLSsrC9OnT8eJEydgaWmJ5ORk2NjY4NWrV2jfvj0WLlzIpcbKgLS0NMyZMwdRUVEAAD09PWRnZ0OhUMDLywuzZs1S+7icSl9CQgKmTp2Kv//+G2ZmZrCyskJSUhIkEgkaNGiAxYsXo3LlymLHrPBOnDiB9evX4/bt25DJZNDV1UXDhg0xatQoeHp6ih2PtBQLXS318uVLbN26FTdv3kRCQgJsbW3RpEkTDBo0CLa2tmLHo/938eJFnDt3TrlChru7O1q3bi12LHrHw4cP1X6W+MlI2XP69Gm196lt27Zix6J3yOVyvHr1CtbW1mrz34mKGwtdIiIiKjXcJIdKE/8P02IpKSm4f/8+nj9/jrZt28LCwgKZmZnQ19fnb9Eii4uLQ1paGurWrQsAkEql2LRpE2JiYuDu7o6ePXuKnJByZGdnY9++fbh16xaeP3+O6dOno2bNmoiIiEC9evU4sluGnD59Wvk+jR49Gvb29rh8+TJq1KjBqQtlADfJITGw2tFCCoUCK1asgI+PD8aOHYuZM2cqnxyfNm0a1q5dK3JCmjdvHg4cOKA8Xr58OYKDgxETE4Mff/wRu3btEjEd5YiNjUW/fv2wfPlyxMbG4sKFC0hPTwcAXLlyBRs3bhQ5IQHAq1evMGrUKHzxxRcIDQ1FaGgokpKSAABhYWFYv369uAEJhw4dwuTJk1GtWjVMnz4dcrlceS5nkxyiksBCVwsFBQVh586d+OKLL7Bnzx6V7RY9PT1x4sQJEdMRANy9exctWrQA8GbEMDw8HJ999hk2bdqEsWPHYvfu3SInJABYtGgRrKysEBoaipUrV6r8LLVs2RKXL18WMR3l+OWXX5CUlIQdO3Zg7969Ku+Tu7s7t5YtA3I2yfnpp5/UPrGqU6cO7t+/L1Iy0nYsdLVQeHg4/P390a9fP1SrVk3lXPXq1REbGytSMsqRnp6ufFr/xo0bSEtLg7e3NwDAxcWFax2XEZcuXcLo0aNhZWWl3BY4R6VKlZCQkCBSMnrbyZMn4e/vDycnJ7X3yc7ODvHx8SIloxzcJIfEwkJXCyUnJ+c5b1AulyM7O7uUE9G7qlSpotwJ6OjRo3ByclKuhpGSkgIjIyMx49H/09XVRV7P6yYmJqqsU03ikclkef7MpKamQl9fv5QT0bu4SQ6JhYWuFqpZsybOnTuX67lLly6hTp06pZyI3tW7d28EBQVh2LBh2L59O/r06aM8d+PGDT7gVEa0bNkSW7ZsUfvlUKFQYM+ePdyIoIxo2rQpwsLCcj13+PBhNG/evJQT0btyNsl5dxpJziY5fBCNSgpXXdBCgwcPxrx586Cnp4fOnTsDAJ4/f45r165h+/btmD17trgBCSNGjICtrS1u3bqF/v37q8xZS0lJQe/evUVMRzk+//xzjBo1CgMGDICnpycEQcCuXbtw//59PH78GL/99pvYEQnAhAkTMH78eHz66afo3LkzBEFAVFQUQkJCcPLkSaxbt07siBXe2LFjcf/+fUycOFG5Rf3kyZOVm+SMGDFC3ICktbiOrpbavHkzgoOD8fr1a+VHr8bGxhg7diyGDh0qcjqi8uPJkydYs2aNcmMPS0tLuLu7Y9y4cahevbrY8ej/Xbt2Db/++iuuXr0KuVwOQRDQrFkzTJ48Gc7OzmLHo//HTXKotLHQ1WLp6em4du2a8i8UZ2dnbldaRjx79qzAazhnjajoMjIykJqaCnNzc851JyIWukRicHNzU3s6/F1cEomocObMmYMxY8bAwcFB7VxcXBzWrFmDWbNmiZCMckyZMgUtW7ZEixYt0KhRI+jq6oodiSoIFrpaaPv27Xjx4gU+++wztXPLly+HnZ0dPvroIxGSUY6oqCi1ttTUVJw5cwY3btzApEmT0K1bt9IPRpgyZQqmTJmCmjVrYsqUKfleKwgCFi9eXErJKC9ubm4ICQlB06ZN1c7dvn0bw4cP5y+OIvv2228RHR2NFy9ewNjYGE2bNkWLFi3QokULODs7w8DAQOyIpKX4MJoW2r17N4YMGZLruZo1a2LLli0sdEXm5eWVa3vPnj2xePFiXL58mYWuSNLT05W7NqWlpRU48k5lQ17v07///qt8+InE89NPPwEAHj9+jMuXLyM6Ohrh4eEIDg6Gnp4eGjduzF07qUSw0NVCcXFxqFGjRq7nHBwclNsBU9nUrl07fPPNN/j666/FjlIhrV69Wvn1mjVrRExC+dm9e7dyB0FBEDBjxgwYGhqqXCOVSvH06VN06dJFjIiUixo1aqBGjRpo1aoVmjdvjkOHDuHSpUu4du2a2NFIS7HQ1UJmZmZ5FrNPnjzhAxpl3LVr1/gxXhmQmZmJESNGYPLkyfDw8BA7Dr3D1tYWjRo1AgDcv38ftWrVgrW1tco1enp66N+/P5frKwMePnyIy5cv4/Lly7hy5QpevnyJ2rVro0WLFujfv79yS3Si4sZCVwu1bt0awcHBcHd3V3ly//nz51i3bh3atm0rYjoCgMDAQLW2rKwsxMTEIDo6mkvAlQGGhoaIj4/n1IUyysvLS2UKUF4Po1HZ8NFHH8HQ0BC+vr74+uuv4eLiAgsLC7FjUQXAh9G00IsXLzBixAgkJSXBzc0NlStXxosXL3DhwgVYW1tj/fr1qFKlitgxK7RevXqptRkYGKBKlSro3Lkz/Pz8+FRyGfDjjz8CAL777juRkxCVb1OmTMHVq1fx+vVrNG7cGC1btkTLli3RvHlzbqVNJYqFrpZKTk7G5s2bcfHiRSQnJysXuR88eDAfzCAqpPDwcKxYsQINGjRAu3btYGNjozbC26lTJ5HS0dtSU1Px559/4tGjR5BKpSrnBEHAl19+KVIyyqFQKPDPP/8opy9ER0cjOTkZ9evXR8uWLfHFF1+IHZG0EAtdLZOZmYlly5ahR48eaNKkidhxiMo1Nze3fM8LgsBlq8qAf//9F6NGjUJWVhZev34Na2trJCcnQyaTwcLCAmZmZggNDRU7Jr0lISEBly5dwp49e3D58mX+LFGJ4RxdLWNoaIiwsDCOMpUDf//9N0JCQhAdHY2UlBRYWFigRYsWGDlyJBo0aCB2PAIQFhYmdgQqhCVLlqBp06ZYsGABPvjgAyxduhT16tXDkSNHsGLFCixYsEDsiBXekydPcOXKFeVo7pMnT6Cnp4cGDRrgk08+QcuWLcWOSFqKha4WcnZ2xvXr19GqVSuxo1Aerly5gokTJ6JSpUrw9vaGjY0NEhMTERUVhVGjRmHFihVwcXERO2aFlpmZiYULF/If4XLg5s2b+P7775WrlWRlZUFXVxfdu3dHUlISAgMDsX79epFTVmx+fn4wMDBAkyZN4O3tjZYtW8LZ2ZmrAFGJY6GrhcaPH48ZM2ZAV1c3z3mFnKcrruXLl6NVq1ZYsmQJ9PT++zGcPHkyvvjiCyxfvhzr1q0TMSEZGhri8uXLeW6+QmWHVCqFqakpdHR0YGFhgRcvXijP1alTB3fv3hUxHQFAcHAwmjRpAn19fbGjUAXDQlcLjRo1CgCwbNkyLF++PNdrOBdKXHfu3MHPP/+sUuQCgK6uLj7++GNMnz5dpGT0Ng8PD5w9exaurq5iR6F81KxZE3FxcQCABg0aYPfu3WjdujV0dXWxZ88eVK5cWeSE9PYnVBkZGZBIJDAzM+OILpU4FrpaaObMmVz7s4wzNjbGq1evcj2XmJgIY2PjUk5EuenZsyfmz5+PtLQ0tGvXDpUqVVL72WrYsKFI6ShHt27dcPfuXfj4+GDChAmYNGmSynMKs2fPFi8cKZ04cQLBwcG4c+cOFAoFBEFAgwYNMHbsWLRv317seKSluOoCkQjmzJmDU6dOYe7cuWjdurWy/dy5c5g5cybatWuHmTNnipiQAPVVF94ucnP+oeanI2XPs2fPcObMGWRmZsLV1RV169YVO1KFFxUVha+++gpNmzZFt27dYGNjg5cvXyIyMhI3btzAzz//rLIBCFFxYaGrxVJSUnDz5k3lOrpNmjThTjRlREpKCj777DPcvn0bpqamsLa2xqtXr5CWlobGjRtj2bJlfK/KgEuXLhV4DR/6FBeXVCwfBg8ejDp16mDu3Llq577//nvcv38fW7duFSEZaTtOXdBCCoUCy5Ytw86dO1UWTjcwMMDHH3+Mzz//XMR0BAAWFhYICQnBiRMnEB0djdTUVFhYWMDFxQXt27eHjo6O2BEJLGLLAy6pWD48evQoz397evTowQ09qMSw0NVC69evx9atWzFs2DDlR0SJiYmIiIjAxo0bYW5ujpEjR4ods8LT0dFBhw4d0KFDB7GjEJVrXFKx7LOwsEBMTAw8PDzUzj169IifYFGJYaGrhUJDQzFmzBh8+umnyrZKlSqhXr16MDAwwN69e1nolhFnz57FjRs38PLlS1SqVAnNmjVTmbNL4jtw4AD27NmDf//9V21rWQA4duyYCKnobVxSsezr2rUrVq5cCSMjI3Tu3Bnm5uaQSCSIjIxEUFAQ/Pz8xI5IWopzdLVQ27ZtsWTJklwLprNnz2Lq1Kk4ffq0CMkoR0JCAr766itcv34dFhYWylH3lJQUNGvWDAsXLoStra3YMSu8gwcPYt68efD19cXevXvRq1cvyOVyHD9+HObm5vDx8VH5hZLE8fZDg3mtOMOHBsUllUrx3XffISoqCoIgQE9PD9nZ2VAoFOjYsSPmzZun3PCDqDhxRFcLVa1aFSdPnsy10D116hSqVq0qQip62/z58/HkyRMEBQWprNF64cIFfP/991iwYAEWLVokYkICgM2bN2P06NEYMWIE9u7diwEDBqBhw4ZIS0vDpEmTuAxcGcElFcs+AwMDBAYG4p9//lFue25paQkXFxeuikElioWuFho8eDAWLFiApKQkdO7cWTla+Oeff+Lw4cP4+uuvxY5Y4Z07dw7ffPON2kYEbm5umDRpEhYsWCBSMnrb48eP0bx5c+jq6kJHRwcSiQQAYGpqiuHDh+OXX37B0KFDRU5JPXv2FDsCFVK9evVQr149sWNQBcJCVwv169cPWVlZWLduHQ4dOgRBEKBQKGBtbY2AgAD07dtX7IgVnoWFRZ4PX1hYWMDc3LyUE1FuzMzMkJWVBQCoUqUKHj58qPzlRCaTITk5Wcx4RGXa5cuXi3R9y5YtSygJVWQsdLXUwIED8dFHHyEmJkb5EVGtWrW4bFUZMWjQIGzYsAGtWrWCiYmJsj0tLQ2//fYbBg4cKGI6ytGoUSP8888/aNOmDTw9PREcHAy5XA49PT389ttvaNq0qdgR6f9dvnwZe/fuxaNHj3J9aHD79u0ipKrYxo0bpxxoAXLfcOVtnEdNJYGFrhY6e/YsWrduDR0dHdSuXVvsOJSLuLg4xMXFoUePHnB1dVVuGHHx4kWYmpoiPj4egYGBAN7848A1JsUxcuRIxMXFAXjzZH9cXBwWL14MuVyOxo0b49tvvxU5IQHAmTNn8MUXX8Dd3R23b99G27ZtkZmZiatXr6JKlSocKRTJli1blF8nJiZi7ty5cHV1VZlSFxkZiUuXLuH7778XMSlpM666oIXc3NxgY2ODrl27wtvbG82aNRM7Er2jV69ehb5WEASEhoaWYBoqCqlUCqlUCjMzM7Gj0P8bMWIEXFxc8Nlnn8HDwwObNm1Cw4YNERcXh0mTJmH06NHo0aOH2DErtK+++goODg6YPHmy2rmlS5ciNjZW+cs9UXHiiK4W2rZtGw4fPowjR45gx44dqFatGrp16wZvb28+3VpGhIWFiR2BNGRgYMBlkMqYhw8fwt/fHzo6OhAEAa9fvwbwZgWasWPHYs2aNSx0RXb27FksXLgw13OtW7fG77//XsqJqKJgoauF6tati7p162LixIm4ceMGDh8+jAMHDuC3336Dk5MTunfvzg0jiArp+vXriIyMxPPnz5GZmalyThAELF68WKRklMPQ0BByuRyCIMDW1haxsbFo0aIFgDcrZDx//lzkhGRsbIwLFy7kujPauXPnuFQflRgWulquadOmaNq0KaZOnYoTJ05gwYIFCAoKYqErstOnTyMlJQXdu3cHADx79gw//PADYmJi4O7ujunTp/Mv/jJg27ZtWLx4MWxsbODg4AB9fX2xI1Eu6tWrh0ePHsHDwwNubm5Yv349rKysoKenh5UrV/KTrDJgwIABWL16NRITE9GhQwflHN2oqCgcPHgQY8eOFTsiaSnO0dVyUqkUx48fR0REBE6dOgWZTAZ3d3csW7ZM7GgV2rBhw9C1a1d88sknAICpU6fi77//Rrdu3fDHH3+ge/fumDJlisgpycfHB15eXggICOCKJWXYyZMn8fTpU3z00UeIj4/HlClTcPfuXQBvloVbtGgRGjVqJHJK2rFjB3777Te8ePFCuRqDra0thg8fzpVmqMSw0NVCMpkMZ86cweHDh3H8+HG8fv0azZs3h7e3N7p06QIrKyuxI1Z4HTt2xPz58+Hh4QGJRIKuXbti7ty56NKlC/bv34/g4GDO4y0DOnXqhAULFsDd3V3sKFQECoUCjx8/RmZmJhwdHTkSX4bI5XLEx8cjISEBtra2qFKlCn+JpBLFqQtaqFu3bkhNTUX9+vUxZswYdOvWDXZ2dmLHorfIZDLlX+5XrlyBQqFA27ZtAQAODg54+fKlmPHo/3Xt2hWnT59moVvOCIKAmjVrih2DcqGjowN7e3vY29uLHYUqCBa6Wujjjz+Gt7c3atWqJXYUyoOjoyP++OMPNGvWDHv27IGzs7Ny44iEhARYWlqKnJAAICAgAHPnzsV3330Hd3f3XHes69SpkwjJ6F33799HSEgIbty4oRwtbNasGYYPH845uiLZvHkzPvzwQ1SqVAmbN2/O91pBEDBkyJBSSkYVCacuEIng2LFj+Prrr5Uju//73/+UTyPPmTMHSUlJWLJkicgp6e7du5g2bRqePn2a63lBELibUxlw8uRJTJs2DVWqVIGXl5fKg045m6+0b99e7JgVjpubG0JCQtC0aVO4ubnley1/lqiksNDVEvzNufyJjY3FnTt3UK9ePZWPWffs2YN69epxo48yYOjQocjOzsakSZNQs2bNXOd6Vq1aVYRk9LYBAwagRo0aWLRokcp8T7lcjoCAAMTGxmLXrl0iJiQisbDQ1RL8zZmo+LVr1w6BgYHK+dNUNuX3Pp0+fRrTpk3DqVOnREhWsUVGRsLFxQW2trZiR6EKjHN0tcSFCxdy/ZrKPplMhjZt2mDjxo1o2LCh2HHoLQ0aNEBiYqLYMagAdevWzXN6ydOnT1GnTp1STkQA8M0330AQBFSrVg3NmzeHi4sLXFxc4OTkJHY0qkBY6BKVAfxgpWyaPn065syZA1tbW7i6ukJPj39llkXTp0/Ht99+CyMjI3h5ecHMzAwSiQRHjx7F5s2b8eOPP4odsULavHkzoqOjce3aNVy6dAkHDx6EIAgwNzeHs7MzXFxc0Lx5czRp0oRLwFGJ4dQFLZOeno6IiAhcu3YNL1++hCAIqFSpEpo3b45u3brByMhI7Ij0Do7oll2enp7Izs5GdnY2BEHI9efn2LFjIiSjt739PgGAnp6eytfvFlF8z8Tx/PlzXL16FVevXsW1a9dw9+5dKBQK6Ovro3HjxggODhY7ImkhDk9okQsXLuCbb75BcnIydHV1YWVlBYVCgeTkZISFhWHFihWYP38+WrZsKXZUegd/3yybhgwZAkEQxI5BBeD7VD7Y2dmhW7du6NatG2QyGa5cuYItW7bg1KlTuHr1qtjxSEtxRFdLPH/+HB9//DHs7Ozw+eefw83NDQYGBgDebAN8/vx5LFu2DC9evMCOHTtQpUoVkRNXbGlpaTA1NRU7BhFRqUhNTVWO5F69ehW3bt1Cdna2coUZZ2dneHt7ix2TtBALXS2xfPlyHDp0CNu3b891UXsASElJwaBBg9CjRw9MnDixlBPS29q1a4cPPvgA3t7eaN++PeenlRPh4eHw9PSEhYWF2FEoD8+ePcPz589Rv359GBsbix2nQgsPD1dOVYiJiYGlpaWyqHV2dkbjxo05nY5KHAtdLTFs2DB4eXlh1KhR+V63fv16HD16FJs2bSqlZJSbHTt2ICIiAtevX4eJiQk6duwIb29vuLu7c9/3Mopzqcu2PXv2IDg4GAkJCRAEQfk+ffnll2jVqhUGDRokdsQKx83NDcbGxvD19cWAAQO42gKJgnN0tURsbCwaN25c4HWNGzcucEMJKnkff/wxPv74Yzx79gyHDx9GREQEwsPDYWNjg86dO8Pb2xvNmzcXOya9g+MCZdPWrVuxfPlyDBkyBG5ubpg0aZLyXKtWrRAZGclCVwSDBw/GtWvXsHfvXuzfvx+NGjVC8+bN0bx5czRr1oxbnVOpYKGrJdLS0mBmZlbgdWZmZkhLSyuFRFQY9vb2GD58OIYPH45Hjx7h8OHD2Lt3L37//XecO3dO7HhE5cKOHTswevRojBkzBjKZTOVcrVq18OjRI5GSVWxTpkwBAGRmZuLmzZvK+bl79uxBamoqatSoAWdnZzRv3hzOzs6oXbu2yIlJG7HQ1RJyubzQTx1zVKrsSUxMxNmzZ3H27FkkJCQU6pcWKlmZmZlYtmwZevTogSZNmkBHRwe+vr6wsrISOxq948WLF3l+AqKnp4fXr1+XciJ6m6GhIVq2bKmy4k9MTAyio6Nx7NgxzJ8/HwD4yz2VCBa6WmTGjBkwNDTM95rMzMxSSkMFSU1NxZ9//omIiAhcunQJ+vr6+OCDDzBs2DC0a9dO7HgVnqGhIcLCwtCpUycAb7bOnjVrlsipKDf29va4efNmrtuf37hxAzVr1hQhFeUmp8DNeUjt8ePHAJDnQ9RE74uFrpbw9fUt9LUuLi4lF4QKZcqUKTh37hwUCgVat26N2bNno0OHDjAxMRE7Gr3F2dkZ169fR6tWrcSOQvno06cP1qxZAysrK+UvJtnZ2Th58iQ2bdoEf39/kRNWTFKpFDdv3lTZJCI1NRUKhQL29vZwcXHB4MGD4eLiwm2aqcRw1QUiEYwbNw7e3t7o3LkzH8gow65fv44ZM2agf//+aNeuHWxsbNSmCPH9KxsCAwOxa9cuCIIAuVyuXL2kf//+mDZtmsjpKiYPDw/ltLq6deuiefPmcHFxgYuLC9dyp1LDQpeIKA9vfxSe1xz48+fPl1YcKkBsbCzOnz+PpKQkWFhYwN3dndMWRLR69WrlCgvcIIfEwkKXqJT8/fffcHR0hJGREf7+++8Cr+dareLbv39/gQ95FmXaEBERlS4WukSlxM3NDSEhIWjatCnc3NzyLKAUCgUEQeBIIVE+srOzkZGRobZCSUJCAjZv3oyHDx/C1tYW/fr1K9Qa40SknVjoEpWSS5cuoVGjRjAxMcGlS5cKvJ4PQJUdKSkpuH//Pp4/f462bdvCwsICmZmZ0NfX5052Ilm4cCHOnj2LPXv2KNuSkpLw8ccfIzExERYWFpBIJNDT08P69evRoEEDEdMSkVi46gJRKXm7cGURWz7I5XIEBQVh+/btyMjIUG4ta2FhgWnTpqFp06YYO3as2DErpOjoaPTo0UOlbcuWLUhMTMR3330HPz8/JCYmwt/fHxs2bFCu1UpEFQuHIohEcPfuXZw8eTLXcydPnsQ///xTyokoN6tWrcLOnTvxxRdfYM+ePSqbrXh6euLEiRMipqvYnj17pjZKe+zYMdSqVQt+fn4AABsbGwwdOhQ3b94UISERlQUsdIlEsHjxYly/fj3Xczdv3sSSJUtKORHlJjw8HP7+/ujXrx+qVaumcq569eqIjY0VKRllZ2fDyMhIeZyamoqYmBi1TSMcHBzw8uXL0o5HRGUEC10iEdy9ezfPLUudnZ1x586dUk5EuUlOToaTk1Ou5+RyObKzs0s5EeWoXr06rl27pjw+c+YMAKgVusnJybCwsCjVbERUdnCOLpEIpFIpsrKy8jzHrZrLhpo1a+LcuXNwd3dXO3fp0iXu5iSiXr164ddffwUAVKpUCevWrYONjQ3atm2rct2lS5fg6OgoQkIiKgtY6BKJoEGDBjhw4AA6dOigdu7AgQOoX7++CKnoXYMHD8a8efOgp6eHzp07AwCeP3+Oa9euYfv27Zg9e7a4ASuwAQMGICYmBmvXrkV2djbs7e3x448/qk1nOHDgAEaMGCFeUCISFZcXIxLByZMnMXXqVLRp0wY9e/ZE5cqV8eLFC+zfvx9nz57FL7/8gvbt24sdkwBs3rwZwcHBeP36tfJhNGNjY4wdOxZDhw4VOR1lZGQgIyMDVlZWaueys7MhkUhgZmYGPT2O6xBVRCx0iUQSERGBpUuXIj4+HoIgQKFQoEqVKvjiiy/QtWtXsePRW9LT03Ht2jXl1rLOzs5qGxUQEVHZw0KXSGQxMTFITk6GpaUl5xKWMf/++y9q1qwpdgwiItIQC12iMuDZs2ewtbXlx6tljJubGxo1agRvb29069YNlStXFjsSEREVAQtdIpHJZDK0adMGGzduRMOGDcWOQ285ceIEIiIicPz4cWRkZMDFxQXdu3dHp06dYGlpKXY8IiIqAAtdIpHJZDJ4eHhg06ZNLHTLqIyMDGXRe/r0acjlcnh4eMDb2xvdu3cXOx4REeWBhS6RyFjoli8SiQR//fUXVq1ahYSEBJw/f17sSERElAdOCCQSmSAIqFq1KvT19cWOQgW4desWIiIicOTIEbx48QK1atUSOxIREeWDI7pERPl48OABDh8+jCNHjuDx48ews7NDt27d4O3tjQYNGogdj4iI8sFCl4goDx9//DEePnwIKysrdO7cGd7e3nBxcRE7FhERFRKnLhCVEjc3NwiCUOjrOfdTfI0bN8aUKVPg5uYGXV1dseMQEVERcUSXqJRs3bpVWejKZDJs27YNenp68PLygo2NDV6+fIljx44hOzsbgwYN4vayRERE74mFLpEIli1bhocPH+KXX36Bjo6Osl0ulyMgIACOjo6YPHmyiAkpR1JSEjZt2oRbt27h+fPnCAwMRJ06dbBt2zY0bdoUzZo1EzsiERHlQafgS4iouIWHh2PAgAEqRS4A6OjooH///ggPDxcpGb3t77//hp+fHyIiIlClShXExsYiKysLABAfH4+tW7eKnJCIiPLDQpdIBJmZmXj69Gmu5+Li4iCVSks5EeVm8eLFcHZ2xt69e/H999/j7Q/AmjZtiuvXr4uYjoiICsJCl0gEHTp0wPLlyxEeHg6JRALgzUYE+/fvx6+//ooOHTqInJAA4ObNmxg4cCD09PTUHiS0trbGq1evREpGRESFwVUXiEQwffp0ZGRk4IcffsAPP/wAPT09ZGdnQ6FQwMvLC1999ZXYEQmAsbEx0tLScj337NkzWFpalnIiIiIqCha6RCIwNTXFwoUL8fDhQ9y8eRMJCQmwtbVFkyZN4OTkJHY8+n9t2rTBunXr4ObmBnNzc2V7RkYGtm/fjnbt2omYjoiICsJVF4iI8hAfH4/Ro0cjLS0NrVq1wrFjx9CmTRs8fPgQgiAgJCQENjY2YsckIqI8sNAlEkl2djb27dunXLZq+vTpqFmzJiIiIlCvXj2O7JYRqamp2Lp1K86dO4ekpCRYWFjA3d0dQ4YM4dQFIqIyjoUukQhiY2MxceJEJCUloUGDBoiOjsbGjRvRsGFD/Pzzz8jIyMCsWbPEjklERFSucdUFIhEsWrQIVlZWCA0NxcqVK1WWrWrZsiUuX74sYjoiIiLtwIfRiERw6dIl/Pjjj7CysoJMJlM5V6lSJSQkJIiUjMaPH1/oawVBQFBQUAmmISKi98FCl0gEurq6yGvWUGJiIkxMTEo5EeUwNTVVWzP3XYmJibh+/XqB1xERkbhY6BKJoGXLltiyZQvatWunUiwpFArs2bMHbm5uIqar2H755Zc8zyUkJGDjxo04e/YszMzM8PHHH5diMiIiKio+jEYkgpiYGIwaNQqWlpbw9PTEtm3b0LNnT9y/fx+PHz/Gb7/9hurVq4sdk/7fixcvsGHDBoSGhsLIyAgDBw7EwIEDYWZmJnY0IiLKBwtdIpE8efIEa9asUS5bZWlpCXd3d4wbN45Fbhnx7NkzhISEIDw8HKamphg8eDA++ugjTi0hIionWOgSEb3jyZMnCAkJwYEDB2BpaYlPPvkE/fr1g5GRkdjRiIioCFjoEhG9Zfbs2Th06BBsbW0xfPhw+Pn5QV9fX+xYRESkARa6RKVkypQpmDJlCmrWrIkpU6bke60gCFi8eHEpJaO35TwIWKNGDRgYGOR7rSAI2LZtW2nEIiIiDXDVBaJSkp6eDrlcDgBIS0vj0lRllI+PD98bIiItwRFdIiIiItJK3AKYqJRlZmZi0KBBOHv2rNhRiIiItBoLXaJSZmhoiPj4eH48TkREVMJY6BKJoFOnToiMjBQ7BhERkVbjHF0iEYSHh2PFihVo0KAB2rVrBxsbG7UR3k6dOomUjoiISDuw0CUSQc4SVnkRBAHnz58vpTRERETaiYUukQji4uIKvKZq1aqlkISIiEh7cR1dolKWmZmJhQsX4pNPPkHLli3FjkNERKS1+DAaUSkzNDTE5cuXlZtHEBERUclgoUskAg8PD66jS0REVMI4dYFIBD179sT8+fORlpaGdu3aoVKlSmqrLjRs2FCkdERERNqBD6MRieDdVRfeLnIVCgVXXSAiIioGLHSJRHDp0qUCr2nVqlUpJCEiItJeLHSJiIiISCvxYTQiIiIi0kp8GI1IJAcOHMCePXvw77//QiqVqp0/duyYCKmIiIi0B0d0iURw8OBB/Pjjj6hTpw6SkpLQpUsXdOrUCXp6erC2tsbQoUPFjkhERFTucUSXSASbN2/G6NGjMWLECOzduxcDBgxAw4YNkZaWhkmTJsHY2FjsiEREROUeR3SJRPD48WM0b94curq60NHRgUQiAQCYmppi+PDh2LFjh8gJiYiIyj8WukQiMDMzQ1ZWFgCgSpUqePjwofKcTCZDcnKyWNGIiIi0BqcuEImgUaNG+Oeff9CmTRt4enoiODgYcrkcenp6+O2339C0aVOxIxIREZV7LHSJRDBy5EjExcUBAMaPH4+4uDgsXrwYcrkcjRs3xrfffityQiIiovKPG0YQlRFSqRRSqRRmZmZiRyEiItIKLHSJiIiISCtx6gJRKQkODi70tYIgYMyYMSWYhoiISPtxRJeolLi5ucHAwACGhoYo6MdOEAT89ddfpZSMiIhIO3FEl6iUVKlSBYmJiXBxcUH37t3h6ekJIyMjsWMRERFpLY7oEpWiK1eu4PDhw/jzzz8hlUrh6ekJb29vtGnTBrq6umLHIyIi0iosdIlEIJPJcObMGURERODYsWPQ19dH586d4efnh0aNGokdj4iISCuw0CUSWWZmJlatWoWtW7fC09MTgYGBYkciIiLSCpyjSySSmJgYHD58GIcPH8aTJ0/g5uaG3r17ix2LiIhIa3BEl6gUxcXFKYvb+/fvo1mzZvD29kaXLl1gY2MjdjwiIiKtwkKXqJSMGjUKN27cQL169eDt7Y1u3brB3t5e7FhERERai4UuUSlxc3ODiYkJ7O3tIQhCvtcKgoBt27aVUjIiIiLtxDm6RKXEx8enwAKXiIiIig9HdImIiIhIK+mIHYCIiIiIqCSw0CUiIiIircRCl4iIiIi0EgtdIiIiItJKLHSJiIiISCux0CUiIiIircRCl4iIiIi0EgtdIiIiItJKLHSJiIiISCux0CUirfD06VO4urpi9erVJdJ/z549MXbs2BLpm4iISgYLXSLSWqmpqVi9ejUuXrwodpRcrV69GlFRUe/dz9atW7F///73D0REpGVY6BKRVqhatSpOnTqF0aNHK9tSU1MRHByMS5cuiZgsb8HBwcVS6G7bto2FLhFRLljoElG5lpaWBgAQBAGGhobQ09MTOREREZUVgkKhUIgdgojKp/3792POnDlYuXIlrl69itDQULx69Qp169bFl19+iWbNmuHSpUtYuXIl7ty5A1NTUwwYMABjxoxR9nH27FmEhobi1q1bSEhIgL6+Ppo0aYJRo0ahVatWKq83duxYxMXFISgoCMuWLcPFixeRkpKCixcv4unTp+jVqxc+/fRTjBs3DhcvXsT48ePVMletWlU5+rlr1y5ERUXhwYMHePXqFSwtLeHu7o4JEyagWrVqKvf17NkTVatWxZo1awr875KZmYkNGzbg8OHDeP78OfT19WFnZ4e2bdti8uTJyqy5yZlmERERgT/++AN3795FYmIiTExM4OLigvHjx6NevXrK611dXXPtJywsDNWqVYOrqyt8fX0xe/ZslfM5792qVauUfSQnJ2Pt2rU4fvw4Xrx4AWNjY1StWhXdunXDsGHDCvy+iYjKGg59ENF7+/XXXyGTyTBw4EBkZ2dj8+bNmDRpEubMmYO5c+eiT58++PDDD3HkyBGsWrUK1apVQ48ePQC8KbiSk5PRo0cP2NnZIT4+HqGhofD398eqVavQokULlddKT0/HuHHj4OzsDH9/fyQmJuaaycnJCVOnTsXixYvRsWNHdOzYEQBgYmKivGbz5s1o2rQpPv74Y1haWuL+/fvYt28fLly4gO3bt8PKykqj/x4///wzwsLC4OPjgyFDhkAmk+Hx48e4cOECAMDa2ho//PADZs6ciRYtWqBPnz5qfezcuROWlpbo06cPbG1tERsbi71792L06NHYvHkzatasCQD44YcfsHjxYlhZWWHUqFHK+62trYuc++uvv8bly5fRr18/1KtXD5mZmXj48CEuXbrEQpeIyiUWukT03mQyGTZs2AB9fX0Ab4rMgIAATJ8+HSEhIWjcuDEAoHfv3vD19cWuXbuUhe6MGTNgbGys0l+/fv3w0UcfISQkRK3QTU5ORr9+/eDv759vpkqVKsHLywuLFy9G3bp1la/3tu3bt6u9tqenJ/z9/REaGorhw4cX7T/E/4uKikLbtm0xZ86cXM8bGxujR48emDlzJhwcHHLNtnz5crVsPj4+GDx4MLZu3Yqvv/4aANCjRw8EBQXBxsYm134KSyKR4MKFC+jfvz+++uorjfshIipLOEeXiN5b//79lUUuAGVx2rRpU2WRC0A5LeHff/9Vtr1dzKWnpyMpKQm6urpo2rQpbt68mevrffLJJ8WSO+e15XI5JBIJkpKSUL9+fZiZmeHGjRsa92tmZoYHDx7g3r17751NoVAos1lbW6NWrVrvlS0vhoaGMDAwwI0bN/D06dNi75+ISAwc0SWi9+bg4KBybGFhAQBq81xzziUnJyuPY2NjsWLFCpw9exapqakq1wqCoHa/tbU1zM3NiyM2Lly4gODgYNy8eROZmZkq597N8i6JRIKMjAy1bLq6upg6dSpmzZqFgQMHwsHBAa6urvjggw/g6ekJHZ3CjS/8/fffWLVqFS5duoTXr1+rnHv3v3dx0NfXx9SpU/HLL7+gV69eqF27NlxdXeHl5QV3d/difz0iotLAQpeI3ltexZuurm6+96Wnp+PTTz/F69evMWjQINStWxempqYQBAEbNmxQzml9m5GRUbFkvnnzJiZNmoTq1atj0qRJqFatGgwNDSEIAr799lvI5fJ871+0aBHCw8NV2nIeAPPy8kJYWBhOnTqFy5cv4/z58wgNDUWLFi2wcuVKldHv3Dx79gxjx46FqakpRo8eDUdHRxgZGUEQBPzyyy9qha8mZDKZWlv//v3h5eWFkydP4tKlS/jzzz+xc+dOdO3aFfPnz3/v1yQiKm0sdIlINOfPn8eLFy8wc+ZMtVUIgoKC3rv/3EaEcxw6dAgymQzLli1TGSF9/fp1gaO5ADBs2DB8+OGHKm2VKlVSfm1paYkePXqgR48eUCgUWL58OTZu3Ihjx46hS5cu+fZ99OhRpKenY/HixWqrKiQnJ8PAwKDQ36elpaXKCHqOJ0+e5Hq9ra0t/Pz84OfnB5lMhpkzZ+Lw4cMYOnQomjRpkm9uIqKyhoUuEYkmZ8T33VUOz549WyzzUHPmuaakpBT6tdevX1/gaC4A1K5dG7Vr11Zrl8lkSE9PV5leIQgCGjRoAAAqRaeJiUmuRWjOCPm72fbu3YuXL1+iatWqKu3Gxsa5fo8AULNmTVy/fh0ZGRnK0fCUlBSEhYWpXJczDePtEXNdXV3Uq1cPhw8fzrN/IqKyjIUuEYnGxcUFlSpVwv/+9z/ExcWhSpUquHv3Lg4ePIi6deu+18NcAGBlZYUaNWogIiIC1atXh42NDYyNjeHp6QkvLy9s3boVkydPRp8+faCvr49z587h3r17Gi8rBryZjtG9e3d4enqiQYMGsLa2xtOnT7F7925YWFjA09NTeW3Tpk1x/vx5bNiwAfb29hAEAd7e3mjXrh2WL1+OmTNn4qOPPoK5uTmuXr2K06dPo3r16mrTDpo1a4bQ0FAEBQXByckJgiDA09MTxsbG+Oijj/D9999j/Pjx6NGjB1JTU7Fv3z5UrVoVL1++VPbx6NEjjB07Fh07dkSdOnVgbm6OmJgY7N69Gw4ODmqrXxARlQcsdIlINObm5vj111+xbNky7NixAzKZDA0bNsTSpUsRGhr63oUuAMydOxeLFy/GihUrkJGRgapVq8LT0xMuLi5YuHAh1q5di1WrVsHQ0BDu7u5Ys2YNPv30U41fz8jICIMGDcL58+dx/vx5pKenw9bWFp6enhg5ciQqV66svPbrr7/Gzz//jJCQEOUOb97e3qhevTqWLVuGFStWICQkBDo6OmjevDlWr16NhQsXIi4uTuU1/f39kZycjF27diE1NRUKhQJhYWEwNjbGhx9+iBcvXmDnzp1YsmQJHBwcMGbMGOjo6KiMmtvZ2aFXr164dOkSoqKikJWVhcqVK6NPnz4YPnx4sc2NJiIqTdwZjYiIiIi0EtfRJSIiIiKtxEKXiIiIiLQSC10iIiIi0kosdImIiIhIK7HQJSIiIiKtxEKXiIiIiLQSC136v3brQAYAAABgkL/1Pb6iCABgSXQBAFgSXQAAlkQXAIAl0QUAYEl0AQBYEl0AAJYC0CXP/vjPD5EAAAAASUVORK5CYII=\"\n },\n \"metadata\": {},\n \"output_type\": \"display_data\",\n \"jetTransient\": {\n \"display_id\": null\n }\n }\n ],\n \"execution_count\": 38\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"For multiclass problems, we can still score overall feature importance (how much each feature contributes to predictive accuracy) using AutoGluon's built-in permutation procedure:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:13.864468Z\",\n \"start_time\": \"2026-01-07T01:37:13.727602Z\"\n }\n },\n \"source\": [\n \"predictor_multi.feature_importance(test_data)\"\n ],\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Computing feature importance via permutation shuffling for 14 features using 50 rows with 5 shuffle sets...\\n\",\n \"\\t0.85s\\t= Expected runtime (0.17s per shuffle set)\\n\",\n \"\\t0.13s\\t= Actual runtime (Completed 5 of 5 shuffle sets)\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \" importance stddev p_value n p99_high p99_low\\n\",\n \"marital-status 0.368 0.030332 0.000005 5 0.430453 0.305547\\n\",\n \"sex 0.208 0.057619 0.000640 5 0.326639 0.089361\\n\",\n \"occupation 0.032 0.022804 0.017460 5 0.078953 -0.014953\\n\",\n \"class 0.028 0.010954 0.002318 5 0.050555 0.005445\\n\",\n \"age 0.020 0.024495 0.070964 5 0.070435 -0.030435\\n\",\n \"education-num 0.016 0.016733 0.049650 5 0.050454 -0.018454\\n\",\n \"education 0.004 0.008944 0.186950 5 0.022416 -0.014416\\n\",\n \"hours-per-week 0.004 0.016733 0.310654 5 0.038454 -0.030454\\n\",\n \"capital-gain 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"workclass 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"capital-loss 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"race 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"native-country 0.000 0.000000 0.500000 5 0.000000 0.000000\\n\",\n \"fnlwgt -0.012 0.017889 0.896000 5 0.024833 -0.048833\"\n ],\n \"text/html\": [\n \"`_.\n\n Examples\n --------\n >>> import pandas as pd\n >>> import numpy as np\n >>> from skrub import SquashingScaler\n\n In the general case, this scale uses a RobustScaler:\n\n >>> X = pd.DataFrame(dict(col=[np.inf, -np.inf, 3, -1, np.nan, 2]))\n >>> SquashingScaler(max_absolute_value=3).fit_transform(X)\n array([[ 3. ],\n [-3. ],\n [ 0.49319696],\n [-1.34164079],\n [ nan],\n [ 0. ]])\n\n When quantile ranges are equal, this scaler uses a customized MinMaxScaler:\n\n >>> X = pd.DataFrame(dict(col=[0, 1, 1, 1, 2, np.nan]))\n >>> SquashingScaler().fit_transform(X)\n array([[-0.9486833],\n [ 0. ],\n [ 0. ],\n [ 0. ],\n [ 0.9486833],\n [ nan]])\n\n Finally, when the min and max are equal, this scaler fills the column with zeros:\n\n >>> X = pd.DataFrame(dict(col=[1, 1, 1, np.nan]))\n >>> SquashingScaler().fit_transform(X)\n array([[ 0.],\n [ 0.],\n [ 0.],\n [nan]])\n \"\"\" # noqa: E501\n\n def __init__(\n self,\n max_absolute_value=3.0,\n quantile_range=(25.0, 75.0),\n ):\n self.max_absolute_value = max_absolute_value\n self.quantile_range = quantile_range\n\n def fit(self, X, y=None):\n \"\"\"Fit the transformer to a column.\n\n Parameters\n ----------\n X : numpy array, Pandas or Polars DataFrame of shape (n_samples, n_features)\n The data to transform.\n y : None\n Unused. Here for compatibility with scikit-learn.\n\n Returns\n -------\n self : SquashingScaler\n The fitted transformer.\n \"\"\"\n del y\n\n self.fit_transform(X)\n return self\n\n def fit_transform(self, X, y=None):\n \"\"\"Fit the transformer and transform a column.\n\n Parameters\n ----------\n X : numpy array, Pandas or Polars DataFrame of shape (n_samples, n_features)\n The data to transform.\n y : None\n Unused. Here for compatibility with scikit-learn.\n\n Returns\n -------\n X_out: numpy array, shape (n_samples, n_features)\n The transformed version of the input.\n \"\"\"\n del y\n\n if not (\n isinstance(self.max_absolute_value, numbers.Number)\n and np.isfinite(self.max_absolute_value)\n and self.max_absolute_value > 0\n ):\n raise ValueError(\n f\"Got max_absolute_value={self.max_absolute_value!r}, but expected a positive finite number.\"\n )\n X = validate_data(\n self,\n X=X,\n reset=True,\n dtype=FLOAT_DTYPES,\n accept_sparse=False,\n ensure_2d=True,\n ensure_all_finite=False,\n )\n # To use sklearn scalers, we need to convert np.inf to np.nan. However, we need\n # to replace the original Âąnp.inf with Âąmax_absolute_value in the final output.\n # mask_inf is a 2D array containing the sign of the np.inf in the input.\n X, mask_inf = _mask_inf(X)\n\n # For each column, we apply 1 out of 3 scaling methods:\n # If the max is equal to the min, then we fill the column with zeros.\n zero_cols = np.nanmax(X, axis=0) == np.nanmin(X, axis=0)\n\n # If the two quantiles defined by quantile_range have the same values, we\n # use a customized MinMaxScaler. We remove from this selection columns that\n # are already selected as zero cols (i.e. columns that have the same min and max\n # also have the same quantile_range values).\n quantiles = np.nanpercentile(X, self.quantile_range, axis=0)\n minmax_cols = quantiles[0, :] == quantiles[1, :]\n minmax_cols = minmax_cols & ~zero_cols\n\n # Otherwise (general case), we use a RobustScaler.\n robust_cols = ~(minmax_cols | zero_cols)\n\n # Copying the input since we change the values in place.\n X_tr = X.copy()\n if robust_cols.any():\n self.robust_scaler_ = RobustScaler(\n with_centering=True,\n with_scaling=True,\n quantile_range=self.quantile_range,\n copy=True,\n )\n X_tr[:, robust_cols] = self.robust_scaler_.fit_transform(X[:, robust_cols])\n else:\n self.robust_scaler_ = None\n self.robust_cols_ = robust_cols\n\n if minmax_cols.any():\n self.minmax_scaler_ = _MinMaxScaler()\n X_tr[:, minmax_cols] = self.minmax_scaler_.fit_transform(X[:, minmax_cols])\n else:\n self.minmax_scaler_ = None\n self.minmax_cols_ = minmax_cols\n\n if zero_cols.any():\n X_tr = _set_zeros(X_tr, zero_cols)\n self.zero_cols_ = zero_cols\n\n return _soft_clip(X_tr, self.max_absolute_value, mask_inf)\n\n def transform(self, X):\n \"\"\"Transform a column.\n\n Parameters\n ----------\n X : numpy array, Pandas or Polars DataFrame of shape (n_samples, n_features)\n The data to transform.\n\n Returns\n -------\n X_out: numpy array of shape (n_samples, n_features)\n The transformed version of the input.\n \"\"\"\n check_is_fitted(self, [\"robust_scaler_\", \"minmax_scaler_\"])\n\n X = validate_data(\n self,\n X=X,\n reset=False,\n dtype=FLOAT_DTYPES,\n accept_sparse=False,\n ensure_2d=True,\n ensure_all_finite=False,\n )\n X, mask_inf = _mask_inf(X)\n\n X_tr = X.copy()\n if self.robust_cols_.any():\n X_tr[:, self.robust_cols_] = self.robust_scaler_.transform(X[:, self.robust_cols_])\n if self.minmax_cols_.any():\n X_tr[:, self.minmax_cols_] = self.minmax_scaler_.transform(X[:, self.minmax_cols_])\n if self.zero_cols_.any():\n # if the scale is 0, we set the values to 0\n X_tr = _set_zeros(X_tr, self.zero_cols_)\n\n return _soft_clip(X_tr, self.max_absolute_value, mask_inf)\n"
},
{
"path": "features/src/autogluon/features/generators/text_ngram.py",
"content": "import copy\nimport logging\nimport traceback\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\nfrom sklearn.feature_selection import SelectKBest, f_classif, f_regression\n\nfrom autogluon.common.features.types import S_IMAGE_BYTEARRAY, S_IMAGE_PATH, S_TEXT, S_TEXT_NGRAM\nfrom autogluon.common.utils.lite import disable_if_lite_mode\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom ..vectorizers import downscale_vectorizer, get_ngram_freq, vectorizer_auto_ml_default\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add argument to define the text preprocessing logic\n# TODO: Add argument to output ngrams as a sparse matrix\n# TODO: Add HashingVectorizer support\n# TODO: Documentation\nclass TextNgramFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Generates ngram features from text features.\n\n Parameters\n ----------\n vectorizer : :class:`sklearn.feature_extraction.text.CountVectorizer` or :class:`sklearn.feature_extraction.text.TfidfVectorizer`, default CountVectorizer(min_df=30, ngram_range=(1, 3), max_features=10000, dtype=np.uint8) # noqa\n sklearn CountVectorizer which is used to generate the ngrams given the text data.\n Can also specify a TfidfVectorizer, but note that memory usage will increase by 4-8x relative to CountVectorizer.\n vectorizer_strategy : str, default 'combined'\n If 'combined', all text features are concatenated together to fit the vectorizer.\n Features generated in this way have their names prepended with '__nlp__.'.\n If 'separate', all text features are fit separately with their own copy of the vectorizer.\n Their ngram features are then concatenated together to form the output.\n If 'both', the outputs of 'combined' and 'separate' are concatenated together to form the output.\n It is generally recommended to keep vectorizer_strategy as 'combined' unless the text features are not associated with each other,\n as fitting separate vectorizers could increase memory usage and model training time.\n Valid values: ['combined', 'separate', 'both']\n max_memory_ratio : float, default 0.15\n Safety measure to avoid out-of-memory errors downstream in model training.\n The number of ngrams generated will be capped to take at most max_memory_ratio proportion of total available memory,\n treating the ngrams as float32 values.\n ngram features will be removed in least frequent to most frequent order.\n Note: For vectorizer_strategy values other than 'combined', the resulting ngrams may use more than this value.\n It is recommended to only increase this value above 0.15 if confident that higher values will not result in out-of-memory errors.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(\n self,\n vectorizer=None,\n vectorizer_strategy=\"combined\",\n max_memory_ratio=0.15,\n prefilter_tokens=False,\n prefilter_token_count=100,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self.vectorizers = []\n # TODO: 0.20 causes OOM error with 64 GB ram on NN with several datasets. LightGBM and CatBoost succeed\n # TODO: Finetune this, or find a better way to ensure stability\n # TODO: adjust max_memory_ratio correspondingly if prefilter_tokens==True\n self.max_memory_ratio = max_memory_ratio # Ratio of maximum memory the output ngram features are allowed to use in dense int32 form.\n\n if vectorizer is None:\n self.vectorizer_default_raw = vectorizer_auto_ml_default()\n else:\n self.vectorizer_default_raw = vectorizer\n\n if vectorizer_strategy not in [\"combined\", \"separate\", \"both\"]:\n raise ValueError(\n f\"vectorizer_strategy must be one of {['combined', 'separate', 'both']}, but value is: {vectorizer_strategy}\"\n )\n self.vectorizer_strategy = vectorizer_strategy\n self.vectorizer_features = None\n self.prefilter_tokens = prefilter_tokens\n self.prefilter_token_count = prefilter_token_count\n self.token_mask = None\n self._feature_names_dict = dict()\n\n def _fit_transform(self, X: DataFrame, y: Series = None, problem_type: str = None, **kwargs) -> (DataFrame, dict):\n X_out = self._fit_transform_ngrams(X)\n\n if self.prefilter_tokens and self.prefilter_token_count >= X_out.shape[1]:\n logger.warning(\n \"`prefilter_tokens` was enabled but `prefilter_token_count` larger than the vocabulary. Disabling `prefilter_tokens`.\"\n )\n self.prefilter_tokens = False\n\n if self.prefilter_tokens and problem_type not in [\"binary\", \"regression\"]:\n logger.warning(\"`prefilter_tokens` was enabled but invalid `problem_type`. Disabling `prefilter_tokens`.\")\n self.prefilter_tokens = False\n\n if self.prefilter_tokens and y is None:\n logger.warning(\n \"`prefilter_tokens` was enabled but `y` values were not provided to fit_transform. Disabling `prefilter_tokens`.\"\n )\n self.prefilter_tokens = False\n\n if self.prefilter_tokens:\n scoring_function = f_classif if problem_type == \"binary\" else f_regression\n selector = SelectKBest(scoring_function, k=self.prefilter_token_count)\n selector.fit(X_out, y)\n self.token_mask = selector.get_support()\n X_out = X_out[X_out.columns[self.token_mask]] # select the columns that are most correlated with y\n\n type_family_groups_special = {S_TEXT_NGRAM: list(X_out.columns)}\n return X_out, type_family_groups_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n # TODO: Optimize for inference\n if not self.features_in:\n return DataFrame(index=X.index)\n try:\n X_out = self._generate_ngrams(X=X)\n if self.prefilter_tokens:\n X_out = X_out[X_out.columns[self.token_mask]] # select the columns identified during training\n except Exception:\n self._log(\n 40,\n \"\\tError: OOM error during NLP feature transform, unrecoverable. Increase memory allocation or reduce data size to avoid this error.\",\n )\n raise\n return X_out\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(required_special_types=[S_TEXT], invalid_special_types=[S_IMAGE_PATH, S_IMAGE_BYTEARRAY])\n\n def _fit_transform_ngrams(self, X):\n if not self.features_in:\n return DataFrame(index=X.index)\n features_nlp_to_remove = []\n if self.vectorizer_strategy == \"combined\":\n self.vectorizer_features = [\"__nlp__\"]\n elif self.vectorizer_strategy == \"separate\":\n self.vectorizer_features = copy.deepcopy(self.features_in)\n elif self.vectorizer_strategy == \"both\":\n self.vectorizer_features = [\"__nlp__\"] + copy.deepcopy(self.features_in)\n else:\n raise ValueError(\n f\"vectorizer_strategy must be one of {['combined', 'separate', 'both']}, but value is: {self.vectorizer_features}\"\n )\n self._log(\n 20,\n f\"Fitting {self.vectorizer_default_raw.__class__.__name__} for text features: \" + str(self.features_in),\n self.log_prefix + \"\\t\",\n )\n self._log(15, f\"{self.vectorizer_default_raw}\", self.log_prefix + \"\\t\\t\")\n for nlp_feature in self.vectorizer_features:\n # TODO: Preprocess text?\n if nlp_feature == \"__nlp__\": # Combine Text Fields\n features_in_str = X[self.features_in].astype(str)\n text_list = list(set([\". \".join(row) for row in features_in_str.values]))\n else:\n text_list = list(X[nlp_feature].astype(str).drop_duplicates().values)\n vectorizer_raw = copy.deepcopy(self.vectorizer_default_raw)\n try:\n # Don't use transform_matrix output because it may contain fewer rows due to drop_duplicates call.\n vectorizer_fit, _ = self._train_vectorizer(text_list, vectorizer_raw)\n self._log(\n 20,\n f\"{vectorizer_fit.__class__.__name__} fit with vocabulary size = {len(vectorizer_fit.vocabulary_)}\",\n self.log_prefix + \"\\t\",\n )\n except ValueError:\n self._log(30, f\"Removing text_ngram feature due to error: '{nlp_feature}'\", self.log_prefix + \"\\t\")\n if nlp_feature == \"__nlp__\":\n self.vectorizer_features = []\n features_nlp_to_remove = self.features_in\n break\n else:\n features_nlp_to_remove.append(nlp_feature)\n else:\n self.vectorizers.append(vectorizer_fit)\n self._remove_features_in(features_nlp_to_remove)\n\n downsample_ratio = None\n nlp_failure_count = 0\n X_text_ngram = None\n keep_trying_nlp = True\n while keep_trying_nlp:\n try:\n X_text_ngram = self._generate_ngrams(X=X, downsample_ratio=downsample_ratio)\n keep_trying_nlp = False\n except Exception as err:\n nlp_failure_count += 1\n traceback.print_tb(err.__traceback__)\n\n X_text_ngram = None\n skip_nlp = False\n for vectorizer in self.vectorizers:\n vocab_size = len(vectorizer.vocabulary_)\n if vocab_size <= 50:\n skip_nlp = True\n break\n else:\n if nlp_failure_count >= 3:\n skip_nlp = True\n\n if skip_nlp:\n self._log(\n 30,\n \"Warning: ngrams generation resulted in OOM error, removing ngrams features. \"\n \"If you want to use ngrams for this problem, increase memory allocation for AutoGluon.\",\n self.log_prefix + \"\\t\",\n )\n self._log(10, str(err))\n self.vectorizers = []\n self.features_in = []\n keep_trying_nlp = False\n else:\n self._log(\n 20,\n \"Warning: ngrams generation resulted in OOM error, attempting to reduce ngram feature count. \"\n \"If you want to optimally use ngrams for this problem, increase memory allocation for AutoGluon.\",\n self.log_prefix + \"\\t\",\n )\n self._log(10, str(err))\n downsample_ratio = 0.25\n if X_text_ngram is None:\n X_text_ngram = DataFrame(index=X.index)\n return X_text_ngram\n\n def _generate_ngrams(self, X, downsample_ratio: int = None):\n X_nlp_features_combined = []\n for nlp_feature, vectorizer_fit in zip(self.vectorizer_features, self.vectorizers):\n if nlp_feature == \"__nlp__\":\n X_str = X.astype(str)\n text_data = [\". \".join(row) for row in X_str.values]\n else:\n nlp_feature_str = X[nlp_feature].astype(str)\n text_data = nlp_feature_str.values\n transform_matrix = vectorizer_fit.transform(text_data)\n\n if not self._is_fit:\n transform_matrix = self._adjust_vectorizer_memory_usage(\n transform_matrix=transform_matrix,\n text_data=text_data,\n vectorizer_fit=vectorizer_fit,\n downsample_ratio=downsample_ratio,\n )\n nlp_features_names = vectorizer_fit.get_feature_names_out()\n nlp_features_names_final = np.array(\n [f\"{nlp_feature}.{x}\" for x in nlp_features_names] + [f\"{nlp_feature}._total_\"]\n )\n self._feature_names_dict[nlp_feature] = nlp_features_names_final\n\n transform_array = transform_matrix.toarray()\n # This count could technically overflow in absurd situations. Consider making dtype a variable that is computed.\n nonzero_count = np.count_nonzero(transform_array, axis=1).astype(np.uint16)\n transform_array = np.append(transform_array, np.expand_dims(nonzero_count, axis=1), axis=1)\n X_nlp_features = pd.DataFrame(\n transform_array, columns=self._feature_names_dict[nlp_feature], index=X.index\n ) # TODO: Consider keeping sparse\n X_nlp_features_combined.append(X_nlp_features)\n\n if X_nlp_features_combined:\n if len(X_nlp_features_combined) == 1:\n X_nlp_features_combined = X_nlp_features_combined[0]\n else:\n X_nlp_features_combined = pd.concat(X_nlp_features_combined, axis=1)\n else:\n X_nlp_features_combined = DataFrame(index=X.index)\n\n return X_nlp_features_combined\n\n # TODO: REMOVE NEED FOR text_data input!\n def _adjust_vectorizer_memory_usage(\n self, transform_matrix, text_data, vectorizer_fit, downsample_ratio: int = None\n ):\n @disable_if_lite_mode(ret=downsample_ratio)\n def _adjust_per_memory_constraints(downsample_ratio: int):\n # This assumes that the ngrams eventually turn into int32/float32 downstream\n predicted_ngrams_memory_usage_bytes = len(text_data) * 4 * (transform_matrix.shape[1] + 1) + 80\n mem_avail = ResourceManager.get_available_virtual_mem()\n mem_rss = ResourceManager.get_memory_rss()\n predicted_rss = mem_rss + predicted_ngrams_memory_usage_bytes\n predicted_percentage = predicted_rss / mem_avail\n if downsample_ratio is None:\n if self.max_memory_ratio is not None and predicted_percentage > self.max_memory_ratio:\n self._log(\n 30,\n \"Warning: Due to memory constraints, ngram feature count is being reduced. Allocate more memory to maximize model quality.\",\n )\n return self.max_memory_ratio / predicted_percentage\n\n downsample_ratio = _adjust_per_memory_constraints(downsample_ratio)\n\n if downsample_ratio is not None:\n if (downsample_ratio >= 1) or (downsample_ratio <= 0):\n raise ValueError(f\"downsample_ratio must be >0 and <1, but downsample_ratio is {downsample_ratio}\")\n vocab_size = len(vectorizer_fit.vocabulary_)\n downsampled_vocab_size = int(np.floor(vocab_size * downsample_ratio))\n self._log(\n 20, f\"Reducing Vectorizer vocab size from {vocab_size} to {downsampled_vocab_size} to avoid OOM error\"\n )\n ngram_freq = get_ngram_freq(vectorizer=vectorizer_fit, transform_matrix=transform_matrix)\n downscale_vectorizer(vectorizer=vectorizer_fit, ngram_freq=ngram_freq, vocab_size=downsampled_vocab_size)\n # TODO: This doesn't have to be done twice, can update transform matrix based on new vocab instead of calling .transform\n # If we have this functionality, simply update transform_matrix each time OOM occurs instead of re-calling .transform\n transform_matrix = vectorizer_fit.transform(text_data)\n\n return transform_matrix\n\n @staticmethod\n def _train_vectorizer(text_data: list, vectorizer):\n # TODO: Consider upgrading to pandas 0.25.0 to benefit from sparse attribute improvements / bug fixes!\n # https://pandas.pydata.org/pandas-docs/stable/whatsnew/v0.25.0.html\n transform_matrix = vectorizer.fit_transform(text_data)\n vectorizer.stop_words_ = None # Reduces object size by 100x+ on large datasets, no effect on usability\n return vectorizer, transform_matrix\n\n def _remove_features_in(self, features):\n super()._remove_features_in(features)\n if features:\n self.vectorizer_features = [feature for feature in self.vectorizer_features if feature not in features]\n"
},
{
"path": "features/src/autogluon/features/generators/text_special.py",
"content": "import logging\nfrom typing import List, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.features.types import S_IMAGE_BYTEARRAY, S_IMAGE_PATH, S_TEXT, S_TEXT_SPECIAL\n\nfrom .abstract import AbstractFeatureGenerator\nfrom .binned import BinnedFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass TextSpecialFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n TextSpecialFeatureGenerator generates text specific features from incoming raw text features.\n These include word counts, character counts, symbol counts, capital letter ratios, and much more.\n Features generated by this generator will have 'text_special' as a special type.\n\n Parameters\n ----------\n symbols : List[str], optional\n List of string symbols to compute counts and ratios for as features.\n If not specified, defaults to ['!', '?', '@', '%', '$', '*', '&', '#', '^', '.', ':', ' ', '/', ';', '-', '=']\n min_occur_ratio : float, default 0.01\n Minimum ratio of symbol occurrence to consider as a feature.\n If a symbol appears in fewer than 1 in 1/min_occur_ratio samples, it will not be used as a feature.\n min_occur_offset : int, default 10\n Minimum symbol occurrences to consider as a feature. This is added to the threshold calculated from min_occur_ratio.\n bin_features : bool, default True\n If True, adds a BinnedFeatureGenerator to the front of post_generators such that all features generated from this generator are then binned.\n This is useful for 'text_special' features because it lowers the chance models will overfit on the features and reduces their memory usage.\n post_drop_duplicates : bool, default True\n Identical to AbstractFeatureGenerator's post_drop_duplicates, except it is defaulted to True instead of False.\n This helps to clean the output of this generator when symbols aren't present in the data.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid keyword arguments.\n \"\"\"\n\n def __init__(\n self,\n symbols: List[str] = None,\n min_occur_ratio=0.01,\n min_occur_offset=10,\n bin_features: bool = True,\n post_drop_duplicates: bool = True,\n **kwargs,\n ):\n super().__init__(post_drop_duplicates=post_drop_duplicates, **kwargs)\n if symbols is None:\n symbols = [\"!\", \"?\", \"@\", \"%\", \"$\", \"*\", \"&\", \"#\", \"^\", \".\", \":\", \" \", \"/\", \";\", \"-\", \"=\"]\n self._symbols = symbols # Symbols to generate count and ratio features for.\n self._symbols_per_feature = {}\n self._min_occur_ratio = min_occur_ratio\n self._min_occur_offset = min_occur_offset\n if bin_features:\n self._post_generators = [BinnedFeatureGenerator()] + self._post_generators\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> Tuple[DataFrame, dict]:\n self._symbols_per_feature = self._filter_symbols(X, self._symbols)\n self._feature_names_dict = self._compute_feature_names_dict()\n X_out = self._transform(X)\n type_family_groups_special = {S_TEXT_SPECIAL: list(X_out.columns)}\n return X_out, type_family_groups_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self._generate_features_text_special(X)\n\n def _compute_feature_names_dict(self) -> dict:\n feature_names = {}\n for feature in self.features_in:\n feature_names_cur = {}\n for feature_name_base in [\n \"char_count\",\n \"word_count\",\n \"capital_ratio\",\n \"lower_ratio\",\n \"digit_ratio\",\n \"special_ratio\",\n ]:\n feature_names_cur[feature_name_base] = f\"{feature}.{feature_name_base}\"\n symbols = self._symbols_per_feature[feature]\n for symbol in symbols:\n feature_names_cur[symbol] = {\n \"count\": f\"{feature}.symbol_count.{symbol}\",\n \"ratio\": f\"{feature}.symbol_ratio.{symbol}\",\n }\n feature_names[feature] = feature_names_cur\n return feature_names\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(required_special_types=[S_TEXT], invalid_special_types=[S_IMAGE_PATH, S_IMAGE_BYTEARRAY])\n\n def _filter_symbols(self, X: DataFrame, symbols: list) -> dict:\n symbols_per_feature = {}\n if self.features_in:\n num_samples = len(X)\n symbol_occur_threshold = min(\n np.ceil(self._min_occur_offset + num_samples * self._min_occur_ratio), np.ceil(num_samples / 2)\n )\n for text_feature_name in self.features_in:\n above_threshold_symbols = []\n text_feature = X[text_feature_name].astype(str)\n for symbol in symbols:\n symbol_occur_count = text_feature.str.contains(symbol, regex=False).sum()\n if symbol_occur_count >= symbol_occur_threshold:\n above_threshold_symbols.append(symbol)\n symbols_per_feature[text_feature_name] = np.array(above_threshold_symbols)\n return symbols_per_feature\n\n def _generate_features_text_special(self, X: DataFrame) -> DataFrame:\n if self.features_in:\n X_text_special_combined = {}\n for text_feature in self.features_in:\n X_text_special_combined = self._generate_text_special(\n X[text_feature],\n text_feature,\n symbols=self._symbols_per_feature[text_feature],\n X_dict=X_text_special_combined,\n )\n X_text_special_combined = pd.DataFrame(X_text_special_combined, index=X.index)\n else:\n X_text_special_combined = pd.DataFrame(index=X.index)\n return X_text_special_combined\n\n def _generate_text_special(self, X: Series, feature: str, symbols: list, X_dict: dict) -> dict:\n fn = self._feature_names_dict[feature]\n X_str = X.astype(str)\n\n X_no_ws = X_str.str.replace(\" \", \"\")\n X_no_ws_text_len = X_no_ws.str.len()\n\n char_count = X_str.str.len()\n\n X_dict[fn[\"char_count\"]] = char_count.to_numpy(dtype=np.uint32)\n X_dict[fn[\"word_count\"]] = X_str.str.split().str.len().to_numpy(dtype=np.uint32)\n X_dict[fn[\"capital_ratio\"]] = (\n X_no_ws.str.count(\"[A-Z]\").divide(X_no_ws_text_len, fill_value=0.0).fillna(0.0).to_numpy(dtype=np.float32)\n )\n X_dict[fn[\"lower_ratio\"]] = (\n X_no_ws.str.count(\"[a-z]\").divide(X_no_ws_text_len, fill_value=0.0).fillna(0.0).to_numpy(dtype=np.float32)\n )\n X_dict[fn[\"digit_ratio\"]] = (\n X_no_ws.str.count(\"[0-9]\").divide(X_no_ws_text_len, fill_value=0.0).fillna(0.0).to_numpy(dtype=np.float32)\n )\n X_dict[fn[\"special_ratio\"]] = (\n X_no_ws.str.count(r\"[^\\w]\").divide(X_no_ws_text_len, fill_value=0.0).fillna(0.0).to_numpy(dtype=np.float32)\n )\n\n for symbol in symbols:\n symbol_count = X_str.str.count(\"\\\\\" + symbol)\n X_dict[fn[symbol][\"count\"]] = symbol_count.to_numpy(dtype=np.uint32)\n X_dict[fn[symbol][\"ratio\"]] = (\n symbol_count.divide(char_count, fill_value=0.0).fillna(0.0).to_numpy(dtype=np.float32)\n )\n\n return X_dict\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self._symbols_per_feature:\n for feature in features:\n if feature in self._symbols_per_feature:\n self._symbols_per_feature.pop(feature)\n"
},
{
"path": "features/src/autogluon/features/registry/__init__.py",
"content": ""
},
{
"path": "features/src/autogluon/features/registry/_ag_feature_generator_registry.py",
"content": "from autogluon.features.generators import REGISTERED_FE_CLS_LST\nfrom autogluon.features.registry._feature_generator_registry import FeatureGeneratorRegistry\n\nag_feature_generator_registry = FeatureGeneratorRegistry(cls_list=REGISTERED_FE_CLS_LST)\n"
},
{
"path": "features/src/autogluon/features/registry/_feature_generator_registry.py",
"content": "from __future__ import annotations\n\nfrom typing import Type\n\nimport pandas as pd\n\nfrom autogluon.features.generators.abstract import AbstractFeatureGenerator\n\n\nclass FeatureGeneratorRegistry:\n def __init__(self, cls_list: list[Type[AbstractFeatureGenerator]] | None = None):\n if cls_list is None:\n cls_list = []\n assert isinstance(cls_list, list)\n self._cls_list = []\n self._key_to_cls_map = dict()\n for fe_cls in cls_list:\n self.add(fe_cls)\n\n def exists(self, fe_cls: Type[AbstractFeatureGenerator]) -> bool:\n return fe_cls in self._cls_list\n\n def add(self, fe_cls: Type[AbstractFeatureGenerator]):\n \"\"\"\n Adds `fe_cls` to the model registry\n \"\"\"\n assert not self.exists(fe_cls), f\"Cannot add fe_cls that is already registered: {fe_cls}\"\n if fe_cls.__name__ is None:\n raise AssertionError(\n f\"Cannot add fe_cls with `ag_key=None`. \"\n f\"Ensure you set class attribute `ag_key` to a string for your fe_cls: {fe_cls}\"\n f'\\n\\tFor example, LightGBModel sets `ag_key = \"GBM\"`'\n )\n assert isinstance(fe_cls.__name__, str)\n if fe_cls.__name__ in self._key_to_cls_map:\n raise AssertionError(\n f\"Cannot register a model class that shares a model key with an already registered model class.\"\n f\"\\n`fe_cls.ag_key` must be unique among registered models:\"\n f\"\\n\\t New Class: {fe_cls}\"\n f\"\\n\\tConflicting Class: {self._key_to_cls_map[fe_cls.__name__]}\"\n f\"\\n\\tConflicting ag_key: {fe_cls.__name__}\"\n )\n self._cls_list.append(fe_cls)\n self._key_to_cls_map[fe_cls.__name__] = fe_cls\n\n def remove(self, fe_cls: Type[AbstractFeatureGenerator]):\n \"\"\"\n Removes `fe_cls` from the model registry\n \"\"\"\n assert self.exists(fe_cls), f\"Cannot remove fe_cls that isn't registered: {fe_cls}\"\n self._cls_list = [m for m in self._cls_list if m != fe_cls]\n self._key_to_cls_map.pop(fe_cls.__name__)\n\n @property\n def cls_list(self) -> list[Type[AbstractFeatureGenerator]]:\n return self._cls_list\n\n @property\n def keys(self) -> list[str]:\n return [self.key(fe_cls) for fe_cls in self.cls_list]\n\n def key_to_cls_map(self) -> dict[str, Type[AbstractFeatureGenerator]]:\n return self._key_to_cls_map\n\n def key_to_cls(self, key: str) -> Type[AbstractFeatureGenerator]:\n if key not in self._key_to_cls_map:\n raise ValueError(\n f\"No registered model exists with provided key: {key}\"\n f\"\\n\\tValid keys: {list(self.key_to_cls_map().keys())}\"\n )\n return self.key_to_cls_map()[key]\n\n def key(self, fe_cls: Type[AbstractFeatureGenerator]) -> str:\n assert self.exists(fe_cls), f\"Model class must be registered: {fe_cls}\"\n return fe_cls.__name__\n\n def docstring(self, fe_cls: Type[AbstractFeatureGenerator]) -> str:\n assert self.exists(fe_cls), f\"Model class must be registered: {fe_cls}\"\n return fe_cls.__doc__\n\n # TODO: Could add a lot of information here to track which features are supported for each model:\n # ag.early_stop support\n # refit_full support\n # GPU support\n # etc.\n def to_frame(self) -> pd.DataFrame:\n model_classes = self.cls_list\n cls_dict = {}\n for fe_cls in model_classes:\n cls_dict[self.key(fe_cls)] = {\n \"fe_cls\": fe_cls.__name__,\n }\n df = pd.DataFrame(cls_dict).T\n df.index.name = \"name\"\n return df\n"
},
{
"path": "features/src/autogluon/features/registry/parse_custom_generator.py",
"content": "from __future__ import annotations\n\nimport importlib\nimport inspect\nfrom dataclasses import dataclass\nfrom typing import Any\n\n\n@dataclass(frozen=True)\nclass ImportTarget:\n module: str\n qualname: str # e.g. \"MyCustomFE\" or \"Outer.Inner\"\n\n\ndef parse_import_target(s: str) -> ImportTarget:\n # Accept \"pkg.mod:Class\" (preferred) and \"pkg.mod.Class\" (optional)\n if \":\" in s:\n module, qualname = s.split(\":\", 1)\n return ImportTarget(module=module, qualname=qualname)\n # dotted fallback: split last token as qualname\n parts = s.split(\".\")\n if len(parts) < 2:\n raise ValueError(f\"Invalid import target: {s!r}\")\n return ImportTarget(module=\".\".join(parts[:-1]), qualname=parts[-1])\n\n\ndef import_by_target(target: ImportTarget) -> Any:\n mod = importlib.import_module(target.module)\n obj = mod\n for attr in target.qualname.split(\".\"):\n obj = getattr(obj, attr)\n return obj\n\n\ndef resolve_fg_class(\n name: str,\n *,\n registry: dict[str, type] | None = None,\n base_class: type | None = None, # e.g. FeatureGenerator\n) -> type:\n # 1) registry\n if registry and name in registry:\n cls = registry[name]\n else:\n # 2) direct import target (optional convenience)\n if \":\" in name or (name.count(\".\") >= 1 and name[0].isalpha()):\n target = parse_import_target(name)\n cls = import_by_target(target)\n else:\n raise KeyError(f\"Unknown feature generator {name!r}. Known registry keys: {sorted(registry or {})}... \")\n\n if not inspect.isclass(cls):\n raise TypeError(f\"Resolved {name!r} to non-class object: {cls!r}\")\n if base_class is not None and not issubclass(cls, base_class):\n raise TypeError(f\"{cls} is not a subclass of {base_class}\")\n return cls\n"
},
{
"path": "features/src/autogluon/features/utils.py",
"content": "import logging\n\nimport numpy as np\nfrom pandas import DataFrame, Series\n\nlogger = logging.getLogger(__name__)\n\n\ndef clip_and_astype(df: DataFrame, columns: list = None, clip_min=0, clip_max=255, dtype: str = \"uint8\") -> DataFrame:\n \"\"\"\n Clips columns in a DataFrame to min and max values, and then converts dtype.\n\n Parameters\n ----------\n df : DataFrame\n Input DataFrame.\n columns : list, optional\n Column subset of df to apply the clip_and_astype logic to. If not specified, all columns of df are used.\n clip_min : int or float, default 0\n Minimum value to clip column values to. All values less than this will be set to clip_min.\n clip_max : int or float, default 255\n Maximum value to clip column values to. All values greater than this will be set to clip_max.\n dtype : dtype, default 'uint8'\n Data type to force after clipping is applied.\n\n Returns\n -------\n df_clipped : DataFrame\n clipped and astyped version of the input df.\n \"\"\"\n if columns is None:\n df = np.clip(df, clip_min, clip_max).astype(dtype)\n elif columns:\n df[columns] = np.clip(df[columns], clip_min, clip_max).astype(dtype)\n return df\n\n\n# TODO: Consider NaN values as a separate value?\ndef is_useless_feature(X: Series) -> bool:\n \"\"\"If a feature has the same value for every row, it carries no useful information\"\"\"\n return len(X.unique()) <= 1\n\n\ndef get_smallest_valid_dtype_int(min_val: int, max_val: int):\n \"\"\"Based on the minimum and maximum values of a feature, returns the smallest valid dtype to represent the feature.\"\"\"\n if min_val < 0:\n dtypes_to_check = [np.int8, np.int16, np.int32, np.int64]\n else:\n dtypes_to_check = [np.uint8, np.uint16, np.uint32, np.uint64]\n for dtype in dtypes_to_check:\n if max_val <= np.iinfo(dtype).max and min_val >= np.iinfo(dtype).min:\n return dtype\n raise ValueError(\n f\"Value is not able to be represented by {dtypes_to_check[-1].__name__}. (min_val, max_val): ({min_val}, {max_val})\"\n )\n"
},
{
"path": "features/src/autogluon/features/vectorizers.py",
"content": "from collections import Counter\n\nimport numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n\ndef vectorizer_auto_ml_default():\n return CountVectorizer(min_df=30, ngram_range=(1, 3), max_features=10000, dtype=np.uint8)\n\n\ndef get_ngram_freq(vectorizer, transform_matrix):\n names = vectorizer.get_feature_names_out()\n frequencies = transform_matrix.sum(axis=0).tolist()[0]\n ngram_freq = {ngram: freq for ngram, freq in zip(names, frequencies)}\n return ngram_freq\n\n\n# Reduces vectorizer vocabulary size to vocab_size, keeping highest frequency ngrams\ndef downscale_vectorizer(vectorizer, ngram_freq, vocab_size):\n counter = Counter(ngram_freq)\n top_n = counter.most_common(vocab_size)\n top_n_names = sorted([name for name, _ in top_n])\n new_vocab = {name: i for i, name in enumerate(top_n_names)}\n vectorizer.vocabulary_ = new_vocab\n"
},
{
"path": "features/tests/__init__.py",
"content": ""
},
{
"path": "features/tests/conftest.py",
"content": "import pytest\n\n\ndef pytest_addoption(parser):\n parser.addoption(\"--runslow\", action=\"store_true\", default=False, help=\"run slow tests\")\n\n\ndef pytest_configure(config):\n config.addinivalue_line(\"markers\", \"slow: mark test as slow to run\")\n\n\ndef pytest_collection_modifyitems(config, items):\n if config.getoption(\"--runslow\"):\n # --runslow given in cli: do not skip slow tests\n return\n skip_slow = pytest.mark.skip(reason=\"need --runslow option to run\")\n for item in items:\n if \"slow\" in item.keywords:\n item.add_marker(skip_slow)\n"
},
{
"path": "features/tests/features/__init__.py",
"content": ""
},
{
"path": "features/tests/features/conftest.py",
"content": "import copy\n\nimport numpy as np\nimport pandas as pd\nimport pytest\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.features.generators import AbstractFeatureGenerator\n\n\nclass GeneratorHelper:\n @staticmethod\n def fit_transform_assert(\n input_data: DataFrame,\n generator: AbstractFeatureGenerator,\n y=None,\n feature_metadata_in: FeatureMetadata = None,\n expected_feature_metadata_in_full: dict = None,\n expected_feature_metadata_full: dict = None,\n can_transform_on_train: bool = True,\n ):\n # Given\n original_input_data = copy.deepcopy(input_data)\n\n # Raise exception\n with pytest.raises(AssertionError):\n # Can't call transform before fit_transform\n generator.transform(input_data)\n\n if len(input_data.columns) > 0:\n # Raise exception\n with pytest.raises(AssertionError):\n input_data_with_duplicate_columns = pd.concat([input_data, input_data], axis=1)\n # Can't call fit_transform with duplicate column names\n generator.fit_transform(\n input_data_with_duplicate_columns, y=y, feature_metadata_in=feature_metadata_in\n )\n\n assert not generator.is_fit()\n output_data = generator.fit_transform(input_data, y=y, feature_metadata_in=feature_metadata_in)\n assert generator.is_fit()\n with pytest.raises(AssertionError):\n # Can't call fit_transform after fit\n generator.fit_transform(input_data, y=y, feature_metadata_in=feature_metadata_in)\n\n # Ensure input_data is not altered inplace by fit_transform\n assert input_data.equals(original_input_data)\n\n # Ensure unchanged row count\n assert len(input_data) == len(output_data)\n\n output_data_og = output_data\n # Ensure transform and fit_transform output are the same for training data\n output_data_transform = generator.transform(input_data)\n if can_transform_on_train:\n assert output_data.equals(output_data_transform)\n else:\n output_data = output_data_transform # Do this for future transform checks\n\n # Ensure input_data is not altered inplace by transform\n assert input_data.equals(original_input_data)\n\n # Ensure transform will be the same if unnecessary columns are removed from input\n input_data_features_in = input_data[generator.features_in]\n original_input_data_features_in = copy.deepcopy(input_data_features_in)\n output_data_transform = generator.transform(input_data_features_in)\n assert output_data.equals(output_data_transform)\n\n # Ensure input_data is not altered inplace by transform when features_in match exactly\n assert input_data_features_in.equals(original_input_data_features_in)\n\n # Ensure transform will be the same if input feature order is not the same as generator.features_in\n reversed_column_names = list(input_data.columns)\n reversed_column_names.reverse()\n input_data_reversed = input_data[reversed_column_names]\n original_input_data_reversed = copy.deepcopy(input_data_reversed)\n output_data_transform = generator.transform(input_data_reversed)\n assert output_data.equals(output_data_transform)\n\n # Ensure input_data is not altered inplace by transform when column order is reversed\n assert input_data_reversed.equals(original_input_data_reversed)\n\n # Ensure output feature order is correct\n assert generator.features_out == list(output_data.columns)\n\n if generator.features_in:\n with pytest.raises(KeyError):\n # Error if missing input feature\n generator.transform(input_data.drop(generator.features_in[0]))\n with pytest.raises(KeyError):\n # Error if missing all input features\n generator.transform(pd.DataFrame())\n\n # Ensure unknown input columns don't affect output\n input_data_with_extra = copy.deepcopy(input_data)\n input_data_with_extra[\"__UNKNOWN_COLUMN__\"] = 0\n original_input_data_with_extra = copy.deepcopy(input_data_with_extra)\n output_data_transform = generator.transform(input_data_with_extra)\n assert output_data.equals(output_data_transform)\n\n # Ensure input_data is not altered inplace by transform when extra columns are present\n assert input_data_with_extra.equals(original_input_data_with_extra)\n\n # Ensure feature_metadata_in is as expected\n if expected_feature_metadata_in_full is not None:\n assert expected_feature_metadata_in_full == generator.feature_metadata_in.to_dict(inverse=True)\n # Ensure feature_metadata is as expected\n if expected_feature_metadata_full is not None:\n assert expected_feature_metadata_full == generator.feature_metadata.to_dict(inverse=True)\n\n return output_data_og\n\n\nclass DataHelper:\n @staticmethod\n def generate_empty() -> DataFrame:\n return DataFrame(index=[0, 1, 2, 3, 4, 5, 6, 7, 8])\n\n @staticmethod\n def generate_obj_feature() -> Series:\n return Series([\"a\", \"b\", \"a\", \"d\", \"d\", \"d\", \"c\", np.nan, np.nan])\n\n @staticmethod\n def generate_int_feature() -> Series:\n return Series([12, 0, -8, 3, 2, 2, 2, 0, 5])\n\n @staticmethod\n def generate_cat_feature() -> Series:\n return DataHelper.generate_obj_feature().astype(\"category\")\n\n @staticmethod\n def generate_float_feature() -> Series:\n return Series([12, 0, np.nan, 3, -0.2, 2.3, 2.3, 0.1, 5])\n\n @staticmethod\n def generate_text_feature() -> Series:\n return Series(\n [\n \"hello world\",\n \"sentence breaks.\",\n \"\",\n \"unique words\",\n \"the end of the sentence\",\n \"goodbye world\",\n \"the end is not the end is not the end is not the end is not the end\",\n \"the end of the world\",\n \"sentence. breaks. sentence. breaks. sentence. breaks. sentence. breaks.\",\n ]\n )\n\n @staticmethod\n def generate_datetime_as_object_feature() -> Series:\n return Series(\n [\n \"8/1/2018 16:27\",\n \"\", # nan\n np.nan, # nan\n \"4/20/2018 15:37\",\n \"4/20/2018 15:37\",\n \"1/01/1800 00:00\",\n \"12/31/2200 23:59\",\n \"8/15/2700 7:12\", # nan\n \"12/18/1000 2:12\", # nan\n ]\n )\n\n @staticmethod\n def generate_datetime_as_object_feature_advanced() -> Series:\n \"\"\"Nightmare input for datetime\"\"\"\n return Series(\n [\n \"8/1/2018 16:27\",\n \"\", # nan\n np.nan, # nan\n \"2021-08-09T17:07:08.659-0400\", # With timezone 4\n \"2021-08-09T17:08:15.541-0400\", # With timezone 4\n \"2021-01-07T12:33:23.938-0500\", # With timezone 5\n ]\n )\n\n @staticmethod\n def generate_datetime_feature() -> Series:\n return pd.to_datetime(DataHelper.generate_datetime_as_object_feature(), errors=\"coerce\", format=\"mixed\")\n\n @staticmethod\n def generate_bool_feature_int(name=\"int_bool\") -> DataFrame:\n return Series([0, 1, 1, 0, 0, 0, 1, 0, 1], name=name).to_frame()\n\n @staticmethod\n def generate_bool_feature_with_nan() -> DataFrame:\n return Series([None, 5, None, 5], name=\"edgecase_with_nan_bool\").to_frame()\n\n @staticmethod\n def generate_bool_feature_edgecase() -> DataFrame:\n return Series([None, None, None, np.nan, np.nan, np.nan, None, None, None], name=\"edgecase_bool\").to_frame()\n\n @staticmethod\n def generate_bool_feature_extreme_edgecase() -> DataFrame:\n return Series([5, \"5\", 5, 5, \"5\"], name=\"edgecase_extreme_bool\").to_frame()\n\n @staticmethod\n def generate_multi_feature_standard() -> DataFrame:\n df = pd.concat(\n [\n DataHelper.generate_int_feature(),\n DataHelper.generate_float_feature(),\n DataHelper.generate_obj_feature(),\n DataHelper.generate_cat_feature(),\n DataHelper.generate_datetime_feature(),\n ],\n axis=1,\n )\n df.columns = [\"int\", \"float\", \"obj\", \"cat\", \"datetime\"]\n return df\n\n @staticmethod\n def generate_useless_category() -> DataFrame:\n return Series(\n [\n \"a\",\n \"b\",\n \"c\",\n \"d\",\n \"e\",\n \"f\",\n \"g\",\n \"h\",\n \"i\",\n ],\n name=\"cat_useless\",\n ).to_frame()\n\n @staticmethod\n def generate_duplicate() -> DataFrame:\n df_bool_dupes = []\n for i in range(20):\n df_bool_dupes.append(\n DataHelper.generate_bool_feature_int(name=f\"int_bool_dup_{i + 1}\"),\n )\n\n df_to_concat = (\n [\n DataHelper.generate_bool_feature_int(),\n DataHelper.generate_multi_feature_special(),\n DataHelper.generate_useless_category(),\n DataHelper.generate_multi_feature_standard(),\n ]\n + df_bool_dupes\n + [\n DataHelper.generate_bool_feature_int(name=\"int_bool_dup_final\"),\n ]\n )\n\n df = pd.concat(\n df_to_concat,\n axis=1,\n )\n return df\n\n @staticmethod\n def generate_multi_feature_special() -> DataFrame:\n df = pd.concat(\n [\n DataHelper.generate_text_feature(),\n DataHelper.generate_datetime_as_object_feature(),\n ],\n axis=1,\n )\n df.columns = [\"text\", \"datetime_as_object\"]\n return df\n\n @staticmethod\n def generate_multi_feature_full() -> DataFrame:\n df = pd.concat(\n [\n DataHelper.generate_bool_feature_int(),\n DataHelper.generate_multi_feature_standard(),\n DataHelper.generate_multi_feature_special(),\n ],\n axis=1,\n )\n return df\n\n\n@pytest.fixture\ndef generator_helper():\n return GeneratorHelper\n\n\n@pytest.fixture\ndef data_helper():\n return DataHelper\n"
},
{
"path": "features/tests/features/generators/__init__.py",
"content": ""
},
{
"path": "features/tests/features/generators/arithmetic/__init__.py",
"content": ""
},
{
"path": "features/tests/features/generators/arithmetic/test_operation.py",
"content": "# tests/test_canonical_key.py\n\nfrom autogluon.features.generators.arithmetic.canonical_key import _expr_to_canonical_key # adjust import path\nfrom autogluon.features.generators.arithmetic.operation import Operation\n\n\ndef test_expr_to_canonical_key_commutative_mul():\n # A*B == B*A\n k1 = _expr_to_canonical_key(Operation(\"A\", \"B\", \"*\"))\n k2 = _expr_to_canonical_key(Operation(\"B\", \"A\", \"*\"))\n assert k1 == k2\n\n\ndef test_expr_to_canonical_key_commutative_add():\n # A+B == B+A\n k1 = _expr_to_canonical_key(Operation(\"A\", \"B\", \"+\"))\n k2 = _expr_to_canonical_key(Operation(\"B\", \"A\", \"+\"))\n assert k1 == k2\n\n\ndef test_expr_to_canonical_key_associative_mul():\n # (A*B)*C == A*(B*C)\n k1 = _expr_to_canonical_key(Operation(Operation(\"A\", \"B\", \"*\"), \"C\", \"*\"))\n k2 = _expr_to_canonical_key(Operation(\"A\", Operation(\"B\", \"C\", \"*\"), \"*\"))\n assert k1 == k2\n\n\ndef test_expr_to_canonical_key_associative_add():\n # (A+B)+C == A+(B+C)\n k1 = _expr_to_canonical_key(Operation(Operation(\"A\", \"B\", \"+\"), \"C\", \"+\"))\n k2 = _expr_to_canonical_key(Operation(\"A\", Operation(\"B\", \"C\", \"+\"), \"+\"))\n assert k1 == k2\n\n\ndef test_expr_to_canonical_key_mul_div_interaction():\n # (A/B)*C == (C*A)/B\n expr1 = Operation(Operation(\"A\", \"B\", \"/\"), \"C\", \"*\")\n expr2 = Operation(Operation(\"C\", \"A\", \"*\"), \"B\", \"/\")\n k1 = _expr_to_canonical_key(expr1)\n k2 = _expr_to_canonical_key(expr2)\n assert k1 == k2\n\n\ndef test_expr_to_canonical_key_subtraction_not_commutative():\n # A-B != B-A\n k1 = _expr_to_canonical_key(Operation(\"A\", \"B\", \"-\"))\n k2 = _expr_to_canonical_key(Operation(\"B\", \"A\", \"-\"))\n assert k1 != k2\n\n\ndef test_expr_to_canonical_key_fallback_non_monomial_division():\n # (A+B)/C cannot be reduced as polynomial-by-monomial division in the implementation\n # but should still produce a stable key via structural fallback.\n expr1 = Operation(Operation(\"A\", \"B\", \"+\"), \"C\", \"/\")\n expr2 = Operation(\"C\", Operation(\"A\", \"B\", \"+\"), \"/\") # different structure\n k1 = _expr_to_canonical_key(expr1)\n k2 = _expr_to_canonical_key(expr2)\n assert k1 != k2\n\n # And commutativity inside the numerator should still be respected by structural key\n expr3 = Operation(Operation(\"B\", \"A\", \"+\"), \"C\", \"/\")\n k3 = _expr_to_canonical_key(expr3)\n assert k1 == k3\n"
},
{
"path": "features/tests/features/generators/test_astype.py",
"content": "from autogluon.features.generators import AsTypeFeatureGenerator\n\n\ndef test_astype_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = AsTypeFeatureGenerator(reset_index=True)\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert not input_data.equals(output_data)\n\n\ndef test_astype_feature_generator_bool(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_bool_feature_int()\n\n generator = AsTypeFeatureGenerator(convert_bool_method=\"v2\") # v2 doesn't edit in-place, so no need to reset_index\n\n expected_feature_metadata_in_full = {\n (\"int\", ()): [\"int_bool\"],\n }\n\n expected_feature_metadata_full = {\n (\"int\", (\"bool\",)): [\"int_bool\"],\n }\n\n # When\n generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n\ndef test_astype_feature_generator_bool_edgecase_with_nan(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_bool_feature_with_nan()\n\n generator = AsTypeFeatureGenerator(reset_index=True)\n\n expected_feature_metadata_in_full = {\n (\"float\", ()): [\"edgecase_with_nan_bool\"],\n }\n\n # Since only NaN, don't convert to boolean\n expected_feature_metadata_full = {\n (\"int\", (\"bool\",)): [\"edgecase_with_nan_bool\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Ensure `NaN` and `None` are mapped to 0, even if they are ordered first.\n assert list(output_data[\"edgecase_with_nan_bool\"]) == [0, 1, 0, 1]\n\n\ndef test_astype_feature_generator_bool_edgecase(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_bool_feature_edgecase()\n\n generator = AsTypeFeatureGenerator(reset_index=True)\n\n expected_feature_metadata_in_full = {\n (\"float\", ()): [\"edgecase_bool\"],\n }\n\n # Since only NaN, don't convert to boolean\n expected_feature_metadata_full = {(\"float\", ()): [\"edgecase_bool\"]}\n\n # When\n generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n\ndef test_astype_feature_generator_bool_extreme_edgecase(generator_helper, data_helper):\n \"\"\"\n Ensure that int 5 and string 5 are considered different values\n Also ensure that AsTypeFeatureGenerator returns the same output regardless of hyperparameters\n \"\"\"\n # Given\n input_data = data_helper.generate_bool_feature_extreme_edgecase()\n\n generator_1 = AsTypeFeatureGenerator(reset_index=True)\n generator_2 = AsTypeFeatureGenerator(convert_bool_method_v2_threshold=1)\n generator_3 = AsTypeFeatureGenerator(convert_bool_method=\"v2\")\n generator_4 = AsTypeFeatureGenerator(convert_bool_method=\"v2\", convert_bool_method_v2_row_threshold=-1)\n expected_feature_metadata_in_full = {\n (\"object\", ()): [\"edgecase_extreme_bool\"],\n }\n expected_feature_metadata_full = {\n (\"int\", (\"bool\",)): [\"edgecase_extreme_bool\"],\n }\n\n out_list = []\n for generator in [generator_1, generator_2, generator_3, generator_4]:\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n out_list.append(output_data)\n\n for i in range(len(out_list) - 1):\n assert out_list[i].equals(out_list[i + 1])\n"
},
{
"path": "features/tests/features/generators/test_auto_ml_pipeline.py",
"content": "import numpy as np\nimport pytest\nfrom packaging.version import Version\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.features.generators import (\n AutoMLPipelineFeatureGenerator,\n IdentityFeatureGenerator,\n TextNgramFeatureGenerator,\n)\n\n\ndef test_auto_ml_pipeline_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n with pytest.raises(KeyError):\n # generators is an invalid argument\n AutoMLPipelineFeatureGenerator(generators=[], vectorizer=toy_vectorizer)\n\n generator = AutoMLPipelineFeatureGenerator(vectorizer=toy_vectorizer)\n\n for generator_stage in generator.generators:\n for generator_inner in generator_stage:\n if isinstance(generator_inner, TextNgramFeatureGenerator):\n # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n generator_inner.max_memory_ratio = None\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.sentence\",\n \"__nlp__.the\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n expected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # int and float checks\n assert output_data[\"int\"].equals(input_data[\"int\"])\n assert output_data[\"float\"].equals(input_data[\"float\"])\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n # datetime checks. There are further checks in test_datetime.py\n assert expected_output_data_feat_datetime == list(output_data[\"datetime\"].values)\n\n # text_special checks\n assert (\n list(map(int, output_data[\"text.lower_ratio\"].values)) == expected_output_data_feat_lower_ratio_np_lt_2_0\n if Version(np.__version__) < Version(\"2.0.0\")\n else expected_output_data_feat_lower_ratio_np_ge_2_0\n )\n\n # text_ngram checks\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\ndef test_auto_ml_pipeline_feature_generator_raw_text(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n generator = AutoMLPipelineFeatureGenerator(enable_raw_text_features=True, vectorizer=toy_vectorizer)\n\n for generator_stage in generator.generators:\n for generator_inner in generator_stage:\n if isinstance(generator_inner, TextNgramFeatureGenerator):\n # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n generator_inner.max_memory_ratio = None\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.sentence\",\n \"__nlp__.the\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n (\"object\", (\"text\",)): [\"text_raw_text\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert list(input_data[\"text\"].values) == list(output_data[\"text_raw_text\"].values)\n\n\ndef test_auto_ml_pipeline_feature_generator_only_raw_text(generator_helper, data_helper):\n \"\"\"\n Specifically tests when only text columns are provided.\n This verifies the edge-case bug in v0.6.2 from https://github.com/autogluon/autogluon/issues/2688 is not present.\n \"\"\"\n\n # Given\n input_data = data_helper.generate_text_feature().to_frame(\"text\")\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n generator = AutoMLPipelineFeatureGenerator(enable_raw_text_features=True, vectorizer=toy_vectorizer)\n\n for generator_stage in generator.generators:\n for generator_inner in generator_stage:\n if isinstance(generator_inner, TextNgramFeatureGenerator):\n # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n generator_inner.max_memory_ratio = None\n\n expected_feature_metadata_in_full = {(\"object\", (\"text\",)): [\"text\"]}\n\n expected_feature_metadata_full = {\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.sentence\",\n \"__nlp__.the\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n (\"object\", (\"text\",)): [\"text_raw_text\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert list(input_data[\"text\"].values) == list(output_data[\"text_raw_text\"].values)\n\n\ndef test_auto_ml_pipeline_feature_generator_duplicates(generator_helper, data_helper):\n \"\"\"\n Test the most complicated situation: Many duplicate features, useless features, and all dtypes at once\n This test ensures the fix in https://github.com/autogluon/autogluon/pull/2986 works, test failed prior to fix\n \"\"\"\n # Given\n input_data = data_helper.generate_duplicate()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n with pytest.raises(KeyError):\n # generators is an invalid argument\n AutoMLPipelineFeatureGenerator(generators=[], vectorizer=toy_vectorizer)\n\n generator = AutoMLPipelineFeatureGenerator(vectorizer=toy_vectorizer)\n\n for generator_stage in generator.generators:\n for generator_inner in generator_stage:\n if isinstance(generator_inner, TextNgramFeatureGenerator):\n # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n generator_inner.max_memory_ratio = None\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.sentence\",\n \"__nlp__.the\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n expected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # int and float checks\n assert output_data[\"int\"].equals(input_data[\"int\"])\n assert output_data[\"float\"].equals(input_data[\"float\"])\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n # datetime checks. There are further checks in test_datetime.py\n assert expected_output_data_feat_datetime == list(output_data[\"datetime_as_object\"].values)\n\n # text_special checks\n assert (\n list(map(int, output_data[\"text.lower_ratio\"].values)) == expected_output_data_feat_lower_ratio_np_lt_2_0\n if Version(np.__version__) < Version(\"2.0.0\")\n else expected_output_data_feat_lower_ratio_np_ge_2_0\n )\n\n # text_ngram checks\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\ndef test_auto_ml_pipeline_feature_generator_duplicates_without_dedupe(generator_helper, data_helper):\n \"\"\"\n This test additionally turns off the drop_duplicates logic\n\n Test the most complicated situation: Many duplicate features, useless features, and all dtypes at once\n This test ensures the fix in https://github.com/autogluon/autogluon/pull/2986 works, test failed prior to fix\n \"\"\"\n # Given\n input_data = data_helper.generate_duplicate()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n with pytest.raises(KeyError):\n # generators is an invalid argument\n AutoMLPipelineFeatureGenerator(generators=[], vectorizer=toy_vectorizer, post_drop_duplicates=False)\n\n generator = AutoMLPipelineFeatureGenerator(vectorizer=toy_vectorizer, post_drop_duplicates=False)\n\n for generator_stage in generator.generators:\n for generator_inner in generator_stage:\n if isinstance(generator_inner, TextNgramFeatureGenerator):\n # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n generator_inner.max_memory_ratio = None\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\n \"int_bool\",\n \"int\",\n \"int_bool_dup_1\",\n \"int_bool_dup_2\",\n \"int_bool_dup_3\",\n \"int_bool_dup_4\",\n \"int_bool_dup_5\",\n \"int_bool_dup_6\",\n \"int_bool_dup_7\",\n \"int_bool_dup_8\",\n \"int_bool_dup_9\",\n \"int_bool_dup_10\",\n \"int_bool_dup_11\",\n \"int_bool_dup_12\",\n \"int_bool_dup_13\",\n \"int_bool_dup_14\",\n \"int_bool_dup_15\",\n \"int_bool_dup_16\",\n \"int_bool_dup_17\",\n \"int_bool_dup_18\",\n \"int_bool_dup_19\",\n \"int_bool_dup_20\",\n \"int_bool_dup_final\",\n ],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\n \"int_bool\",\n \"int_bool_dup_1\",\n \"int_bool_dup_2\",\n \"int_bool_dup_3\",\n \"int_bool_dup_4\",\n \"int_bool_dup_5\",\n \"int_bool_dup_6\",\n \"int_bool_dup_7\",\n \"int_bool_dup_8\",\n \"int_bool_dup_9\",\n \"int_bool_dup_10\",\n \"int_bool_dup_11\",\n \"int_bool_dup_12\",\n \"int_bool_dup_13\",\n \"int_bool_dup_14\",\n \"int_bool_dup_15\",\n \"int_bool_dup_16\",\n \"int_bool_dup_17\",\n \"int_bool_dup_18\",\n \"int_bool_dup_19\",\n \"int_bool_dup_20\",\n \"int_bool_dup_final\",\n ],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.end of the\",\n \"__nlp__.of\",\n \"__nlp__.of the\",\n \"__nlp__.sentence\",\n \"__nlp__.sentence breaks\",\n \"__nlp__.the\",\n \"__nlp__.the end\",\n \"__nlp__.the end of\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n expected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # int and float checks\n assert output_data[\"int\"].equals(input_data[\"int\"])\n assert output_data[\"float\"].equals(input_data[\"float\"])\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n # datetime checks. There are further checks in test_datetime.py\n assert list(output_data[\"datetime\"].values) == list(output_data[\"datetime_as_object\"].values)\n assert expected_output_data_feat_datetime == list(output_data[\"datetime\"].values)\n\n # text_special checks\n assert (\n list(map(int, output_data[\"text.lower_ratio\"].values)) == expected_output_data_feat_lower_ratio_np_lt_2_0\n if Version(np.__version__) < Version(\"2.0.0\")\n else expected_output_data_feat_lower_ratio_np_ge_2_0\n )\n\n # text_ngram checks\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\ndef test_add_custom_feature_generators():\n \"\"\"Test the _add_custom_feature_generators method of AutoMLPipelineFeatureGenerator.\n\n Ensures that the custom feature generator insertion logic works as expected\n for different use cases.\n \"\"\"\n gen_1 = IdentityFeatureGenerator()\n gen_2 = TextNgramFeatureGenerator()\n fg = AutoMLPipelineFeatureGenerator(custom_feature_generators=[gen_1, gen_2])\n\n fg_main_stage = fg.generators[2]\n assert fg_main_stage[-2] == gen_1\n assert fg_main_stage[-1] == gen_2\n"
},
{
"path": "features/tests/features/generators/test_bulk.py",
"content": "import numpy as np\nfrom packaging.version import Version\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.common.features.types import R_FLOAT, R_INT\nfrom autogluon.features.generators import (\n AsTypeFeatureGenerator,\n BulkFeatureGenerator,\n CategoryFeatureGenerator,\n DatetimeFeatureGenerator,\n DropUniqueFeatureGenerator,\n FillNaFeatureGenerator,\n IdentityFeatureGenerator,\n TextNgramFeatureGenerator,\n TextSpecialFeatureGenerator,\n)\n\n\ndef test_bulk_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n text_ngram_feature_generator = TextNgramFeatureGenerator(vectorizer=toy_vectorizer)\n text_ngram_feature_generator.max_memory_ratio = (\n None # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n )\n\n generator = BulkFeatureGenerator(\n generators=[\n [AsTypeFeatureGenerator(convert_bool_method=\"v2\")],\n [FillNaFeatureGenerator()],\n [\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT])),\n CategoryFeatureGenerator(),\n DatetimeFeatureGenerator(),\n TextSpecialFeatureGenerator(),\n text_ngram_feature_generator,\n ],\n [DropUniqueFeatureGenerator()],\n ],\n reset_index=True,\n )\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.end of the\",\n \"__nlp__.of\",\n \"__nlp__.of the\",\n \"__nlp__.sentence\",\n \"__nlp__.sentence breaks\",\n \"__nlp__.the\",\n \"__nlp__.the end\",\n \"__nlp__.the end of\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n expected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert list(output_data.columns) == [\n \"int_bool\",\n \"int\",\n \"float\",\n \"obj\",\n \"cat\",\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.end of the\",\n \"__nlp__.of\",\n \"__nlp__.of the\",\n \"__nlp__.sentence\",\n \"__nlp__.sentence breaks\",\n \"__nlp__.the\",\n \"__nlp__.the end\",\n \"__nlp__.the end of\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ]\n\n # int and float checks\n assert output_data[\"int\"].equals(input_data[\"int\"])\n assert output_data[\"float\"].equals(input_data[\"float\"])\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n # datetime checks. There are further checks in test_datetime.py\n assert list(output_data[\"datetime\"].values) == list(output_data[\"datetime_as_object\"].values)\n assert expected_output_data_feat_datetime == list(output_data[\"datetime\"].values)\n\n # text_special checks\n assert (\n list(map(int, output_data[\"text.lower_ratio\"].values)) == expected_output_data_feat_lower_ratio_np_lt_2_0\n if Version(np.__version__) < Version(\"2.0.0\")\n else expected_output_data_feat_lower_ratio_np_ge_2_0\n )\n\n # text_ngram checks\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\n# Test that bulk does not create dummy output\ndef test_bulk_feature_generator_no_dummy(generator_helper, data_helper):\n input_data = data_helper.generate_multi_feature_full()\n\n input_data_transform = data_helper.generate_empty()\n\n generator = BulkFeatureGenerator(\n generators=[\n [\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[\"unknown_raw_type\"])),\n ]\n ]\n )\n\n expected_feature_metadata_in_full = {}\n expected_feature_metadata_full = {}\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Then\n assert output_data.shape == (9, 0)\n\n assert generator.transform(input_data_transform).shape == (9, 0)\n assert generator.transform(input_data_transform.head(5)).shape == (5, 0)\n"
},
{
"path": "features/tests/features/generators/test_category.py",
"content": "import numpy as np\n\nfrom autogluon.features.generators import CategoryFeatureGenerator\n\n\ndef test_category_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_standard()\n category_input_data = input_data[[\"obj\", \"cat\"]].astype(\"category\")\n\n generator_1 = CategoryFeatureGenerator(minimum_cat_count=None)\n generator_2 = CategoryFeatureGenerator(minimum_cat_count=None, maximum_num_cat=2)\n generator_3 = CategoryFeatureGenerator(minimum_cat_count=3)\n generator_4 = CategoryFeatureGenerator(minimum_cat_count=None, cat_order=\"count\")\n generator_5 = CategoryFeatureGenerator(minimum_cat_count=None, fillna=\"mode\")\n generator_6 = CategoryFeatureGenerator(minimum_cat_count=None, minimize_memory=False)\n\n expected_feature_metadata_in_full = {\n (\"object\", ()): [\"obj\"],\n (\"category\", ()): [\"cat\"],\n }\n expected_feature_metadata_full = {(\"category\", ()): [\"obj\", \"cat\"]}\n\n expected_cat_categories_lst = [\n [0, 1, 2, 3],\n [0, 1],\n [0],\n [0, 1, 2, 3],\n [0, 1, 2, 3],\n ]\n\n expected_cat_values_lst = [\n [[0, 1, 0, 3, 3, 3, 2, np.nan, np.nan]],\n [[0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]],\n [[np.nan, np.nan, np.nan, 0, 0, 0, np.nan, np.nan, np.nan]],\n # the outputs order for this case varies between python 3.6 vs 3.7 and numpy versions\n [[2, 1, 2, 3, 3, 3, 0, np.nan, np.nan], [2, 0, 2, 3, 3, 3, 1, np.nan, np.nan]],\n [[0, 1, 0, 3, 3, 3, 2, 3, 3]],\n ]\n\n expected_cat_codes_lst = [\n [[0, 1, 0, 3, 3, 3, 2, -1, -1]],\n [[0, -1, 0, 1, 1, 1, -1, -1, -1]],\n [[-1, -1, -1, 0, 0, 0, -1, -1, -1]],\n # the outputs order for this case varies between python 3.6 vs 3.7 and numpy versions\n [[2, 1, 2, 3, 3, 3, 0, -1, -1], [2, 0, 2, 3, 3, 3, 1, -1, -1]],\n [[0, 1, 0, 3, 3, 3, 2, 3, 3]],\n ]\n\n # When\n output_datas = []\n for generator in [generator_1, generator_2, generator_3, generator_4, generator_5, generator_6]:\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n output_datas.append(output_data)\n\n # Therefore\n assert category_input_data.equals(output_datas[5])\n output_datas = output_datas[:5]\n\n for i in range(len(output_datas)):\n output_data = output_datas[i]\n for col in [\"obj\", \"cat\"]:\n assert output_data[col].dtype.name == \"category\"\n assert list(output_data[col].cat.categories) == expected_cat_categories_lst[i]\n assert list(output_data[col]) in expected_cat_values_lst[i]\n assert list(output_data[col].cat.codes) in expected_cat_codes_lst[i]\n"
},
{
"path": "features/tests/features/generators/test_datetime.py",
"content": "from autogluon.features.generators import DatetimeFeatureGenerator\n\n\ndef test_datetime_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator_1 = DatetimeFeatureGenerator()\n generator_2 = DatetimeFeatureGenerator(features=[\"hour\"])\n\n expected_feature_metadata_in_full = {\n (\"datetime\", ()): [\"datetime\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n }\n\n expected_feature_metadata_full_1 = {\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n ]\n }\n\n expected_feature_metadata_full_2 = {\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.hour\",\n \"datetime_as_object\",\n \"datetime_as_object.hour\",\n ]\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_datetime_year = [\n 2018,\n 2011, # blank and nan values are set to the mean of good values = 2011\n 2011,\n 2018,\n 2018,\n 1800,\n 2200,\n 2011, # 2700 and 1000 are out of range for a pandas datetime so they are set to the mean\n 2011, # see limits at https://pandas.pydata.org/docs/reference/api/pandas.Timestamp.max.html\n ]\n\n expected_output_data_feat_datetime_hour = [16, 14, 14, 15, 15, 0, 23, 14, 14]\n\n # When\n output_data_1 = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator_1,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full_1,\n )\n\n assert list(output_data_1[\"datetime\"].values) == list(output_data_1[\"datetime_as_object\"].values)\n assert expected_output_data_feat_datetime == list(output_data_1[\"datetime\"].values)\n assert expected_output_data_feat_datetime_year == list(output_data_1[\"datetime.year\"].values)\n\n output_data_2 = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator_2,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full_2,\n )\n\n assert list(output_data_2[\"datetime\"].values) == list(output_data_2[\"datetime_as_object\"].values)\n assert expected_output_data_feat_datetime == list(output_data_2[\"datetime\"].values)\n assert expected_output_data_feat_datetime_hour == list(output_data_2[\"datetime.hour\"].values)\n\n\n# This covers the nightmare input scenario for a datetime column:\n# multiple formats, multiple NaN's of different types, multiple time zones (including no time zone), all as strings.\n# This is just about as bad as it could get. If we work here, we should work with practically anything.\ndef test_datetime_feature_generator_advanced(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_datetime_as_object_feature_advanced().to_frame(name=\"datetime_as_object\")\n\n generator = DatetimeFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n }\n\n expected_feature_metadata_full = {\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime_as_object\",\n \"datetime_as_object.year\",\n \"datetime_as_object.month\",\n \"datetime_as_object.day\",\n \"datetime_as_object.dayofweek\",\n ]\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1600067037034500096,\n 1600067037034500096,\n 1628543228659000000,\n 1628543295541000000,\n 1610040803938000000,\n ]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data_feat_datetime == list(output_data[\"datetime_as_object\"].values)\n"
},
{
"path": "features/tests/features/generators/test_drop_duplicates.py",
"content": "import numpy as np\nimport pandas as pd\n\nfrom autogluon.common import FeatureMetadata\nfrom autogluon.features.generators import DropDuplicatesFeatureGenerator\n\n\ndef test_drop_duplicates_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = DropDuplicatesFeatureGenerator()\n\n og_feature_metadata_in = FeatureMetadata.from_df(input_data).to_dict(inverse=True)\n\n expected_og_feature_metadata_in = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n assert expected_og_feature_metadata_in == og_feature_metadata_in\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n # When\n generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n\ndef test_drop_duplicates_feature_generator_with_dupes(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_duplicate()\n\n generator = DropDuplicatesFeatureGenerator()\n\n og_feature_metadata_in = FeatureMetadata.from_df(input_data).to_dict(inverse=True)\n\n expected_og_feature_metadata_in = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\n \"int_bool\",\n \"int\",\n \"int_bool_dup_1\",\n \"int_bool_dup_2\",\n \"int_bool_dup_3\",\n \"int_bool_dup_4\",\n \"int_bool_dup_5\",\n \"int_bool_dup_6\",\n \"int_bool_dup_7\",\n \"int_bool_dup_8\",\n \"int_bool_dup_9\",\n \"int_bool_dup_10\",\n \"int_bool_dup_11\",\n \"int_bool_dup_12\",\n \"int_bool_dup_13\",\n \"int_bool_dup_14\",\n \"int_bool_dup_15\",\n \"int_bool_dup_16\",\n \"int_bool_dup_17\",\n \"int_bool_dup_18\",\n \"int_bool_dup_19\",\n \"int_bool_dup_20\",\n \"int_bool_dup_final\",\n ],\n (\"object\", ()): [\"cat_useless\", \"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n assert expected_og_feature_metadata_in == og_feature_metadata_in\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"cat_useless\", \"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"cat_useless\", \"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n # When\n generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n\ndef test_drop_duplicates_numeric_edge_cases(generator_helper):\n df = pd.DataFrame(\n {\n \"A\": [0, 1, 2, 3, 4],\n \"B\": [\"0\", \"1\", \"2\", \"3\", \"4\"],\n \"C\": [np.nan, 1, 2, 3, 4],\n \"D\": [np.nan, 1, 2, 3, 4],\n \"E\": [4, 3, 2, 1, 4],\n \"F\": [4, 3, 2, 1, 0],\n \"G\": [4, 3, 2, 1, 4],\n \"H\": [4.0, 3.0, 2.0, 1.0, 4.0],\n \"I\": [4.2, 3.6, -4.3, 1.7, 4.2],\n \"J\": [4.2, 3.6, -4.3, 1.7, 4.2],\n }\n )\n\n feature_generator = DropDuplicatesFeatureGenerator()\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop D because C and D are identical and both numerical, and C comes earlier\n # Drop G and H because E and G are identical and both numerical, and G comes earlier\n # Drop J because I and J are identical and both numerical, and I comes earlier\n expected_dropped_1 = [\"D\", \"G\", \"H\", \"J\"]\n actual_dropped_1 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_1 == actual_dropped_1\n\n og_feature_metadata_in = FeatureMetadata.from_df(df).to_dict(inverse=True)\n\n expected_og_feature_metadata_in = {\n (\"float\", ()): [\"C\", \"D\", \"H\", \"I\", \"J\"],\n (\"int\", ()): [\"A\", \"E\", \"F\", \"G\"],\n (\"object\", ()): [\"B\"],\n }\n\n assert expected_og_feature_metadata_in == og_feature_metadata_in\n\n expected_feature_metadata_in_full = {\n (\"float\", ()): [\"C\", \"I\"],\n (\"int\", ()): [\"A\", \"E\", \"F\"],\n (\"object\", ()): [\"B\"],\n }\n\n expected_feature_metadata_full = {(\"float\", ()): [\"C\", \"I\"], (\"int\", ()): [\"A\", \"E\", \"F\"], (\"object\", ()): [\"B\"]}\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=df,\n generator=feature_generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n for removed_feature in expected_dropped_1:\n assert removed_feature not in output_data.columns\n\n\ndef test_drop_duplicates_category_edge_cases():\n df = pd.DataFrame(\n {\n \"A\": [0, 1, 2, 3, 4],\n \"B\": [0, 3, \"b\", 4, np.nan],\n \"C\": [\"a\", \"b\", 3, \"d\", \"e\"],\n \"D\": [0, 3, \"b\", 4, np.nan],\n \"E\": [5, np.nan, \"b\", 4, 3],\n \"F\": [4, 3, 2, 1, 0],\n \"G\": [4, 3, 2, 1, 4],\n }\n )\n feature_generator = DropDuplicatesFeatureGenerator()\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop D because B and D are identical with same dtype, and B comes earlier\n expected_dropped_1 = [\"D\"]\n actual_dropped_1 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_1 == actual_dropped_1\n\n df2 = pd.DataFrame(\n {\n \"A\": [0, 1, 2, 3, 4],\n \"D\": [0, 3, \"b\", 4, np.nan],\n \"C\": [\"a\", \"b\", 3, \"d\", \"e\"],\n \"B\": [0, 3, \"b\", 4, np.nan],\n \"E\": [5, np.nan, \"b\", 4, 3],\n \"F\": [4, 3, 2, 1, 0],\n \"G\": [4, 3, 2, 1, 4],\n }\n )\n feature_metadata_in2 = FeatureMetadata.from_df(df2)\n # Drop B because B and D are identical with same dtype, and D comes earlier\n expected_dropped_2 = [\"B\"]\n actual_dropped_2 = feature_generator._drop_duplicate_features(X=df2, feature_metadata_in=feature_metadata_in2)\n assert expected_dropped_2 == actual_dropped_2\n\n df[\"D\"] = df[\"D\"].astype(\"category\")\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop nothing because even though B and D have same values, they aren't the same dtype and thus aren't safe to drop.\n expected_dropped_3 = []\n actual_dropped_3 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_3 == actual_dropped_3\n\n df[\"B\"] = df[\"B\"].astype(\"category\")\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop D because B and D are identical with same dtype, and B comes earlier\n expected_dropped_4 = [\"D\"]\n actual_dropped_4 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_4 == actual_dropped_4\n\n df[\"E\"] = df[\"E\"].astype(\"category\")\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop D and E because B and D are identical with same dtype, and B comes earlier, and E is functionally identical and comes later\n expected_dropped_5 = [\"D\", \"E\"]\n actual_dropped_5 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_5 == actual_dropped_5\n\n # Make everything category type\n df[\"A\"] = df[\"A\"].astype(\"category\")\n df[\"C\"] = df[\"C\"].astype(\"category\")\n df[\"F\"] = df[\"F\"].astype(\"category\")\n df[\"G\"] = df[\"G\"].astype(\"category\")\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop all except A and G, because all others are equal to A functionally\n expected_dropped_6 = [\"B\", \"C\", \"D\", \"E\", \"F\"]\n actual_dropped_6 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_6 == actual_dropped_6\n\n # Make everything object type\n df = df.astype(\"object\")\n feature_metadata_in = FeatureMetadata.from_df(df)\n\n # Drop only D, because B and D are identical with same dtype, and B is earlier\n expected_dropped_7 = [\"D\"]\n actual_dropped_7 = feature_generator._drop_duplicate_features(X=df, feature_metadata_in=feature_metadata_in)\n assert expected_dropped_7 == actual_dropped_7\n"
},
{
"path": "features/tests/features/generators/test_dummy.py",
"content": "from pandas import DataFrame\n\nfrom autogluon.features.generators import DummyFeatureGenerator\n\n\ndef test_dummy_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = DummyFeatureGenerator()\n\n expected_feature_metadata_in_full = {}\n expected_feature_metadata_full = {(\"int\", ()): [\"__dummy__\"]}\n\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert output_data.equals(DataFrame(data=[0, 0, 0, 0, 0, 0, 0, 0, 0], columns=[\"__dummy__\"]))\n"
},
{
"path": "features/tests/features/generators/test_fillna.py",
"content": "import copy\n\nimport numpy as np\n\nfrom autogluon.features.generators import FillNaFeatureGenerator\n\n\n# TODO: Consider adding test of loading from csv\ndef test_fillna_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n expected_output_data = input_data.fillna(\n {\n \"int\": np.nan,\n \"float\": np.nan,\n \"obj\": \"\",\n \"cat\": np.nan,\n \"datetime\": np.nan,\n \"text\": \"\",\n \"datetime_as_object\": \"\",\n },\n downcast=False,\n )\n\n generator = FillNaFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = expected_feature_metadata_in_full\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data.equals(output_data)\n\n\n# Edge case when input is all integers but is of object type without NaN values. Without downcast=False, it will be converted to int type.\n# This test confirms that this unwanted conversion does not happen.\ndef test_fillna_object_edgecase_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_int_feature().to_frame(\"int_as_object\")\n input_data = input_data.astype(\"object\")\n edge_case_input = copy.deepcopy(input_data)\n input_data.loc[0] = np.nan\n expected_output_data = copy.deepcopy(input_data)\n expected_output_data.loc[0] = \"\"\n\n generator = FillNaFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"object\", ()): [\"int_as_object\"],\n }\n\n expected_feature_metadata_full = expected_feature_metadata_in_full\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n edge_case_output = generator.transform(edge_case_input)\n\n assert expected_output_data.equals(output_data)\n assert edge_case_input.equals(edge_case_output)\n"
},
{
"path": "features/tests/features/generators/test_identity.py",
"content": "from autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.types import R_FLOAT, R_INT\nfrom autogluon.features.generators import IdentityFeatureGenerator\n\n\ndef test_identity_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = IdentityFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = expected_feature_metadata_in_full\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert input_data.equals(output_data)\n\n\ndef test_identity_feature_generator_int_float(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT]))\n\n expected_feature_metadata_in_full = {(\"float\", ()): [\"float\"], (\"int\", ()): [\"int_bool\", \"int\"]}\n\n expected_feature_metadata_full = expected_feature_metadata_in_full\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert input_data[output_data.columns].equals(output_data)\n\n\ndef test_identity_feature_generator_int_float_with_banned_features(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = IdentityFeatureGenerator(\n infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT]),\n banned_feature_special_types=[\"my_banned_feature_type\"],\n )\n\n expected_feature_metadata_in_full = {(\"int\", ()): [\"int_bool\"], (\"int\", (\"my_valid_feature_type\",)): [\"int\"]}\n\n expected_feature_metadata_full = {(\"int\", ()): [\"int_bool\"], (\"int\", (\"my_valid_feature_type\",)): [\"int\"]}\n\n feature_metadata_in = FeatureMetadata.from_df(input_data)\n\n feature_metadata_in = feature_metadata_in.add_special_types(\n {\n \"float\": [\"my_banned_feature_type\"],\n \"int\": [\"my_valid_feature_type\"],\n }\n )\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n feature_metadata_in=feature_metadata_in,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert input_data[output_data.columns].equals(output_data)\n"
},
{
"path": "features/tests/features/generators/test_isnan.py",
"content": "import numpy as np\nimport pandas as pd\n\nfrom autogluon.features.generators import IsNanFeatureGenerator\n\n\ndef test_isnan_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n expected_output_data = pd.DataFrame(\n [\n [0, 0, 0, 0, 0, 0, 0, 0],\n [0, 0, 0, 0, 0, 1, 0, 1],\n [0, 0, 1, 0, 0, 1, 1, 0],\n [0, 0, 0, 0, 0, 0, 0, 0],\n [0, 0, 0, 0, 0, 0, 0, 0],\n [0, 0, 0, 0, 0, 0, 0, 0],\n [0, 0, 0, 0, 0, 0, 0, 0],\n [0, 0, 0, 0, 1, 1, 0, 0],\n [0, 0, 0, 0, 1, 1, 0, 0],\n ],\n columns=[\n \"__nan__.int_bool\",\n \"__nan__.int\",\n \"__nan__.float\",\n \"__nan__.obj\",\n \"__nan__.cat\",\n \"__nan__.datetime\",\n \"__nan__.text\",\n \"__nan__.datetime_as_object\",\n ],\n dtype=np.uint8,\n )\n\n generator = IsNanFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"int\", (\"bool\",)): [\n \"__nan__.int_bool\",\n \"__nan__.int\",\n \"__nan__.float\",\n \"__nan__.obj\",\n \"__nan__.cat\",\n \"__nan__.datetime\",\n \"__nan__.text\",\n \"__nan__.datetime_as_object\",\n ]\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data.equals(output_data)\n"
},
{
"path": "features/tests/features/generators/test_label_encoder.py",
"content": "import numpy as np\n\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\n\n\ndef test_label_encoder_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_standard()\n\n generator = LabelEncoderFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n }\n expected_feature_metadata_full = {(\"int\", ()): [\"cat\"]}\n\n expected_output_data_cat_val = [0, 1, 0, 3, 3, 3, 2, -1, -1]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Therefore\n assert output_data[\"cat\"].dtype == np.int8\n assert list(output_data[\"cat\"].values) == expected_output_data_cat_val\n"
},
{
"path": "features/tests/features/generators/test_one_hot_encoder.py",
"content": "import numpy as np\n\nfrom autogluon.features.generators import OneHotEncoderFeatureGenerator\n\n\ndef test_one_hot_encoder_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_standard()\n\n generator = OneHotEncoderFeatureGenerator()\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"int\", ()): [\"int\"],\n }\n expected_feature_metadata_full = {\n (\"int\", (\"bool\", \"sparse\")): [\n \"_ohe_0\",\n \"_ohe_1\",\n \"_ohe_2\",\n \"_ohe_3\",\n \"_ohe_4\",\n \"_ohe_5\",\n \"_ohe_6\",\n \"_ohe_7\",\n \"_ohe_8\",\n \"_ohe_9\",\n \"_ohe_10\",\n ]\n }\n\n expected_output_data_int_0_val = [0, 1, 0, 0, 0, 0, 0, 1, 0]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Therefore\n assert len(output_data.columns) == 11\n assert output_data[\"_ohe_1\"].dtype.subtype == np.uint8\n assert list(output_data[\"_ohe_1\"].values) == expected_output_data_int_0_val\n\n\ndef test_one_hot_encoder_feature_generator_advanced(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_standard()\n\n generator = OneHotEncoderFeatureGenerator(max_levels=3, sparse=False, dtype=np.uint16)\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"int\", ()): [\"int\"],\n }\n # TODO: improve readability of names when max_levels is specified\n expected_feature_metadata_full = {\n (\"int\", (\"bool\",)): [\"_ohe_0\", \"_ohe_1\", \"_ohe_2\", \"_ohe_3\", \"_ohe_4\", \"_ohe_5\", \"_ohe_6\", \"_ohe_7\"]\n }\n\n expected_output_data_int_0_val = [0, 1, 0, 0, 0, 0, 0, 1, 0]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Therefore\n assert len(output_data.columns) == 8\n assert output_data[\"_ohe_0\"].dtype == np.uint16\n assert list(output_data[\"_ohe_0\"].values) == expected_output_data_int_0_val\n"
},
{
"path": "features/tests/features/generators/test_oof_target_encoder.py",
"content": "import numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.features.generators import OOFTargetEncodingFeatureGenerator\n\ncan_transform_on_train = False\n\n\ndef _assert_all_between(series: pd.Series, low: float, high: float):\n assert ((series >= low) & (series <= high)).all()\n\n\ndef test_oof_target_encoding_regression(generator_helper, data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard()\n # Simple regression target\n y = pd.Series([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"regression\",\n keep_original=False,\n n_splits=3,\n alpha=10.0,\n random_state=0,\n )\n\n # When\n X_out = generator_helper.fit_transform_assert(\n input_data=X,\n generator=generator,\n y=y,\n can_transform_on_train=can_transform_on_train,\n )\n\n # Then\n assert generator.is_fit()\n # Categorical columns detected\n assert generator.cols_ == [\"obj\", \"cat\"]\n assert set(generator.passthrough_cols_) == {\"int\", \"float\", \"datetime\"}\n\n expected_encoded_cols = [\"obj__te\", \"cat__te\"]\n expected_columns = generator.passthrough_cols_ + expected_encoded_cols\n assert list(X_out.columns) == expected_columns\n\n # Original categorical features removed\n assert \"obj\" not in X_out.columns\n assert \"cat\" not in X_out.columns\n\n # Encoded columns are float and non-null\n for col in expected_encoded_cols:\n assert np.issubdtype(X_out[col].dtype, np.floating)\n assert not X_out[col].isna().any()\n\n # Encoded values between min/max of target\n y_min, y_max = float(y.min()), float(y.max())\n for col in expected_encoded_cols:\n _assert_all_between(X_out[col], y_min, y_max)\n\n # FeatureMetadata: in = original features, out = transformed features\n fm_in = generator.feature_metadata_in\n fm_out = generator.feature_metadata\n assert set(fm_in.get_features()) == set(X.columns)\n assert set(fm_out.get_features()) == set(X_out.columns)\n for col in expected_encoded_cols:\n assert fm_out.get_feature_type_raw(col) == \"float\"\n\n\ndef test_oof_target_encoding_binary(generator_helper, data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard()\n y = pd.Series([0, 1, 0, 1, 0, 1, 0, 1, 0])\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"binary\",\n keep_original=False,\n n_splits=3,\n alpha=5.0,\n random_state=1,\n )\n\n # When\n X_out = generator_helper.fit_transform_assert(\n input_data=X,\n generator=generator,\n y=y,\n can_transform_on_train=can_transform_on_train,\n )\n\n # Then\n assert generator.is_fit()\n assert generator.cols_ == [\"obj\", \"cat\"]\n assert set(generator.passthrough_cols_) == {\"int\", \"float\", \"datetime\"}\n\n expected_encoded_cols = [\"obj__te\", \"cat__te\"]\n expected_columns = generator.passthrough_cols_ + expected_encoded_cols\n assert list(X_out.columns) == expected_columns\n\n # classes_ must have exactly 2 classes\n assert hasattr(generator, \"classes_\")\n assert len(generator.classes_) == 2\n assert set(generator.classes_) == {0, 1}\n\n # Encoded values are probabilities in [0, 1]\n for col in expected_encoded_cols:\n _assert_all_between(X_out[col], 0.0, 1.0)\n\n\ndef test_oof_target_encoding_binary_raises_if_more_than_two_classes(data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard()\n # 3 unique classes but target_type=\"binary\"\n y = pd.Series([0, 1, 2, 0, 1, 2, 0, 1, 2])\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"binary\",\n keep_original=False,\n n_splits=3,\n alpha=5.0,\n random_state=1,\n )\n\n # Then\n with pytest.raises(AssertionError):\n generator.fit_transform(X, y=y)\n\n\ndef test_oof_target_encoding_multiclass(generator_helper, data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard()\n # 3-class labels\n y = pd.Series([0, 1, 2, 0, 1, 2, 0, 1, 2])\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"multiclass\",\n keep_original=False,\n n_splits=3,\n alpha=5.0,\n random_state=2,\n )\n\n # When\n X_out = generator_helper.fit_transform_assert(\n input_data=X,\n generator=generator,\n y=y,\n can_transform_on_train=can_transform_on_train,\n )\n\n # Then\n assert generator.is_fit()\n assert generator.cols_ == [\"obj\", \"cat\"]\n assert set(generator.passthrough_cols_) == {\"int\", \"float\", \"datetime\"}\n\n n_classes = len(np.unique(y))\n expected_encoded_cols = [\n \"obj__te_class0\",\n \"obj__te_class1\",\n \"obj__te_class2\",\n \"cat__te_class0\",\n \"cat__te_class1\",\n \"cat__te_class2\",\n ]\n assert len(expected_encoded_cols) == 2 * n_classes\n\n expected_columns = generator.passthrough_cols_ + expected_encoded_cols\n assert list(X_out.columns) == expected_columns\n\n # classes_ must match unique classes\n assert hasattr(generator, \"classes_\")\n assert set(generator.classes_) == set(np.unique(y))\n\n # Each encoded column must be in [0, 1]\n for col in expected_encoded_cols:\n _assert_all_between(X_out[col], 0.0, 1.0)\n\n\ndef test_oof_target_encoding_keep_original_true(generator_helper, data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard()\n y = pd.Series(range(len(X))) # regression\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"regression\",\n keep_original=True,\n n_splits=3,\n alpha=10.0,\n random_state=0,\n )\n\n # When\n X_out = generator_helper.fit_transform_assert(\n input_data=X,\n generator=generator,\n y=y,\n can_transform_on_train=can_transform_on_train,\n )\n\n # Then\n assert generator.is_fit()\n # All original columns must still be present\n for col in X.columns:\n assert col in X_out.columns\n\n expected_encoded_cols = [\"obj__te\", \"cat__te\"]\n for col in expected_encoded_cols:\n assert col in X_out.columns\n\n # Total columns = original + encoded\n assert len(X_out.columns) == len(X.columns) + len(expected_encoded_cols)\n\n # FeatureMetadata: original raw types preserved for original features\n fm_in = generator.feature_metadata_in\n fm_out = generator.feature_metadata\n for col in X.columns:\n assert fm_out.get_feature_type_raw(col) == fm_in.get_feature_type_raw(col)\n for col in expected_encoded_cols:\n assert fm_out.get_feature_type_raw(col) == \"float\"\n\n\ndef test_oof_target_encoding_no_categorical_columns(generator_helper):\n # Given: X has only numeric\n X = pd.DataFrame({\"int\": [1, 2, 3, 4, 5]})\n y = pd.Series([10.0, 20.0, 30.0, 40.0, 50.0])\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"regression\",\n keep_original=False,\n n_splits=2,\n alpha=1.0,\n random_state=0,\n )\n\n # When\n X_out = generator_helper.fit_transform_assert(\n input_data=X,\n generator=generator,\n y=y,\n can_transform_on_train=can_transform_on_train,\n )\n\n # Then\n assert generator.is_fit()\n assert generator.cols_ == []\n assert generator.encodings_ == {}\n # Output should be identical to input\n assert X_out.equals(X)\n\n\ndef test_oof_target_encoding_unseen_and_nan_categories():\n # Given: simple single-categorical, binary\n X_train = pd.DataFrame({\"feat\": [\"a\", \"b\", \"a\", \"c\"]})\n y = pd.Series([0, 1, 0, 1])\n\n X_test = pd.DataFrame(\n {\n \"feat\": [\"a\", \"d\", np.nan, \"b\"], # \"d\" unseen, np.nan missing\n }\n )\n\n generator = OOFTargetEncodingFeatureGenerator(\n target_type=\"binary\",\n keep_original=False,\n n_splits=2,\n alpha=5.0,\n random_state=0,\n )\n\n # When\n _ = generator.fit_transform(X_train, y=y)\n X_test_enc = generator.transform(X_test)\n\n # Then\n encoded_col = \"feat__te\"\n assert encoded_col in X_test_enc.columns\n\n # All encodings are probabilities\n _assert_all_between(X_test_enc[encoded_col], 0.0, 1.0)\n\n # Unseen category \"d\" and NaN should map to same global-mean encoding\n val_unseen = X_test_enc.loc[1, encoded_col]\n val_nan = X_test_enc.loc[2, encoded_col]\n assert val_unseen == pytest.approx(val_nan)\n\n\ndef test_oof_target_encoding_estimate_no_of_new_features(data_helper):\n # Given\n X = data_helper.generate_multi_feature_standard() # has \"obj\" (object) and \"cat\" (category)\n num_cat_cols = 2\n num_classes = 3\n\n gen_reg = OOFTargetEncodingFeatureGenerator(target_type=\"regression\")\n gen_bin = OOFTargetEncodingFeatureGenerator(target_type=\"binary\")\n gen_multi = OOFTargetEncodingFeatureGenerator(target_type=\"multiclass\")\n\n # When\n n_reg, cols_reg = gen_reg.estimate_no_of_new_features(X, num_classes=num_classes)\n n_bin, cols_bin = gen_bin.estimate_no_of_new_features(X, num_classes=num_classes)\n n_multi, cols_multi = gen_multi.estimate_no_of_new_features(X, num_classes=num_classes)\n\n # Then\n # Regression & binary: one encoded feature per categorical column\n assert n_reg == num_cat_cols\n assert n_bin == num_cat_cols\n assert cols_reg == [\"obj\", \"cat\"]\n assert cols_bin == [\"obj\", \"cat\"]\n\n # Multiclass: num_cat_cols * num_classes\n assert n_multi == num_cat_cols * num_classes\n assert cols_multi == [\"obj\", \"cat\"]\n"
},
{
"path": "features/tests/features/generators/test_pipeline.py",
"content": "import numpy as np\nfrom packaging.version import Version\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.common.features.types import R_CATEGORY, R_FLOAT, R_INT\nfrom autogluon.features.generators import (\n CategoryFeatureGenerator,\n DatetimeFeatureGenerator,\n IdentityFeatureGenerator,\n PipelineFeatureGenerator,\n TextNgramFeatureGenerator,\n TextSpecialFeatureGenerator,\n)\n\n\ndef test_pipeline_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n text_ngram_feature_generator = TextNgramFeatureGenerator(vectorizer=toy_vectorizer)\n text_ngram_feature_generator.max_memory_ratio = (\n None # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n )\n\n generator = PipelineFeatureGenerator(\n generators=[\n [\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT])),\n CategoryFeatureGenerator(),\n DatetimeFeatureGenerator(),\n TextSpecialFeatureGenerator(),\n text_ngram_feature_generator,\n ]\n ]\n )\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"obj\", \"cat\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int\"],\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ],\n (\"int\", (\"bool\",)): [\"int_bool\"],\n (\"int\", (\"datetime_as_int\",)): [\n \"datetime\",\n \"datetime.year\",\n \"datetime.month\",\n \"datetime.day\",\n \"datetime.dayofweek\",\n ],\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.sentence\",\n \"__nlp__.the\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ],\n }\n\n expected_output_data_feat_datetime = [\n 1533140820000000000,\n 1301322000000000000,\n 1301322000000000000,\n 1524238620000000000,\n 1524238620000000000,\n -5364662400000000000,\n 7289654340000000000,\n 1301322000000000000,\n 1301322000000000000,\n ]\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n expected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # int and float checks\n assert output_data[\"int\"].equals(input_data[\"int\"])\n assert output_data[\"float\"].equals(input_data[\"float\"])\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n # datetime checks. There are further checks in test_datetime.py\n assert expected_output_data_feat_datetime == list(output_data[\"datetime\"].values)\n\n # text_special checks\n assert (\n list(map(int, output_data[\"text.lower_ratio\"].values)) == expected_output_data_feat_lower_ratio_np_lt_2_0\n if Version(np.__version__) < Version(\"2.0.0\")\n else expected_output_data_feat_lower_ratio_np_ge_2_0\n )\n\n # text_ngram checks\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\ndef test_pipeline_feature_generator_dummy(generator_helper, data_helper):\n input_data = data_helper.generate_multi_feature_full()\n\n input_data_transform = data_helper.generate_empty()\n\n generator = PipelineFeatureGenerator(\n generators=[\n [\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[\"unknown_raw_type\"])),\n ]\n ]\n )\n\n expected_feature_metadata_in_full = {}\n expected_feature_metadata_full = {(\"int\", ()): [\"__dummy__\"]}\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert list(output_data[\"__dummy__\"].values) == [0, 0, 0, 0, 0, 0, 0, 0, 0]\n\n assert list(generator.transform(input_data_transform)[\"__dummy__\"].values) == [0, 0, 0, 0, 0, 0, 0, 0, 0]\n assert list(generator.transform(input_data_transform.head(5))[\"__dummy__\"].values) == [0, 0, 0, 0, 0]\n\n\ndef test_pipeline_feature_generator_removal_advanced(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=10, dtype=np.uint8)\n\n text_ngram_feature_generator = TextNgramFeatureGenerator(vectorizer=toy_vectorizer)\n text_ngram_feature_generator.max_memory_ratio = (\n None # Necessary in test to avoid CI non-deterministically pruning ngram counts.\n )\n\n generator = PipelineFeatureGenerator(\n generators=[\n [\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT])),\n CategoryFeatureGenerator(),\n DatetimeFeatureGenerator(),\n TextSpecialFeatureGenerator(),\n text_ngram_feature_generator,\n ],\n [IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_CATEGORY]))],\n ]\n )\n\n expected_feature_metadata_in_full = {(\"category\", ()): [\"cat\"], (\"object\", ()): [\"obj\"]}\n\n expected_feature_metadata_full = {(\"category\", ()): [\"obj\", \"cat\"]}\n\n expected_feature_metadata_in_unused_full = {\n \"datetime\": (\"datetime\", ()),\n \"datetime_as_object\": (\"object\", (\"datetime_as_object\",)),\n \"float\": (\"float\", ()),\n \"int\": (\"int\", ()),\n \"int_bool\": (\"int\", ()),\n \"text\": (\"object\", (\"text\",)),\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n feature_metadata_in_unused_full = generator._feature_metadata_in_unused.to_dict()\n\n # object and category checks\n assert list(output_data[\"obj\"].values) == [1, np.nan, 1, 2, 2, 2, np.nan, 0, 0]\n assert list(output_data[\"cat\"].values) == [0, np.nan, 0, 1, 1, 1, np.nan, np.nan, np.nan]\n\n assert feature_metadata_in_unused_full == expected_feature_metadata_in_unused_full\n"
},
{
"path": "features/tests/features/generators/test_rename.py",
"content": "from autogluon.features.generators import RenameFeatureGenerator\n\n\ndef test_rename(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = RenameFeatureGenerator(name_prefix=\"pre_\", name_suffix=\"_suf\")\n\n expected_feature_metadata_in_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"pre_cat_suf\"],\n (\"datetime\", ()): [\"pre_datetime_suf\"],\n (\"float\", ()): [\"pre_float_suf\"],\n (\"int\", ()): [\"pre_int_bool_suf\", \"pre_int_suf\"],\n (\"object\", ()): [\"pre_obj_suf\"],\n (\"object\", (\"datetime_as_object\",)): [\"pre_datetime_as_object_suf\"],\n (\"object\", (\"text\",)): [\"pre_text_suf\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n # Therefore\n output_data.columns = input_data.columns\n\n assert input_data.equals(output_data)\n"
},
{
"path": "features/tests/features/generators/test_text_ngram.py",
"content": "import numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.features.generators import TextNgramFeatureGenerator\n\nexpected_feature_metadata_in_full = {\n (\"object\", (\"text\",)): [\"text\"],\n}\nexpected_feature_metadata_full = {\n (\"int\", (\"text_ngram\",)): [\n \"__nlp__.breaks\",\n \"__nlp__.end\",\n \"__nlp__.end of\",\n \"__nlp__.end of the\",\n \"__nlp__.of\",\n \"__nlp__.of the\",\n \"__nlp__.sentence\",\n \"__nlp__.sentence breaks\",\n \"__nlp__.the\",\n \"__nlp__.the end\",\n \"__nlp__.the end of\",\n \"__nlp__.world\",\n \"__nlp__._total_\",\n ]\n}\n\nexpected_output_data_feat_total = [1, 3, 0, 0, 9, 1, 3, 9, 3]\n\n\ndef test_text_ngram_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n # max_memory_ratio=None in test to avoid CI reducing ngrams non-deterministically.\n generator = TextNgramFeatureGenerator(max_memory_ratio=None, vectorizer=toy_vectorizer)\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n\n\ndef test_text_ngram_feature_generator_categorical_nan(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n input_data.loc[2, \"text\"] = None\n input_data[\"text\"] = input_data[\"text\"].astype(\"category\")\n\n type_map_raw = {\n \"int\": \"int\",\n \"float\": \"float\",\n \"obj\": \"object\",\n \"cat\": \"category\",\n \"datetime\": \"datetime\",\n \"text\": \"category\",\n \"datetime_as_object\": \"object\",\n }\n type_map_special = {\n \"text\": [\"text\"],\n }\n feature_metadata = FeatureMetadata(\n type_map_raw,\n type_map_special=type_map_special,\n )\n\n toy_vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 3), max_features=1000, dtype=np.uint8)\n\n # max_memory_ratio=None in test to avoid CI reducing ngrams non-deterministically.\n generator = TextNgramFeatureGenerator(max_memory_ratio=None, vectorizer=toy_vectorizer)\n\n expected_feature_metadata_in_full = {\n (\"category\", (\"text\",)): [\"text\"],\n }\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n feature_metadata_in=feature_metadata,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data_feat_total == list(output_data[\"__nlp__._total_\"].values)\n"
},
{
"path": "features/tests/features/generators/test_text_special.py",
"content": "import numpy as np\nfrom packaging.version import Version\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.features.generators import TextSpecialFeatureGenerator\n\nexpected_feature_metadata_full_np_lt_2_0 = {\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.capital_ratio\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_ratio. \",\n ]\n}\n\nexpected_feature_metadata_full_np_ge_2_0 = {\n (\"int\", (\"binned\", \"text_special\")): [\n \"text.char_count\",\n \"text.word_count\",\n \"text.capital_ratio\",\n \"text.lower_ratio\",\n \"text.special_ratio\",\n \"text.symbol_count..\",\n \"text.symbol_ratio. \",\n ]\n}\n\n\ndef test_text_special_feature_generator(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n generator = TextSpecialFeatureGenerator(min_occur_ratio=0, min_occur_offset=0)\n\n expected_feature_metadata_in_full = {\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [3, 2, 0, 3, 3, 3, 3, 3, 1]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [2, 2, 0, 2, 2, 2, 2, 2, 1]\n\n if Version(np.__version__) < Version(\"2.0.0\"):\n expected_feature_metadata_full = expected_feature_metadata_full_np_lt_2_0\n expected_output_data_feat_lower_ratio = expected_output_data_feat_lower_ratio_np_lt_2_0\n else:\n expected_feature_metadata_full = expected_feature_metadata_full_np_ge_2_0\n expected_output_data_feat_lower_ratio = expected_output_data_feat_lower_ratio_np_ge_2_0\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data_feat_lower_ratio == list(map(int, output_data[\"text.lower_ratio\"].values))\n\n\ndef test_text_special_feature_generator_categorical_nan(generator_helper, data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n input_data.loc[2, \"text\"] = None\n input_data[\"text\"] = input_data[\"text\"].astype(\"category\")\n\n type_map_raw = {\n \"int\": \"int\",\n \"float\": \"float\",\n \"obj\": \"object\",\n \"cat\": \"category\",\n \"datetime\": \"datetime\",\n \"text\": \"category\",\n \"datetime_as_object\": \"object\",\n }\n type_map_special = {\n \"text\": [\"text\"],\n }\n feature_metadata = FeatureMetadata(\n type_map_raw,\n type_map_special=type_map_special,\n )\n\n generator = TextSpecialFeatureGenerator(min_occur_ratio=0, min_occur_offset=0)\n\n expected_feature_metadata_in_full = {\n (\"category\", (\"text\",)): [\"text\"],\n }\n\n expected_output_data_feat_lower_ratio_np_lt_2_0 = [2, 1, 2, 2, 2, 2, 2, 2, 0]\n expected_output_data_feat_lower_ratio_np_ge_2_0 = [1, 1, 1, 1, 1, 1, 1, 1, 0]\n\n if Version(np.__version__) < Version(\"2.0.0\"):\n expected_feature_metadata_full = expected_feature_metadata_full_np_lt_2_0\n expected_output_data_feat_lower_ratio = expected_output_data_feat_lower_ratio_np_lt_2_0\n else:\n expected_feature_metadata_full = expected_feature_metadata_full_np_ge_2_0\n expected_output_data_feat_lower_ratio = expected_output_data_feat_lower_ratio_np_ge_2_0\n\n # When\n output_data = generator_helper.fit_transform_assert(\n input_data=input_data,\n generator=generator,\n feature_metadata_in=feature_metadata,\n expected_feature_metadata_in_full=expected_feature_metadata_in_full,\n expected_feature_metadata_full=expected_feature_metadata_full,\n )\n\n assert expected_output_data_feat_lower_ratio == list(map(int, output_data[\"text.lower_ratio\"].values))\n"
},
{
"path": "features/tests/features/test_feature_metadata.py",
"content": "import itertools\n\nimport pytest\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\n\n\n# TODO: This test should technically be in autogluon.common, but currently exists in autogluon.features to avoid code duplication of the data_helper code\ndef test_feature_metadata(data_helper):\n # Given\n input_data = data_helper.generate_multi_feature_full()\n\n expected_feature_metadata_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\", \"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n expected_feature_metadata_get_features = [\n \"int_bool\",\n \"int\",\n \"float\",\n \"obj\",\n \"cat\",\n \"datetime\",\n \"text\",\n \"datetime_as_object\",\n ]\n\n expected_type_map_raw = {\n \"cat\": \"category\",\n \"datetime\": \"datetime\",\n \"datetime_as_object\": \"object\",\n \"float\": \"float\",\n \"int\": \"int\",\n \"int_bool\": \"int\",\n \"obj\": \"object\",\n \"text\": \"object\",\n }\n\n expected_type_group_map_special = {\"datetime_as_object\": [\"datetime_as_object\"], \"text\": [\"text\"]}\n\n expected_feature_metadata_renamed_full = {\n (\"category\", ()): [\"cat\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"obj\"],\n (\"int\", ()): [\"int_bool\", \"int_renamed\"],\n (\"object\", ()): [\"float\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text_renamed\"],\n }\n\n expected_feature_metadata_recombined_full_full = {\n (\"category\", ()): [\"cat\"],\n (\"custom_raw_type\", (\"custom_special_type\",)): [\"new_feature\"],\n (\"datetime\", ()): [\"datetime\"],\n (\"float\", ()): [\"float\"],\n (\"int\", ()): [\"int_bool\"],\n (\"int\", (\"custom_special_type\",)): [\"int\"],\n (\"object\", ()): [\"obj\"],\n (\"object\", (\"datetime_as_object\",)): [\"datetime_as_object\"],\n (\"object\", (\"text\",)): [\"text\"],\n }\n\n # When\n feature_metadata = FeatureMetadata.from_df(input_data)\n feature_metadata_renamed = feature_metadata.rename_features(\n rename_map={\"text\": \"text_renamed\", \"int\": \"int_renamed\", \"obj\": \"float\", \"float\": \"obj\"}\n )\n feature_metadata_remove = feature_metadata.remove_features(features=[\"text\", \"obj\", \"float\"])\n feature_metadata_keep = feature_metadata.keep_features(features=[\"text\", \"obj\", \"float\"])\n feature_metadata_custom = FeatureMetadata(\n type_map_raw={\"int\": \"int\", \"new_feature\": \"custom_raw_type\"},\n type_group_map_special={\"custom_special_type\": [\"int\", \"new_feature\"]},\n )\n feature_metadata_recombined = feature_metadata_keep.join_metadata(feature_metadata_remove)\n feature_metadata_recombined_alternate = FeatureMetadata.join_metadatas(\n metadata_list=[feature_metadata_keep, feature_metadata_remove]\n )\n feature_metadata_recombined_full = FeatureMetadata.join_metadatas(\n metadata_list=[feature_metadata_keep, feature_metadata_remove, feature_metadata_custom],\n shared_raw_features=\"error_if_diff\",\n )\n\n # Therefore\n with pytest.raises(AssertionError):\n # Error because special contains feature not in raw\n FeatureMetadata(\n type_map_raw={\"int\": \"int\"}, type_group_map_special={\"custom_special_type\": [\"int\", \"new_feature\"]}\n )\n with pytest.raises(AssertionError):\n # Error because renaming to another existing feature without also renaming that feature\n feature_metadata.rename_features(rename_map={\"text\": \"obj\"})\n with pytest.raises(KeyError):\n # Error if removing unknown feature\n feature_metadata_remove.remove_features(features=[\"text\"])\n with pytest.raises(KeyError):\n # Error if getting unknown feature type\n feature_metadata_remove.get_feature_type_raw(\"text\")\n with pytest.raises(KeyError):\n # Error if getting unknown feature type\n feature_metadata_remove.get_feature_types_special(\"text\")\n with pytest.raises(AssertionError):\n # Error because feature_metadata_remove and feature_metadata_custom share a raw feature\n FeatureMetadata.join_metadatas(\n metadata_list=[feature_metadata_keep, feature_metadata_remove, feature_metadata_custom]\n )\n\n assert feature_metadata.to_dict(inverse=True) == expected_feature_metadata_full\n assert feature_metadata.get_features() == expected_feature_metadata_get_features\n assert feature_metadata.type_map_raw == expected_type_map_raw\n assert dict(feature_metadata.type_group_map_special) == expected_type_group_map_special\n\n assert feature_metadata.get_feature_type_raw(\"text\") == \"object\"\n assert feature_metadata.get_feature_types_special(\"text\") == [\"text\"]\n assert feature_metadata.get_feature_type_raw(\"int\") == \"int\"\n assert feature_metadata.get_feature_types_special(\"int\") == []\n assert feature_metadata_recombined_full.get_feature_types_special(\"int\") == [\"custom_special_type\"]\n assert feature_metadata_recombined_full.get_feature_type_raw(\"new_feature\") == \"custom_raw_type\"\n\n assert feature_metadata_renamed.to_dict(inverse=True) == expected_feature_metadata_renamed_full\n assert feature_metadata_recombined.to_dict() == feature_metadata.to_dict()\n assert feature_metadata_recombined_alternate.to_dict() == feature_metadata.to_dict()\n assert feature_metadata_recombined_full.to_dict(inverse=True) == expected_feature_metadata_recombined_full_full\n\n\ndef test_feature_metadata_get_features():\n type_map_raw = dict(\n a=\"1\",\n b=\"2\",\n c=\"3\",\n d=\"1\",\n e=\"1\",\n f=\"4\",\n )\n\n type_group_map_special = {\"s1\": [\"a\", \"b\", \"d\"], \"s2\": [\"a\", \"e\"], \"s3\": [\"a\", \"b\"], \"s4\": [\"f\"]}\n\n expected_get_features = [\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"]\n\n feature_metadata = FeatureMetadata(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n\n assert feature_metadata.get_features() == expected_get_features\n\n assert feature_metadata.get_features(valid_raw_types=[\"1\"]) == [\"a\", \"d\", \"e\"]\n assert feature_metadata.get_features(valid_raw_types=[\"1\", \"3\"]) == [\"a\", \"c\", \"d\", \"e\"]\n assert feature_metadata.get_features(valid_raw_types=[\"UNKNOWN\"]) == []\n\n assert feature_metadata.get_features(valid_special_types=[\"s2\", \"s3\"]) == [\"a\", \"b\", \"c\", \"e\"]\n assert feature_metadata.get_features(valid_special_types=[\"s4\"]) == [\"c\", \"f\"]\n assert feature_metadata.get_features(valid_special_types=[]) == [\"c\"]\n assert feature_metadata.get_features(valid_special_types=[\"UNKNOWN\"]) == [\"c\"]\n\n assert feature_metadata.get_features(invalid_raw_types=[]) == expected_get_features\n assert feature_metadata.get_features(invalid_raw_types=[\"1\", \"3\"]) == [\"b\", \"f\"]\n assert feature_metadata.get_features(invalid_raw_types=[\"UNKNOWN\"]) == expected_get_features\n\n assert feature_metadata.get_features(invalid_special_types=[\"UNKNOWN\"]) == expected_get_features\n assert feature_metadata.get_features(invalid_special_types=[]) == expected_get_features\n assert feature_metadata.get_features(invalid_special_types=[\"s2\", \"s4\"]) == [\"b\", \"c\", \"d\"]\n\n assert feature_metadata.get_features(required_special_types=[\"s2\"]) == [\"a\", \"e\"]\n assert feature_metadata.get_features(required_special_types=[\"s2\", \"s3\"]) == [\"a\"]\n assert feature_metadata.get_features(required_special_types=[\"s2\", \"s4\"]) == []\n assert feature_metadata.get_features(required_special_types=[\"UNKNOWN\"]) == []\n\n assert feature_metadata.get_features(required_special_types=[\"s2\"], required_exact=True) == [\"e\"]\n assert feature_metadata.get_features(required_special_types=[\"s1\", \"s2\", \"s3\"], required_exact=True) == [\"a\"]\n\n assert feature_metadata.get_features(required_at_least_one_special=True) == [\"a\", \"b\", \"d\", \"e\", \"f\"]\n\n assert feature_metadata.get_features(\n required_raw_special_pairs=[\n (\"1\", [\"s2\"]),\n ]\n ) == [\"a\", \"e\"]\n assert feature_metadata.get_features(\n required_raw_special_pairs=[\n (\"1\", None),\n ]\n ) == [\"a\", \"d\", \"e\"]\n assert feature_metadata.get_features(\n required_raw_special_pairs=[\n (\"1\", [\"s2\"]),\n (None, [\"s4\"]),\n (\"3\", None),\n ]\n ) == [\"a\", \"c\", \"e\", \"f\"]\n assert feature_metadata.get_features(\n required_raw_special_pairs=[\n (\"1\", [\"s2\"]),\n (None, [\"s4\"]),\n (\"3\", None),\n ],\n required_exact=True,\n ) == [\"c\", \"e\", \"f\"]\n\n # Assert that valid_raw_types is the opposite of invalid_raw_types through all combinations\n raw_types_to_check = [\"1\", \"2\", \"3\", \"4\", \"UNKNOWN\"]\n for L in range(0, len(raw_types_to_check) + 1):\n for subset in itertools.combinations(raw_types_to_check, L):\n valid_raw_types = list(subset)\n invalid_raw_types = [raw_type for raw_type in raw_types_to_check if raw_type not in valid_raw_types]\n assert feature_metadata.get_features(valid_raw_types=valid_raw_types) == feature_metadata.get_features(\n invalid_raw_types=invalid_raw_types\n )\n\n # Combined arguments\n assert feature_metadata.get_features(invalid_special_types=[\"s2\", \"s3\"], required_special_types=[\"s1\"]) == [\"d\"]\n assert feature_metadata.get_features(valid_raw_types=[\"2\", \"3\"], valid_special_types=[\"s1\"]) == [\"b\", \"c\"]\n assert feature_metadata.get_features(\n valid_raw_types=[\"2\", \"3\"], valid_special_types=[\"s1\"], required_at_least_one_special=True\n ) == [\"b\"]\n assert feature_metadata.get_features(valid_raw_types=[\"2\", \"3\"], required_special_types=[\"s1\"]) == [\"b\"]\n assert (\n feature_metadata.get_features(valid_raw_types=[\"2\", \"3\"], required_special_types=[\"s1\"], required_exact=True)\n == []\n )\n assert feature_metadata.get_features(valid_raw_types=[\"2\", \"3\"], required_special_types=[\"s1\", \"s3\"]) == [\"b\"]\n assert feature_metadata.get_features(\n valid_raw_types=[\"2\", \"3\"], required_special_types=[\"s1\", \"s3\"], required_exact=True\n ) == [\"b\"]\n\n\ndef test_feature_metadata_equals():\n type_map_raw = dict(\n a=\"1\",\n b=\"2\",\n c=\"3\",\n d=\"1\",\n e=\"1\",\n f=\"4\",\n )\n type_group_map_special = {\"s1\": [\"a\", \"b\", \"d\"], \"s2\": [\"a\", \"e\"], \"s3\": [\"a\", \"b\"], \"s4\": [\"f\"]}\n feature_metadata = FeatureMetadata(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n feature_metadata_2 = FeatureMetadata(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n assert feature_metadata == feature_metadata_2\n feature_metadata_2 = feature_metadata_2.remove_features(features=[\"a\"])\n assert feature_metadata != feature_metadata_2\n\n feature_metadata_3 = FeatureMetadata(type_map_raw=dict(a=\"1\"))\n feature_metadata_2 = feature_metadata_2.join_metadata(feature_metadata_3)\n assert feature_metadata != feature_metadata_2\n\n feature_metadata_2 = feature_metadata_2.add_special_types(type_map_special={\"a\": [\"s1\"]})\n assert feature_metadata != feature_metadata_2\n\n feature_metadata_2 = feature_metadata_2.add_special_types(type_map_special={\"a\": [\"s2\", \"s3\"]})\n assert feature_metadata == feature_metadata_2\n"
},
{
"path": "features/tests/features/test_utils.py",
"content": "import numpy as np\nimport pytest\n\nfrom autogluon.features.utils import get_smallest_valid_dtype_int\n\n\n@pytest.mark.parametrize(\n \"min_val, max_val, expected_dtype\",\n [\n # Purely non-negative values â unsigned branch\n (0, 0, np.uint8),\n (0, 10, np.uint8),\n (0, 255, np.uint8), # max exactly at uint8 max\n (0, 256, np.uint16), # just above uint8, needs uint16\n (0, np.iinfo(np.uint16).max, np.uint16),\n (0, np.iinfo(np.uint16).max + 1, np.uint32),\n # Large but still in uint32\n (0, np.iinfo(np.uint32).max, np.uint32),\n ],\n)\ndef test_get_smallest_valid_dtype_int_unsigned(min_val, max_val, expected_dtype):\n result = get_smallest_valid_dtype_int(min_val=min_val, max_val=max_val)\n assert np.dtype(result) == np.dtype(expected_dtype)\n\n\n@pytest.mark.parametrize(\n \"min_val, max_val, expected_dtype\",\n [\n # Negative min â signed branch\n (-1, 0, np.int8),\n (-1, 1, np.int8),\n (-127, 127, np.int8),\n (-128, 127, np.int8), # full int8 range\n (-129, 127, np.int16), # just below int8.min\n (np.iinfo(np.int16).min, np.iinfo(np.int16).max, np.int16),\n (np.iinfo(np.int16).min - 1, 0, np.int32),\n # Mixed negative/positive within int32\n (-10_000, 10_000, np.int16),\n (-100_000, 100_000, np.int32),\n ],\n)\ndef test_get_smallest_valid_dtype_int_signed(min_val, max_val, expected_dtype):\n result = get_smallest_valid_dtype_int(min_val=min_val, max_val=max_val)\n assert np.dtype(result) == np.dtype(expected_dtype)\n\n\ndef test_min_zero_prefers_unsigned():\n \"\"\"When min_val == 0, the function should use unsigned dtypes.\"\"\"\n # Could fit in int8, but branch is unsigned so we expect uint8\n result = get_smallest_valid_dtype_int(min_val=0, max_val=np.iinfo(np.int8).max)\n assert np.dtype(result) == np.dtype(np.uint8)\n\n\n@pytest.mark.parametrize(\n \"min_val, max_val\",\n [\n # Out of range for all signed dtypes\n (np.iinfo(np.int64).min - 1, 0),\n (np.iinfo(np.int64).min - 10, np.iinfo(np.int64).max),\n ],\n)\ndef test_get_smallest_valid_dtype_int_raises_for_signed_overflow(min_val, max_val):\n with pytest.raises(ValueError):\n get_smallest_valid_dtype_int(min_val=min_val, max_val=max_val)\n\n\n@pytest.mark.parametrize(\n \"min_val, max_val\",\n [\n # Out of range for all unsigned dtypes\n (0, np.iinfo(np.uint64).max + 1),\n (5, np.iinfo(np.uint64).max + 12345),\n ],\n)\ndef test_get_smallest_valid_dtype_int_raises_for_unsigned_overflow(min_val, max_val):\n with pytest.raises(ValueError):\n get_smallest_valid_dtype_int(min_val=min_val, max_val=max_val)\n"
},
{
"path": "features/tests/test_check_style.py",
"content": "import json\nimport logging\nimport warnings\nfrom collections import Counter\nfrom subprocess import PIPE, Popen\n\n\ndef test_check_style():\n logging.getLogger().setLevel(logging.INFO)\n logging.info(\"Ruff style check\")\n\n ruff_proc = Popen(\n [\"ruff\", \"check\", \".\", \"--exit-zero\", \"--output-format\", \"json\"],\n stdout=PIPE,\n stderr=PIPE,\n )\n\n stdout, stderr = ruff_proc.communicate()\n stdout_str = stdout.decode()\n stderr_str = stderr.decode()\n\n if stderr_str.strip():\n print(\"\\n=== Ruff stderr ===\")\n print(stderr_str)\n\n try:\n diagnostics = json.loads(stdout_str) if stdout_str.strip() else []\n except json.JSONDecodeError as e:\n print(\"\\n=== Ruff stdout (failed to parse as JSON) ===\")\n print(stdout_str)\n raise AssertionError(f\"Failed to parse Ruff JSON output: {e}\") from e\n\n total_count = len(diagnostics)\n\n if total_count > 0:\n print(\"\\n=== Ruff warnings ===\")\n per_file_counts = Counter()\n\n for d in diagnostics:\n file_path = d.get(\"filename\", \"\")\n per_file_counts[file_path] += 1\n\n loc = d.get(\"location\", {}) or {}\n row = loc.get(\"row\", \"?\")\n col = loc.get(\"column\", \"?\")\n\n code = d.get(\"code\", \"\")\n message = d.get(\"message\", \"\")\n print(f\"{file_path}:{row}:{col}: {code} {message}\")\n\n print(\"\\n=== Ruff warnings per file ===\")\n for file_path, count in per_file_counts.most_common():\n print(f\"{file_path}: {count}\")\n\n warnings.warn(f\"{total_count} Ruff warnings remaining\")\n\n assert total_count < 10, \"Too many Ruff warnings found, improve code quality to pass test.\"\n"
},
{
"path": "full_install.bat",
"content": "\r\npython -m pip install -e common/[tests]\r\npython -m pip install -e features/\r\npython -m pip install -e core/[all,tests]\r\npython -m pip install -e tabular/[all,tests]\r\npython -m pip install -e multimodal/[tests]\r\npython -m pip install -e timeseries/[all,tests]\r\npython -m pip install -e eda/\r\npython -m pip install -e autogluon/\r\n"
},
{
"path": "full_install.sh",
"content": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Get the directory of the script and always change to it\nscript_dir=\"$(cd -- \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\ncd \"$script_dir\"\n\nPY=\"${PYTHON:-python}\"\n\n# Check if we're in Colab\nIN_COLAB=\"$(\"$PY\" - <<'PYCODE'\ntry:\n import google.colab # type: ignore\n print(\"true\")\nexcept ImportError:\n print(\"false\")\nPYCODE\n)\"\n\n# Set installation type based on environment\nif [ \"$IN_COLAB\" == \"true\" ]; then\n EDITABLE=\"false\"\n echo \"Colab detected - forcing non-editable install\"\nelse\n EDITABLE=\"true\"\nfi\n\n# Handle user override of editable setting\nwhile test $# -gt 0\ndo\n case \"$1\" in\n --non-editable) EDITABLE=\"false\";;\n *) echo \"Error: Unused argument: $1\" >&2\n exit 1;;\n esac\n shift\ndone\n\n# --- UV discovery (standalone or pip-installed) ---\n# Priority: 1) $UV env var 2) uv on PATH 3) python -m uv\nUV_BIN=\"${UV:-}\"\n\n# Helper: set UV_LAUNCH to how we should invoke uv.\n_detect_uv() {\n # 1) Explicit path via $UV\n if [[ -n \"$UV_BIN\" && -x \"$UV_BIN\" ]]; then\n UV_LAUNCH=(\"$UV_BIN\")\n return 0\n fi\n\n # 2) Standalone uv on PATH\n if command -v uv >/dev/null 2>&1; then\n UV_LAUNCH=(\"$(command -v uv)\")\n return 0\n fi\n\n # 3) Pip/conda-installed module importable by Python\n if \"$PY\" - <<'PYCODE' >/dev/null 2>&1\nimport importlib.util, sys\nsys.exit(0 if importlib.util.find_spec(\"uv\") else 1)\nPYCODE\n then\n UV_LAUNCH=(\"$PY\" -m uv)\n return 0\n fi\n\n return 1\n}\n\n# Try to detect uv first\nif ! _detect_uv; then\n # 4) If pip exists, install uv automatically (like the original script), then retry detection\n if command -v pip >/dev/null 2>&1 || \"$PY\" -m pip --version >/dev/null 2>&1; then\n echo \"[INFO] 'uv' not found. Installing via pip...\"\n \"$PY\" -m pip install \"uv\"\n # Retry detection after install\n if ! _detect_uv; then\n echo \"[ERROR] 'uv' still not found after pip install. Check your Python environment.\" >&2\n exit 1\n fi\n else\n echo \"[ERROR] 'uv' not found and pip is unavailable.\" >&2\n echo \"Install standalone uv: curl -LsSf https://astral.sh/uv/install.sh | sh\" >&2\n echo \"OR install via pip once pip is available: $PY -m pip install uv\" >&2\n echo \"OR set UV=/full/path/to/uv and retry.\" >&2\n exit 1\n fi\nfi\n\n# Convenience wrapper\nuvpip() { \"${UV_LAUNCH[@]}\" pip \"$@\"; }\n\n# Use uv to install packages\n# TODO: We should simplify this by having a single setup.py at project root, and let user call `pip install -e .`\nif [ \"$EDITABLE\" == \"true\" ]; then\n # Editable install (used outside Colab)\n uvpip install --refresh -e \"common/[tests]\"\n uvpip install -e \"features/\" -e \"core/[all,tests]\" -e \"tabular/[all,tests]\" -e \"multimodal/[tests]\" -e \"timeseries/[all,tests]\" -e \"eda/\" -e \"autogluon/\"\nelse\n # Non-editable install (forced in Colab)\n uvpip install --refresh \"common/[tests]\"\n uvpip install \"features/\" \"core/[all,tests]\" \"tabular/[all,tests]\" \"multimodal/[tests]\" \"timeseries/[all,tests]\" \"eda/\" \"autogluon/\"\nfi\n"
},
{
"path": "multimodal/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport importlib.util\nimport os\nimport platform\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec)\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag)\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = \"multimodal\"\ninstall_requires = [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"scipy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"scikit-learn\", # version range defined in `core/_setup_utils.py`\n \"Pillow\", # version range defined in `core/_setup_utils.py`\n \"tqdm\", # version range defined in `core/_setup_utils.py`\n \"boto3\", # version range defined in `core/_setup_utils.py`\n \"torch\", # version range defined in `core/_setup_utils.py`\n \"lightning\", # version range defined in `core/_setup_utils.py`\n \"transformers[sentencepiece]\", # version range defined in `core/_setup_utils.py`\n \"accelerate\", # version range defined in `core/_setup_utils.py`\n \"fsspec[http]<=2025.3\", # pin version to avoid conflicts with `datasets`\n \"requests>=2.30,<3\",\n \"jsonschema>=4.18,<4.24\",\n \"seqeval>=1.2.2,<1.3.0\",\n \"evaluate>=0.4.0,<0.5.0\",\n \"timm>=0.9.5,<1.0.7\",\n \"torchvision>=0.21.0,<0.25.0\",\n \"scikit-image>=0.19.1,<0.26.0\",\n \"text-unidecode>=1.3,<1.4\",\n \"torchmetrics>=1.2.0,<1.8\",\n \"omegaconf>=2.1.1,<2.4.0\",\n f\"autogluon.core[raytune]=={version}\",\n f\"autogluon.features=={version}\",\n f\"autogluon.common=={version}\",\n \"pytorch-metric-learning>=1.3.0,<2.9\",\n \"nlpaug>=1.1.10,<1.2.0\",\n \"nltk>=3.4.5,<3.10\", # Updated upper bound to address CVE-2024-39705\n \"openmim>=0.3.7,<0.4.0\",\n \"defusedxml>=0.7.1,<0.7.2\",\n \"jinja2>=3.0.3,<3.2\",\n \"tensorboard>=2.9,<3\",\n \"pytesseract>=0.3.9,<0.4\",\n \"nvidia-ml-py3>=7.352.0, <8.0\",\n \"pdf2image>=1.17.0,<1.19\",\n]\n\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\n\ntests_require = [\n \"ruff\",\n \"datasets>=2.16.0,<3.6.0\",\n \"tensorrt>=8.6.0,<10.9.1;platform_system=='Linux' and python_version<'3.11'\",\n # Sync ONNX requirements with tabular/setup.py\n \"onnx>=1.13.0,!=1.16.2,<1.21.0;platform_system=='Windows'\", # exclude 1.16.2 for issue https://github.com/onnx/onnx/issues/6267\n \"onnx>=1.13.0,<1.21.0;platform_system!='Windows'\",\n # For macOS, there isn't a onnxruntime-gpu package installed with skl2onnx.\n # Therefore, we install onnxruntime explicitly here just for macOS.\n \"onnxruntime>=1.17.0,<1.24.0\",\n \"onnxruntime-gpu>=1.17.0,<1.24.0; platform_system != 'Darwin' and platform_machine != 'aarch64'\",\n]\n\nextras_require = {\"tests\": tests_require}\n\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup_args[\"package_data\"][\"autogluon.multimodal\"] = [\n \"configs/data/*.yaml\",\n \"configs/model/*.yaml\",\n \"configs/optim/*.yaml\",\n \"configs/env/*.yaml\",\n \"configs/distiller/*.yaml\",\n \"configs/matcher/*.yaml\",\n ]\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/__init__.py",
"content": "from autogluon.common.utils.log_utils import _add_stream_handler\n\ntry:\n from .version import __version__\nexcept ImportError:\n pass\n\nfrom .predictor import MultiModalPredictor\n\n_add_stream_handler()\n"
},
{
"path": "multimodal/src/autogluon/multimodal/cli/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/cli/prepare_detection_dataset.py",
"content": "import argparse\nimport os\nimport shutil\n\nimport requests\n\n\ndef get_root_dir(output_dir=None, new_folder_name=None):\n if not output_dir:\n root_dir = \"./\"\n elif os.path.isdir(output_dir):\n if new_folder_name:\n root_dir = os.path.join(output_dir, new_folder_name)\n if not os.path.exists(root_dir):\n os.mkdir(root_dir)\n else:\n root_dir = output_dir\n else:\n raise ValueError(f\"{output_dir} is not a valid directory\")\n return root_dir\n\n\ndef get_fname_from_path_or_url(path_or_url):\n fname_start = path_or_url.rfind(\"/\") + 1\n fname = path_or_url[fname_start:]\n return fname\n\n\ndef download_one_url(url, root_dir, fname=None):\n if not fname:\n fname = get_fname_from_path_or_url(url)\n output_path = os.path.join(root_dir, fname)\n print(f\"Downloading {fname}...\")\n\n r = requests.get(url, timeout=(10, 1000))\n with open(output_path, \"wb\") as f:\n f.write(r.content)\n\n return output_path\n\n\ndef download_urls(urls, root_dir, fnames=[]):\n output_paths = []\n if isinstance(urls, str):\n urls = [urls]\n for i, url in enumerate(urls):\n output_path = download_one_url(url, root_dir, fname=fnames[i] if fnames else None)\n output_paths.append(output_path)\n return output_paths\n\n\ndef unpack(archived_file_paths, root_dir):\n for archived_file_path in archived_file_paths:\n fname = get_fname_from_path_or_url(archived_file_path)\n print(f\"extracting {fname}...\")\n shutil.unpack_archive(archived_file_path, root_dir)\n\n\ndef remove_archived_file_paths(archived_file_paths):\n for archived_file_path in archived_file_paths:\n fname = get_fname_from_path_or_url(archived_file_path)\n print(f\"removing {fname}...\")\n os.remove(archived_file_path)\n\n\ndef prepare_dataset(output_dir, new_folder_name, urls, fnames=[]):\n root_dir = get_root_dir(output_dir=output_dir, new_folder_name=new_folder_name)\n archived_file_paths = download_urls(urls, root_dir, fnames=fnames)\n unpack(archived_file_paths, root_dir)\n remove_archived_file_paths(archived_file_paths)\n\n\ndef prepare_coco17(output_dir):\n print(\"Preparing COCO17 dataset...\")\n urls = [\n \"http://images.cocodataset.org/zips/train2017.zip\",\n \"http://images.cocodataset.org/zips/val2017.zip\",\n \"http://images.cocodataset.org/zips/test2017.zip\",\n \"http://images.cocodataset.org/zips/unlabeled2017.zip\",\n \"http://images.cocodataset.org/annotations/annotations_trainval2017.zip\",\n \"http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip\",\n \"http://images.cocodataset.org/annotations/image_info_test2017.zip\",\n \"http://images.cocodataset.org/annotations/image_info_unlabeled2017.zip\",\n ]\n prepare_dataset(output_dir=output_dir, new_folder_name=\"coco17\", urls=urls)\n\n\ndef prepare_voc07(output_dir):\n print(\"Preparing VOC07 dataset...\")\n urls = [\n \"http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar\",\n \"http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar\",\n ]\n prepare_dataset(output_dir=output_dir, new_folder_name=None, urls=urls)\n\n\ndef prepare_voc12(output_dir):\n print(\"Preparing VOC12 dataset...\")\n urls = [\n \"http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar\",\n ]\n prepare_dataset(output_dir=output_dir, new_folder_name=None, urls=urls)\n\n\ndef prepare_voc0712(output_dir):\n prepare_voc07(output_dir)\n prepare_voc12(output_dir)\n\n\ndef prepare_watercolor(output_dir):\n print(\"Preparing Watercolor dataset...\")\n urls = [\n \"http://www.hal.t.u-tokyo.ac.jp/~inoue/projects/cross_domain_detection/datasets/watercolor.zip\",\n ]\n prepare_dataset(output_dir=output_dir, new_folder_name=None, urls=urls)\n\n\ndef prepare_pothole(output_dir):\n print(\"Preparing Pothole dataset...\")\n # urls = [\"https://www.kaggle.com/datasets/andrewmvd/pothole-detection/download?datasetVersionNumber=1\"]\n urls = [\"https://automl-mm-bench.s3.amazonaws.com/object_detection/dataset/pothole.zip\"]\n prepare_dataset(output_dir=output_dir, new_folder_name=None, urls=urls, fnames=[])\n\n\ndef main(dataset_name, output_dir):\n if dataset_name.lower() in [\"coco\", \"coco17\", \"coco2017\"]:\n prepare_coco17(output_dir=output_dir)\n elif dataset_name.lower() in [\"voc\", \"voc0712\"]:\n prepare_voc0712(output_dir=output_dir)\n elif dataset_name.lower() in [\"voc07\", \"voc2007\"]:\n prepare_voc07(output_dir=output_dir)\n elif dataset_name.lower() in [\"voc12\", \"voc2012\"]:\n prepare_voc12(output_dir=output_dir)\n elif dataset_name.lower() in [\"watercolor\"]:\n prepare_watercolor(output_dir=output_dir)\n elif dataset_name.lower() in [\"pothole\"]:\n prepare_pothole(output_dir=output_dir)\n else:\n raise ValueError(f\"This dataset name is not supported: {dataset_name}\")\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\"-d\", \"--dataset_name\", type=str)\n parser.add_argument(\"-o\", \"--output_dir\", default=\"./\", type=str)\n args = parser.parse_args()\n\n main(args.dataset_name, args.output_dir)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/cli/voc2coco.py",
"content": "\"\"\"\nConvert VOC format dataset to COCO.\nReference: https://github.com/yukkyo/voc2coco/blob/master/voc2coco.py\nWith changes:\n1. id stored as int by default\n2. provide only root_dir, and corresponding simplification\n3. split train/val/test\n4. Use defusedxml.ElementTree for security concern\n5. remove invalid images\n\nTo use:\n If you'd like to customize train/val/test ratio. Note test_ratio = 1 - train_ratio - val_ratio.\n python3 -m autogluon.multimodal.cli.voc2coco --root_dir --train_ratio --val_ratio \n If you'd like to use the dataset provided train/val/test splits:\n python3 -m autogluon.multimodal.cli.voc2coco --root_dir \n\"\"\"\n\nimport argparse\nimport json\nimport os\nimport random\nimport re\nimport subprocess\nfrom typing import Dict, List\n\nimport defusedxml.ElementTree as ET\nfrom tqdm import tqdm\n\nfrom autogluon.multimodal.utils.object_detection import dump_voc_classes, dump_voc_xml_files, process_voc_annotations\n\nDEFAULT_EXT = \".jpg\"\nMIN_AREA = 4\n\n\ndef get_label2id(labels_path: str) -> Dict[str, int]:\n \"\"\"id is 1 start\"\"\"\n with open(labels_path, \"r\") as f:\n labels_str = f.read().split()\n labels_ids = list(range(1, len(labels_str) + 1))\n return dict(zip(labels_str, labels_ids))\n\n\ndef get_annpaths(\n root_dir: str,\n annpaths_list_path: str = None,\n train_ratio=0.6,\n val_ratio=0.2,\n) -> Dict:\n if annpaths_list_path is not None:\n with open(annpaths_list_path, \"r\") as f:\n ann_paths = f.read().split()\n random.shuffle(ann_paths)\n N = len(ann_paths)\n num_train = int(N * train_ratio)\n num_val = int(N * val_ratio)\n return {\n \"usersplit_train\": ann_paths[:num_train],\n \"usersplit_val\": ann_paths[num_train : num_train + num_val],\n \"usersplit_test\": ann_paths[num_train + num_val :],\n }\n else:\n ann_ids_folder = os.path.join(root_dir, \"ImageSets\", \"Main\")\n ann_dir_path = os.path.join(root_dir, \"Annotations\")\n ann_paths = {}\n for ann_ids_filename in os.listdir(ann_ids_folder):\n ann_ids_path = os.path.join(ann_ids_folder, ann_ids_filename)\n if os.path.isfile(ann_ids_path) and ann_ids_filename[-4:] == \".txt\":\n ann_ids_name = ann_ids_filename[:-4]\n\n with open(ann_ids_path, \"r\") as f:\n rows = f.readlines()\n if not rows:\n print(f\"Skipping {ann_ids_path}: file is empty\")\n else:\n ann_ids = []\n for r in rows:\n data = r.strip().split()\n if len(data) == 1: # Each row is an annotation id\n ann_ids.append(data[0])\n elif (\n len(data) == 2\n ): # Each row contains an annotation id and a flag (0 if we do not use this annotation in this split, and 1 if we use it)\n ann_id, used = data\n if int(used) == 1:\n ann_ids.append(ann_id)\n else:\n print(\n f\"Skipping {ann_ids_path}: file format not recognized. Make sure your annotation follows \"\n f\"VOC format!\"\n )\n break\n\n ann_paths[ann_ids_name] = [os.path.join(ann_dir_path, aid + \".xml\") for aid in ann_ids]\n return ann_paths\n\n\ndef get_image_info(annotation_root, extract_num_from_imgid=True):\n path = annotation_root.findtext(\"path\")\n if path is None:\n filename = annotation_root.findtext(\"filename\")\n else:\n filename = os.path.basename(path)\n img_name = os.path.basename(filename)\n if not img_name[-4:] in [\".jpg\", \".png\"]:\n img_name = img_name + DEFAULT_EXT\n img_id = os.path.splitext(img_name)[0]\n if extract_num_from_imgid and isinstance(img_id, str):\n img_id = int(\"\".join(re.findall(r\"\\d+\", img_id)))\n\n size = annotation_root.find(\"size\")\n width = int(size.findtext(\"width\"))\n height = int(size.findtext(\"height\"))\n\n image_info = {\"file_name\": os.path.join(\"JPEGImages\", img_name), \"height\": height, \"width\": width, \"id\": img_id}\n return image_info\n\n\ndef get_coco_annotation_from_obj(obj, label2id):\n label = obj.findtext(\"name\")\n assert label in label2id, f\"Error: {label} is not in label2id !\"\n category_id = label2id[label]\n bndbox = obj.find(\"bndbox\")\n xmin = int(float(bndbox.findtext(\"xmin\")))\n ymin = int(float(bndbox.findtext(\"ymin\")))\n xmax = int(float(bndbox.findtext(\"xmax\")))\n ymax = int(float(bndbox.findtext(\"ymax\")))\n if xmin >= xmax or ymin >= ymax:\n return {}\n o_width = xmax - xmin\n o_height = ymax - ymin\n area = o_width * o_height\n if area <= MIN_AREA:\n return {}\n ann = {\n \"area\": o_width * o_height,\n \"iscrowd\": 0,\n \"bbox\": [xmin, ymin, o_width, o_height],\n \"category_id\": category_id,\n \"ignore\": 0,\n \"segmentation\": [], # This script is not for segmentation\n }\n return ann\n\n\ndef convert_xmls_to_cocojson(\n annotation_paths: List[str],\n label2id: Dict[str, int],\n output_jsonpath: str,\n extract_num_from_imgid: bool = True,\n root_dir: str = \"./\",\n):\n output_json_dict = {\"images\": [], \"type\": \"instances\", \"annotations\": [], \"categories\": []}\n bnd_id = 1 # START_BOUNDING_BOX_ID, TODO input as args ?\n print(\"Start converting !\")\n for a_path in tqdm(annotation_paths):\n # Read annotation xml\n try:\n ann_tree = ET.parse(a_path)\n except:\n ann_tree = ET.parse(os.path.join(root_dir, a_path))\n ann_root = ann_tree.getroot()\n\n img_info = get_image_info(annotation_root=ann_root, extract_num_from_imgid=extract_num_from_imgid)\n img_id = img_info[\"id\"]\n\n valid_image = False # remove image without bounding box to speed up mAP calculation\n for obj in ann_root.findall(\"object\"):\n ann = get_coco_annotation_from_obj(obj=obj, label2id=label2id)\n if ann:\n ann.update({\"image_id\": img_id, \"id\": bnd_id})\n output_json_dict[\"annotations\"].append(ann)\n bnd_id = bnd_id + 1\n valid_image = True\n\n if valid_image:\n output_json_dict[\"images\"].append(img_info)\n\n for label, label_id in label2id.items():\n category_info = {\"supercategory\": \"none\", \"id\": label_id, \"name\": label}\n output_json_dict[\"categories\"].append(category_info)\n\n with open(output_jsonpath, \"w\") as f:\n output_json = json.dumps(output_json_dict)\n f.write(output_json)\n print(f\"The COCO format annotation is saved to {output_jsonpath}\")\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"This script support converting voc format xmls to coco format json\")\n parser.add_argument(\"--root_dir\", type=str, default=None, help=\"path to VOC format dataset root\")\n parser.add_argument(\"--train_ratio\", type=float, default=None, help=\"training set ratio\")\n parser.add_argument(\"--val_ratio\", type=float, default=None, help=\"validation set ratio\")\n parser.add_argument(\"--ext\", type=str, default=\".jpg\", help=\"default extension for image file\")\n parser.add_argument(\"--min_area\", type=int, default=4, help=\"min area for a valid bounding box\")\n parser.add_argument(\n \"--not_extract_num_from_imgid\", action=\"store_true\", help=\"Extract image number from the image filename\"\n )\n args = parser.parse_args()\n\n annpaths_list_path = None\n DEFAULT_EXT = args.ext\n MIN_AREA = args.min_area\n assert DEFAULT_EXT in [\".jpg\", \".png\"]\n\n if not args.root_dir:\n raise ValueError(\"Must specify the root of the VOC format dataset.\")\n if args.train_ratio is not None:\n assert args.train_ratio >= 0\n assert args.val_ratio >= 0\n assert args.train_ratio + args.val_ratio <= 1\n annpaths_list_path = os.path.join(args.root_dir, \"pathlist.txt\")\n ## generate pathlist.txt containing all xml file paths\n dump_voc_xml_files(\n voc_annotation_path=os.path.join(args.root_dir, \"Annotations\"),\n voc_annotation_xml_output_path=annpaths_list_path,\n )\n\n assert os.path.exists(annpaths_list_path), \"FatalError: pathlist.txt does not exist!\"\n\n labels_path = os.path.join(args.root_dir, \"labels.txt\")\n ## generate labels.txt containing all unique class names\n dump_voc_classes(\n voc_annotation_path=os.path.join(args.root_dir, \"Annotations\"), voc_class_names_output_path=labels_path\n )\n\n assert os.path.exists(labels_path), \"FatalError: labels.txt does not exist!\"\n\n output_path_fmt = os.path.join(args.root_dir, \"Annotations\", \"%s_cocoformat.json\")\n\n label2id = get_label2id(labels_path=labels_path)\n ann_paths = get_annpaths(\n root_dir=args.root_dir,\n annpaths_list_path=annpaths_list_path,\n train_ratio=args.train_ratio,\n val_ratio=args.val_ratio,\n )\n for mode in ann_paths.keys():\n convert_xmls_to_cocojson(\n annotation_paths=ann_paths[mode],\n label2id=label2id,\n output_jsonpath=output_path_fmt % mode,\n extract_num_from_imgid=(not args.not_extract_num_from_imgid),\n root_dir=args.root_dir,\n )\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/data/default.yaml",
"content": "data:\n image:\n missing_value_strategy: \"zero\" # How to deal with missing images. By default, we use a zero image to replace a missing image. We also support \"skip\", i.e., skipping a sample with missing images.\n text:\n normalize_text: False # Whether to normalize text\n categorical:\n minimum_cat_count: 100 # The minimum number of occurrences a category must have in the training data to avoid being considered a rare category.\n maximum_num_cat: 20 # The maximum amount of categories that can be considered non-rare.\n convert_to_text: False # Whether to convert the feature to text.\n convert_to_text_template: \"latex\" # The template used to convert categorical to text. Choices are: \"direct\", \"list\", \"text\", \"latex\".\n numerical:\n convert_to_text: False # Whether to convert the feature to text.\n scaler_with_mean: True # Whether to normalize with mean.\n scaler_with_std: True # Whether to normalize with std.\n document:\n missing_value_strategy: \"zero\" # How to deal with missing documents. By default, we use a zero document image to replace a missing document. We also support \"skip\", i.e., skipping a sample with missing documents.\n label:\n numerical_preprocessing: \"standardscaler\" # The mode of numerical label preprocessing for . Support \"standardscaler\" or \"minmaxscaler\" or None (means no transform).\n pos_label: # The name of binary classification's positive class. It's used in computing some metrics, e.g., roc_auc. If not provided, then use label_encoder.classes_[1],\n column_features_pooling_mode: \"concat\" # How to pool multi-column features into one feature vector. Currently only support \"concat\" or \"mean\" for few shot classification.\n mixup:\n turn_on: False # The total control of mixup.\n mixup_alpha: 0.8 # Mixup alpha.\n cutmix_alpha: 1.0 # Cutmix alpha.\n cutmix_minmax: # Cutmix min/max ratio, it will override cutmix alpha if set, a list/tuple with size two.\n prob: 1.0 # The probability of conducting mixup/cutmix if enabled.\n switch_prob: 0.5 # The probability of switching mixup to cutmix if both enable.\n mode: \"batch\" # Perform mixup/cutmix on \"batch\" or \"pair\" or \"elem\".\n turn_off_epoch: 5 # The epoch when the mixup will be turned off.\n label_smoothing: 0.1 # Label smoothing.\n modality_dropout: 0\n templates:\n turn_on: False\n num_templates: 30 # The number of templates to sample from uniformly.\n template_length: 2048 # Truncation of jinja template variables\n preset_templates: [\"super_glue\", \"rte\"] # Select templates from a dataset template collection in the form (Dataset, Subset). For full list see data/templates. \n custom_templates: # Specify your own template in jinja format as well as answer choices for the model to select from.\n # 1:\n # template: \"{{premise}} {{hypothesis}}. Yes, or no? ||| {{answer_choices[label]}}\"\n # answer_choices: \"Yes ||| No\"\n # 2:\n # template: \"{{premise}} \\n\\nQuestion: Does this imply that '{{hypothesis}}'? Yes, no, or maybe? ||| {{answer_choices[label]}}\"\n # answer_choices: \"Yes ||| Maybe ||| No\"\n # 3:\n # template: \"{{premise}} \\n\\nQuestion: Are we justified in saying that '{{hypothesis}}'? Yes, no, or maybe? ||| {{answer_choices[label]}}\"\n # answer_choices: \"Yes ||| Maybe ||| No\"\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/distiller/default.yaml",
"content": "distiller:\n soft_label_loss_type: \"\"\n softmax_regression_loss_type: \"mse\"\n output_feature_loss_type: \"mse\"\n temperature: 5.\n hard_label_weight: 0.1\n soft_label_weight: 1.\n # Softmax Regression Paper: https://www.adrianbulat.com/downloads/ICLR2021/knowledge_distillation_via_softmax_regression_representation_learning.pdf\n softmax_regression_weight: 0.\n output_feature_loss_weight: 0.01\n # RKD Paper: https://arxiv.org/abs/1904.05068\n rkd_distance_loss_weight: 0.\n rkd_angle_loss_weight: 0.\n matches:\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/matcher/default.yaml",
"content": "matcher:\n loss:\n type: \"contrastive_loss\"\n pos_margin: 0.8\n neg_margin: 0.2\n distance:\n type: \"cosine_similarity\"\n miner:\n type: \"pair_margin_miner\"\n pos_margin: 0.6\n neg_margin: 0.4\n matches:\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/model/default.yaml",
"content": "model:\n names:\n categorical_mlp:\n hidden_size: 64\n activation: \"leaky_relu\"\n num_layers: 1\n dropout: 0.1\n normalization: \"layer_norm\"\n data_types:\n - \"categorical\"\n\n numerical_mlp:\n hidden_size: 128\n activation: \"leaky_relu\"\n num_layers: 1\n dropout: 0.1\n normalization: \"layer_norm\"\n token_dim: 8\n embedding_arch:\n data_types:\n - \"numerical\"\n merge: \"concat\"\n\n hf_text:\n checkpoint_name: \"google/electra-base-discriminator\"\n gradient_checkpointing: False\n pooling_mode: 'cls' # The pooling mode, can be 'cls' or 'mean'\n data_types:\n - \"text\"\n tokenizer_name: \"hf_auto\"\n use_fast: True # Use a fast Rust-based tokenizer if it is supported for a given model. If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead.\n max_text_len: 512 # If None or <=0, then use the max length of pretrained models.\n insert_sep: True\n low_cpu_mem_usage: False\n text_segment_num: 2\n stochastic_chunk: False\n text_aug_detect_length: 10 # We perform text augmentation only if a text has more than text_detection_length words. It is used to differentiate text columns versus tabular columns that are treated as text.\n text_trivial_aug_maxscale: 0.1 # augmentation magnitude randomly drawn from [0, text_trivial_aug_maxscale]\n text_train_augment_types: # specify augmentation space manually, will randomly select one from the following and identity\n # - \"random_swap(0.05)\" # less than 0.1 based on eda paper\n # - \"random_delete(0.05)\" # less than 0.1 based on eda paper\n # - \"syn_replacement(0.05)\" # less than 0.1 based on eda paper\n # - \"insert_punc(0.05)\"\n\n ner_text:\n checkpoint_name: \"bert-base-cased\"\n max_text_len: 512\n gradient_checkpointing: False\n low_cpu_mem_usage: False\n data_types:\n - \"text_ner\"\n tokenizer_name: \"hf_auto\"\n insert_sep: False\n text_segment_num: 2\n stochastic_chunk: False\n special_tags: \n - X # CLS, SEP, and non-first tokens of a word will be labelled as X\n - O # Outside of a named entity\n\n document_transformer:\n checkpoint_name: \"microsoft/layoutlmv3-base\" # document foundation models\n gradient_checkpointing: False\n pooling_mode: 'cls' # The pooling mode, can be 'cls' or 'mean'\n data_types:\n - \"document\"\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"imagenet\"\n image_size: 224\n tokenizer_name: \"hf_auto\"\n max_text_len: 512 # If None or <=0, then use the max length of pretrained models.\n insert_sep: True\n low_cpu_mem_usage: False\n text_segment_num: 2\n stochastic_chunk: False\n text_aug_detect_length: 10 # We perform text augmentation only if a text has more than text_detection_length words. It is used to differentiate text columns versus tabular columns that are treated as text.\n text_trivial_aug_maxscale: 0.0 # augmentation magnitude randomly drawn from [0, text_trivial_aug_maxscale]\n\n t_few:\n checkpoint_name: \"t5-small\" #\"bigscience/T0_3B\"\n gradient_checkpointing: False\n data_types:\n - \"text\"\n tokenizer_name: \"hf_auto\"\n length_norm: 1.0 # Normalizes length to adjust for length bias in target template\n unlikely_loss: 1.0 # Adds loss term that lowers probability of incorrect outputs\n mc_loss: 1.0 # Adds multiple choice cross entropy loss\n max_text_len: 512 # If None or <=0, then use the max length of pretrained models.\n text_segment_num: 2\n insert_sep: True\n low_cpu_mem_usage: False\n stochastic_chunk: False\n text_aug_detect_length: 10 # We perform text augmentation only if a text has more than text_detection_length words. It is used to differentiate text columns versus tabular columns that are treated as text.\n text_trivial_aug_maxscale: 0.0 # augmentation magnititude randomly drawn from [0, text_trivial_aug_maxscale]\n text_train_augment_types:\n\n timm_image:\n checkpoint_name: \"swin_base_patch4_window7_224\"\n mix_choice: \"all_logits\"\n data_types:\n - \"image\"\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n - \"trivial_augment\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"imagenet\"\n image_size: null\n image_chan_num: 3\n use_learnable_image: False\n max_image_num_per_column: 1\n\n mmdet_image:\n checkpoint_name: \"yolov3_mobilenetv2_8xb24-320-300e_coco\"\n config_file: \"\"\n data_types:\n - \"image\"\n max_img_num_per_col: 1\n output_bbox_format: \"xyxy\" # now support xyxy or xywh, for bbox format details see https://keras.io/api/keras_cv/bounding_box/formats/\n frozen_layers: null\n coco_root: null\n\n mmocr_text_detection:\n checkpoint_name: \"TextSnake\"\n data_types:\n - \"image\"\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n - \"trivial_augment\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"imagenet\"\n image_size: 224\n max_img_num_per_col: 2\n\n mmocr_text_recognition:\n checkpoint_name: \"ABINet\"\n data_types:\n - \"image\"\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n - \"trivial_augment\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"imagenet\"\n image_size: 224\n max_img_num_per_col: 2\n\n clip:\n checkpoint_name: \"openai/clip-vit-base-patch32\"\n data_types:\n - \"image\"\n - \"text\"\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n - \"trivial_augment\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"clip\"\n image_size: 224\n image_chan_num: 3\n use_learnable_image: False\n max_image_num_per_column: 1\n tokenizer_name: \"clip\"\n max_text_len: 77 # The maximum possible length.\n insert_sep: False\n text_segment_num: 1\n stochastic_chunk: False\n text_aug_detect_length: 10 # We perform text augmentation only if a text has more than text_detection_length words. It is used to differentiate text columns versus tabular columns that are treated as text.\n text_trivial_aug_maxscale: 0.0 # scale randomly drawn from [0, text_trivial_aug_maxscale]\n text_train_augment_types: # specify augmentation space manually, will randomly select one from the following and identity\n # - \"random_swap(0.05)\" # less than 0.1 based on eda paper\n # - \"random_delete(0.05)\" # less than 0.1 based on eda paper\n # - \"syn_replacement(0.05)\" # less than 0.1 based on eda paper\n # - \"insert_punc(0.05)\"\n\n fusion_mlp:\n aux_loss_weight:\n adapt_in_features: \"max\"\n hidden_sizes:\n - 128\n activation: \"leaky_relu\"\n dropout: 0.1\n normalization: \"layer_norm\"\n data_types:\n\n fusion_ner:\n weight:\n adapt_in_features: \"max\"\n hidden_sizes:\n - 128\n activation: \"leaky_relu\"\n drop_rate: 0.1\n normalization: \"layer_norm\"\n data_types:\n\n fusion_transformer:\n aux_loss_weight:\n hidden_size: 192\n num_blocks: 3\n attention_num_heads: 8\n adapt_in_features: \"max\"\n attention_dropout: 0.2\n residual_dropout: 0.0\n ffn_dropout: 0.1\n ffn_hidden_size: 192\n normalization: \"layer_norm\"\n ffn_activation: \"geglu\"\n head_activation: \"relu\"\n data_types:\n additive_attention: False # Whether to use lightweight additive attention, can be True, False or \"auto\"\n share_qv_weights: False # Whether to share weight for query and value, can be True, False or \"auto\"\n\n ft_transformer:\n data_types:\n - \"categorical\"\n - \"numerical\"\n embedding_arch:\n - 'linear'\n token_dim: 192\n hidden_size: 192\n num_blocks: 3\n attention_num_heads: 8\n attention_dropout: 0.2\n residual_dropout: 0.0\n ffn_dropout: 0.1\n ffn_hidden_size: 192\n ffn_activation: \"geglu\"\n head_activation: \"relu\"\n normalization: \"layer_norm\"\n merge: \"concat\"\n requires_all_dtypes: False\n additive_attention: False # Whether to use lightweight additive attention, can be True, False or \"auto\"\n share_qv_weights: False # Whether to share weight for query and value, can be True, False or \"auto\"\n pooling_mode: \"cls\"\n checkpoint_name: null\n\n sam:\n checkpoint_name: \"facebook/sam-vit-huge\"\n image_norm: \"imagenet\"\n data_types:\n - \"semantic_segmentation_img\"\n train_transforms:\n - \"random_horizontal_flip\"\n val_transforms: []\n img_transforms:\n - \"resize_to_square\"\n gt_transforms:\n - \"resize_gt_to_square\"\n max_img_num_per_col: 1\n frozen_layers: [\"mask_decoder.iou_prediction_head\", \"prompt_encoder\"]\n num_mask_tokens: 1\n ignore_label: 255\n\n meta_transformer:\n data_types:\n - \"image\"\n - \"text\"\n - \"categorical\"\n - \"numerical\"\n checkpoint_path: null\n model_version: \"base\"\n requires_all_dtypes: False\n train_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n - \"trivial_augment\"\n val_transforms:\n - \"resize_shorter_side\"\n - \"center_crop\"\n image_norm: \"imagenet\"\n image_size: 224\n image_chan_num: 3\n use_learnable_image: False\n max_image_num_per_column: 1\n tokenizer_name: \"hf_auto\"\n max_text_len: 512 # If None or <=0, then use the max length of pretrained models.\n insert_sep: True\n text_segment_num: 2\n stochastic_chunk: False\n text_aug_detect_length: 10 # We perform text augmentation only if a text has more than text_detection_length words. It is used to differentiate text columns versus tabular columns that are treated as text.\n text_trivial_aug_maxscale: 0.1 # augmentation magnitude randomly drawn from [0, text_trivial_aug_maxscale]\n text_train_augment_types:\n merge: \"concat\"\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/optim/default.yaml",
"content": "optim:\n optim_type: \"adamw\"\n lr: 1.0e-4\n weight_decay: 0.001\n lr_choice: \"layerwise_decay\"\n lr_decay: 0.9\n lr_schedule: \"cosine_decay\"\n max_epochs: 20\n max_steps: -1\n warmup_steps: 0.1\n end_lr: 0\n lr_mult: 1 # multiply lr for downstream heads\n patience: 10\n val_check_interval: 0.5 # Check validation score a fraction of epoch if float and every n steps if integer with n < total step in epoch.\n check_val_every_n_epoch: 1 # Check validation score every n epochs.\n skip_final_val: False # Flag to skip the last validation\n gradient_clip_val: 1\n gradient_clip_algorithm: \"norm\"\n track_grad_norm: -1 # Whether to check gradient norm. We can set it to 2 to check for gradient norm.\n log_every_n_steps: 10\n label_smoothing: 0\n top_k: 3\n top_k_average_method:\n \"greedy_soup\" # We support averaging method described in https://arxiv.org/pdf/2203.05482.pdf.\n # Currently support \"uniform_soup\", \"greedy_soup\", and \"best\".\n peft: null # Can be 'bit_fit' (only finetune bias), 'norm_fit' (finetune the normalization terms + bias terms), lora (LoRA Adaptations only), lora_bias (LoRA Adaptation + bit_fit), lora_norm (LoRA Adaptation + norm_fit), or null\n lora:\n module_filter: null # Specify which module (if any) to adapt(e.g. \".*EncDecAttention|.*DenseReluDense\"). Default consider all modules in a model (i.e. empty filter).\n filter: # Specify which layer in a module to adapt. Default fine-tune only query and value attention weights, recommended in https://arxiv.org/abs/2106.09685\n - \"query\"\n - \"value\"\n - \"^q$\" # The filter can also be a regex. We will use re.match(filter, name) to determine if LoRA should be applied.\n - \"^v$\"\n - \"^k$\"\n - \"^o$\"\n r: 8\n alpha: 8\n conv_lora_expert_num: 8 # default setting for Conv-LoRA\n loss_func:\n \"auto\" # The replaced loss for regression. Can only support loss function in torch.nn.\n # example\n # \"BCEWithLogitsLoss\" or \"nn.BCEWithLogitsloss\"\n focal_loss:\n alpha: null\n gamma: 2.0\n reduction: \"mean\"\n mask2former_loss:\n loss_cross_entropy_weight: 10.0\n loss_mask_weight: 5.0\n loss_dice_weight: 5.0\n extra_trainable_params: []\n cross_modal_align: null\n cross_modal_align_weight: 0\n automatic_optimization: True\n lemda:\n turn_on: False\n arch_type: \"mlp_vae\"\n z_dim: 8\n num_layers: 6\n kld_weight: 0.1\n mse_weight: 0.1\n adv_weight: 1.0e-4\n consist_weight: 0.01\n consist_threshold: 0.5\n lr: 1.0e-4\n optim_type: \"adamw\"\n weight_decay: 1.0e-5\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/coco_detection.py",
"content": "# dataset settings\ndataset_type = \"CocoDataset\"\ndata_root = \"data/coco/\"\n\n# Example to use different file client\n# Method 1: simply set the data root and let the file I/O module\n# automatically infer from prefix (not support LMDB and Memcache yet)\n\n# data_root = 's3://openmmlab/datasets/detection/coco/'\n\n# Method 2: Use `backend_args`, `file_client_args` in versions before 3.0.0rc6\n# backend_args = dict(\n# backend='petrel',\n# path_mapping=dict({\n# './data/': 's3://openmmlab/datasets/detection/',\n# 'data/': 's3://openmmlab/datasets/detection/'\n# }))\nbackend_args = None\n\ntrain_pipeline = [\n dict(type=\"LoadImageFromFile\", backend_args=backend_args),\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"Resize\", scale=(1333, 800), keep_ratio=True),\n dict(type=\"RandomFlip\", prob=0.5),\n dict(type=\"PackDetInputs\"),\n]\ntest_pipeline = [\n dict(type=\"LoadImageFromFile\", backend_args=backend_args),\n dict(type=\"Resize\", scale=(1333, 800), keep_ratio=True),\n # If you don't have a gt annotation, delete the pipeline\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"PackDetInputs\", meta_keys=(\"img_id\", \"img_path\", \"ori_shape\", \"img_shape\", \"scale_factor\")),\n]\ntrain_dataloader = dict(\n batch_size=2,\n num_workers=2,\n persistent_workers=True,\n sampler=dict(type=\"DefaultSampler\", shuffle=True),\n batch_sampler=dict(type=\"AspectRatioBatchSampler\"),\n dataset=dict(\n type=dataset_type,\n data_root=data_root,\n ann_file=\"annotations/instances_train2017.json\",\n data_prefix=dict(img=\"train2017/\"),\n filter_cfg=dict(filter_empty_gt=True, min_size=32),\n pipeline=train_pipeline,\n backend_args=backend_args,\n ),\n)\nval_dataloader = dict(\n batch_size=1,\n num_workers=2,\n persistent_workers=True,\n drop_last=False,\n sampler=dict(type=\"DefaultSampler\", shuffle=False),\n dataset=dict(\n type=dataset_type,\n data_root=data_root,\n ann_file=\"annotations/instances_val2017.json\",\n data_prefix=dict(img=\"val2017/\"),\n test_mode=True,\n pipeline=test_pipeline,\n backend_args=backend_args,\n ),\n)\ntest_dataloader = val_dataloader\n\nval_evaluator = dict(\n type=\"CocoMetric\",\n ann_file=data_root + \"annotations/instances_val2017.json\",\n metric=\"bbox\",\n format_only=False,\n backend_args=backend_args,\n)\ntest_evaluator = val_evaluator\n\n# inference on test dataset and\n# format the output results for submission.\n# test_dataloader = dict(\n# batch_size=1,\n# num_workers=2,\n# persistent_workers=True,\n# drop_last=False,\n# sampler=dict(type='DefaultSampler', shuffle=False),\n# dataset=dict(\n# type=dataset_type,\n# data_root=data_root,\n# ann_file=data_root + 'annotations/image_info_test-dev2017.json',\n# data_prefix=dict(img='test2017/'),\n# test_mode=True,\n# pipeline=test_pipeline))\n# test_evaluator = dict(\n# type='CocoMetric',\n# metric='bbox',\n# format_only=True,\n# ann_file=data_root + 'annotations/image_info_test-dev2017.json',\n# outfile_prefix='./work_dirs/coco_detection/test')\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/default_runtime.py",
"content": "checkpoint_config = dict(interval=1)\n# yapf:disable\nlog_config = dict(\n interval=50,\n hooks=[\n dict(type=\"TextLoggerHook\"),\n # dict(type='TensorboardLoggerHook')\n ],\n)\n# yapf:enable\ncustom_hooks = [dict(type=\"NumClassCheckHook\")]\n\ndist_params = dict(backend=\"nccl\")\nlog_level = \"INFO\"\nload_from = None\nresume_from = None\nworkflow = [(\"train\", 1)]\n\n# disable opencv multithreading to avoid system being overloaded\nopencv_num_threads = 0\n# set multi-process start method as `fork` to speed up the training\nmp_start_method = \"fork\"\n\n# Default setting for scaling LR automatically\n# - `enable` means enable scaling LR automatically\n# or not by default.\n# - `base_batch_size` = (8 GPUs) x (2 samples per GPU).\nauto_scale_lr = dict(enable=False, base_batch_size=16)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/dino/dino-4scale_r50_8xb2-12e_coco.py",
"content": "_base_ = [\"../coco_detection.py\", \"../default_runtime.py\"]\nmodel = dict(\n type=\"DINO\",\n num_queries=900, # num_matching_queries\n with_box_refine=True,\n as_two_stage=True,\n data_preprocessor=dict(\n type=\"DetDataPreprocessor\",\n mean=[123.675, 116.28, 103.53],\n std=[58.395, 57.12, 57.375],\n bgr_to_rgb=True,\n pad_size_divisor=1,\n ),\n backbone=dict(\n type=\"ResNet\",\n depth=50,\n num_stages=4,\n out_indices=(1, 2, 3),\n frozen_stages=1,\n norm_cfg=dict(type=\"BN\", requires_grad=False),\n norm_eval=True,\n style=\"pytorch\",\n init_cfg=dict(type=\"Pretrained\", checkpoint=\"torchvision://resnet50\"),\n ),\n neck=dict(\n type=\"ChannelMapper\",\n in_channels=[512, 1024, 2048],\n kernel_size=1,\n out_channels=256,\n act_cfg=None,\n norm_cfg=dict(type=\"GN\", num_groups=32),\n num_outs=4,\n ),\n encoder=dict(\n num_layers=6,\n layer_cfg=dict(\n self_attn_cfg=dict(embed_dims=256, num_levels=4, dropout=0.0), # 0.1 for DeformDETR\n ffn_cfg=dict(embed_dims=256, feedforward_channels=2048, ffn_drop=0.0), # 1024 for DeformDETR\n ),\n ), # 0.1 for DeformDETR\n decoder=dict(\n num_layers=6,\n return_intermediate=True,\n layer_cfg=dict(\n self_attn_cfg=dict(embed_dims=256, num_heads=8, dropout=0.0), # 0.1 for DeformDETR\n cross_attn_cfg=dict(embed_dims=256, num_levels=4, dropout=0.0), # 0.1 for DeformDETR\n ffn_cfg=dict(embed_dims=256, feedforward_channels=2048, ffn_drop=0.0), # 1024 for DeformDETR\n ), # 0.1 for DeformDETR\n post_norm_cfg=None,\n ),\n positional_encoding=dict(\n num_feats=128,\n normalize=True,\n offset=0.0,\n temperature=20, # -0.5 for DeformDETR\n ), # 10000 for DeformDETR\n bbox_head=dict(\n type=\"DINOHead\",\n num_classes=80,\n sync_cls_avg_factor=True,\n loss_cls=dict(type=\"FocalLoss\", use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), # 2.0 in DeformDETR\n loss_bbox=dict(type=\"L1Loss\", loss_weight=5.0),\n loss_iou=dict(type=\"GIoULoss\", loss_weight=2.0),\n ),\n dn_cfg=dict( # TODO: Move to model.train_cfg ?\n label_noise_scale=0.5,\n box_noise_scale=1.0, # 0.4 for DN-DETR\n group_cfg=dict(dynamic=True, num_groups=None, num_dn_queries=100),\n ), # TODO: half num_dn_queries\n # training and testing settings\n train_cfg=dict(\n assigner=dict(\n type=\"HungarianAssigner\",\n match_costs=[\n dict(type=\"FocalLossCost\", weight=2.0),\n dict(type=\"BBoxL1Cost\", weight=5.0, box_format=\"xywh\"),\n dict(type=\"IoUCost\", iou_mode=\"giou\", weight=2.0),\n ],\n )\n ),\n test_cfg=dict(max_per_img=300),\n) # 100 for DeformDETR\n\n# train_pipeline, NOTE the img_scale and the Pad's size_divisor is different\n# from the default setting in mmdet.\ntrain_pipeline = [\n dict(type=\"LoadImageFromFile\", backend_args={{_base_.backend_args}}), # nosec\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"RandomFlip\", prob=0.5),\n dict(\n type=\"RandomChoice\",\n transforms=[\n [\n dict(\n type=\"RandomChoiceResize\",\n scales=[\n (480, 1333),\n (512, 1333),\n (544, 1333),\n (576, 1333),\n (608, 1333),\n (640, 1333),\n (672, 1333),\n (704, 1333),\n (736, 1333),\n (768, 1333),\n (800, 1333),\n ],\n keep_ratio=True,\n )\n ],\n [\n dict(\n type=\"RandomChoiceResize\",\n # The radio of all image in train dataset < 7\n # follow the original implement\n scales=[(400, 4200), (500, 4200), (600, 4200)],\n keep_ratio=True,\n ),\n dict(type=\"RandomCrop\", crop_type=\"absolute_range\", crop_size=(384, 600), allow_negative_crop=True),\n dict(\n type=\"RandomChoiceResize\",\n scales=[\n (480, 1333),\n (512, 1333),\n (544, 1333),\n (576, 1333),\n (608, 1333),\n (640, 1333),\n (672, 1333),\n (704, 1333),\n (736, 1333),\n (768, 1333),\n (800, 1333),\n ],\n keep_ratio=True,\n ),\n ],\n ],\n ),\n dict(type=\"PackDetInputs\"),\n]\ntrain_dataloader = dict(dataset=dict(filter_cfg=dict(filter_empty_gt=False), pipeline=train_pipeline))\n\n# optimizer\noptim_wrapper = dict(\n type=\"OptimWrapper\",\n optimizer=dict(type=\"AdamW\", lr=0.0001, weight_decay=0.0001), # 0.0002 for DeformDETR\n clip_grad=dict(max_norm=0.1, norm_type=2),\n paramwise_cfg=dict(custom_keys={\"backbone\": dict(lr_mult=0.1)}),\n) # custom_keys contains sampling_offsets and reference_points in DeformDETR # noqa\n\n# learning policy\nmax_epochs = 12\ntrain_cfg = dict(type=\"EpochBasedTrainLoop\", max_epochs=max_epochs, val_interval=1)\n\nval_cfg = dict(type=\"ValLoop\")\ntest_cfg = dict(type=\"TestLoop\")\n\nparam_scheduler = [dict(type=\"MultiStepLR\", begin=0, end=max_epochs, by_epoch=True, milestones=[11], gamma=0.1)]\n\n# NOTE: `auto_scale_lr` is for automatically scaling LR,\n# USER SHOULD NOT CHANGE ITS VALUES.\n# base_batch_size = (8 GPUs) x (2 samples per GPU)\nauto_scale_lr = dict(base_batch_size=16)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/dino/dino-5scale_swin-l_8xb2-12e_coco.py",
"content": "_base_ = \"./dino-4scale_r50_8xb2-12e_coco.py\"\n\npretrained = (\n \"https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window12_384_22k.pth\" # noqa\n)\nnum_levels = 5\nmodel = dict(\n num_feature_levels=num_levels,\n backbone=dict(\n _delete_=True,\n type=\"SwinTransformer\",\n pretrain_img_size=384,\n embed_dims=192,\n depths=[2, 2, 18, 2],\n num_heads=[6, 12, 24, 48],\n window_size=12,\n mlp_ratio=4,\n qkv_bias=True,\n qk_scale=None,\n drop_rate=0.0,\n attn_drop_rate=0.0,\n drop_path_rate=0.2,\n patch_norm=True,\n out_indices=(0, 1, 2, 3),\n # Please only add indices that would be used\n # in FPN, otherwise some parameter will not be used\n with_cp=True,\n convert_weights=True,\n init_cfg=dict(type=\"Pretrained\", checkpoint=pretrained),\n ),\n neck=dict(in_channels=[192, 384, 768, 1536], num_outs=num_levels),\n encoder=dict(layer_cfg=dict(self_attn_cfg=dict(num_levels=num_levels))),\n decoder=dict(layer_cfg=dict(cross_attn_cfg=dict(num_levels=num_levels))),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/dino/dino-5scale_swin-l_8xb2-36e_coco.py",
"content": "_base_ = \"./dino-5scale_swin-l_8xb2-12e_coco.py\"\nmax_epochs = 36\ntrain_cfg = dict(type=\"EpochBasedTrainLoop\", max_epochs=max_epochs, val_interval=1)\nparam_scheduler = [dict(type=\"MultiStepLR\", begin=0, end=max_epochs, by_epoch=True, milestones=[27, 33], gamma=0.1)]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/dino/dino_swinl_tta.py",
"content": "_base_ = [\"./dino-5scale_swin-l_8xb2-36e_coco.py\", \"./dino_tta.py\"]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/dino/dino_tta.py",
"content": "tta_model = dict(type=\"DetTTAModel\", tta_cfg=dict(nms=dict(type=\"nms\", iou_threshold=0.65), max_per_img=100))\n\nimg_scales = [\n (480, 1333),\n (512, 1333),\n (544, 1333),\n (576, 1333),\n (608, 1333),\n (640, 1333),\n (672, 1333),\n (704, 1333),\n (736, 1333),\n (768, 1333),\n (800, 1333),\n]\ntta_pipeline = [\n dict(type=\"LoadImageFromFile\", backend_args=None),\n dict(\n type=\"TestTimeAug\",\n transforms=[\n [dict(type=\"Resize\", scale=s, keep_ratio=True) for s in img_scales],\n [\n # ``RandomFlip`` must be placed before ``Pad``, otherwise\n # bounding box coordinates after flipping cannot be\n # recovered correctly.\n dict(type=\"RandomFlip\", prob=1.0),\n dict(type=\"RandomFlip\", prob=0.0),\n ],\n [\n dict(type=\"Pad\", pad_to_square=True, pad_val=dict(img=(114.0, 114.0, 114.0))),\n ],\n [dict(type=\"LoadAnnotations\", with_bbox=True)],\n [\n dict(\n type=\"PackDetInputs\",\n meta_keys=(\n \"img_id\",\n \"img_path\",\n \"ori_shape\",\n \"img_shape\",\n \"scale_factor\",\n \"flip\",\n \"flip_direction\",\n ),\n )\n ],\n ],\n ),\n]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/faster_rcnn/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/faster_rcnn/faster_rcnn_r50_fpn.py",
"content": "# model settings\nmodel = dict(\n type=\"FasterRCNN\",\n backbone=dict(\n type=\"ResNet\",\n depth=50,\n num_stages=4,\n out_indices=(0, 1, 2, 3),\n frozen_stages=1,\n norm_cfg=dict(type=\"BN\", requires_grad=True),\n norm_eval=True,\n style=\"pytorch\",\n init_cfg=dict(type=\"Pretrained\", checkpoint=\"torchvision://resnet50\"),\n ),\n neck=dict(type=\"FPN\", in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5),\n rpn_head=dict(\n type=\"RPNHead\",\n in_channels=256,\n feat_channels=256,\n anchor_generator=dict(type=\"AnchorGenerator\", scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]),\n bbox_coder=dict(\n type=\"DeltaXYWHBBoxCoder\", target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[1.0, 1.0, 1.0, 1.0]\n ),\n loss_cls=dict(type=\"CrossEntropyLoss\", use_sigmoid=True, loss_weight=1.0),\n loss_bbox=dict(type=\"L1Loss\", loss_weight=1.0),\n ),\n roi_head=dict(\n type=\"StandardRoIHead\",\n bbox_roi_extractor=dict(\n type=\"SingleRoIExtractor\",\n roi_layer=dict(type=\"RoIAlign\", output_size=7, sampling_ratio=0),\n out_channels=256,\n featmap_strides=[4, 8, 16, 32],\n ),\n bbox_head=dict(\n type=\"Shared2FCBBoxHead\",\n in_channels=256,\n fc_out_channels=1024,\n roi_feat_size=7,\n num_classes=80,\n bbox_coder=dict(\n type=\"DeltaXYWHBBoxCoder\", target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]\n ),\n reg_class_agnostic=False,\n loss_cls=dict(type=\"CrossEntropyLoss\", use_sigmoid=False, loss_weight=1.0),\n loss_bbox=dict(type=\"L1Loss\", loss_weight=1.0),\n ),\n ),\n # model training and testing settings\n train_cfg=dict(\n rpn=dict(\n assigner=dict(\n type=\"MaxIoUAssigner\",\n pos_iou_thr=0.7,\n neg_iou_thr=0.3,\n min_pos_iou=0.3,\n match_low_quality=True,\n ignore_iof_thr=-1,\n ),\n sampler=dict(type=\"RandomSampler\", num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False),\n allowed_border=-1,\n pos_weight=-1,\n debug=False,\n ),\n rpn_proposal=dict(nms_pre=2000, max_per_img=1000, nms=dict(type=\"nms\", iou_threshold=0.7), min_bbox_size=0),\n rcnn=dict(\n assigner=dict(\n type=\"MaxIoUAssigner\",\n pos_iou_thr=0.5,\n neg_iou_thr=0.5,\n min_pos_iou=0.5,\n match_low_quality=False,\n ignore_iof_thr=-1,\n ),\n sampler=dict(type=\"RandomSampler\", num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True),\n pos_weight=-1,\n debug=False,\n ),\n ),\n test_cfg=dict(\n rpn=dict(nms_pre=1000, max_per_img=1000, nms=dict(type=\"nms\", iou_threshold=0.7), min_bbox_size=0),\n rcnn=dict(score_thr=0.05, nms=dict(type=\"nms\", iou_threshold=0.5), max_per_img=100),\n # soft-nms is also supported for rcnn testing\n # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)\n ),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/schedule_1x.py",
"content": "# optimizer\noptimizer = dict(type=\"SGD\", lr=0.02, momentum=0.9, weight_decay=0.0001)\noptimizer_config = dict(grad_clip=None)\n# learning policy\nlr_config = dict(policy=\"step\", warmup=\"linear\", warmup_iters=500, warmup_ratio=0.001, step=[8, 11])\nrunner = dict(type=\"EpochBasedRunner\", max_epochs=12)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/voc/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/voc/faster_rcnn_r50_fpn_1x_voc0712.py",
"content": "_base_ = [\"../faster_rcnn/faster_rcnn_r50_fpn.py\", \"./voc0712.py\", \"../default_runtime.py\"]\nmodel = dict(roi_head=dict(bbox_head=dict(num_classes=20)))\n# optimizer\noptimizer = dict(type=\"SGD\", lr=0.01, momentum=0.9, weight_decay=0.0001)\noptimizer_config = dict(grad_clip=None)\n# learning policy\n# actual epoch = 3 * 3 = 9\nlr_config = dict(policy=\"step\", step=[3])\n# runtime settings\nrunner = dict(type=\"EpochBasedRunner\", max_epochs=4) # actual epoch = 4 * 3 = 12\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/voc/voc0712.py",
"content": "# dataset settings\ndataset_type = \"VOCDataset\"\ndata_root = \"data/VOCdevkit/\"\nimg_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)\ntrain_pipeline = [\n dict(type=\"LoadImageFromFile\"),\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"Resize\", img_scale=(1000, 600), keep_ratio=True),\n dict(type=\"RandomFlip\", flip_ratio=0.5),\n dict(type=\"Normalize\", **img_norm_cfg),\n dict(type=\"Pad\", size_divisor=32),\n dict(type=\"DefaultFormatBundle\"),\n dict(type=\"Collect\", keys=[\"img\", \"gt_bboxes\", \"gt_labels\"]),\n]\ntest_pipeline = [\n dict(type=\"LoadImageFromFile\"),\n dict(\n type=\"MultiScaleFlipAug\",\n img_scale=(1000, 600),\n flip=False,\n transforms=[\n dict(type=\"Resize\", keep_ratio=True),\n dict(type=\"RandomFlip\"),\n dict(type=\"Normalize\", **img_norm_cfg),\n dict(type=\"Pad\", size_divisor=32),\n dict(type=\"ImageToTensor\", keys=[\"img\"]),\n dict(type=\"Collect\", keys=[\"img\"]),\n ],\n ),\n]\ndata = dict(\n samples_per_gpu=2,\n workers_per_gpu=2,\n train=dict(\n type=\"RepeatDataset\",\n times=3,\n dataset=dict(\n type=dataset_type,\n ann_file=[\n data_root + \"VOC2007/ImageSets/Main/trainval.txt\",\n data_root + \"VOC2012/ImageSets/Main/trainval.txt\",\n ],\n img_prefix=[data_root + \"VOC2007/\", data_root + \"VOC2012/\"],\n pipeline=train_pipeline,\n ),\n ),\n val=dict(\n type=dataset_type,\n ann_file=data_root + \"VOC2007/ImageSets/Main/test.txt\",\n img_prefix=data_root + \"VOC2007/\",\n pipeline=test_pipeline,\n ),\n test=dict(\n type=dataset_type,\n ann_file=data_root + \"VOC2007/ImageSets/Main/test.txt\",\n img_prefix=data_root + \"VOC2007/\",\n pipeline=test_pipeline,\n ),\n)\nevaluation = dict(interval=1, metric=\"mAP\")\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/__init__.py",
"content": ""
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_l_8xb8-300e_coco.py",
"content": "_base_ = \"./yolox_s_8xb8-300e_coco.py\"\n\n# model settings\nmodel = dict(\n backbone=dict(deepen_factor=1.0, widen_factor=1.0),\n neck=dict(in_channels=[256, 512, 1024], out_channels=256, num_csp_blocks=3),\n bbox_head=dict(in_channels=256, feat_channels=256),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_m_8xb8-300e_coco.py",
"content": "_base_ = \"./yolox_s_8xb8-300e_coco.py\"\n\n# model settings\nmodel = dict(\n backbone=dict(deepen_factor=0.67, widen_factor=0.75),\n neck=dict(in_channels=[192, 384, 768], out_channels=192, num_csp_blocks=2),\n bbox_head=dict(in_channels=192, feat_channels=192),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_nano_8xb8-300e_coco.py",
"content": "_base_ = \"./yolox_tiny_8xb8-300e_coco.py\"\n\n# model settings\nmodel = dict(\n backbone=dict(deepen_factor=0.33, widen_factor=0.25, use_depthwise=True),\n neck=dict(in_channels=[64, 128, 256], out_channels=64, num_csp_blocks=1, use_depthwise=True),\n bbox_head=dict(in_channels=64, feat_channels=64, use_depthwise=True),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_s_8xb8-300e_coco.py",
"content": "_base_ = [\"../schedule_1x.py\", \"../default_runtime.py\", \"./yolox_tta.py\"]\n\nimg_scale = (640, 640) # width, height\n\n# model settings\nmodel = dict(\n type=\"YOLOX\",\n data_preprocessor=dict(\n type=\"DetDataPreprocessor\",\n pad_size_divisor=32,\n batch_augments=[\n dict(type=\"BatchSyncRandomResize\", random_size_range=(480, 800), size_divisor=32, interval=10)\n ],\n ),\n backbone=dict(\n type=\"CSPDarknet\",\n deepen_factor=0.33,\n widen_factor=0.5,\n out_indices=(2, 3, 4),\n use_depthwise=False,\n spp_kernal_sizes=(5, 9, 13),\n norm_cfg=dict(type=\"BN\", momentum=0.03, eps=0.001),\n act_cfg=dict(type=\"Swish\"),\n ),\n neck=dict(\n type=\"YOLOXPAFPN\",\n in_channels=[128, 256, 512],\n out_channels=128,\n num_csp_blocks=1,\n use_depthwise=False,\n upsample_cfg=dict(scale_factor=2, mode=\"nearest\"),\n norm_cfg=dict(type=\"BN\", momentum=0.03, eps=0.001),\n act_cfg=dict(type=\"Swish\"),\n ),\n bbox_head=dict(\n type=\"YOLOXHead\",\n num_classes=80,\n in_channels=128,\n feat_channels=128,\n stacked_convs=2,\n strides=(8, 16, 32),\n use_depthwise=False,\n norm_cfg=dict(type=\"BN\", momentum=0.03, eps=0.001),\n act_cfg=dict(type=\"Swish\"),\n loss_cls=dict(type=\"CrossEntropyLoss\", use_sigmoid=True, reduction=\"sum\", loss_weight=1.0),\n loss_bbox=dict(type=\"IoULoss\", mode=\"square\", eps=1e-16, reduction=\"sum\", loss_weight=5.0),\n loss_obj=dict(type=\"CrossEntropyLoss\", use_sigmoid=True, reduction=\"sum\", loss_weight=1.0),\n loss_l1=dict(type=\"L1Loss\", reduction=\"sum\", loss_weight=1.0),\n ),\n train_cfg=dict(assigner=dict(type=\"SimOTAAssigner\", center_radius=2.5)),\n # In order to align the source code, the threshold of the val phase is\n # 0.01, and the threshold of the test phase is 0.001.\n test_cfg=dict(score_thr=0.01, nms=dict(type=\"nms\", iou_threshold=0.65)),\n)\n\nloading_pipeline = [\n dict(type=\"LoadImageFromFile\"),\n dict(type=\"LoadAnnotations\", with_bbox=True),\n]\n\nmulti_image_mix_dataset = dict(\n mosaic=dict(\n img_scale=img_scale,\n center_ratio_range=(0.5, 1.5),\n bbox_clip_border=True,\n pad_val=114.0,\n prob=0.5,\n ),\n # TODO: add random affine dict(\n # RandomAffine=dict(\n # scaling_ratio_range=(0.1, 2),\n # # img_scale is (width, height)\n # border=(-img_scale[0] // 2, -img_scale[1] // 2),\n # ),\n mixup=dict(\n img_scale=img_scale,\n ratio_range=(0.8, 1.6),\n flip_ratio=0.5,\n pad_val=114.0,\n max_iters=15,\n bbox_clip_border=True,\n ),\n)\n\ntrain_pipeline = [\n dict(type=\"YOLOXHSVRandomAug\"),\n dict(type=\"RandomFlip\", prob=0.5),\n # According to the official implementation, multi-scale\n # training is not considered here but in the\n # 'mmdet/models/detectors/yolox.py'.\n # Resize and Pad are for the last 15 epochs when Mosaic,\n # RandomAffine, and MixUp are closed by YOLOXModeSwitchHook.\n dict(type=\"Resize\", scale=img_scale, keep_ratio=True),\n dict(\n type=\"Pad\",\n pad_to_square=True,\n # If the image is three-channel, the pad value needs\n # to be set separately for each channel.\n pad_val=dict(img=(114.0, 114.0, 114.0)),\n ),\n dict(type=\"FilterAnnotations\", min_gt_bbox_wh=(1, 1), keep_empty=False),\n dict(type=\"PackDetInputs\"),\n]\n\ntest_pipeline = [\n dict(type=\"LoadImageFromFile\"),\n dict(type=\"Resize\", scale=img_scale, keep_ratio=True),\n dict(type=\"Pad\", pad_to_square=True, pad_val=dict(img=(114.0, 114.0, 114.0))),\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"PackDetInputs\", meta_keys=(\"img_id\", \"img_path\", \"ori_shape\", \"img_shape\", \"scale_factor\")),\n]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_tiny_8xb8-300e_coco.py",
"content": "_base_ = \"./yolox_s_8xb8-300e_coco.py\"\n\n# model settings\nmodel = dict(\n data_preprocessor=dict(\n batch_augments=[dict(type=\"BatchSyncRandomResize\", random_size_range=(320, 640), size_divisor=32, interval=10)]\n ),\n backbone=dict(deepen_factor=0.33, widen_factor=0.375),\n neck=dict(in_channels=[96, 192, 384], out_channels=96),\n bbox_head=dict(in_channels=96, feat_channels=96),\n)\n\nimg_scale = (640, 640) # width, height\n\ntest_pipeline = [\n dict(type=\"LoadImageFromFile\"),\n dict(type=\"Resize\", scale=(416, 416), keep_ratio=True),\n dict(type=\"Pad\", pad_to_square=True, pad_val=dict(img=(114.0, 114.0, 114.0))),\n dict(type=\"LoadAnnotations\", with_bbox=True),\n dict(type=\"PackDetInputs\", meta_keys=(\"img_id\", \"img_path\", \"ori_shape\", \"img_shape\", \"scale_factor\")),\n]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_tta.py",
"content": "tta_model = dict(type=\"DetTTAModel\", tta_cfg=dict(nms=dict(type=\"nms\", iou_threshold=0.65), max_per_img=100))\n\nimg_scales = [(640, 640), (320, 320), (960, 960)]\ntta_pipeline = [\n dict(type=\"LoadImageFromFile\", backend_args=None),\n dict(\n type=\"TestTimeAug\",\n transforms=[\n [dict(type=\"Resize\", scale=s, keep_ratio=True) for s in img_scales],\n [\n # ``RandomFlip`` must be placed before ``Pad``, otherwise\n # bounding box coordinates after flipping cannot be\n # recovered correctly.\n dict(type=\"RandomFlip\", prob=1.0),\n dict(type=\"RandomFlip\", prob=0.0),\n ],\n [\n dict(type=\"Pad\", pad_to_square=True, pad_val=dict(img=(114.0, 114.0, 114.0))),\n ],\n [dict(type=\"LoadAnnotations\", with_bbox=True)],\n [\n dict(\n type=\"PackDetInputs\",\n meta_keys=(\n \"img_id\",\n \"img_path\",\n \"ori_shape\",\n \"img_shape\",\n \"scale_factor\",\n \"flip\",\n \"flip_direction\",\n ),\n )\n ],\n ],\n ),\n]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/configs/pretrain/detection/yolox/yolox_x_8xb8-300e_coco.py",
"content": "_base_ = \"./yolox_s_8xb8-300e_coco.py\"\n\n# model settings\nmodel = dict(\n backbone=dict(deepen_factor=1.33, widen_factor=1.25),\n neck=dict(in_channels=[320, 640, 1280], out_channels=320, num_csp_blocks=4),\n bbox_head=dict(in_channels=320, feat_channels=320),\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/constants.py",
"content": "\"\"\"Storing the constants\"\"\"\n\nfrom autogluon.core.metrics import METRICS\n\n# Column/Label Types\nNULL = \"null\"\nCATEGORICAL = \"categorical\"\nTEXT = \"text\"\nTEXT_NER = \"text_ner\" # Added for NER text column\nNUMERICAL = \"numerical\"\nIMAGE_PATH = \"image_path\"\nIMAGE_BYTEARRAY = \"image_bytearray\"\nIMAGE_BASE64_STR = \"image_base64_str\"\nIDENTIFIER = \"identifier\"\nDOCUMENT = \"document\"\nDOCUMENT_IMAGE = \"document_image\"\nDOCUMENT_PDF = \"document_pdf\"\n\n# Scarcity modes\nFEW_SHOT = \"few_shot\"\nDEFAULT_SHOT = \"default_shot\"\nZERO_SHOT = \"zero_shot\"\n\n# Problem types\nCLASSIFICATION = \"classification\"\nBINARY = \"binary\"\nMULTICLASS = \"multiclass\"\nREGRESSION = \"regression\"\nNER = \"ner\"\nNAMED_ENTITY_RECOGNITION = \"named_entity_recognition\"\nFEATURE_EXTRACTION = \"feature_extraction\"\nZERO_SHOT_IMAGE_CLASSIFICATION = \"zero_shot_image_classification\"\nOBJECT_DETECTION = \"object_detection\"\nOCR = \"ocr\"\nOCR_TEXT_DETECTION = f\"{OCR}_text_detection\"\nOCR_TEXT_RECOGNITION = f\"{OCR}_text_recognition\"\nIMAGE_SIMILARITY = \"image_similarity\"\nTEXT_SIMILARITY = \"text_similarity\"\nIMAGE_TEXT_SIMILARITY = \"image_text_similarity\"\nFEW_SHOT_CLASSIFICATION = \"few_shot_classification\"\nSEMANTIC_SEGMENTATION = \"semantic_segmentation\"\n\n# Input keys\nIMAGE = \"image\"\nIMAGE_META = \"image_meta\"\nIMAGE_VALID_NUM = \"image_valid_num\"\nLABEL = \"label\"\nTEXT_TOKEN_IDS = \"text_token_ids\"\nCHOICES_IDS = \"choices_ids\"\nTEXT_VALID_LENGTH = \"text_valid_length\"\nTEXT_SEGMENT_IDS = \"text_segment_ids\"\nCOLUMN = \"column\"\nATTENTION_MASK = \"attention_mask\"\nTOKEN_TYPE_IDS = \"token_type_ids\"\nPIXEL_VALUES = \"pixel_values\"\nINPUT_IDS = \"input_ids\"\nSEMANTIC_SEGMENTATION_IMG = \"semantic_segmentation_img\"\nSEMANTIC_SEGMENTATION_GT = \"semantic_segmentation_gt\"\n\n# Output keys\nLOGITS = \"logits\"\nORI_LOGITS = \"ori_logits\"\nAUG_LOGITS = \"aug_logits\"\nTEMPLATE_LOGITS = \"template_logits\"\nLM_TARGET = \"lm_target\"\nLOSS = \"loss\"\nOUTPUT = \"output\"\nWEIGHT = \"weight\"\nFEATURES = \"features\"\nMULTIMODAL_FEATURES = \"multimodal_features\" # used for the adapted multimodal features before the fusion module\nMULTIMODAL_FEATURES_PRE_AUG = \"multimodal_features_pre_aug\"\nMULTIMODAL_FEATURES_POST_AUG = \"multimodal_features_post_aug\"\nRAW_FEATURES = \"raw_features\"\nMASKS = \"masks\"\nPROBABILITY = \"probability\"\nCOLUMN_FEATURES = \"column_features\"\nBBOX = \"bbox\"\nROIS = \"rois\"\nSCORE = \"score\"\nLOGIT_SCALE = \"logit_scale\"\nVAE_MEAN = \"vae_mean\"\nVAE_VAR = \"vae_var\"\n\n# Loss\nMOE_LOSS = \"moe_loss\"\n\n# Metric for Object Detection\nMAP = \"map\"\nMEAN_AVERAGE_PRECISION = \"mean_average_precision\"\nMAP_50 = \"map_50\"\nMAP_75 = \"map_75\"\nMAP_SMALL = \"map_small\"\nMAP_MEDIUM = \"map_medium\"\nMAP_LARGE = \"map_large\"\nMAR_1 = \"mar_1\"\nMAR_10 = \"mar_10\"\nMAR_100 = \"mar_100\"\nMAR_SMALL = \"mar_small\"\nMAR_MEDIUM = \"mar_medium\"\nMAR_LARGE = \"mar_large\"\nDETECTION_METRICS = [\n MAP,\n MEAN_AVERAGE_PRECISION,\n MAP_50,\n MAP_75,\n MAP_SMALL,\n MAP_MEDIUM,\n MAP_LARGE,\n MAR_1,\n MAR_10,\n MAR_100,\n MAR_SMALL,\n MAR_MEDIUM,\n MAR_LARGE,\n]\n\n# Metric\nMAX = \"max\"\nMIN = \"min\"\nACCURACY = \"accuracy\"\nACC = \"acc\"\nOVERALL_ACCURACY = \"overall_accuracy\"\nRMSE = \"rmse\"\nROOT_MEAN_SQUARED_ERROR = \"root_mean_squared_error\"\nR2 = \"r2\"\nPEARSONR = \"pearsonr\"\nSPEARMANR = \"spearmanr\"\nQUADRATIC_KAPPA = \"quadratic_kappa\"\nROC_AUC = \"roc_auc\"\nAVERAGE_PRECISION = \"average_precision\"\nLOG_LOSS = \"log_loss\"\nCROSS_ENTROPY = \"cross_entropy\"\nCOSINE_EMBEDDING_LOSS = \"cosine_embedding_loss\"\nF1 = \"f1\"\nOVERALL_F1 = \"overall_f1\"\nF1_MACRO = \"f1_macro\"\nF1_MICRO = \"f1_micro\"\nF1_WEIGHTED = \"f1_weighted\"\nNER_TOKEN_F1 = \"ner_token_f1\"\nDIRECT_LOSS = \"direct_loss\"\nHIT_RATE = \"hit_rate\"\nNDCG = \"ndcg\"\nPRECISION = \"precision\"\nRECALL = \"recall\"\nMRR = \"mrr\"\nSM = \"sm\"\nEM = \"em\"\nFM = \"fm\"\nMAE = \"mae\"\nBER = \"ber\"\nIOU = \"iou\"\nCOVERAGE = \"coverage\"\nRETRIEVAL_METRICS = [NDCG, PRECISION, RECALL, MRR]\nMETRIC_MODE_MAP = {\n ACC: MAX,\n ACCURACY: MAX,\n DIRECT_LOSS: MIN,\n RMSE: MIN,\n ROOT_MEAN_SQUARED_ERROR: MIN,\n R2: MAX,\n QUADRATIC_KAPPA: MAX,\n ROC_AUC: MAX,\n LOG_LOSS: MIN,\n CROSS_ENTROPY: MIN,\n PEARSONR: MAX,\n SPEARMANR: MAX,\n F1: MAX,\n F1_MACRO: MAX,\n F1_MICRO: MAX,\n F1_WEIGHTED: MAX,\n MAP: MAX,\n MEAN_AVERAGE_PRECISION: MAX,\n NER_TOKEN_F1: MAX,\n OVERALL_F1: MAX,\n RECALL: MAX,\n SM: MAX,\n IOU: MAX,\n BER: MIN,\n COVERAGE: MAX,\n}\n\nMATCHING_METRICS = {\n BINARY: [ROC_AUC, ROC_AUC],\n MULTICLASS: [SPEARMANR, SPEARMANR],\n REGRESSION: [SPEARMANR, SPEARMANR],\n}\nMATCHING_METRICS_WITHOUT_PROBLEM_TYPE = [RECALL, NDCG]\n\nEVALUATION_METRICS = {\n # Use evaluation metrics from METRICS for these types\n BINARY: list(METRICS[BINARY].keys()) + [COVERAGE],\n MULTICLASS: METRICS[MULTICLASS].keys(),\n REGRESSION: METRICS[REGRESSION].keys(),\n OBJECT_DETECTION: DETECTION_METRICS,\n SEMANTIC_SEGMENTATION: [IOU, BER, SM],\n NER: [OVERALL_F1, NER_TOKEN_F1],\n NAMED_ENTITY_RECOGNITION: [OVERALL_F1, NER_TOKEN_F1],\n FEW_SHOT_CLASSIFICATION: METRICS[MULTICLASS].keys(),\n}\n\nVALIDATION_METRICS = {\n problem_type: [metric for metric in metrics if metric in METRIC_MODE_MAP] + [DIRECT_LOSS]\n for problem_type, metrics in EVALUATION_METRICS.items()\n}\n\n# Training status\nTRAIN = \"train\"\nVALIDATE = \"validate\"\nVAL = \"val\"\nTEST = \"test\"\nPREDICT = \"predict\"\n\n# Model sources\nHUGGINGFACE = \"huggingface\"\nTIMM = \"timm\"\nMMDET = \"mmdet\"\nMMOCR = \"mmocr\"\n\n# Modality keys. may need to update here if new modality keys are added in above.\nALL_MODALITIES = [IMAGE, TEXT, CATEGORICAL, NUMERICAL, TEXT_NER, DOCUMENT, SEMANTIC_SEGMENTATION_IMG]\n\n# Keys to compute metrics\nY_PRED = \"y_pred\"\nY_PRED_PROB = \"y_pred_prob\"\nY_TRUE = \"y_true\"\n\n# Configuration keys\nMODEL = \"model\"\nDATA = \"data\"\nOPTIM = \"optim\"\nENV = \"env\"\nDISTILLER = \"distiller\"\nMATCHER = \"matcher\"\nVALID_CONFIG_KEYS = [MODEL, DATA, OPTIM, ENV, DISTILLER, MATCHER]\n\n# Image normalization mean and std. This is only to normalize images for the CLIP model.\nCLIP_IMAGE_MEAN = (0.48145466, 0.4578275, 0.40821073)\nCLIP_IMAGE_STD = (0.26862954, 0.26130258, 0.27577711)\n\n# Logger name\nAUTOMM = \"automm\"\n\n# environment variables\nAUTOMM_TUTORIAL_MODE = \"AUTOMM_TUTORIAL_MODE\"\n\n# error try\nGET_ITEM_ERROR_RETRY = 50\n\n# top-k checkpoint average methods\nUNIFORM_SOUP = \"uniform_soup\"\nGREEDY_SOUP = \"greedy_soup\"\nBEST = \"best\"\n\n# efficient finetuning strategies\nNORM_FIT = \"norm_fit\"\nBIT_FIT = \"bit_fit\"\nLORA = \"lora\"\nLORA_BIAS = \"lora_bias\"\nLORA_NORM = \"lora_norm\"\nIA3 = \"ia3\"\nIA3_BIAS = \"ia3_bias\"\nIA3_NORM = \"ia3_norm\"\nIA3_LORA = \"ia3_lora\"\nIA3_LORA_BIAS = \"ia3_lora_bias\"\nIA3_LORA_NORM = \"Ia3_lora_norm\"\nCONV_LORA = \"conv_lora\"\nPEFT_ADDITIVE_STRATEGIES = [\n LORA,\n LORA_BIAS,\n LORA_NORM,\n IA3,\n IA3_BIAS,\n IA3_NORM,\n IA3_LORA,\n IA3_LORA_BIAS,\n IA3_LORA_NORM,\n CONV_LORA,\n]\nPEFT_NON_ADDITIVE_STRATEGIES = [\n BIT_FIT,\n NORM_FIT,\n]\nPEFT_STRATEGIES = list(set(PEFT_ADDITIVE_STRATEGIES) | set(PEFT_NON_ADDITIVE_STRATEGIES))\n\n# DeepSpeed constants\nDEEPSPEED_OFFLOADING = \"deepspeed_stage_3_offload\"\nDEEPSPEED_STRATEGY = \"deepspeed\"\nDEEPSPEED_MODULE = \"autogluon.multimodal.optim.deepspeed\"\nDEEPSPEED_MIN_PL_VERSION = \"1.7.1\"\n\n# registered model keys. TODO: document how to add new models.\nCLIP = \"clip\"\nTIMM_IMAGE = \"timm_image\"\nHF_TEXT = \"hf_text\"\nT_FEW = \"t_few\"\nNUMERICAL_MLP = \"numerical_mlp\"\nCATEGORICAL_MLP = \"categorical_mlp\"\nFUSION = \"fusion\"\nFUSION_MLP = f\"{FUSION}_mlp\"\nFUSION_TRANSFORMER = f\"{FUSION}_transformer\"\nFT_TRANSFORMER = \"ft_transformer\"\nFUSION_NER = f\"{FUSION}_{NER}\"\nMMDET_IMAGE = \"mmdet_image\"\nMMOCR_TEXT_DET = \"mmocr_text_detection\"\nMMOCR_TEXT_RECOG = \"mmocr_text_recognition\"\nNER_TEXT = \"ner_text\"\nDOCUMENT_TRANSFORMER = \"document_transformer\"\nHF_MODELS = (HF_TEXT, T_FEW, CLIP, NER_TEXT, DOCUMENT_TRANSFORMER)\nMMLAB_MODELS = (MMDET_IMAGE, MMOCR_TEXT_DET, MMOCR_TEXT_RECOG)\nSAM = \"sam\"\nMETA_TRANSFORMER = \"meta_transformer\"\n\n# matcher loss type\nCONTRASTIVE_LOSS = \"contrastive_loss\"\nMULTI_NEGATIVES_SOFTMAX_LOSS = \"multi_negatives_softmax_loss\"\n\n# matcher distance type\nCOSINE_SIMILARITY = \"cosine_similarity\"\n\n# matcher miner type\nPAIR_MARGIN_MINER = \"pair_margin_miner\"\n\n# checkpoints\nRAY_TUNE_CHECKPOINT = \"ray_tune_checkpoint.ckpt\"\nBEST_K_MODELS_FILE = \"best_k_models.yaml\"\nLAST_CHECKPOINT = \"last.ckpt\"\nMODEL_CHECKPOINT = \"model.ckpt\"\n\n# url\nS3_PREFIX = \"s3://\"\nSOURCEPROMPT_URL = \"https://automl-mm-bench.s3.amazonaws.com/few_shot/templates.zip\"\nSOURCEPROMPT_SHA1 = \"c25cdf3730ff96ab4859b72e18d46ff117b62bd6\"\n\n# ner\nENTITY_GROUP = \"entity_group\"\nSTART_OFFSET = \"start\"\nEND_OFFSET = \"end\"\nTOKEN_WORD_MAPPING = \"token_word_mapping\"\nWORD_OFFSETS = \"word_offsets\"\nNER_RET = \"ner_ret\"\nNER_ANNOTATION = \"ner_annotation\"\n\n# matcher\nQUERY = \"query\"\nRESPONSE = \"response\"\nQUERY_RESPONSE = f\"{QUERY}_{RESPONSE}\"\nPAIR = \"pair\"\nTRIPLET = \"triplet\"\n\n# mmdet\nXYWH = \"xywh\"\nXYXY = \"xyxy\"\nBBOX_FORMATS = [XYWH, XYXY]\n\n# sam (multi-class)\nCLASS_LOGITS = \"class_logits\"\nMASK_LABEL = \"mask_label\"\nCLASS_LABEL = \"class_label\"\nSEMANTIC_MASK = \"semantic_mask\"\n\n# presets\nDEFAULT = \"default\"\nHIGH_QUALITY = \"high_quality\"\nMEDIUM_QUALITY = \"medium_quality\"\nBEST_QUALITY = \"best_quality\"\nALL_MODEL_QUALITIES = [HIGH_QUALITY, MEDIUM_QUALITY, BEST_QUALITY, DEFAULT]\n\n# datasets\nDEFAULT_DATASET = \"default_dataset\"\nMULTI_IMAGE_MIX_DATASET = \"multi_image_mix_dataset\"\n\n# strategies\nDDP = \"ddp\"\nDDP_FIND_UNUSED_PARAMETERS_FALSE = \"ddp_find_unused_parameters_false\"\nDDP_FIND_UNUSED_PARAMETERS_TRUE = \"ddp_find_unused_parameters_true\"\nDDP_STRATEGIES = [DDP, DDP_FIND_UNUSED_PARAMETERS_FALSE, DDP_FIND_UNUSED_PARAMETERS_TRUE]\n\n# torch constants\nTORCH_COMPILE_MIN_VERSION = \"2.2.0.dev20230908\"\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/__init__.py",
"content": "from .datamodule import BaseDataModule\nfrom .dataset import BaseDataset\nfrom .dataset_mmlab import MultiImageMixDataset\nfrom .infer_types import (\n infer_column_types,\n infer_ner_column_type,\n infer_output_shape,\n infer_problem_type,\n infer_rois_column_type,\n is_image_column,\n)\nfrom .label_encoder import CustomLabelEncoder, NerLabelEncoder\nfrom .mixup import MixupModule\nfrom .preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom .process_categorical import CategoricalProcessor\nfrom .process_document import DocumentProcessor\nfrom .process_image import ImageProcessor\nfrom .process_label import LabelProcessor\nfrom .process_mmlab import MMDetProcessor, MMOcrProcessor\nfrom .process_ner import NerProcessor\nfrom .process_numerical import NumericalProcessor\nfrom .process_semantic_seg_img import SemanticSegImageProcessor\nfrom .process_text import TextProcessor\nfrom .utils import (\n create_data_processor,\n create_fusion_data_processors,\n data_to_df,\n get_detected_data_types,\n get_mixup,\n infer_dtypes_by_model_names,\n infer_scarcity_mode_by_data_size,\n init_df_preprocessor,\n split_train_tuning_data,\n turn_on_off_feature_column_info,\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/collator.py",
"content": "import math\nimport warnings\n\nimport numpy as np\nimport torch\n\n\ndef _pad_arrs_to_max_length(arrs, pad_axis, pad_val, round_to=None, max_length=None):\n \"\"\"\n Inner Implementation of the Pad collate.\n\n Parameters\n ----------\n arrs (list)\n pad_axis (int)\n pad_val (number)\n round_to (int, optional). (default: ``None``)\n max_length (int, optional). (default: ``None``)\n\n Returns\n -------\n ret : torch.Tensor\n original_length : torch.Tensor\n \"\"\"\n if not isinstance(arrs[0], torch.Tensor):\n arrs = [torch.as_tensor(ele) for ele in arrs]\n\n original_length = [ele.size(pad_axis) for ele in arrs]\n max_arr_len = max(original_length)\n\n if round_to is not None:\n max_arr_len = round_to * math.ceil(max_arr_len / round_to)\n elif max_length is not None:\n if max_length < max_arr_len:\n raise ValueError(\n f\"If max_length is specified, max_length={max_length} must be larger \"\n f\"than the maximum length {max_arr_len} of the given arrays at axis={pad_axis}\"\n )\n max_arr_len = max_length\n\n size = arrs[0].size()\n prev_trailing_dims = size[:pad_axis]\n after_trailing_dims = size[pad_axis + 1 :]\n\n out_dims = (len(arrs),) + prev_trailing_dims + (max_arr_len,) + after_trailing_dims\n out_tensor = arrs[0].new_full(out_dims, pad_val)\n for i, tensor in enumerate(arrs):\n length = tensor.size(pad_axis)\n out_tensor[i].narrow(pad_axis, 0, length)[:] = tensor\n\n original_length = torch.as_tensor(original_length)\n\n return out_tensor, original_length\n\n\ndef _stack_arrs(arrs):\n if isinstance(arrs[0], torch.Tensor):\n return torch.stack(arrs, 0)\n else:\n return _stack_arrs([torch.as_tensor(x) for x in arrs])\n\n\nclass StackCollator:\n \"\"\"\n Stack the input data samples to construct the batch.\n The N input samples must have the same shape/length and will be stacked to construct a batch.\n \"\"\"\n\n def __call__(self, data):\n \"\"\"\n Collate the input data.\n\n Parameters\n ----------\n data (list): The input data samples.\n\n Returns\n -------\n batch_data (torch.Tensor)\n \"\"\"\n return _stack_arrs(data)\n\n\nclass PadCollator:\n \"\"\"\n Returns a callable that pads and stacks data.\n\n Parameters\n ----------\n axis (int, optional): The axis to pad the arrays.\n The arrays will be padded to the largest dimension at :attr:`axis`.\n For example, assume the input arrays have shape (10, 8, 5), (6, 8, 5), (3, 8, 5)\n and the `axis` is 0.\n Each input will be padded into (10, 8, 5) and then stacked to form the final output,\n which has shapeīŧ3, 10, 8, 5). (default ``0``)\n pad_val (float or int, optional): The padding value. (default ``0``)\n round_to (int, optional):\n If specified, the padded dimension will be rounded to be multiple of this argument.\n Mutually exclusive with :attr:`max_length`. (default ``None``)\n max_length (int, optional):\n If specified, the padded dimension will have length :attr:`max_length`,\n and it must be larger than the maximum length in the arrays at :attr:`axis`.\n Mutually exclusive with :attr:`round_to`. (default ``None``)\n ret_length (bool, optional): Whether to return the valid length in the output.\n (default ``False``)\n \"\"\"\n\n def __init__(self, axis=0, pad_val=0, round_to=None, max_length=None, ret_length=False):\n self._axis = axis\n if not isinstance(axis, int):\n raise ValueError(f\"axis must be an integer! Received axis={axis}, type={type(axis)}.\")\n\n if round_to is not None and max_length is not None:\n raise ValueError(f\"Only either round_to={round_to} or max_length={max_length} can be specified.\")\n\n self._pad_val = 0 if pad_val is None else pad_val\n self._round_to = round_to\n self._max_length = max_length\n self._ret_length = ret_length\n\n if pad_val is None:\n warnings.warn(\n \"Padding value is not given and will be set automatically to 0 \"\n \"in data.Pad(). \"\n \"Please check whether this is intended \"\n \"(e.g. value of padding index in the tokenizer).\"\n )\n\n def __call__(self, data):\n \"\"\"\n Collate the input data.\n\n The arrays will be padded to the largest dimension at `axis` and then\n stacked to form the final output. In addition, the function will output\n the original dimensions at the `axis` if ret_length is turned on.\n\n Parameters\n ----------\n data : List[np.ndarray] or List[List[dtype]] or List[torch.Tensor]\n List of samples to pad and stack.\n\n Returns\n -------\n batch_data (torch.Tensor): Data in the minibatch. Shape is (N, ...)\n valid_length (NDArray, optional):\n The sequences' original lengths at the padded axis. Shape is (N,). This will only be\n returned if `ret_length` is True.\n\n \"\"\"\n if isinstance(data[0], (torch.Tensor, np.ndarray, list, tuple)):\n padded_arr, original_length = _pad_arrs_to_max_length(\n data,\n pad_axis=self._axis,\n pad_val=self._pad_val,\n round_to=self._round_to,\n max_length=self._max_length,\n )\n if self._ret_length:\n return padded_arr, original_length\n else:\n return padded_arr\n else:\n raise NotImplementedError\n\n\nclass TupleCollator:\n \"\"\"\n Wrap multiple data collator functions together. The input functions will be applied\n to the corresponding input fields.\n\n Each data sample should be a list or tuple containing multiple attributes. The `i`th collate\n function stored in `Tuple` will be applied on the `i`th attribute. For example, each\n data sample is (nd_data, label). You can wrap two collate functions using\n `Tuple(DataCollate, LabelCollate)` to collate nd_data and label correspondingly.\n\n Parameters\n ----------\n fn (list or tuple or callable): The collate functions to wrap.\n *args (tuple of callable, optional): The additional collate functions to wrap.\n\n \"\"\"\n\n def __init__(self, fn, *args):\n if isinstance(fn, (list, tuple)):\n if len(args) != 0:\n raise ValueError(\n \"Input pattern not understood. \"\n \"The input of Tuple can be Tuple(A, B, C) \"\n \"or Tuple([A, B, C]) or Tuple((A, B, C)). \"\n f\"Received fn={str(fn)}, args={str(args)}\"\n )\n self._fn = fn\n else:\n self._fn = (fn,) + args\n for i, ele_fn in enumerate(self._fn):\n if not hasattr(ele_fn, \"__call__\"):\n raise ValueError(f\"Collate functions must be callable! type(fn[{i}])={type(ele_fn)}\")\n\n def __call__(self, data):\n \"\"\"\n Collate the input data.\n\n Parameters\n ----------\n data (list): The samples to collate. Each sample should contain N attributes.\n\n Returns\n -------\n ret (tuple):\n A tuple of length N. Contains the collated result of each attribute in the input.\n \"\"\"\n if len(data[0]) != len(self._fn):\n raise ValueError(f\"The number of attributes in each data sample should contains {len(self._fn)} elements\")\n ret = []\n for i, ele_fn in enumerate(self._fn):\n ret.append(ele_fn([ele[i] for ele in data]))\n return tuple(ret)\n\n\nclass ListCollator:\n \"\"\"\n Simply forward the list of input data.\n\n This is particularly useful when the Dataset contains textual data\n and in conjunction with the `Tuple` collate function.\n\n \"\"\"\n\n def __call__(self, data):\n \"\"\"\n Parameters\n ----------\n data (list): The list of samples\n\n Returns\n -------\n ret (list): The input list\n \"\"\"\n return list(data)\n\n\nclass DictCollator:\n \"\"\"\n Wrap multiple collate functions together and apply it to merge inputs from a dict.\n\n The generated batch samples are stored as a dict with the same keywords.\n\n Each data sample should be a dict and the fn corresponds to `key` will be applied on the\n input with the keyword `key`.\n For example, each data sample is {'data': nd_data, 'label': nd_label}.\n You can merge the data and labels using\n `Dict({'data': DataCollate, 'label': LabelCollate})` to collate the nd_data and nd_label.\n\n Parameters\n ----------\n fn_dict (dict): A dictionary that contains the key-->collate function mapping.\n\n \"\"\"\n\n def __init__(self, fn_dict):\n self._fn_dict = fn_dict\n if not isinstance(fn_dict, dict):\n raise ValueError(f\"Input must be a dictionary! type of input={type(fn_dict)}\")\n for fn in fn_dict.values():\n if not hasattr(fn, \"__call__\"):\n raise ValueError(\"Elements of the dictionary must be callable!\")\n self._fn_dict = fn_dict\n\n def __call__(self, data):\n \"\"\"\n\n Parameters\n ----------\n data (dict): The samples to collate. Each sample should be a dictionary\n\n Returns\n -------\n ret (dict): The resulting dictionary that stores the merged samples.\n \"\"\"\n ret = dict()\n for k, ele_fn in self._fn_dict.items():\n ret[k] = ele_fn([ele[k] for ele in data])\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/datamodule.py",
"content": "from typing import Dict, List, Optional, Union\n\nimport pandas as pd\nfrom lightning.pytorch import LightningDataModule\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom ..constants import PREDICT, TEST, TRAIN, VALIDATE\nfrom .dataset import BaseDataset\nfrom .preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom .utils import get_collate_fn\n\n\nclass BaseDataModule(LightningDataModule):\n \"\"\"\n Set up Pytorch DataSet and DataLoader objects to prepare data for single-modal/multimodal training,\n validation, testing, and prediction. We organize the multimodal data using pd.DataFrame.\n For some modalities, e.g, image, that cost much memory, we only store their disk path to do lazy loading.\n This class inherits from the Pytorch Lightning's LightningDataModule:\n https://lightning.ai/docs/pytorch/stable/data/datamodule.html\n \"\"\"\n\n def __init__(\n self,\n df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],\n data_processors: Union[dict, List[dict]],\n per_gpu_batch_size: int,\n num_workers: int,\n train_data: Optional[pd.DataFrame] = None,\n train_dataset: Optional[Dataset] = None,\n validate_data: Optional[pd.DataFrame] = None,\n test_data: Optional[pd.DataFrame] = None,\n predict_data: Optional[pd.DataFrame] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n val_use_training_mode: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n df_preprocessor\n One or a list of dataframe preprocessors. The preprocessing of one modality is generic so that\n the preprocessed data can be used by different models requiring the modality.\n For example, formatting input data as strings is a valid preprocessing operation for text.\n However, tokenizing strings into ids is invalid since different models generally\n use different tokenizers.\n data_processors\n The data processors to prepare customized data for each model. Each processor is only charge of\n one modality of one model. This helps scale up training arbitrary combinations of models.\n per_gpu_batch_size\n Mini-batch size for each GPU.\n num_workers\n Number of workers for Pytorch DataLoader.\n train_data\n Training data.\n train_dataset\n Training dataset.\n validate_data\n Validation data.\n test_data\n Test data.\n predict_data\n Prediction data. No labels required in it.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n val_use_training_mode\n whether we are triggering is_training when creating the dataset for validation.\n This is used when we want to use val_loss as val metric, and thus we'll use data pipeline\n for training instead of for inference during validation.\n \"\"\"\n super().__init__()\n self.prepare_data_per_node = True\n\n if isinstance(df_preprocessor, MultiModalFeaturePreprocessor):\n df_preprocessor = [df_preprocessor]\n if isinstance(data_processors, dict):\n data_processors = [data_processors]\n\n self.df_preprocessor = df_preprocessor\n self.data_processors = data_processors\n self.per_gpu_batch_size = per_gpu_batch_size\n self.num_workers = num_workers\n self.train_data = train_data\n self.train_dataset = train_dataset\n self.validate_data = validate_data\n self.test_data = test_data\n self.predict_data = predict_data\n self.id_mappings = id_mappings\n self.val_use_training_mode = val_use_training_mode\n\n def set_dataset(self, split):\n if self.val_use_training_mode:\n is_training = split in [TRAIN, VALIDATE]\n else:\n is_training = split == TRAIN\n\n if is_training and self.train_dataset is not None:\n return\n\n data_split = getattr(self, f\"{split}_data\")\n dataset = BaseDataset(\n data=data_split,\n preprocessor=self.df_preprocessor,\n processors=self.data_processors,\n id_mappings=self.id_mappings,\n is_training=is_training,\n )\n\n setattr(self, f\"{split}_dataset\", dataset)\n\n def setup(self, stage):\n \"\"\"\n Set up datasets for different stages: \"fit\" (training and validation), \"test\", and \"predict\".\n This method is registered by Pytorch Lightning's LightningDataModule.\n Refer to: https://lightning.ai/docs/pytorch/stable/data/datamodule.html#setup\n\n Parameters\n ----------\n stage\n Stage name including choices:\n - fit (For the fitting stage)\n - test (For the test stage)\n - predict (For the prediction stage)\n \"\"\"\n if stage == \"fit\":\n self.set_dataset(TRAIN)\n self.set_dataset(VALIDATE)\n elif stage == \"validate\":\n self.set_dataset(VALIDATE)\n elif stage == \"test\":\n self.set_dataset(TEST)\n elif stage == \"predict\":\n self.set_dataset(PREDICT)\n else:\n raise ValueError(f\"Unknown stage {stage}\")\n\n def train_dataloader(self):\n \"\"\"\n Create the dataloader for training.\n This method is registered by Pytorch Lightning's LightningDataModule.\n Refer to: https://lightning.ai/docs/pytorch/stable/data/datamodule.html#train-dataloader\n\n Returns\n -------\n A Pytorch DataLoader object.\n \"\"\"\n loader = DataLoader(\n self.train_dataset,\n batch_size=self.per_gpu_batch_size,\n num_workers=self.num_workers,\n shuffle=True,\n pin_memory=False,\n collate_fn=get_collate_fn(\n df_preprocessor=self.df_preprocessor,\n data_processors=self.data_processors,\n per_gpu_batch_size=self.per_gpu_batch_size,\n ),\n persistent_workers=self.num_workers > 0,\n )\n return loader\n\n def val_dataloader(self):\n \"\"\"\n Create the dataloader for validation.\n This method is registered by Pytorch Lightning's LightningDataModule.\n Refer to: https://lightning.ai/docs/pytorch/stable/data/datamodule.html#val-dataloader\n\n Returns\n -------\n A Pytorch DataLoader object.\n \"\"\"\n loader = DataLoader(\n self.validate_dataset,\n batch_size=self.per_gpu_batch_size,\n num_workers=self.num_workers,\n pin_memory=False,\n collate_fn=get_collate_fn(\n df_preprocessor=self.df_preprocessor,\n data_processors=self.data_processors,\n per_gpu_batch_size=self.per_gpu_batch_size,\n ),\n persistent_workers=self.num_workers > 0,\n )\n return loader\n\n def test_dataloader(self):\n \"\"\"\n Create the dataloader for test.\n This method is registered by Pytorch Lightning's LightningDataModule.\n Refer to: https://lightning.ai/docs/pytorch/stable/data/datamodule.html#test-dataloader\n\n Returns\n -------\n A Pytorch DataLoader object.\n \"\"\"\n loader = DataLoader(\n self.test_dataset,\n batch_size=self.per_gpu_batch_size,\n num_workers=self.num_workers,\n pin_memory=False,\n collate_fn=get_collate_fn(\n df_preprocessor=self.df_preprocessor,\n data_processors=self.data_processors,\n per_gpu_batch_size=self.per_gpu_batch_size,\n ),\n persistent_workers=self.num_workers > 0,\n )\n return loader\n\n def predict_dataloader(self):\n \"\"\"\n Create the dataloader for prediction.\n This method is registered by Pytorch Lightning's LightningDataModule.\n Refer to: https://lightning.ai/docs/pytorch/stable/data/datamodule.html#predict-dataloader\n\n Returns\n -------\n A Pytorch DataLoader object.\n \"\"\"\n loader = DataLoader(\n self.predict_dataset,\n batch_size=self.per_gpu_batch_size,\n num_workers=self.num_workers,\n pin_memory=False,\n collate_fn=get_collate_fn(\n df_preprocessor=self.df_preprocessor,\n data_processors=self.data_processors,\n per_gpu_batch_size=self.per_gpu_batch_size,\n ),\n persistent_workers=self.num_workers > 0,\n )\n return loader\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/dataset.py",
"content": "import logging\nfrom typing import Dict, List, Optional, Union\n\nimport pandas as pd\nimport torch\n\nfrom ..constants import GET_ITEM_ERROR_RETRY\nfrom .preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom .utils import apply_data_processor, apply_df_preprocessor, get_per_sample_features\n\nlogger = logging.getLogger(__name__)\n\n\nclass BaseDataset(torch.utils.data.Dataset):\n \"\"\"\n A Pytorch DataSet class to process a multimodal pd.DataFrame. It first uses a preprocessor to\n produce model-agnostic features. Then, each processor prepares customized data for one modality\n per model. For code simplicity, here we treat ground-truth label as one modality. This class is\n independent of specific data modalities and models.\n \"\"\"\n\n def __init__(\n self,\n data: pd.DataFrame,\n preprocessor: List[MultiModalFeaturePreprocessor],\n processors: List[dict],\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n is_training: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n data\n A pd.DataFrame containing multimodal features.\n preprocessor\n A list of multimodal feature preprocessors generating model-agnostic features.\n processors\n Data processors customizing data for each modality per model.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n is_training\n Whether in training mode. Some data processing may be different between training\n and validation/testing/prediction, e.g., image data augmentation is used only in\n training.\n \"\"\"\n super().__init__()\n self.processors = processors\n self.is_training = is_training\n self._consecutive_errors = 0\n\n self.lengths = []\n for i, (per_preprocessor, per_processors_group) in enumerate(zip(preprocessor, processors)):\n modality_features, modality_types, length = apply_df_preprocessor(\n data=data,\n df_preprocessor=per_preprocessor,\n modalities=per_processors_group.keys(),\n )\n self.lengths.append(length)\n setattr(self, f\"modality_features_{i}\", modality_features)\n setattr(self, f\"modality_types_{i}\", modality_types)\n\n assert len(set(self.lengths)) == 1\n\n self.id_mappings = id_mappings\n\n def __len__(self):\n \"\"\"\n Assume that all modalities have the same sample number.\n\n Returns\n -------\n Sample number in this dataset.\n \"\"\"\n return self.lengths[0]\n\n def __getitem__(self, idx):\n \"\"\"\n Iterate through all data processors to prepare model inputs. The data processors are\n organized first by modalities and then by models.\n\n Parameters\n ----------\n idx\n Index of sample to process.\n\n Returns\n -------\n Input data formatted as a dictionary.\n \"\"\"\n ret = dict()\n try:\n for group_id, per_processors_group in enumerate(self.processors):\n per_sample_features = get_per_sample_features(\n modality_features=getattr(self, f\"modality_features_{group_id}\"),\n modality_types=getattr(self, f\"modality_types_{group_id}\"),\n idx=idx,\n id_mappings=self.id_mappings,\n )\n per_ret = apply_data_processor(\n per_sample_features=per_sample_features,\n data_processors=per_processors_group,\n data_types=getattr(self, f\"modality_types_{group_id}\"),\n is_training=self.is_training,\n )\n ret.update(per_ret)\n except Exception as e:\n logger.debug(f\"Skipping sample {idx} due to '{e}'\")\n self._consecutive_errors += 1\n if self._consecutive_errors < GET_ITEM_ERROR_RETRY:\n return self.__getitem__((idx + 1) % self.__len__())\n else:\n raise e\n self._consecutive_errors = 0\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/dataset_mmlab/__init__.py",
"content": "try:\n from .multi_image_mix_dataset import MultiImageMixDataset\nexcept ImportError:\n MultiImageMixDataset = None\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/dataset_mmlab/multi_image_mix_dataset.py",
"content": "import collections\nimport copy\nimport logging\nimport math\nfrom typing import Dict, List, Optional, Sequence, Tuple, Union\n\nimport mmcv\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom mmcv.transforms import BaseTransform\nfrom mmcv.transforms.utils import cache_randomness\nfrom mmdet.structures.bbox import autocast_box_type\nfrom mmdet.utils import log_img_scale\nfrom mmengine.config import Config as MMConfig\nfrom mmengine.dataset import BaseDataset\nfrom numpy import random\n\nfrom ...constants import GET_ITEM_ERROR_RETRY, MULTI_IMAGE_MIX_DATASET, ROIS\nfrom ..preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom ..utils import apply_data_processor, apply_df_preprocessor, get_per_sample_features\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultiImageMixDataset(torch.utils.data.Dataset):\n \"\"\"\n A Pytorch DataSet class to process a multimodal pd.DataFrame. It first uses a preprocessor to\n produce model-agnostic features. Then, each processor prepares customized data for one modality\n per model. For code simplicity, here we treat ground-truth label as one modality. This class is\n independent of specific data modalities and models.\n \"\"\"\n\n def __init__(\n self,\n data: pd.DataFrame,\n preprocessor: List[MultiModalFeaturePreprocessor],\n processors: List[dict],\n model_config: MMConfig,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n is_training: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n data\n A pd.DataFrame containing multimodal features.\n preprocessor\n A list of multimodal feature preprocessors generating model-agnostic features.\n processors\n Data processors customizing data for each modality per model.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n is_training\n Whether in training mode. Some data processing may be different between training\n and validation/testing/prediction, e.g., image data augmentation is used only in\n training.\n model_config\n Model config used to decided dataset type. e.g. if multi_image_mix_dataset is used in detection model,\n MultiImageMixDataset will be used instead of BaseDataset\n \"\"\"\n super().__init__()\n self.processors = processors\n self.is_training = is_training\n self._consecutive_errors = 0\n\n mix_config = model_config[MULTI_IMAGE_MIX_DATASET]\n self.mix_data_key = \"mmdet_image_image\" # the key of the data to mix, TODO: remove hardcoding\n self.mix_result_key = \"mix_results\" # the key of the mix result to store\n\n self.mix_transforms = []\n self.mix_transforms_types = [] # TODO: remove hardcode\n if \"mosaic\" in mix_config:\n self.mix_transforms.append(Mosaic(**mix_config[\"mosaic\"]))\n self.mix_transforms_types.append(\"mosaic\")\n if \"mixup\" in mix_config:\n self.mix_transforms.append(MixUp(**mix_config[\"mixup\"]))\n self.mix_transforms_types.append(\"mixup\")\n\n self._skip_type_keys = None # TODO: remove hardcode, we need to disable multi image mix in late epochs\n self.max_refetch = 15 # TODO: remove hardcode (do we need refetch?)\n\n self.lengths = []\n\n for i, (per_preprocessor, per_processors_group) in enumerate(zip(preprocessor, processors)):\n modality_features, modality_types, length = apply_df_preprocessor(\n data=data,\n df_preprocessor=per_preprocessor,\n modalities=per_processors_group.keys(),\n )\n self.lengths.append(length)\n setattr(self, f\"modality_features_{i}\", modality_features)\n setattr(self, f\"modality_types_{i}\", modality_types)\n\n assert len(set(self.lengths)) == 1\n\n self.id_mappings = id_mappings\n\n def __len__(self):\n \"\"\"\n Assume that all modalities have the same sample number.\n\n Returns\n -------\n Sample number in this dataset.\n \"\"\"\n return self.lengths[0]\n\n def _load_item(self, idx):\n \"\"\"\n Get a single item without mix_results.\n Iterate through all data processors to prepare model inputs. The data processors are\n organized first by modalities and then by models.\n\n Parameters\n ----------\n idx\n Index of sample to process.\n\n Returns\n -------\n Input data formatted as a dictionary.\n \"\"\"\n ret = dict()\n try:\n for group_id, per_processors_group in enumerate(self.processors):\n per_sample_features = get_per_sample_features(\n modality_features=getattr(self, f\"modality_features_{group_id}\"),\n modality_types=getattr(self, f\"modality_types_{group_id}\"),\n idx=idx,\n id_mappings=self.id_mappings,\n )\n per_ret = apply_data_processor(\n per_sample_features=per_sample_features,\n data_processors=per_processors_group,\n data_types=getattr(self, f\"modality_types_{group_id}\"),\n is_training=self.is_training,\n load_only=True,\n )\n ret.update(per_ret)\n except Exception as e:\n logger.debug(f\"Skipping sample {idx} due to '{e}'\")\n self._consecutive_errors += 1\n if self._consecutive_errors < GET_ITEM_ERROR_RETRY:\n return self.__getitem__((idx + 1) % self.__len__())\n else:\n raise e\n self._consecutive_errors = 0\n\n return ret\n\n def __getitem__(self, idx):\n \"\"\"\n Iterate through all data processors to prepare model inputs. The data processors are\n organized first by modalities and then by models.\n\n Parameters\n ----------\n idx\n Index of sample to process.\n\n Returns\n -------\n Input data formatted as a dictionary.\n \"\"\"\n results = copy.deepcopy(self._load_item(idx))\n\n for transform, transform_type in zip(self.mix_transforms, self.mix_transforms_types):\n assert hasattr(transform, \"get_indexes\")\n\n if self._skip_type_keys is not None and transform_type in self._skip_type_keys:\n continue\n\n for i in range(self.max_refetch):\n # Make sure the results passed the loading pipeline\n # of the original dataset is not None.\n indexes = transform.get_indexes(self)\n if not isinstance(indexes, collections.abc.Sequence):\n indexes = [indexes]\n mix_results = [copy.deepcopy(self._load_item(index)[self.mix_data_key]) for index in indexes]\n if None not in mix_results:\n results[self.mix_data_key][self.mix_result_key] = mix_results\n break\n else:\n raise RuntimeError(\n \"The loading pipeline of the original dataset\"\n \" always return None. Please check the correctness \"\n \"of the dataset and its pipeline.\"\n )\n\n for i in range(self.max_refetch):\n # To confirm the results passed the training pipeline\n # of the wrapper is not None.\n updated_results = transform(copy.deepcopy(results[self.mix_data_key]))\n if updated_results is not None:\n results[self.mix_data_key] = updated_results\n break\n else:\n raise RuntimeError(\n \"The training pipeline of the dataset wrapper\"\n \" always return None.Please check the correctness \"\n \"of the dataset and its pipeline.\"\n )\n\n if self.mix_result_key in results[self.mix_data_key]:\n results[self.mix_data_key].pop(self.mix_result_key)\n\n rois_processor = self.processors[0][ROIS][0] # TODO: remove hardcode\n results.update(\n rois_processor.process_one_loaded_sample(\n results,\n is_training=True, # This dataset is used only in training\n )\n )\n\n return results\n\n\nclass Mosaic(BaseTransform):\n \"\"\"Mosaic augmentation.\n\n Given 4 images, mosaic transform combines them into\n one output image. The output image is composed of the parts from each sub-\n image.\n\n .. code:: text\n\n mosaic transform\n center_x\n +------------------------------+\n | pad | pad |\n | +-----------+ |\n | | | |\n | | image1 |--------+ |\n | | | | |\n | | | image2 | |\n center_y |----+-------------+-----------|\n | | cropped | |\n |pad | image3 | image4 |\n | | | |\n +----|-------------+-----------+\n | |\n +-------------+\n\n The mosaic transform steps are as follows:\n\n 1. Choose the mosaic center as the intersections of 4 images\n 2. Get the left top image according to the index, and randomly\n sample another 3 images from the custom dataset.\n 3. Sub image will be cropped if image is larger than mosaic patch\n\n Required Keys:\n\n - img\n - gt_bboxes (BaseBoxes[torch.float32]) (optional)\n - gt_bboxes_labels (np.int64) (optional)\n - gt_ignore_flags (bool) (optional)\n - mix_results (List[dict])\n\n Modified Keys:\n\n - img\n - img_shape\n - gt_bboxes (optional)\n - gt_bboxes_labels (optional)\n - gt_ignore_flags (optional)\n\n Args:\n img_scale (Sequence[int]): Image size after mosaic pipeline of single\n image. The shape order should be (width, height).\n Defaults to (640, 640).\n center_ratio_range (Sequence[float]): Center ratio range of mosaic\n output. Defaults to (0.5, 1.5).\n bbox_clip_border (bool, optional): Whether to clip the objects outside\n the border of the image. In some dataset like MOT17, the gt bboxes\n are allowed to cross the border of images. Therefore, we don't\n need to clip the gt bboxes in these cases. Defaults to True.\n pad_val (int): Pad value. Defaults to 114.\n prob (float): Probability of applying this transformation.\n Defaults to 1.0.\n \"\"\"\n\n def __init__(\n self,\n img_scale: Tuple[int, int] = (640, 640),\n center_ratio_range: Tuple[float, float] = (0.5, 1.5),\n bbox_clip_border: bool = True,\n pad_val: float = 114.0,\n prob: float = 1.0,\n ) -> None:\n assert isinstance(img_scale, tuple)\n assert 0 <= prob <= 1.0, f\"The probability should be in range [0,1]. got {prob}.\"\n\n log_img_scale(img_scale, skip_square=True, shape_order=\"wh\")\n self.img_scale = img_scale\n self.center_ratio_range = center_ratio_range\n self.bbox_clip_border = bbox_clip_border\n self.pad_val = pad_val\n self.prob = prob\n\n @cache_randomness\n def get_indexes(self, dataset: BaseDataset) -> int:\n \"\"\"Call function to collect indexes.\n\n Args:\n dataset (:obj:`MultiImageMixDataset`): The dataset.\n\n Returns:\n list: indexes.\n \"\"\"\n\n indexes = [random.randint(0, len(dataset)) for _ in range(3)]\n return indexes\n\n @autocast_box_type()\n def transform(self, results: dict) -> dict:\n \"\"\"Mosaic transform function.\n\n Args:\n results (dict): Result dict.\n\n Returns:\n dict: Updated result dict.\n \"\"\"\n if random.uniform(0, 1) > self.prob:\n return results\n\n assert \"mix_results\" in results\n mosaic_bboxes = []\n mosaic_bboxes_labels = []\n mosaic_ignore_flags = []\n if len(results[\"img\"].shape) == 3:\n mosaic_img = np.full(\n (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2), 3),\n self.pad_val,\n dtype=results[\"img\"].dtype,\n )\n else:\n mosaic_img = np.full(\n (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2)), self.pad_val, dtype=results[\"img\"].dtype\n )\n\n # mosaic center x, y\n center_x = int(random.uniform(*self.center_ratio_range) * self.img_scale[0])\n center_y = int(random.uniform(*self.center_ratio_range) * self.img_scale[1])\n center_position = (center_x, center_y)\n\n loc_strs = (\"top_left\", \"top_right\", \"bottom_left\", \"bottom_right\")\n for i, loc in enumerate(loc_strs):\n if loc == \"top_left\":\n results_patch = copy.deepcopy(results)\n else:\n results_patch = copy.deepcopy(results[\"mix_results\"][i - 1])\n\n img_i = results_patch[\"img\"]\n h_i, w_i = img_i.shape[:2]\n # keep_ratio resize\n scale_ratio_i = min(self.img_scale[1] / h_i, self.img_scale[0] / w_i)\n img_i = mmcv.imresize(img_i, (int(w_i * scale_ratio_i), int(h_i * scale_ratio_i)))\n\n # compute the combine parameters\n paste_coord, crop_coord = self._mosaic_combine(loc, center_position, img_i.shape[:2][::-1])\n x1_p, y1_p, x2_p, y2_p = paste_coord\n x1_c, y1_c, x2_c, y2_c = crop_coord\n\n # crop and paste image\n mosaic_img[y1_p:y2_p, x1_p:x2_p] = img_i[y1_c:y2_c, x1_c:x2_c]\n\n # adjust coordinate\n gt_bboxes_i = results_patch[\"gt_bboxes\"]\n gt_bboxes_labels_i = results_patch[\"gt_bboxes_labels\"]\n gt_ignore_flags_i = results_patch[\"gt_ignore_flags\"]\n\n padw = x1_p - x1_c\n padh = y1_p - y1_c\n gt_bboxes_i.rescale_([scale_ratio_i, scale_ratio_i])\n gt_bboxes_i.translate_([padw, padh])\n mosaic_bboxes.append(gt_bboxes_i)\n mosaic_bboxes_labels.append(gt_bboxes_labels_i)\n mosaic_ignore_flags.append(gt_ignore_flags_i)\n\n mosaic_bboxes = mosaic_bboxes[0].cat(mosaic_bboxes, 0)\n mosaic_bboxes_labels = np.concatenate(mosaic_bboxes_labels, 0)\n mosaic_ignore_flags = np.concatenate(mosaic_ignore_flags, 0)\n\n if self.bbox_clip_border:\n mosaic_bboxes.clip_([2 * self.img_scale[1], 2 * self.img_scale[0]])\n # remove outside bboxes\n inside_inds = mosaic_bboxes.is_inside([2 * self.img_scale[1], 2 * self.img_scale[0]]).numpy()\n mosaic_bboxes = mosaic_bboxes[inside_inds]\n mosaic_bboxes_labels = mosaic_bboxes_labels[inside_inds]\n mosaic_ignore_flags = mosaic_ignore_flags[inside_inds]\n\n results[\"img\"] = mosaic_img\n results[\"img_shape\"] = mosaic_img.shape[:2]\n results[\"gt_bboxes\"] = mosaic_bboxes\n results[\"gt_bboxes_labels\"] = mosaic_bboxes_labels\n results[\"gt_ignore_flags\"] = mosaic_ignore_flags\n return results\n\n def _mosaic_combine(\n self, loc: str, center_position_xy: Sequence[float], img_shape_wh: Sequence[int]\n ) -> Tuple[Tuple[int], Tuple[int]]:\n \"\"\"Calculate global coordinate of mosaic image and local coordinate of\n cropped sub-image.\n\n Args:\n loc (str): Index for the sub-image, loc in ('top_left',\n 'top_right', 'bottom_left', 'bottom_right').\n center_position_xy (Sequence[float]): Mixing center for 4 images,\n (x, y).\n img_shape_wh (Sequence[int]): Width and height of sub-image\n\n Returns:\n tuple[tuple[float]]: Corresponding coordinate of pasting and\n cropping\n - paste_coord (tuple): paste corner coordinate in mosaic image.\n - crop_coord (tuple): crop corner coordinate in mosaic image.\n \"\"\"\n assert loc in (\"top_left\", \"top_right\", \"bottom_left\", \"bottom_right\")\n if loc == \"top_left\":\n # index0 to top left part of image\n x1, y1, x2, y2 = (\n max(center_position_xy[0] - img_shape_wh[0], 0),\n max(center_position_xy[1] - img_shape_wh[1], 0),\n center_position_xy[0],\n center_position_xy[1],\n )\n crop_coord = img_shape_wh[0] - (x2 - x1), img_shape_wh[1] - (y2 - y1), img_shape_wh[0], img_shape_wh[1]\n\n elif loc == \"top_right\":\n # index1 to top right part of image\n x1, y1, x2, y2 = (\n center_position_xy[0],\n max(center_position_xy[1] - img_shape_wh[1], 0),\n min(center_position_xy[0] + img_shape_wh[0], self.img_scale[0] * 2),\n center_position_xy[1],\n )\n crop_coord = 0, img_shape_wh[1] - (y2 - y1), min(img_shape_wh[0], x2 - x1), img_shape_wh[1]\n\n elif loc == \"bottom_left\":\n # index2 to bottom left part of image\n x1, y1, x2, y2 = (\n max(center_position_xy[0] - img_shape_wh[0], 0),\n center_position_xy[1],\n center_position_xy[0],\n min(self.img_scale[1] * 2, center_position_xy[1] + img_shape_wh[1]),\n )\n crop_coord = img_shape_wh[0] - (x2 - x1), 0, img_shape_wh[0], min(y2 - y1, img_shape_wh[1])\n\n else:\n # index3 to bottom right part of image\n x1, y1, x2, y2 = (\n center_position_xy[0],\n center_position_xy[1],\n min(center_position_xy[0] + img_shape_wh[0], self.img_scale[0] * 2),\n min(self.img_scale[1] * 2, center_position_xy[1] + img_shape_wh[1]),\n )\n crop_coord = 0, 0, min(img_shape_wh[0], x2 - x1), min(y2 - y1, img_shape_wh[1])\n\n paste_coord = x1, y1, x2, y2\n return paste_coord, crop_coord\n\n def __repr__(self):\n repr_str = self.__class__.__name__\n repr_str += f\"(img_scale={self.img_scale}, \"\n repr_str += f\"center_ratio_range={self.center_ratio_range}, \"\n repr_str += f\"pad_val={self.pad_val}, \"\n repr_str += f\"prob={self.prob})\"\n return repr_str\n\n\nclass MixUp(BaseTransform):\n \"\"\"MixUp data augmentation.\n\n .. code:: text\n\n mixup transform\n +------------------------------+\n | mixup image | |\n | +--------|--------+ |\n | | | | |\n |---------------+ | |\n | | | |\n | | image | |\n | | | |\n | | | |\n | |-----------------+ |\n | pad |\n +------------------------------+\n\n The mixup transform steps are as follows:\n\n 1. Another random image is picked by dataset and embedded in\n the top left patch(after padding and resizing)\n 2. The target of mixup transform is the weighted average of mixup\n image and origin image.\n\n Required Keys:\n\n - img\n - gt_bboxes (BaseBoxes[torch.float32]) (optional)\n - gt_bboxes_labels (np.int64) (optional)\n - gt_ignore_flags (bool) (optional)\n - mix_results (List[dict])\n\n\n Modified Keys:\n\n - img\n - img_shape\n - gt_bboxes (optional)\n - gt_bboxes_labels (optional)\n - gt_ignore_flags (optional)\n\n\n Args:\n img_scale (Sequence[int]): Image output size after mixup pipeline.\n The shape order should be (width, height). Defaults to (640, 640).\n ratio_range (Sequence[float]): Scale ratio of mixup image.\n Defaults to (0.5, 1.5).\n flip_ratio (float): Horizontal flip ratio of mixup image.\n Defaults to 0.5.\n pad_val (int): Pad value. Defaults to 114.\n max_iters (int): The maximum number of iterations. If the number of\n iterations is greater than `max_iters`, but gt_bbox is still\n empty, then the iteration is terminated. Defaults to 15.\n bbox_clip_border (bool, optional): Whether to clip the objects outside\n the border of the image. In some dataset like MOT17, the gt bboxes\n are allowed to cross the border of images. Therefore, we don't\n need to clip the gt bboxes in these cases. Defaults to True.\n \"\"\"\n\n def __init__(\n self,\n img_scale: Tuple[int, int] = (640, 640),\n ratio_range: Tuple[float, float] = (0.5, 1.5),\n flip_ratio: float = 0.5,\n pad_val: float = 114.0,\n max_iters: int = 15,\n bbox_clip_border: bool = True,\n ) -> None:\n assert isinstance(img_scale, tuple)\n log_img_scale(img_scale, skip_square=True, shape_order=\"wh\")\n self.dynamic_scale = img_scale\n self.ratio_range = ratio_range\n self.flip_ratio = flip_ratio\n self.pad_val = pad_val\n self.max_iters = max_iters\n self.bbox_clip_border = bbox_clip_border\n\n @cache_randomness\n def get_indexes(self, dataset: BaseDataset) -> int:\n \"\"\"Call function to collect indexes.\n\n Args:\n dataset (:obj:`MultiImageMixDataset`): The dataset.\n\n Returns:\n list: indexes.\n \"\"\"\n\n index = [np.random.randint(0, len(dataset)) for _ in range(1)]\n\n return index\n\n @autocast_box_type()\n def transform(self, results: dict) -> dict:\n \"\"\"MixUp transform function.\n\n Args:\n results (dict): Result dict.\n\n Returns:\n dict: Updated result dict.\n \"\"\"\n\n assert \"mix_results\" in results\n assert len(results[\"mix_results\"]) == 1, \"MixUp only support 2 images now !\"\n\n if results[\"mix_results\"][0][\"gt_bboxes\"].shape[0] == 0:\n # empty bbox\n return results\n\n retrieve_results = results[\"mix_results\"][0]\n retrieve_img = retrieve_results[\"img\"]\n\n jit_factor = random.uniform(*self.ratio_range)\n is_filp = random.uniform(0, 1) > self.flip_ratio\n\n if len(retrieve_img.shape) == 3:\n out_img = (\n np.ones((self.dynamic_scale[1], self.dynamic_scale[0], 3), dtype=retrieve_img.dtype) * self.pad_val\n )\n else:\n out_img = np.ones(self.dynamic_scale[::-1], dtype=retrieve_img.dtype) * self.pad_val\n\n # 1. keep_ratio resize\n scale_ratio = min(self.dynamic_scale[1] / retrieve_img.shape[0], self.dynamic_scale[0] / retrieve_img.shape[1])\n retrieve_img = mmcv.imresize(\n retrieve_img, (int(retrieve_img.shape[1] * scale_ratio), int(retrieve_img.shape[0] * scale_ratio))\n )\n\n # 2. paste\n out_img[: retrieve_img.shape[0], : retrieve_img.shape[1]] = retrieve_img\n\n # 3. scale jit\n scale_ratio *= jit_factor\n out_img = mmcv.imresize(out_img, (int(out_img.shape[1] * jit_factor), int(out_img.shape[0] * jit_factor)))\n\n # 4. flip\n if is_filp:\n out_img = out_img[:, ::-1, :]\n\n # 5. random crop\n ori_img = results[\"img\"]\n origin_h, origin_w = out_img.shape[:2]\n target_h, target_w = ori_img.shape[:2]\n padded_img = np.ones((max(origin_h, target_h), max(origin_w, target_w), 3)) * self.pad_val\n padded_img = padded_img.astype(np.uint8)\n padded_img[:origin_h, :origin_w] = out_img\n\n x_offset, y_offset = 0, 0\n if padded_img.shape[0] > target_h:\n y_offset = random.randint(0, padded_img.shape[0] - target_h)\n if padded_img.shape[1] > target_w:\n x_offset = random.randint(0, padded_img.shape[1] - target_w)\n padded_cropped_img = padded_img[y_offset : y_offset + target_h, x_offset : x_offset + target_w]\n\n # 6. adjust bbox\n retrieve_gt_bboxes = retrieve_results[\"gt_bboxes\"]\n retrieve_gt_bboxes.rescale_([scale_ratio, scale_ratio])\n if self.bbox_clip_border:\n retrieve_gt_bboxes.clip_([origin_h, origin_w])\n\n if is_filp:\n retrieve_gt_bboxes.flip_([origin_h, origin_w], direction=\"horizontal\")\n\n # 7. filter\n cp_retrieve_gt_bboxes = retrieve_gt_bboxes.clone()\n cp_retrieve_gt_bboxes.translate_([-x_offset, -y_offset])\n if self.bbox_clip_border:\n cp_retrieve_gt_bboxes.clip_([target_h, target_w])\n\n # 8. mix up\n ori_img = ori_img.astype(np.float32)\n mixup_img = 0.5 * ori_img + 0.5 * padded_cropped_img.astype(np.float32)\n\n retrieve_gt_bboxes_labels = retrieve_results[\"gt_bboxes_labels\"]\n retrieve_gt_ignore_flags = retrieve_results[\"gt_ignore_flags\"]\n\n mixup_gt_bboxes = cp_retrieve_gt_bboxes.cat((results[\"gt_bboxes\"], cp_retrieve_gt_bboxes), dim=0)\n mixup_gt_bboxes_labels = np.concatenate((results[\"gt_bboxes_labels\"], retrieve_gt_bboxes_labels), axis=0)\n mixup_gt_ignore_flags = np.concatenate((results[\"gt_ignore_flags\"], retrieve_gt_ignore_flags), axis=0)\n\n # remove outside bbox\n inside_inds = mixup_gt_bboxes.is_inside([target_h, target_w]).numpy()\n mixup_gt_bboxes = mixup_gt_bboxes[inside_inds]\n mixup_gt_bboxes_labels = mixup_gt_bboxes_labels[inside_inds]\n mixup_gt_ignore_flags = mixup_gt_ignore_flags[inside_inds]\n\n results[\"img\"] = mixup_img.astype(np.uint8)\n results[\"img_shape\"] = mixup_img.shape[:2]\n results[\"gt_bboxes\"] = mixup_gt_bboxes\n results[\"gt_bboxes_labels\"] = mixup_gt_bboxes_labels\n results[\"gt_ignore_flags\"] = mixup_gt_ignore_flags\n\n return results\n\n def __repr__(self):\n repr_str = self.__class__.__name__\n repr_str += f\"(dynamic_scale={self.dynamic_scale}, \"\n repr_str += f\"ratio_range={self.ratio_range}, \"\n repr_str += f\"flip_ratio={self.flip_ratio}, \"\n repr_str += f\"pad_val={self.pad_val}, \"\n repr_str += f\"max_iters={self.max_iters}, \"\n repr_str += f\"bbox_clip_border={self.bbox_clip_border})\"\n return repr_str\n\n\nclass RandomAffine(BaseTransform):\n \"\"\"Random affine transform data augmentation.\n\n This operation randomly generates affine transform matrix which including\n rotation, translation, shear and scaling transforms.\n\n Required Keys:\n\n - img\n - gt_bboxes (BaseBoxes[torch.float32]) (optional)\n - gt_bboxes_labels (np.int64) (optional)\n - gt_ignore_flags (bool) (optional)\n\n Modified Keys:\n\n - img\n - img_shape\n - gt_bboxes (optional)\n - gt_bboxes_labels (optional)\n - gt_ignore_flags (optional)\n\n Args:\n max_rotate_degree (float): Maximum degrees of rotation transform.\n Defaults to 10.\n max_translate_ratio (float): Maximum ratio of translation.\n Defaults to 0.1.\n scaling_ratio_range (tuple[float]): Min and max ratio of\n scaling transform. Defaults to (0.5, 1.5).\n max_shear_degree (float): Maximum degrees of shear\n transform. Defaults to 2.\n border (tuple[int]): Distance from width and height sides of input\n image to adjust output shape. Only used in mosaic dataset.\n Defaults to (0, 0).\n border_val (tuple[int]): Border padding values of 3 channels.\n Defaults to (114, 114, 114).\n bbox_clip_border (bool, optional): Whether to clip the objects outside\n the border of the image. In some dataset like MOT17, the gt bboxes\n are allowed to cross the border of images. Therefore, we don't\n need to clip the gt bboxes in these cases. Defaults to True.\n \"\"\"\n\n def __init__(\n self,\n max_rotate_degree: float = 10.0,\n max_translate_ratio: float = 0.1,\n scaling_ratio_range: Tuple[float, float] = (0.5, 1.5),\n max_shear_degree: float = 2.0,\n border: Tuple[int, int] = (0, 0),\n border_val: Tuple[int, int, int] = (114, 114, 114),\n bbox_clip_border: bool = True,\n ) -> None:\n assert 0 <= max_translate_ratio <= 1\n assert scaling_ratio_range[0] <= scaling_ratio_range[1]\n assert scaling_ratio_range[0] > 0\n self.max_rotate_degree = max_rotate_degree\n self.max_translate_ratio = max_translate_ratio\n self.scaling_ratio_range = scaling_ratio_range\n self.max_shear_degree = max_shear_degree\n self.border = border\n self.border_val = border_val\n self.bbox_clip_border = bbox_clip_border\n\n @cache_randomness\n def _get_random_homography_matrix(self, height, width):\n # Rotation\n rotation_degree = random.uniform(-self.max_rotate_degree, self.max_rotate_degree)\n rotation_matrix = self._get_rotation_matrix(rotation_degree)\n\n # Scaling\n scaling_ratio = random.uniform(self.scaling_ratio_range[0], self.scaling_ratio_range[1])\n scaling_matrix = self._get_scaling_matrix(scaling_ratio)\n\n # Shear\n x_degree = random.uniform(-self.max_shear_degree, self.max_shear_degree)\n y_degree = random.uniform(-self.max_shear_degree, self.max_shear_degree)\n shear_matrix = self._get_shear_matrix(x_degree, y_degree)\n\n # Translation\n trans_x = random.uniform(-self.max_translate_ratio, self.max_translate_ratio) * width\n trans_y = random.uniform(-self.max_translate_ratio, self.max_translate_ratio) * height\n translate_matrix = self._get_translation_matrix(trans_x, trans_y)\n\n warp_matrix = translate_matrix @ shear_matrix @ rotation_matrix @ scaling_matrix\n return warp_matrix\n\n @autocast_box_type()\n def transform(self, results: dict) -> dict:\n import cv2 # TODO: support random affine requires cv2\n\n img = results[\"img\"]\n height = img.shape[0] + self.border[1] * 2\n width = img.shape[1] + self.border[0] * 2\n\n warp_matrix = self._get_random_homography_matrix(height, width)\n\n img = cv2.warpPerspective(img, warp_matrix, dsize=(width, height), borderValue=self.border_val)\n results[\"img\"] = img\n results[\"img_shape\"] = img.shape[:2]\n\n bboxes = results[\"gt_bboxes\"]\n num_bboxes = len(bboxes)\n if num_bboxes:\n bboxes.project_(warp_matrix)\n if self.bbox_clip_border:\n bboxes.clip_([height, width])\n # remove outside bbox\n valid_index = bboxes.is_inside([height, width]).numpy()\n results[\"gt_bboxes\"] = bboxes[valid_index]\n results[\"gt_bboxes_labels\"] = results[\"gt_bboxes_labels\"][valid_index]\n results[\"gt_ignore_flags\"] = results[\"gt_ignore_flags\"][valid_index]\n\n if \"gt_masks\" in results:\n raise NotImplementedError(\"RandomAffine only supports bbox.\")\n return results\n\n def __repr__(self):\n repr_str = self.__class__.__name__\n repr_str += f\"(max_rotate_degree={self.max_rotate_degree}, \"\n repr_str += f\"max_translate_ratio={self.max_translate_ratio}, \"\n repr_str += f\"scaling_ratio_range={self.scaling_ratio_range}, \"\n repr_str += f\"max_shear_degree={self.max_shear_degree}, \"\n repr_str += f\"border={self.border}, \"\n repr_str += f\"border_val={self.border_val}, \"\n repr_str += f\"bbox_clip_border={self.bbox_clip_border})\"\n return repr_str\n\n @staticmethod\n def _get_rotation_matrix(rotate_degrees: float) -> np.ndarray:\n radian = math.radians(rotate_degrees)\n rotation_matrix = np.array(\n [[np.cos(radian), -np.sin(radian), 0.0], [np.sin(radian), np.cos(radian), 0.0], [0.0, 0.0, 1.0]],\n dtype=np.float32,\n )\n return rotation_matrix\n\n @staticmethod\n def _get_scaling_matrix(scale_ratio: float) -> np.ndarray:\n scaling_matrix = np.array(\n [[scale_ratio, 0.0, 0.0], [0.0, scale_ratio, 0.0], [0.0, 0.0, 1.0]], dtype=np.float32\n )\n return scaling_matrix\n\n @staticmethod\n def _get_shear_matrix(x_shear_degrees: float, y_shear_degrees: float) -> np.ndarray:\n x_radian = math.radians(x_shear_degrees)\n y_radian = math.radians(y_shear_degrees)\n shear_matrix = np.array(\n [[1, np.tan(x_radian), 0.0], [np.tan(y_radian), 1, 0.0], [0.0, 0.0, 1.0]], dtype=np.float32\n )\n return shear_matrix\n\n @staticmethod\n def _get_translation_matrix(x: float, y: float) -> np.ndarray:\n translation_matrix = np.array([[1, 0.0, x], [0.0, 1, y], [0.0, 0.0, 1.0]], dtype=np.float32)\n return translation_matrix\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/infer_types.py",
"content": "import base64\nimport collections\nimport json\nimport logging\nimport warnings\nfrom io import BytesIO\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport pandas as pd\nimport PIL\nimport pytesseract\n\nfrom autogluon.core.utils import infer_problem_type as infer_basic_problem_type\n\nfrom ..constants import (\n BINARY,\n CATEGORICAL,\n CLASSIFICATION,\n DOCUMENT_IMAGE,\n DOCUMENT_PDF,\n IDENTIFIER,\n IMAGE_BASE64_STR,\n IMAGE_BYTEARRAY,\n IMAGE_PATH,\n MULTICLASS,\n NER,\n NER_ANNOTATION,\n NULL,\n NUMERICAL,\n OBJECT_DETECTION,\n REGRESSION,\n ROIS,\n SEMANTIC_SEGMENTATION,\n SEMANTIC_SEGMENTATION_GT,\n SEMANTIC_SEGMENTATION_IMG,\n TEXT,\n TEXT_NER,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef is_categorical_column(\n data: pd.Series,\n valid_data: pd.Series,\n threshold: int = None,\n ratio: Optional[float] = None,\n oov_ratio_threshold: Optional[float] = None,\n is_label: bool = False,\n) -> bool:\n \"\"\"\n Identify whether a column is one categorical column.\n If the number of unique elements in the column is smaller than\n\n min(#Total Sample * ratio, threshold),\n\n it will be treated as a categorical column.\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n valid_data\n One column of a multimodal pd.DataFrame for validation.\n threshold\n The threshold for detecting categorical column.\n ratio\n The ratio detecting categorical column.\n oov_ratio_threshold\n The out-of-vocabulary ratio between training and validation.\n This is used to determine if the column is a categorical column.\n Usually, a categorical column can tolerate a small OOV ratio.\n is_label\n Whether the column is a label column.\n\n Returns\n -------\n Whether the column is a categorical column.\n \"\"\"\n if data.dtype.name == \"category\":\n return True\n else:\n if threshold is None:\n if is_label:\n # TODO(?) The following logic will be problematic if the task is few-shot learning.\n threshold = 100\n oov_ratio_threshold = 0\n ratio = 0.1\n else:\n threshold = 20\n oov_ratio_threshold = 0\n ratio = 0.1\n threshold = min(int(len(data) * ratio), threshold)\n try:\n data_value_counts = data.value_counts(dropna=False)\n key_set = set(data_value_counts.keys())\n if len(data_value_counts) < threshold:\n valid_value_counts = valid_data.value_counts(dropna=False)\n total_valid_num = len(valid_data)\n oov_num = 0\n for k, v in zip(valid_value_counts.keys(), valid_value_counts.values):\n if k not in key_set:\n oov_num += v\n if is_label and oov_num != 0:\n return False\n if oov_num / total_valid_num > oov_ratio_threshold:\n return False\n return True\n else:\n return False\n except:\n return False\n\n\ndef is_rois_input(sample):\n \"\"\"\n check if a sample is rois for object detection\n\n Parameters\n ----------\n sample\n The sampled data.\n\n Returns\n -------\n bool, whether a sample is rois for object detection\n \"\"\"\n return isinstance(sample, list) and len(sample) and isinstance(sample[0], list) and len(sample[0]) == 5\n\n\ndef is_rois_column(data: pd.Series) -> bool:\n \"\"\"\n Identify if a column is one rois column.\n\n Parameters\n ----------\n X\n One column of a multimodal pd.DataFrame for training.\n\n Returns\n -------\n Whether the column is a rois column.\n \"\"\"\n idx = data.first_valid_index()\n if isinstance(data[idx], str):\n try:\n rois = json.loads(data[idx][0])\n # TODO: better infer logic / input format to not confuse with other modality\n return rois and isinstance(rois, list) and len(data[idx][0]) == 5\n except:\n return False\n else:\n return is_rois_input(data[idx])\n\n\ndef is_numerical_column(\n data: pd.Series,\n valid_data: Optional[pd.Series] = None,\n) -> bool:\n \"\"\"\n Identify if a column is a numerical column.\n Here it uses a very simple rule to verify if this is a numerical column.\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n valid_data\n One column of a multimodal pd.DataFrame for validation.\n\n Returns\n -------\n Whether the column is a numerical column.\n \"\"\"\n try:\n numerical_data = pd.to_numeric(data)\n if valid_data is not None:\n numerical_valid_data = pd.to_numeric(valid_data)\n return True\n except:\n return False\n\n\ndef is_image_column(\n data: pd.Series,\n col_name: str,\n image_type: str,\n sample_n: Optional[int] = 500,\n) -> bool:\n \"\"\"\n Identify if a column is one image-path column.\n Here it counts the failures when trying PIL.Image.open() on a sampled subset.\n If over 90% attempts fail, this column isn't an image-path column.\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n col_name\n Name of column.\n image_type\n The image type to check.\n sample_n\n Number of sample images to open for sanity check.\n\n Returns\n -------\n Whether the column is an image-path column.\n \"\"\"\n sample_num = min(len(data), sample_n)\n data = data.sample(n=sample_num, random_state=0)\n if image_type == IMAGE_PATH:\n data = data.apply(lambda ele: str(ele).split(\";\")).tolist()\n elif image_type in [IMAGE_BYTEARRAY, IMAGE_BASE64_STR]:\n data = data.tolist()\n else:\n raise ValueError(f\"Unsupported image type: {image_type}\")\n\n failure_count = 0\n for images in data:\n success = False\n if not isinstance(images, list):\n images = [images]\n for per_image in images:\n try:\n if image_type == IMAGE_PATH:\n with PIL.Image.open(per_image) as img:\n pass\n elif image_type == IMAGE_BYTEARRAY:\n with PIL.Image.open(BytesIO(per_image)) as img:\n pass\n elif image_type == IMAGE_BASE64_STR:\n with PIL.Image.open(BytesIO(base64.b64decode(per_image))) as img:\n pass\n else:\n raise ValueError(f\"Unsupported image type: {image_type}\")\n except:\n success = False\n break\n\n success = True\n if not success:\n failure_count += 1\n failure_ratio = failure_count / sample_num\n # Tolerate high failure rate in case that many image files may be corrupted.\n if failure_ratio <= 0.9:\n if failure_ratio > 0:\n warnings.warn(\n f\"Among {sample_num} sampled images in column '{col_name}', \"\n f\"{failure_ratio:.0%} images can't be open. \"\n \"You may need to thoroughly check your data to see the percentage of missing images, \"\n \"and estimate the potential influence. By default, we use an image with zero pixels. \"\n \"You can also set hyperparameter 'data.image.missing_value_strategy' to be 'skip', \"\n \"which skips samples that contain a missing image.\",\n UserWarning,\n )\n return True\n else:\n return False\n\n\ndef is_document_pdf_column(\n data: pd.Series,\n col_name: str,\n sample_m: Optional[int] = 500,\n) -> bool:\n \"\"\"\n Identify if a column is a pdf document column.\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n col_name\n Name of column.\n sample_m\n Number of sample images used to check if images are documents images.\n\n Returns\n -------\n Whether the column is a pdf document column.\n \"\"\"\n\n if len(data) > sample_m:\n # Sample to speed-up type inference\n data = data.sample(n=sample_m, random_state=0)\n\n for docs in data:\n try:\n if not isinstance(docs, list):\n docs = [docs]\n for per_doc in docs:\n # If there is non-pdf document, return False\n if not per_doc.endswith(\".pdf\"):\n return False\n except:\n return False\n\n return True\n\n\ndef is_document_image_column(\n data: pd.Series,\n col_name: str,\n image_type: Optional[str] = IMAGE_PATH,\n sample_m: Optional[int] = 10,\n min_text_len_threshold: Optional[int] = 200,\n text_density_threshold: Optional[float] = 0.001,\n min_line_count: Optional[int] = 3,\n min_document_ratio: Optional[float] = 0.8,\n) -> bool:\n \"\"\"\n Identify if a column is a document image column.\n\n Document images are images that primarily contain text (e.g., scanned documents,\n screenshots of text, PDFs rendered as images). Regular photographs, maps,\n charts with labels, or images with watermarks/captions should NOT be\n classified as document images.\n\n The detection uses multiple heuristics:\n 1. Minimum absolute text length (short text like watermarks is ignored)\n 2. Text density relative to image size (documents have high text-to-pixel ratio)\n 3. Line count (documents typically have multiple lines of text)\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n col_name\n Name of column.\n image_type\n The image type to check. Set to IMAGE_PATH by default.\n sample_m\n Number of sample images used to check if images are documents images.\n min_text_len_threshold\n Minimum text length to even consider an image as a potential document.\n This filters out images with just watermarks or short captions.\n text_density_threshold\n Minimum ratio of (text_characters / image_pixels) to consider as document.\n Documents typically have much higher text density than photos with labels.\n min_line_count\n Minimum number of non-empty text lines expected in a document.\n min_document_ratio\n Minimum ratio of images that must be classified as documents for the\n entire column to be treated as a document column.\n\n Returns\n -------\n Whether the column is a document image column.\n \"\"\"\n if data.empty:\n return False\n\n if len(data) > sample_m:\n data = data.sample(n=sample_m, random_state=0)\n\n document_count = 0\n total_processed = 0\n\n for images in data:\n if images is None:\n continue\n\n if not isinstance(images, list):\n images = [images]\n\n for per_image in images:\n if not isinstance(per_image, str):\n total_processed += 1\n continue\n\n try:\n with PIL.Image.open(per_image) as img:\n width, height = img.size\n total_pixels = width * height\n\n ocr_text = pytesseract.image_to_string(img)\n text_length = len(ocr_text.strip())\n\n total_processed += 1\n\n # Heuristic 1: Minimum absolute text length\n # Filters out watermarks, copyright notices, short captions\n if text_length < min_text_len_threshold:\n continue\n\n # Heuristic 2: Text density (characters per pixel)\n # Documents have dense text; photos with small labels don't\n text_density = text_length / total_pixels\n if text_density < text_density_threshold:\n continue\n\n # Heuristic 3: Line count\n # Documents have multiple lines; watermarks are 1-2 lines\n lines = [line for line in ocr_text.split(\"\\n\") if line.strip()]\n if len(lines) < min_line_count:\n continue\n\n # Passed all heuristics - this looks like a document\n document_count += 1\n\n except Exception as e:\n logger.debug(f\"Failed to process image {per_image}: {e}\")\n total_processed += 1\n\n if total_processed == 0:\n return False\n\n document_ratio = document_count / total_processed\n is_document_column = document_ratio >= min_document_ratio\n\n logger.debug(\n f\"Column '{col_name}': {document_count}/{total_processed} images \"\n f\"({document_ratio:.1%}) classified as documents. \"\n f\"Column type: {'document' if is_document_column else 'regular'} images.\"\n )\n\n return is_document_column\n\n\ndef is_text_column(data: pd.Series) -> bool:\n \"\"\"\n Identify if a column is one text column.\n\n Parameters\n ----------\n data\n One column of a multimodal pd.DataFrame for training.\n\n Returns\n -------\n Whether the column is a text column.\n \"\"\"\n if len(data) > 5000:\n # Sample to speed-up type inference\n data = data.sample(n=5000, random_state=0)\n try:\n data_unique = data.unique()\n num_unique = len(data_unique)\n num_rows = len(data)\n unique_ratio = num_unique / num_rows\n if unique_ratio <= 0.01:\n return False\n try:\n avg_words = pd.Series(data_unique).str.split().str.len().mean()\n except AttributeError:\n return False\n if avg_words < 3:\n return False\n except:\n return False\n return True\n\n\ndef is_identifier_column(data: pd.Series, col_name: str, id_mappings: Dict[str, Dict]) -> bool:\n \"\"\"\n Check if a column is one identifier column.\n\n Parameters\n ----------\n data\n One column of multimodal pd.DataFrame.\n col_name\n Name of the column.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n\n Returns\n -------\n Whether the column is an identifier column.\n \"\"\"\n if not id_mappings or col_name not in id_mappings:\n return False\n\n sample_num = min(len(data), 500)\n data = data.sample(n=sample_num, random_state=0).tolist()\n failure_count = 0\n for index in data:\n try:\n per_value = id_mappings[col_name][index]\n except:\n failure_count += 1\n\n if failure_count == 0:\n return True\n elif 0 < failure_count < sample_num:\n warnings.warn(\n f\"Among {sample_num} sampled indexes in column {col_name}, \"\n f\"we can't index all their values from the id_mappings ({failure_count} failures). \"\n f\"You may need to assure that all the indexes of column {col_name} exist in your id_mappings.\",\n UserWarning,\n )\n else:\n return False\n\n\ndef infer_id_mappings_types(id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]]) -> Dict:\n \"\"\"\n Infer the data types in id_mappings.\n\n Parameters\n ----------\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n\n Returns\n -------\n A dictionary containing the data types in id_mappings.\n \"\"\"\n id_mappings_types = collections.OrderedDict()\n if id_mappings is None:\n return id_mappings_types\n\n for per_name, per_id_mappings in id_mappings.items():\n if isinstance(per_id_mappings, dict):\n per_id_mappings = pd.Series(per_id_mappings.values())\n elif isinstance(per_id_mappings, pd.Series):\n pass\n else:\n raise ValueError(\n f\"Invalid per_id_mappings type: {type(per_id_mappings)}. Make sure the id_mappings is a dict of dicts or a dict of pd.Series.\"\n )\n if is_image_column(per_id_mappings, col_name=per_name, image_type=IMAGE_PATH):\n id_mappings_types[per_name] = IMAGE_PATH\n elif is_text_column(per_id_mappings):\n id_mappings_types[per_name] = TEXT\n elif is_image_column(per_id_mappings, col_name=per_name, image_type=IMAGE_BYTEARRAY):\n id_mappings_types[per_name] = IMAGE_BYTEARRAY\n elif is_image_column(per_id_mappings, col_name=per_name, image_type=IMAGE_BASE64_STR):\n id_mappings_types[per_name] = IMAGE_BASE64_STR\n else:\n raise ValueError(\n f\"{per_name} in the id_mappings has an invalid type. Currently, we only support image and text types.\"\n )\n return id_mappings_types\n\n\ndef infer_column_types(\n data: pd.DataFrame,\n valid_data: Optional[pd.DataFrame] = None,\n label_columns: Union[str, List[str]] = None,\n provided_column_types: Optional[Dict] = None,\n allowable_column_types: Optional[List[str]] = None,\n fallback_column_type: Optional[str] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n problem_type: Optional[str] = None,\n) -> Dict:\n \"\"\"\n Infer the column types of a multimodal pd.DataFrame.\n\n Parameters\n ----------\n data\n The multimodal pd.DataFrame for training.\n valid_data\n The multimodal pd.DataFrame for validation.\n label_columns\n The label column names.\n provided_column_types\n Additional dictionary that you can use to specify the columns types that you know.\n {'col_name': TYPE, ...}\n allowable_column_types\n What column types are allowed. This is the prior knowledge inferred from the model type.\n fallback_column_type\n What's the fallback column type if the detected type if out of the allowable_column_types.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n problem_type\n The problem type used to update some column types.\n\n Returns\n -------\n column_types\n A dictionary containing the mappings from column names to their modality types.\n If the column does not contain any useful information, we will set its column type NULL.\n \"\"\"\n if label_columns is None:\n label_columns = []\n if isinstance(label_columns, str):\n label_columns = [label_columns]\n\n column_types = collections.OrderedDict()\n\n if valid_data is None:\n valid_data = data\n is_training = False\n else:\n is_training = True\n\n id_mappings_types = infer_id_mappings_types(id_mappings)\n\n for col_name in data.columns:\n if provided_column_types is not None and col_name in provided_column_types:\n column_types[col_name] = provided_column_types[col_name]\n continue\n # Identify columns that provide no information\n idx = data[col_name].first_valid_index()\n if idx is None:\n # No valid index, thus, we will just ignore the column\n column_types[col_name] = NULL\n continue\n if (\n (not isinstance(data[col_name][idx], (list, dict, set, bytearray)))\n and len(data[col_name].unique()) == 1\n and is_training\n ):\n column_types[col_name] = NULL\n continue\n # TODO: valid check for collections\n\n if is_rois_column(data[col_name]):\n column_types[col_name] = ROIS\n # keep the elif here because ROIS need to skip the categorical check\n # where error occurs due to List type input\n elif is_identifier_column(data[col_name], col_name=col_name, id_mappings=id_mappings):\n column_types[col_name] = f\"{id_mappings_types[col_name]}_{IDENTIFIER}\"\n elif is_categorical_column(\n data[col_name], valid_data[col_name], is_label=col_name in label_columns\n ): # Infer categorical column\n column_types[col_name] = CATEGORICAL\n elif is_numerical_column(data[col_name], valid_data[col_name]): # Infer numerical column\n column_types[col_name] = NUMERICAL\n elif is_image_column(data[col_name], col_name=col_name, image_type=IMAGE_PATH): # Infer image-path column\n # Check if it is document image or not.\n if is_document_image_column(data[col_name], col_name=col_name):\n column_types[col_name] = DOCUMENT_IMAGE\n elif problem_type == SEMANTIC_SEGMENTATION:\n column_types[col_name] = SEMANTIC_SEGMENTATION_IMG\n else:\n column_types[col_name] = IMAGE_PATH\n elif is_document_pdf_column(data[col_name], col_name=col_name):\n column_types[col_name] = DOCUMENT_PDF\n elif is_text_column(data[col_name]): # Infer text column\n column_types[col_name] = TEXT\n elif is_image_column(\n data[col_name], col_name=col_name, image_type=IMAGE_BYTEARRAY\n ): # Infer image-bytearray column\n column_types[col_name] = IMAGE_BYTEARRAY\n elif is_image_column(\n data[col_name], col_name=col_name, image_type=IMAGE_BASE64_STR\n ): # Infer image-base64str column\n column_types[col_name] = IMAGE_BASE64_STR\n else: # All the other columns are treated as categorical\n column_types[col_name] = CATEGORICAL\n\n if allowable_column_types and fallback_column_type:\n column_types = set_fallback_column_type(\n column_types=column_types,\n allowable_column_types=allowable_column_types,\n fallback_column_type=fallback_column_type,\n )\n if problem_type == NER:\n column_types = infer_ner_column_type(column_types=column_types)\n\n if problem_type and label_columns:\n column_types = infer_label_column_type_by_problem_type(\n column_types=column_types,\n label_columns=label_columns,\n problem_type=problem_type,\n data=data,\n valid_data=valid_data if is_training else None,\n )\n\n return column_types\n\n\ndef check_missing_values(\n data: pd.DataFrame,\n column_name: str,\n split: Optional[str] = \"\",\n):\n num_missing_values = data[column_name].isnull().sum()\n if num_missing_values > 0:\n raise ValueError(\n f\"Label column '{column_name}' contains missing values in the \"\n f\"{split} dataframe. You may want to filter your data because \"\n \"missing label is currently not supported.\"\n )\n\n\ndef infer_label_column_type_by_problem_type(\n column_types: Dict,\n label_columns: Union[str, List[str]],\n problem_type: Optional[str],\n data: Optional[pd.DataFrame] = None,\n valid_data: Optional[pd.DataFrame] = None,\n allowable_label_types: Optional[List[str]] = (\n CATEGORICAL,\n NUMERICAL,\n NER_ANNOTATION,\n ROIS,\n SEMANTIC_SEGMENTATION_GT,\n ),\n fallback_label_type: Optional[str] = CATEGORICAL,\n):\n \"\"\"\n Infer the label column types based on problem type.\n\n Parameters\n ----------\n column_types\n Types of columns in a pd.DataFrame.\n label_columns\n The label columns in a pd.DataFrame.\n problem_type\n Type of problem.\n data\n A pd.DataFrame.\n valid_data\n A validation pd.DataFrame.\n allowable_label_types\n Which label types are allowed.\n fallback_label_type\n If a label type is not within the allowable_label_types, replace it with this fallback_label_type.\n\n Returns\n -------\n Column types with the label columns' types inferred from the problem type.\n \"\"\"\n if label_columns is None:\n return column_types\n\n if isinstance(label_columns, str):\n label_columns = [label_columns]\n\n for col_name in label_columns:\n # For zero-shot inference, label column is unnecessary\n if col_name not in column_types:\n continue\n if data is not None:\n check_missing_values(data=data, column_name=col_name, split=\"training\")\n if valid_data is not None:\n check_missing_values(data=valid_data, column_name=col_name, split=\"validation\")\n if column_types[col_name] == NULL:\n raise ValueError(\n f\"Label column '{col_name}' contains only one label class. Make sure it has at least two label classes.\"\n )\n if problem_type in [MULTICLASS, BINARY, CLASSIFICATION]:\n column_types[col_name] = CATEGORICAL\n elif problem_type == REGRESSION:\n column_types[col_name] = NUMERICAL\n elif problem_type == NER:\n column_types[col_name] = NER_ANNOTATION\n elif problem_type == OBJECT_DETECTION:\n column_types[col_name] = ROIS\n elif problem_type == SEMANTIC_SEGMENTATION:\n column_types[col_name] = SEMANTIC_SEGMENTATION_GT\n\n if column_types[col_name] not in allowable_label_types:\n column_types[col_name] = fallback_label_type\n\n return column_types\n\n\ndef infer_rois_column_type(\n column_types: Dict,\n data: Optional[pd.DataFrame] = None,\n):\n for col_name in column_types.keys():\n if is_rois_column(data[col_name]):\n column_types[col_name] = ROIS\n return column_types\n\n\ndef infer_problem_type(\n y_train_data: Optional[pd.DataFrame] = None,\n provided_problem_type: Optional[str] = None,\n) -> str:\n \"\"\"\n Infer the basic (binary, multiclass, and regression) problem types if problem type is None or classification.\n Only training data are used. Assume users provide valid validation data.\n\n Parameters\n ----------\n y_train_data\n The label column of training data.\n provided_problem_type\n Provided problem type\n\n Returns\n -------\n Inferred problem type\n \"\"\"\n if provided_problem_type in [None, CLASSIFICATION]:\n problem_type = infer_basic_problem_type(y=y_train_data)\n # trust users' prior knowledge\n if provided_problem_type == CLASSIFICATION and problem_type == REGRESSION:\n problem_type = MULTICLASS\n else:\n problem_type = provided_problem_type\n\n return problem_type\n\n\ndef infer_output_shape(\n label_column: str,\n problem_type: str,\n data: pd.DataFrame,\n) -> int:\n \"\"\"\n Infer the output shape based on the label column type, training data, and problem type.\n Binary classification should have class number 2, while multi-class classification's class\n number should be larger than 2. For regression, the output is restricted to 1 scalar.\n\n Parameters\n ----------\n label_column\n The label column in a multimodal pd.DataFrame.\n data\n The multimodal pd.DataFrame for training.\n problem_type\n Problem type which can be inferred or provided.\n\n Returns\n -------\n problem_type\n Type of problem.\n output_shape\n Shape of output.\n \"\"\"\n if label_column is None:\n return None\n\n if problem_type in [BINARY, MULTICLASS, REGRESSION, CLASSIFICATION]:\n class_num = len(data[label_column].unique())\n err_msg = (\n f\"Problem type is '{problem_type}' while the number of unique values in the label column is {class_num}.\"\n )\n if problem_type == BINARY:\n if class_num != 2:\n raise AssertionError(err_msg)\n return 2\n elif problem_type == MULTICLASS:\n if class_num <= 2:\n raise AssertionError(err_msg)\n return class_num\n elif problem_type == REGRESSION:\n if class_num < 1:\n raise AssertionError(err_msg)\n return 1\n else:\n return class_num\n elif problem_type in [NER, OBJECT_DETECTION, SEMANTIC_SEGMENTATION]:\n return None\n else:\n raise ValueError(\n f\"Problem type '{problem_type}' doesn't have a valid output shape \"\n f\"for training. The supported problem types are\"\n f\" '{BINARY}', '{MULTICLASS}', '{REGRESSION}',\"\n f\" '{CLASSIFICATION}', '{NER}',\"\n f\" '{OBJECT_DETECTION}', \"\n f\" '{SEMANTIC_SEGMENTATION}\"\n )\n\n\ndef set_fallback_column_type(column_types: Dict, allowable_column_types: List[str], fallback_column_type: str) -> Dict:\n \"\"\"\n Filter the auto-detected column types to make sure that all column types are allowable.\n Use the fallback type to replace those out of the allowable_column_types.\n\n Parameters\n ----------\n column_types\n The inferred column types.\n allowable_column_types\n The column types which are allowed by the model type.\n fallback_column_type\n Fallback to this type if some invalid column type is found.\n\n Returns\n -------\n The filtered column types.\n \"\"\"\n for col_name, col_type in column_types.items():\n if not col_type.startswith(tuple(allowable_column_types)):\n column_types[col_name] = fallback_column_type\n\n return column_types\n\n\ndef infer_ner_column_type(column_types: Dict):\n \"\"\"\n Replace the first text with text_ner for the ner problem type\n because no text_ner is detected in infer_column_types.\n\n Parameters\n ----------\n column_types\n The column types of a dataframe.\n\n Returns\n -------\n The columns types with one text_ner inside it.\n \"\"\"\n # if there is already one column has type text_ner, no action is taken.\n if any([col_type.startswith(TEXT_NER) for col_type in column_types.values()]):\n return column_types\n\n for column in (\n column_types.keys()\n ): # column_types is an ordered dict, so column_types.keys() returns the keys in the order of insertions.\n if column_types[column].startswith(TEXT):\n column_types[column] = column_types[column].replace(TEXT, TEXT_NER)\n break\n\n return column_types\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/label_encoder.py",
"content": "import json\nimport logging\nimport re\nfrom typing import Any, Dict, List, Optional, Union\n\nimport jsonschema\nimport numpy as np\nimport pandas as pd\nfrom omegaconf import DictConfig, OmegaConf\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom ..constants import END_OFFSET, ENTITY_GROUP, PROBABILITY, START_OFFSET\n\nlogger = logging.getLogger(__name__)\n\n\nclass NerLabelEncoder:\n \"\"\"\n Label Encoder for the named entity recognition task.\n \"\"\"\n\n def __init__(self, config: DictConfig, entity_map: Optional[dict] = None):\n self.entity_map = entity_map\n self.config = config\n model_config = config.model.ner_text\n self.ner_special_tags = OmegaConf.to_object(model_config.special_tags)\n self.prefix = config.model.names[0]\n self.b_prefix = \"B-\"\n self.i_prefix = \"I-\"\n\n def fit(self, y: pd.Series, x: pd.Series):\n \"\"\"\n Extract the annotations, check the unique entity groups, and build entity to index mappings.\n \"\"\"\n _, entity_groups = self.extract_ner_annotations(y)\n self.unique_entity_groups = self.ner_special_tags + entity_groups\n self.entity_map = {entity: index for index, entity in enumerate(self.unique_entity_groups)}\n self.config.entity_map = self.entity_map\n self.inverse_entity_map = {index: entity for index, entity in enumerate(self.unique_entity_groups)}\n logger.debug(f\"Unique entity groups in the data: {entity_groups}\")\n\n def extract_ner_annotations(self, y: pd.Series):\n \"\"\"\n Validate the JSON annotations with predefined JSON schema and convert it into list.\n\n Parameters\n ----------\n y\n The raw json annotations.\n\n Returns\n -------\n all_annotations\n A list of NER annotations.\n unique_entity_groups\n A list of unique entity groups.\n \"\"\"\n\n # Predefined Json schema\n schema = {\n \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n ENTITY_GROUP: {\"type\": \"string\"},\n START_OFFSET: {\"type\": \"integer\"},\n END_OFFSET: {\"type\": \"integer\"},\n },\n \"required\": [ENTITY_GROUP, START_OFFSET, END_OFFSET],\n },\n }\n all_annotations = []\n all_entity_groups = []\n for _, ner_annotations in y.items():\n json_ner_annotations = json.loads(ner_annotations) # load the json annotations\n try:\n jsonschema.validate(json_ner_annotations, schema) # verify the json schema\n except jsonschema.ValidationError as e:\n # Raise an error if the provided json annotations do not match the predefined schema\n raise ValueError(f\"The provided json annotations are invalid: {e.message}\")\n sentence_annotations = []\n for annot in json_ner_annotations:\n entity_group = annot[ENTITY_GROUP]\n if not (\n re.match(self.b_prefix, entity_group, re.IGNORECASE)\n or re.match(self.i_prefix, entity_group, re.IGNORECASE)\n ):\n all_entity_groups.append(self.b_prefix + entity_group)\n all_entity_groups.append(self.i_prefix + entity_group)\n else:\n all_entity_groups.append(entity_group)\n sentence_annotations.append(\n (\n (annot[START_OFFSET], annot[END_OFFSET]),\n entity_group,\n )\n )\n all_annotations.append(sentence_annotations)\n unique_entity_groups = list(set(all_entity_groups))\n return all_annotations, unique_entity_groups\n\n def transform(self, y: pd.Series):\n \"\"\"\n Transform raw JSON annotations to list annotations.\n\n Parameters\n ----------\n y\n The raw json annotations.\n\n Returns\n -------\n all_annotations\n A list of NER annotations.\n \"\"\"\n all_annotations, _ = self.extract_ner_annotations(y)\n return all_annotations\n\n def transform_label_for_metric(self, y: pd.Series, x: pd.Series, tokenizer):\n \"\"\"\n Transform raw JSON annotations to word level annotations which can be used\n as references of seqeval evaluation.\n For more information: https://huggingface.co/spaces/evaluate-metric/seqeval\n\n Parameters\n ----------\n y\n The raw json annotations.\n x\n The raw text data\n tokenizer\n The tokenizer to be used for text tokenization.\n\n Returns\n -------\n transformed_y\n A list of word level annotations.\n \"\"\"\n from .process_ner import NerProcessor\n\n all_annotations, _ = self.extract_ner_annotations(y)\n transformed_y = []\n for annotation, text_snippet in zip(all_annotations, x.items()):\n word_label, _, _, _ = NerProcessor.process_ner_annotations(\n annotation, text_snippet[-1], self.entity_map, tokenizer, is_eval=True\n )\n word_label_invers = []\n for l in word_label:\n entity_group = self.inverse_entity_map[l]\n word_label_invers.append(entity_group)\n transformed_y.append(word_label_invers)\n return transformed_y\n\n def inverse_transform(self, y: List):\n \"\"\"\n Inverse Transform NER model predictions into human readable dictionary annotations\n which have the same format as the original annotations.\n\n Parameters\n ----------\n y\n NER model predictions.\n\n Returns\n -------\n pred_label_only\n Predictions which only have labels.\n pred_with_offset\n Predictions with both labels and the position (character offset) of the corresponding words.\n \"\"\"\n pred_label_only, pred_with_offset, pred_with_proba = [], [], []\n for token_preds, offsets, pred_proba in y:\n temp_pred, temp_offset, temp_proba = [], [], []\n for token_pred, offset, probability in zip(token_preds, offsets, pred_proba):\n inverse_pred_label = self.inverse_entity_map[token_pred]\n temp_pred.append(inverse_pred_label)\n temp_proba.append(\n {\n ENTITY_GROUP: inverse_pred_label,\n START_OFFSET: offset[0],\n END_OFFSET: offset[1],\n PROBABILITY: {e: p for e, p in zip(list(self.entity_map.keys())[1:], probability[1:])},\n }\n )\n if inverse_pred_label not in self.ner_special_tags:\n temp_offset.append(\n {\n ENTITY_GROUP: inverse_pred_label,\n START_OFFSET: offset[0],\n END_OFFSET: offset[1],\n }\n )\n pred_label_only.append(temp_pred)\n pred_with_offset.append(temp_offset)\n pred_with_proba.append(temp_proba)\n return pred_label_only, pred_with_offset, pred_with_proba\n\n\nclass CustomLabelEncoder:\n \"\"\"\n A label encoder that supports specifying a positive class on top of sklearn's LabelEncoder.\n \"\"\"\n\n def __init__(self, positive_class=None):\n self.positive_class = positive_class\n self._le = LabelEncoder()\n self._mapping = None\n self._inverse_mapping = None\n\n def _set_attributes(self):\n if self.positive_class:\n assert self.positive_class in self._le.classes_\n\n if self.positive_class:\n classes, class_labels = [], []\n for i, c in enumerate(self._le.classes_):\n if c == self.positive_class:\n positive_label = i\n else:\n classes.append(c)\n class_labels.append(i)\n classes.append(self.positive_class)\n class_labels.append(positive_label)\n self.classes_ = np.array(classes)\n pairs = {label: i for i, label in enumerate(class_labels)}\n sorted_pairs = sorted(pairs.items(), key=lambda item: item[0])\n self._mapping = np.array([item[1] for item in sorted_pairs])\n sorted_pairs = sorted(pairs.items(), key=lambda item: item[1])\n self._inverse_mapping = np.array([item[0] for item in sorted_pairs])\n else:\n self.classes_ = self._le.classes_\n\n def fit(self, y):\n \"\"\"\n Fit label encoder.\n\n Parameters\n ----------\n y : array-like of shape (n_samples,)\n Target values.\n\n Returns\n -------\n self : returns an instance of self.\n Fitted label encoder.\n \"\"\"\n self._le.fit(y)\n self._set_attributes()\n\n return self\n\n def fit_transform(self, y):\n \"\"\"\n Fit label encoder and return encoded labels.\n\n Parameters\n ----------\n y : array-like of shape (n_samples,)\n Target values.\n\n Returns\n -------\n y : array-like of shape (n_samples,)\n Encoded labels.\n \"\"\"\n y = self._le.fit_transform(y)\n self._set_attributes()\n\n if self._mapping is not None:\n return self._mapping[y]\n else:\n return y\n\n def transform(self, y):\n \"\"\"\n Transform labels to normalized encoding.\n\n Parameters\n ----------\n y : array-like of shape (n_samples,)\n Target values.\n\n Returns\n -------\n y : array-like of shape (n_samples,)\n Labels as normalized encodings.\n \"\"\"\n y = self._le.transform(y)\n\n if self._mapping is not None:\n return self._mapping[y]\n else:\n return y\n\n def inverse_transform(self, y):\n \"\"\"\n Transform labels back to original encoding.\n\n Parameters\n ----------\n y : ndarray of shape (n_samples,)\n Target values.\n\n Returns\n -------\n y : ndarray of shape (n_samples,)\n Original encoding.\n \"\"\"\n if self._inverse_mapping is not None:\n y = self._inverse_mapping[y]\n\n return self._le.inverse_transform(y)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/mixup.py",
"content": "import numpy as np\nimport torch\nfrom timm.data.mixup import Mixup, cutmix_bbox_and_lam, mixup_target\n\n\nclass MixupModule(Mixup):\n \"\"\"\n Mixup class from timm.\n https://github.com/rwightman/pytorch-image-models/blob/d30685c283137b4b91ea43c4e595c964cd2cb6f0/timm/data/mixup.py\n The parameters here are correspond to the mixup config in data.\n The mixup in timm only produce image mixup and cutmix with one-hot class target.\n This module helps to take the lambda from the Mixup.\n Lambda is added to the function to produce the mixup with specific lambda.\n \"\"\"\n\n def __init__(\n self,\n mixup_alpha=1.0,\n cutmix_alpha=0.0,\n cutmix_minmax=None,\n prob=1.0,\n switch_prob=0.5,\n mode=\"batch\",\n correct_lam=True,\n label_smoothing=0.1,\n num_classes=1000,\n ):\n \"\"\"\n Parameters\n ----------\n mixup_alpha\n The mixup alpha value, it is active if > 0.\n cutmix_alpha\n The cutmix alpha value, cutmix is active if > 0.\n cutmix_minmax\n cutmix min/max image ratio. The para should be a list/tuple of float with size 2.\n prob\n The probability of conducting mixup/cutmix if enable.\n switch_prob\n The probability of switching mixup to cutmix if both enable.\n mode\n Perform mixup/cutmix on \"batch\" or \"pair\" or \"elem\".\n correct_lam\n Apply lambda correction when cutmix bbox clipped by image borders.\n label_smoothing\n Apply label smoothing to the mixed target.\n num_classes\n Number of classes for target.\n \"\"\"\n super().__init__(\n mixup_alpha,\n cutmix_alpha,\n cutmix_minmax,\n prob,\n switch_prob,\n mode,\n correct_lam,\n label_smoothing,\n num_classes,\n )\n self.lam = None\n self.target_a = None\n self.target_b = None\n\n def _mix_elem(self, x, lam_batch):\n batch_size = len(x)\n if lam_batch is None:\n lam_batch, use_cutmix = self._params_per_elem(batch_size)\n else:\n _, use_cutmix = self._params_per_elem(batch_size)\n x_orig = x.clone()\n for i in range(batch_size):\n j = batch_size - i - 1\n lam = lam_batch[i]\n if lam != 1.0:\n if use_cutmix[i]:\n (yl, yh, xl, xh), lam = cutmix_bbox_and_lam(\n x[i].shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam\n )\n x[i][:, yl:yh, xl:xh] = x_orig[j][:, yl:yh, xl:xh]\n lam_batch[i] = lam\n else:\n x[i] = x[i] * lam + x_orig[j] * (1 - lam)\n return torch.tensor(lam_batch, device=x.device, dtype=x.dtype).unsqueeze(1)\n\n def _mix_pair(self, x, lam_batch):\n batch_size = len(x)\n if lam_batch is None:\n lam_batch, use_cutmix = self._params_per_elem(batch_size // 2)\n else:\n _, use_cutmix = self._params_per_elem(batch_size // 2)\n x_orig = x.clone()\n for i in range(batch_size // 2):\n j = batch_size - i - 1\n lam = lam_batch[i]\n if lam != 1.0:\n if use_cutmix[i]:\n (yl, yh, xl, xh), lam = cutmix_bbox_and_lam(\n x[i].shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam\n )\n x[i][:, yl:yh, xl:xh] = x_orig[j][:, yl:yh, xl:xh]\n x[j][:, yl:yh, xl:xh] = x_orig[i][:, yl:yh, xl:xh]\n lam_batch[i] = lam\n else:\n x[i] = x[i] * lam + x_orig[j] * (1 - lam)\n x[j] = x[j] * lam + x_orig[i] * (1 - lam)\n lam_batch = np.concatenate((lam_batch, lam_batch[::-1]))\n return torch.tensor(lam_batch, device=x.device, dtype=x.dtype).unsqueeze(1)\n\n def _mix_batch(self, x, lam):\n if lam is None:\n lam, use_cutmix = self._params_per_batch()\n else:\n _, use_cutmix = self._params_per_batch()\n if lam == 1.0:\n return 1.0\n if use_cutmix:\n (yl, yh, xl, xh), lam = cutmix_bbox_and_lam(\n x.shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam\n )\n x[:, :, yl:yh, xl:xh] = x.flip(0)[:, :, yl:yh, xl:xh]\n else:\n x_flipped = x.flip(0).mul_(1.0 - lam)\n x.mul_(lam).add_(x_flipped)\n return lam\n\n def __call__(self, x, target, lam=None):\n if self.mode == \"elem\":\n lam = self._mix_elem(x, lam)\n elif self.mode == \"pair\":\n lam = self._mix_pair(x, lam)\n else:\n lam = self._mix_batch(x, lam)\n target = mixup_target(target, self.num_classes, lam, self.label_smoothing, x.device)\n return x, target, lam\n\n\ndef mixup_others(x, lam):\n \"\"\"\n Mixup special types of data, especially for tuple.\n It is the simplest way of mixup for non image data.\n If lam >=0.5: choose the origin, else: choose the other one.\n\n Parameters\n -------\n x\n The target need to be mixed-up.\n lam\n The mixup lambda.\n\n Returns\n -------\n The mixed-up batch data with specific model.\n \"\"\"\n if lam is None:\n lam = 1\n else:\n lam = round(lam)\n if isinstance(x, tuple):\n target = (pertarget * lam + pertarget.flip(0) * (1.0 - lam) for pertarget in x)\n else:\n target = x * lam + x.flip(0) * (1.0 - lam)\n return target\n\n\ndef multimodel_mixup(batch, model, mixup_fn):\n \"\"\"\n Mixup for different models.\n For image data, use the mixup_fn from timm.\n For other types of data, the simplest way as choosing will be used.\n\n Parameters\n -------\n batch\n The origin data need to be mixed-up.\n model\n The model used on the task.It is used to get the useful column in batch.\n mixup_fn\n The mixup_fn from timm. It can mixup image and produce target label with lambda.\n\n Returns\n -------\n batch\n The mixed-up batch.\n mixup_label\n The mixed-up labels.\n \"\"\"\n mixup_label = batch[model.label_key]\n if hasattr(model, \"image_key\"):\n batch[model.image_key], mixup_label = mixup_fn(batch[model.image_key], batch[model.label_key])\n else:\n lam = None\n for permodel in model.model:\n if hasattr(permodel, \"image_key\"):\n if lam is None:\n batch[permodel.image_key], mixup_label, lam = mixup_fn(\n batch[permodel.image_key], batch[permodel.label_key]\n )\n else:\n batch[permodel.image_key], _, _ = mixup_fn(\n batch[permodel.image_key], batch[permodel.label_key], lam\n )\n for permodel in model.model:\n if hasattr(permodel, \"categorical_key\"):\n mixup_others(batch[permodel.categorical_key], lam)\n if hasattr(permodel, \"numerical_key\"):\n mixup_others(batch[permodel.numerical_key], lam)\n if hasattr(permodel, \"text_token_ids_key\"):\n mixup_others(batch[permodel.text_token_ids_key], lam)\n\n return batch, mixup_label\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/nlpaug.py",
"content": "\"\"\"Utilities built on top of nlpaug. This file is meant to be lazy-imported because importing nlpaug can take time.\nSee the discussion in https://github.com/autogluon/autogluon/issues/2706\n\"\"\"\n\nimport random\n\nfrom nlpaug import Augmenter\nfrom nlpaug.util import Method\n\n\nclass InsertPunctuation(Augmenter):\n \"\"\"\n Inherit nlpaug basic augmenter to support insert random punction at random location https://arxiv.org/pdf/2108.13230.pdf\n\n example:\n a healthy ,clean , sweet little girl in Mantin . send me message if you can give her a nice home\n ? a ! healthy ,clean , sweet little : girl , in Mantin . send me message . if you ; can give her ? a nice home\n \"\"\"\n\n def __init__(\n self,\n name=\"Insert_Punc\",\n aug_min=1,\n aug_max=50,\n aug_p=0.3,\n ):\n \"\"\"\n Parameters\n ----------\n name\n name used when print out augmentation function\n aug_min\n minimum number of punctuation to insert\n aug_max\n maximum number of punctuation to insert\n aug_p\n how many punctuation to insert calculated as aug_p * sentence length\n \"\"\"\n super().__init__(\n name=name,\n method=Method.WORD,\n action=\"insert\",\n aug_min=aug_min,\n aug_max=aug_max,\n aug_p=aug_p,\n device=\"cpu\",\n include_detail=False,\n verbose=0,\n )\n self.punc_list = [\".\", \",\", \"!\", \"?\", \";\", \":\"]\n\n def insert(self, data):\n \"\"\"\n Random insert random punctuation at random location https://arxiv.org/pdf/2108.13230.pdf\n\n Parameters\n --------\n data: text\n\n\n Returns\n --------\n The augmented text\n\n \"\"\"\n words = data.split(\" \")\n cnt = random.randint(1, int(self.aug_p * len(words)))\n loc = random.sample(range(0, len(words)), cnt)\n new = []\n\n for i, word in enumerate(words):\n if i in loc:\n new.append(self.punc_list[random.randint(0, len(self.punc_list) - 1)])\n new.append(word)\n else:\n new.append(word)\n\n new = \" \".join(new)\n return new\n\n @staticmethod\n def clean(data):\n if isinstance(data, list):\n return [d.strip() if d else d for d in data]\n return data.strip()\n\n @staticmethod\n def is_duplicate(dataset, data):\n for d in dataset:\n if d == data:\n return True\n return False\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/preprocess_dataframe.py",
"content": "import base64\nimport logging\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nimport numpy as np\nimport pandas as pd\nfrom numpy.typing import NDArray\nfrom omegaconf import DictConfig, OmegaConf\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\n\nfrom autogluon.features import CategoryFeatureGenerator\n\nfrom ..constants import (\n CATEGORICAL,\n DOCUMENT,\n IDENTIFIER,\n IMAGE,\n IMAGE_BASE64_STR,\n IMAGE_BYTEARRAY,\n IMAGE_PATH,\n LABEL,\n NER_ANNOTATION,\n NULL,\n NUMERICAL,\n ROIS,\n SEMANTIC_SEGMENTATION,\n SEMANTIC_SEGMENTATION_GT,\n SEMANTIC_SEGMENTATION_IMG,\n TEXT,\n TEXT_NER,\n)\nfrom .label_encoder import CustomLabelEncoder\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):\n \"\"\"\n Preprocess one multimodal pd.DataFrame including image paths, image bytearrays, texts, numerical features,\n and categorical features. Each modality may have multiple columns.\n The preprocessor is designed to output model-agnostic features.\n \"\"\"\n\n def __init__(\n self,\n config: DictConfig,\n column_types: Dict,\n label_column: Optional[str] = None,\n label_generator: Optional[object] = None,\n ):\n \"\"\"\n Parameters\n ----------\n config\n Configurations regarding data preprocessing.\n column_types\n The mappings from pd.DataFrame's column names to modality types, e.g., image paths and text.\n label_column\n Name of the label column in pd.DataFrame. Can be None to support zero-short learning.\n label_generator\n A sklearn CustomLabelEncoder instance, or a customized encoder, e.g. NerPreprocessor.\n \"\"\"\n self._column_types = column_types\n self._label_column = label_column\n self._config = config\n self._feature_generators = dict()\n\n if label_column:\n if label_generator is None:\n self._label_generator = CustomLabelEncoder(positive_class=config.pos_label)\n else:\n self._label_generator = label_generator\n\n # Scaler used for numerical labels\n numerical_label_preprocessing = config.label.numerical_preprocessing\n if numerical_label_preprocessing == \"minmaxscaler\":\n self._label_scaler = MinMaxScaler()\n elif numerical_label_preprocessing == \"standardscaler\":\n self._label_scaler = StandardScaler()\n elif numerical_label_preprocessing is None:\n self._label_scaler = StandardScaler(with_mean=False, with_std=False)\n else:\n raise ValueError(\n f\"The numerical_label_preprocessing={numerical_label_preprocessing} is currently not supported\"\n )\n else:\n self._label_generator = None\n self._label_scaler = None\n\n for col_name, col_type in self._column_types.items():\n if col_name == self._label_column:\n continue\n if col_type.startswith((TEXT, IMAGE, ROIS, TEXT_NER, DOCUMENT, SEMANTIC_SEGMENTATION)) or col_type == NULL:\n continue\n elif col_type == CATEGORICAL:\n generator = CategoryFeatureGenerator(\n cat_order=\"count\",\n minimum_cat_count=config.categorical.minimum_cat_count,\n maximum_num_cat=config.categorical.maximum_num_cat,\n verbosity=0,\n )\n self._feature_generators[col_name] = generator\n elif col_type == NUMERICAL:\n generator = Pipeline(\n [\n (\"imputer\", SimpleImputer()),\n (\n \"scaler\",\n StandardScaler(\n with_mean=config.numerical.scaler_with_mean,\n with_std=config.numerical.scaler_with_std,\n ),\n ),\n ]\n )\n self._feature_generators[col_name] = generator\n\n else:\n raise NotImplementedError(\n f\"Type of the column is not supported currently. Received {col_name}={col_type}.\"\n )\n\n self._fit_called = False\n self._fit_x_called = False\n self._fit_y_called = False\n\n # Some columns will be ignored\n self._ignore_columns_set = set()\n self._text_feature_names = []\n self._categorical_num_categories = dict()\n self._numerical_feature_names = []\n self._image_feature_names = []\n self._rois_feature_names = []\n self._ner_feature_names = []\n self._document_feature_names = []\n self._semantic_segmentation_feature_names = []\n\n @property\n def label_column(self):\n return self._label_column\n\n @property\n def column_types(self):\n return self._column_types\n\n @property\n def image_path_names(self):\n return [col_name for col_name in self._image_feature_names if self._column_types[col_name] == IMAGE_PATH]\n\n @property\n def rois_feature_names(self):\n return self._rois_feature_names\n\n @property\n def image_bytearray_names(self):\n return [col_name for col_name in self._image_feature_names if self._column_types[col_name] == IMAGE_BYTEARRAY]\n\n @property\n def image_base64_str_names(self):\n return [col_name for col_name in self._image_feature_names if self._column_types[col_name] == IMAGE_BASE64_STR]\n\n @property\n def image_feature_names(self):\n return self._image_feature_names\n\n @property\n def text_feature_names(self):\n return self._text_feature_names\n\n @property\n def categorical_feature_names(self):\n return list(self.categorical_num_categories.keys())\n\n @property\n def numerical_feature_names(self):\n return self._numerical_feature_names\n\n @property\n def numerical_fill_values(self):\n ret = dict()\n for col_name in self._numerical_feature_names:\n generator = self._feature_generators[col_name]\n ret[col_name] = generator.transform(np.full([1, 1], np.nan))[:, 0][0]\n\n return ret\n\n @property\n def document_feature_names(self):\n # Added for backward compatibility.\n if hasattr(self, \"_document_feature_names\"):\n return self._document_feature_names\n else:\n return []\n\n @property\n def ner_feature_names(self):\n # Added for backward compatibility for v0.6.0 where column_type is not specified.\n if hasattr(self, \"_ner_feature_names\"):\n return self._ner_feature_names\n else:\n if len(self.text_feature_names) > 0:\n return self.text_feature_names[:1]\n else:\n return []\n\n @property\n def semantic_segmentation_feature_names(self):\n # Added for backward compatibility.\n if hasattr(self, \"_semantic_segmentation_feature_names\"):\n return self._semantic_segmentation_feature_names\n else:\n return []\n\n @property\n def required_feature_names(self):\n return (\n self._image_feature_names\n + self._text_feature_names\n + self._numerical_feature_names\n + self.categorical_feature_names\n + self._rois_feature_names\n )\n\n @property\n def all_column_names(self):\n return self._column_types.keys()\n\n @property\n def categorical_num_categories(self):\n \"\"\"We will always include the unknown category\"\"\"\n return self._categorical_num_categories\n\n @property\n def config(self):\n return self._config\n\n @property\n def label_type(self):\n return self._column_types[self._label_column]\n\n @property\n def label_scaler(self):\n return self._label_scaler\n\n @property\n def label_generator(self):\n return self._label_generator\n\n @property\n def fit_called(self):\n return self._fit_x_called and self._fit_y_called\n\n @property\n def fit_x_called(self):\n return self._fit_x_called\n\n @property\n def fit_y_called(self):\n return self._fit_y_called\n\n def get_column_names(self, modality: str):\n if modality.startswith(IMAGE):\n return self._image_feature_names\n elif modality == ROIS:\n return self._rois_feature_names\n elif modality == TEXT:\n return self._text_feature_names\n elif modality == CATEGORICAL:\n return self.categorical_feature_names\n elif modality == NUMERICAL:\n return self._numerical_feature_names\n elif modality.startswith(DOCUMENT):\n return self._document_feature_names\n elif modality == LABEL:\n return [self._label_column] # as a list to be consistent with others\n elif modality == SEMANTIC_SEGMENTATION_IMG:\n return self._semantic_segmentation_feature_names\n elif self.label_type == NER_ANNOTATION:\n return self.ner_feature_names + [self._label_column]\n else:\n raise ValueError(f\"Unknown modality: {modality}.\")\n\n def _fit_x(self, X: pd.DataFrame):\n \"\"\"\n Fit the pd.DataFrame by grouping column names by their modality types. For example, all the\n names of text columns will be put in a list. The CategoryFeatureGenerator, SimpleImputer, and\n StandardScaler will also be initialized.\n\n Parameters\n ----------\n X\n A multimodal pd.DataFrame.\n \"\"\"\n\n if self._fit_x_called:\n raise RuntimeError(\"fit_x() has been called. Please create a new preprocessor and call it again!\")\n self._fit_x_called = True\n # Creating deep copy of the DataFrame, which allows writable buffer to be created for the new df\n # This is needed for 1.4.1 < scikit-learn < 1.5.0, the versions 1.4.0 and 1.5.1 do not need a writable buffer\n X = X.copy(deep=True)\n X.flags.writeable = True\n\n for col_name in sorted(X.columns):\n # Just in case X accidentally contains the label column\n if col_name == self._label_column:\n continue\n col_type = self._column_types[col_name]\n logger.debug(f'Process col \"{col_name}\" with type \"{col_type}\"')\n col_value = X[col_name]\n if col_type == NULL:\n self._ignore_columns_set.add(col_name)\n elif col_type.startswith(TEXT_NER):\n self._ner_feature_names.append(col_name)\n elif col_type.startswith(TEXT):\n self._text_feature_names.append(col_name)\n elif col_type == CATEGORICAL:\n if self._config.categorical.convert_to_text:\n # Convert categorical column as text column\n col_value = col_value.astype(\"object\")\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else str(ele))\n if len(processed_data.unique()) == 1:\n self._ignore_columns_set.add(col_name)\n continue\n self._text_feature_names.append(col_name)\n else:\n processed_data = col_value.astype(\"category\")\n generator = self._feature_generators[col_name]\n processed_data = generator.fit_transform(pd.DataFrame({col_name: processed_data}))[\n col_name\n ].cat.codes.to_numpy(np.int32, copy=True)\n if len(np.unique(processed_data)) == 1:\n self._ignore_columns_set.add(col_name)\n continue\n num_categories = len(generator.category_map[col_name])\n # Add one unknown category\n self._categorical_num_categories[col_name] = num_categories + 1\n elif col_type == NUMERICAL:\n processed_data = pd.to_numeric(col_value)\n if len(processed_data.unique()) == 1:\n self._ignore_columns_set.add(col_name)\n continue\n if self._config.numerical.convert_to_text:\n self._text_feature_names.append(col_name)\n else:\n generator = self._feature_generators[col_name]\n generator.fit(np.expand_dims(processed_data.to_numpy(), axis=-1))\n self._numerical_feature_names.append(col_name)\n elif col_type.startswith(IMAGE):\n self._image_feature_names.append(col_name)\n elif col_type.startswith(DOCUMENT):\n self._document_feature_names.append(col_name)\n elif col_type == ROIS:\n self._rois_feature_names.append(col_name)\n elif col_type == SEMANTIC_SEGMENTATION_IMG:\n self._semantic_segmentation_feature_names.append(col_name)\n else:\n raise NotImplementedError(\n f\"Type of the column is not supported currently. Received {col_name}={col_type}.\"\n )\n\n def _fit_y(self, y: pd.Series, X: Optional[pd.DataFrame] = None):\n \"\"\"\n Fit the label column data to initialize the label encoder or scalar.\n\n Parameters\n ----------\n y\n The Label column data.\n \"\"\"\n if self._fit_y_called:\n raise RuntimeError(\"fit_y() has been called. Please create a new preprocessor and call it again!\")\n self._fit_y_called = True\n # Creating deep copy of the DataFrame, which allows writable buffer to be created for the new df\n # This is needed for 1.4.1 < scikit-learn < 1.5.0, the versions 1.4.0 and 1.5.1 do not need a writable buffer\n y = y.copy(deep=True)\n y.flags.writeable = True\n logger.debug(f'Process col \"{self._label_column}\" with type label')\n if self.label_type == CATEGORICAL:\n self._label_generator.fit(y)\n elif self.label_type == NUMERICAL:\n y = pd.to_numeric(y).to_numpy()\n self._label_scaler.fit(np.expand_dims(y, axis=-1))\n elif self.label_type in [ROIS, SEMANTIC_SEGMENTATION_GT]:\n pass # Do nothing. TODO: Shall we call fit here?\n elif self.label_type == NER_ANNOTATION:\n # If there are ner annotations and text columns but no NER feature columns,\n # we will convert the first text column into a ner column.\n # Added for backward compatibility for v0.6.0 where column_type is not specified.\n if len(self._ner_feature_names) == 0:\n if len(self._text_feature_names) != 0:\n self._ner_feature_names.append(self._text_feature_names.pop(0))\n self.column_types[self._ner_feature_names[0]] = TEXT_NER\n else:\n raise NotImplementedError(\n f\"Text column is necessary for named entity recognition, however, no text column is detected.\"\n )\n self._label_generator.fit(y, X[self.ner_feature_names[0]])\n else:\n raise NotImplementedError(f\"Type of label column is not supported. Label column type={self.label_type}\")\n\n def fit(self, X: Optional[pd.DataFrame] = None, y: Optional[pd.Series] = None):\n \"\"\"\n Fit the dataframe preprocessor with features X and labels y.\n\n Parameters\n ----------\n X\n The multimodal features in the format of pd.DataFrame.\n y\n The Label data in the format of pd.Series.\n \"\"\"\n if X is not None:\n self._fit_x(X=X)\n if y is not None:\n self._fit_y(y=y, X=X)\n\n @staticmethod\n def convert_categorical_to_text(col_value: pd.Series, template: str, col_name: str):\n # TODO: do we need to consider whether categorical values are valid text?\n col_value = col_value.astype(\"object\")\n if template == \"direct\":\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else str(ele))\n elif template == \"list\":\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else col_name + \": \" + str(ele))\n elif template == \"text\":\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else col_name + \" is \" + str(ele))\n elif template == \"latex\":\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else str(ele) + \" & \")\n else:\n raise ValueError(\n f\"Unsupported template {template} for converting categorical data into text. Select one from: ['direct', 'list', 'text', 'latex'].\"\n )\n return processed_data\n\n def transform_text(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, List[str]], Dict[str, str]]:\n \"\"\"\n Preprocess text data by collecting them together. May need to format\n the categorical and numerical data into strings if using them so.\n This function needs to be called preceding the text processor in \"process_text.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n text_features\n All the text data stored in a dictionary.\n text_types\n The column types of these text data, e.g., text or text_identifier.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_text.\"\n )\n text_features = {}\n text_types = {}\n for col_name in self._text_feature_names:\n col_value = df[col_name]\n col_type = self._column_types[col_name]\n if col_type == TEXT:\n col_value = col_value.astype(\"object\")\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else str(ele))\n elif col_type == CATEGORICAL:\n processed_data = self.convert_categorical_to_text(\n col_value=col_value,\n template=self._config.categorical.convert_to_text_template,\n col_name=col_name,\n )\n elif col_type == NUMERICAL:\n processed_data = pd.to_numeric(col_value).apply(\"{:.3f}\".format)\n elif col_type == f\"{TEXT}_{IDENTIFIER}\":\n processed_data = col_value\n else:\n raise ValueError(f\"Column {col_name} has type {col_type}, which can't be converted to text.\")\n\n text_features[col_name] = processed_data.values.tolist()\n text_types[col_name] = col_type\n\n return text_features, text_types\n\n def transform_rois(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, List[List[str]]], Dict[str, str]]:\n \"\"\"\n Preprocess image + rois data.\n For image data we preprocess them by collecting their paths together. If one sample has multiple images\n in an image column, assume that their image paths are separated by \";\".\n For rois data we simply convert them from a column of pandas dataframe to a list.\n This function needs to be called preceding the rois processor in \"process_rois.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n image_features\n All the image data stored in a dictionary.\n image_types\n The column types of these image data, e.g., image_path or image_identifier.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_rois.\"\n )\n\n x = self.transform_image(df)\n ret_data = x[0]\n ret_type = x[1]\n\n for col_name in self._rois_feature_names:\n col_type = self._column_types[col_name]\n\n if col_type == ROIS:\n processed_data = df[col_name].tolist()\n else:\n raise ValueError(f\"Unknown image type {col_type} for column {col_name}\")\n\n ret_data[col_name] = processed_data\n ret_type[col_name] = self._column_types[col_name]\n\n return ret_data, ret_type\n\n def transform_semantic_segmentation_img(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, List[List[str]]], Dict[str, str]]:\n \"\"\"\n Preprocess semantic segmentation data.\n For image data we preprocess them by collecting their paths together. If one sample has multiple images\n in an image column, assume that their image paths are separated by \";\".\n For ground truth image data we simply convert them from a column of pandas dataframe to a list.\n This function needs to be called preceding the image processor in \"process_semantic_seg_img.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n image_features\n All the image data stored in a dictionary.\n image_types\n The column types of these image data, e.g., image_path or image_identifier.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_semantic_segmentation_img.\"\n )\n\n ret_data = {}\n ret_type = {}\n for col_name in self._semantic_segmentation_feature_names:\n col_value = df[col_name]\n col_type = self._column_types[col_name]\n\n if col_type in [SEMANTIC_SEGMENTATION_IMG]:\n processed_data = col_value.apply(lambda ele: str(ele).split(\";\")).tolist()\n else:\n raise ValueError(f\"Unknown image type {col_type} for column {col_name}\")\n\n ret_data[col_name] = processed_data\n ret_type[col_name] = self._column_types[col_name]\n\n if self._label_column in df:\n if self.label_type == SEMANTIC_SEGMENTATION_GT:\n y = self.transform_label(df)\n ret_data.update(y[0])\n ret_type.update(y[1])\n return ret_data, ret_type\n\n def transform_image(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, List[List[str]]], Dict[str, str]]:\n \"\"\"\n Preprocess image data by collecting their paths together. If one sample has multiple images\n in an image column, assume that their image paths are separated by \";\".\n This function needs to be called preceding the image processor in \"process_image.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n image_features\n All the image data stored in a dictionary.\n image_types\n The column types of these image data, e.g., image_path, image_bytearray or image_identifier.\"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_image.\"\n )\n\n image_features = {}\n image_types = {}\n for col_name in self._image_feature_names:\n col_value = df[col_name]\n col_type = self._column_types[col_name]\n\n if col_type == ROIS:\n processed_data = df[col_name].tolist()\n elif col_type in [IMAGE_PATH, IMAGE]:\n processed_data = col_value.apply(lambda ele: str(ele).split(\";\")).tolist()\n elif col_type == IMAGE_BYTEARRAY:\n processed_data = col_value.apply(lambda ele: ele if isinstance(ele, list) else [ele]).tolist()\n elif col_type == IMAGE_BASE64_STR:\n processed_data = col_value.apply(\n lambda ele: (\n [base64.b64decode(e) for e in ele] if isinstance(ele, list) else [base64.b64decode(ele)]\n )\n ).tolist()\n elif col_type == f\"{IMAGE}_{IDENTIFIER}\":\n processed_data = col_value\n else:\n raise ValueError(f\"Unknown image type {col_type} for column {col_name}\")\n\n image_features[col_name] = processed_data\n image_types[col_name] = self._column_types[col_name]\n\n return image_features, image_types\n\n def transform_document(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, List[List[str]]], Dict[str, str]]:\n \"\"\"\n Preprocess document data by collecting their paths together. The current version does not\n support cases where one document column has multiple documents.\n This function needs to be called preceding the document processor in \"process_document.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n document_features\n All the image data stored in a dictionary.\n document_types\n The column types of these document data.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_document.\"\n )\n document_features = {}\n document_types = {}\n for col_name in self._document_feature_names:\n col_value = df[col_name]\n col_type = self._column_types[col_name]\n\n processed_data = col_value.apply(lambda ele: str(ele).split(\";\")).tolist()\n\n document_features[col_name] = processed_data\n document_types[col_name] = self._column_types[col_name]\n\n return document_features, document_types\n\n def transform_numerical(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, NDArray], None]:\n \"\"\"\n Preprocess numerical data by using SimpleImputer to fill possible missing values\n and StandardScaler to standardize the values (z = (x - mean) / std).\n This function needs to be called preceding the numerical processor in \"process_numerical.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n numerical_features\n All the numerical features (a dictionary of np.ndarray).\n None\n The column types of numerical data, which is None currently since only one numerical type exists.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit before calling preprocessor.transform_numerical.\"\n )\n numerical_features = {}\n for col_name in self._numerical_feature_names:\n generator = self._feature_generators[col_name]\n col_value = pd.to_numeric(df[col_name]).to_numpy()\n processed_data = generator.transform(np.expand_dims(col_value, axis=-1))[:, 0]\n numerical_features[col_name] = processed_data.astype(np.float32)\n\n return numerical_features, None\n\n def transform_categorical(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, NDArray], None]:\n \"\"\"\n Preprocess categorical data by using CategoryFeatureGenerator to generate\n categorical encodings, i.e., integers. This function needs to be called\n preceding the categorical processor in \"process_categorical.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n categorical_features\n All the categorical encodings (a dictionary of np.ndarray).\n None\n The column types of categorical data, which is None currently since only one categorical type exists.\n \"\"\"\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit before calling preprocessor.transform_categorical.\"\n )\n categorical_features = {}\n for col_name, num_category in self._categorical_num_categories.items():\n col_value = df[col_name]\n processed_data = col_value.astype(\"category\")\n generator = self._feature_generators[col_name]\n processed_data = generator.transform(pd.DataFrame({col_name: processed_data}))[\n col_name\n ].cat.codes.to_numpy(np.int32, copy=True)\n processed_data[processed_data < 0] = num_category - 1\n categorical_features[col_name] = processed_data\n\n return categorical_features, None\n\n def transform_label(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, NDArray], Dict[str, str]]:\n \"\"\"\n Preprocess ground-truth labels by using CustomLabelEncoder to generate class labels for\n classification tasks or using StandardScaler to standardize numerical values\n (z = (x - mean) / std) for regression tasks. This function needs to be called\n preceding the label processor in \"process_label.py\".\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame.\n\n Returns\n -------\n labels\n All the labels (a dictionary of np.ndarray).\n label_types\n The label column types.\n \"\"\"\n assert self._fit_called or self._fit_y_called, (\n \"You will need to first call preprocessor.fit_y() before calling preprocessor.transform_label.\"\n )\n # Creating deep copy of the DataFrame, which allows writable buffer to be created for the new df\n # This is needed for 1.4.1 < scikit-learn < 1.5.0, versions <=1.4.0 and >=1.5.1 do not need a writable buffer\n df = df.copy(deep=True)\n df.flags.writeable = True\n y_df = df[self._label_column]\n if self.label_type == CATEGORICAL:\n y = self._label_generator.transform(y_df).astype(np.int64)\n elif self.label_type == NUMERICAL:\n y = pd.to_numeric(y_df).to_numpy()\n y = self._label_scaler.transform(np.expand_dims(y, axis=-1))[:, 0].astype(np.float32)\n elif self.label_type in [ROIS, SEMANTIC_SEGMENTATION_GT]:\n y = y_df.to_list()\n elif self.label_type == NER_ANNOTATION:\n y = self._label_generator.transform(y_df)\n else:\n raise NotImplementedError\n\n return {self._label_column: y}, {self._label_column: self.label_type}\n\n def transform_text_ner(\n self,\n df: pd.DataFrame,\n ) -> Tuple[Dict[str, NDArray], Dict[str, str]]:\n assert self._fit_called or self._fit_x_called, (\n \"You will need to first call preprocessor.fit_x() before calling preprocessor.transform_ner.\"\n )\n ret_data, ret_type = {}, {}\n ner_text_features = {}\n ner_text_types = {}\n for col_name in self.ner_feature_names:\n col_value = df[col_name]\n col_type = self._column_types[col_name]\n if col_type.startswith((TEXT_NER, TEXT)):\n col_value = col_value.astype(\"object\")\n processed_data = col_value.apply(lambda ele: \"\" if pd.isnull(ele) else str(ele))\n else:\n raise ValueError(f\"Column {col_name} has type {col_type}, which can't be converted to text.\")\n ner_text_features[col_name] = processed_data.values.tolist()\n ner_text_types[col_name] = col_type\n if self.label_type == NER_ANNOTATION:\n ret_data.update(ner_text_features)\n ret_type.update(ner_text_types)\n if self._label_column in df:\n y = self.transform_label(df)\n ret_data.update(y[0])\n ret_type.update(y[1])\n else:\n raise NotImplementedError\n\n return ret_data, ret_type\n\n def transform_label_for_metric(\n self,\n df: pd.DataFrame,\n tokenizer: Optional[Any] = None,\n ) -> NDArray:\n \"\"\"\n Prepare ground-truth labels to compute metric scores in evaluation. Note that\n numerical values are not normalized since we want to evaluate the model performance\n on the raw numerical values.\n\n Parameters\n ----------\n df\n The multimodal pd.DataFrame for evaluation.\n\n Returns\n -------\n Ground-truth labels ready to compute metric scores.\n \"\"\"\n assert self._fit_called or self._fit_y_called, (\n \"You will need to first call preprocessor.fit_y() before calling preprocessor.transform_label_for_metric.\"\n )\n assert self._label_column in df.columns, (\n f\"Label {self._label_column} is not in the data. Cannot perform evaluation without ground truth labels.\"\n )\n y_df = df[self._label_column]\n if self.label_type == CATEGORICAL:\n # need to encode to integer labels\n y = self._label_generator.transform(y_df)\n elif self.label_type == NUMERICAL:\n # need to compute the metric on the raw numerical values (no normalization)\n y = pd.to_numeric(y_df).to_numpy()\n elif self.label_type == NER_ANNOTATION:\n x_df = df[self.ner_feature_names[0]]\n y = self._label_generator.transform_label_for_metric(y_df, x_df, tokenizer)\n else:\n raise NotImplementedError\n\n return y\n\n def transform_prediction(\n self,\n y_pred: Union[np.ndarray, dict],\n inverse_categorical: bool = True,\n return_proba: bool = False,\n ) -> NDArray:\n \"\"\"\n Transform model's output logits/probability into class labels for classification\n or raw numerical values for regression.\n\n Parameters\n ----------\n y_pred\n The model's output logits/probability.\n inverse_categorical\n Whether to transform categorical value back to the original space, e.g., string values.\n return_proba\n Whether return the probability or not.\n\n Returns\n -------\n Predicted labels ready to compute metric scores.\n \"\"\"\n assert self._fit_called or self._fit_y_called, (\n \"You will need to first call preprocessor.fit_y() before calling preprocessor.transform_prediction.\"\n )\n\n if self.label_type == CATEGORICAL:\n assert len(y_pred.shape) <= 2\n if len(y_pred.shape) == 2 and y_pred.shape[1] >= 2:\n y_pred = y_pred.argmax(axis=1)\n else:\n y_pred = (y_pred > 0.5).astype(int)\n # Transform the predicted label back to the original space (e.g., string values)\n if inverse_categorical:\n y_pred = self._label_generator.inverse_transform(y_pred)\n elif self.label_type == NUMERICAL:\n y_pred = self._label_scaler.inverse_transform(y_pred)\n y_pred = np.squeeze(y_pred)\n # Convert nan to 0\n y_pred = np.nan_to_num(y_pred)\n elif self.label_type == NER_ANNOTATION:\n y_pred = self._label_generator.inverse_transform(y_pred)\n\n if return_proba:\n y_pred = y_pred[-1]\n else:\n if inverse_categorical:\n # Return annotations and offsets\n y_pred = y_pred[1]\n else:\n y_pred = y_pred[0]\n else:\n raise NotImplementedError\n\n return y_pred\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_categorical.py",
"content": "import logging\nimport random\nfrom typing import Any, Dict, List, Optional, Union\n\nfrom torch import nn\n\nfrom ..constants import CATEGORICAL, COLUMN\nfrom .collator import StackCollator, TupleCollator\n\nlogger = logging.getLogger(__name__)\n\n\nclass CategoricalProcessor:\n \"\"\"\n Prepare categorical data for the model specified by \"prefix\".\n For multiple models requiring categorical data, we need to create a CategoricalProcessor\n for each related model so that they will have independent input.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n requires_column_info: bool = False,\n dropout: Optional[float] = 0,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n logger.debug(f\"initializing categorical processor for model {model.prefix}\")\n self.prefix = model.prefix\n self.requires_column_info = requires_column_info\n self.num_categories = model.num_categories\n self.dropout = dropout\n assert 0 <= self.dropout <= 1\n if self.dropout > 0:\n logger.debug(f\"categorical value dropout probability: {self.dropout}\")\n fill_values = {k: v - 1 for k, v in self.num_categories.items()}\n logger.debug(f\"dropped values will be replaced by {fill_values}\")\n\n @property\n def categorical_key(self):\n return f\"{self.prefix}_{CATEGORICAL}\"\n\n @property\n def categorical_column_prefix(self):\n return f\"{self.categorical_key}_{COLUMN}\"\n\n def collate_fn(self, categorical_column_names: Optional[List] = None) -> Dict:\n \"\"\"\n Collate individual samples into a batch. It stacks categorical features of\n each column independently. This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for categorical features.\n \"\"\"\n fn = {}\n if self.requires_column_info:\n assert categorical_column_names, \"Empty categorical column names.\"\n for col_name in categorical_column_names:\n fn[f\"{self.categorical_column_prefix}_{col_name}\"] = StackCollator()\n\n fn[self.categorical_key] = TupleCollator([StackCollator() for _ in range(len(categorical_column_names))])\n\n return fn\n\n def process_one_sample(\n self,\n categorical_features: Dict[str, int],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Process one sample's categorical features. Assume the categorical features\n are the encoded labels from sklearn' LabelEncoder().\n\n Parameters\n ----------\n categorical_features\n Categorical features of one sample.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing the processed categorical features.\n \"\"\"\n ret = {}\n if self.requires_column_info:\n # TODO: consider moving this for loop into __init__() since each sample has the same information.\n for i, col_name in enumerate(categorical_features.keys()):\n ret[f\"{self.categorical_column_prefix}_{col_name}\"] = i\n\n if is_training and self.dropout > 0:\n categorical_features_copy = dict()\n for k, v in categorical_features.items():\n if random.uniform(0, 1) <= self.dropout:\n categorical_features_copy[k] = self.num_categories[k] - 1\n else:\n categorical_features_copy[k] = v\n categorical_features = categorical_features_copy\n\n # make sure keys are in the same order\n assert list(categorical_features.keys()) == list(self.num_categories.keys())\n ret[self.categorical_key] = list(categorical_features.values())\n\n return ret\n\n def __call__(\n self,\n categorical_features: Dict[str, int],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Extract one sample's categorical features and customize it for a specific model.\n\n Parameters\n ----------\n categorical_features\n Categorical features of one sample.\n sub_dtypes\n The sub data types of all categorical columns.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's processed categorical features.\n \"\"\"\n return self.process_one_sample(\n categorical_features=categorical_features,\n is_training=is_training,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_document.py",
"content": "import logging\nimport os\nimport warnings\nfrom typing import Any, Callable, Dict, List, Optional, Union\n\nimport numpy as np\nimport PIL\nimport pytesseract\nfrom numpy.typing import NDArray\nfrom torch import nn\nfrom torchvision import transforms\n\nfrom ..constants import BBOX, DOCUMENT_PDF\nfrom ..models.utils import get_pretrained_tokenizer\nfrom .collator import PadCollator\nfrom .process_image import ImageProcessor\n\nlogger = logging.getLogger(__name__)\n\n\nclass DocumentProcessor(ImageProcessor):\n \"\"\"\n Prepare document data for Document Classification.\n OCR (Optical character recognition) is applied to get the document texts and bounding boxes.\n Both texts and images will be processed.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n train_transforms: Union[List[str], Callable, List[Callable]],\n val_transforms: Union[List[str], Callable, List[Callable]],\n size: Optional[int] = None,\n text_max_len: Optional[int] = 512,\n missing_value_strategy: Optional[str] = \"zero\",\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model using this data processor.\n train_transforms\n A list of image transforms used in training. Note that the transform order matters.\n val_transforms\n A list of image transforms used in validation/test/prediction. Note that the transform order matters.\n size\n The width / height of a square image.\n text_max_len\n The max text length of tokenizer. Default: 512.\n missing_value_strategy\n How to deal with a missing document. We now support:\n - skip\n Skip this sample\n -zero\n Use a document image with zero pixels.\n \"\"\"\n self.prefix = model.prefix\n self.text_token_ids_key = model.text_token_ids_key\n self.text_attention_mask_key = model.text_attention_mask_key\n self.text_bbox_key = model.text_bbox_key\n self.text_segment_ids_key = model.text_segment_ids_key\n self.document_pixel_value_key = model.document_pixel_value_key\n\n # For document image processing.\n self.size = size\n self.train_transforms = train_transforms\n self.val_transforms = val_transforms\n self.mean = model.image_mean\n self.std = model.image_std\n self.normalization = transforms.Normalize(self.mean, self.std)\n self.train_processor = self.construct_image_processor(\n size=self.size, normalization=self.normalization, image_transforms=self.train_transforms\n )\n self.val_processor = self.construct_image_processor(\n size=self.size, normalization=self.normalization, image_transforms=self.val_transforms\n )\n\n self.missing_value_strategy = missing_value_strategy\n\n # Store OCR results\n self.documents = {}\n\n # Whether only text is used (automatically detected).\n # If True, normal text foundation models (e.g., bert-base) can be used.\n self.is_text_only_flag = model.is_text_only_flag\n\n self.pad_token_box = [0, 0, 0, 0] # cls box token\n self.sep_token_box = [1000, 1000, 1000, 1000] # sep box token\n\n # For document text processing.\n # Disable tokenizer parallelism\n os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n self.tokenizer_name = model.tokenizer_name\n self.tokenizer = model.tokenizer\n if text_max_len is None or text_max_len <= 0:\n self.text_max_len = self.tokenizer.model_max_length\n else:\n if text_max_len < self.tokenizer.model_max_length:\n warnings.warn(\n f\"provided max length: {text_max_len} \"\n f\"is smaller than {model.checkpoint_name}'s default: {self.tokenizer.model_max_length}\"\n )\n self.text_max_len = min(text_max_len, self.tokenizer.model_max_length)\n\n self.tokenizer.model_max_length = self.text_max_len\n\n def collate_fn(self, text_column_names: Optional[List] = None) -> Dict:\n \"\"\"\n Collate multimodal features into a batch.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for multimodal data.\n \"\"\"\n fn = {}\n\n fn.update(\n {\n self.text_token_ids_key: PadCollator(pad_val=self.tokenizer.pad_token_id),\n self.text_attention_mask_key: PadCollator(pad_val=0),\n self.text_bbox_key: PadCollator(pad_val=0),\n self.text_segment_ids_key: PadCollator(pad_val=0),\n }\n )\n # If not text only, document images will be used.\n if not self.is_text_only_flag:\n fn.update(\n {\n self.document_pixel_value_key: PadCollator(pad_val=0),\n }\n )\n return fn\n\n @staticmethod\n def normalize_box(box, width, height):\n \"\"\"\n Normalize the bounding boxes.\n\n Parameters\n ----------\n box\n The bounding box to be processed.\n width\n Width of the document image.\n height\n Height of the document image.\n\n Returns\n -------\n A normalized bounding box.\n \"\"\"\n return [\n int(1000 * (box[0] / width)),\n int(1000 * (box[1] / height)),\n int(1000 * (box[2] / width)),\n int(1000 * (box[3] / height)),\n ]\n\n def apply_ocr(self, doc_image):\n \"\"\"\n Apply OCR on document images to ecognize and âreadâ the text embedded in document images.\n Specifically, Python-tesseract, a wrapper for Google's Tesseract-OCR Engine, will be used.\n\n Parameters\n ----------\n doc_image\n The document image.\n\n Returns\n -------\n A dictionary with recognized text and corresponding bounding boxes.\n \"\"\"\n results = {}\n width, height = doc_image.size\n # apply ocr to the document image.\n ocr_df = pytesseract.image_to_data(doc_image, output_type=\"data.frame\")\n float_cols = ocr_df.select_dtypes(\"float\").columns\n ocr_df = ocr_df.dropna().reset_index(drop=True)\n ocr_df[float_cols] = ocr_df[float_cols].round(0).astype(int)\n ocr_df = ocr_df.replace(r\"^\\s*$\", np.nan, regex=True)\n ocr_df = ocr_df.dropna().reset_index(drop=True)\n\n # get the words and actual (unnormalized) bounding boxes.\n words = list(ocr_df.text)\n words = [str(w) for w in words]\n coordinates = ocr_df[[\"left\", \"top\", \"width\", \"height\"]]\n actual_boxes = []\n for idx, row in coordinates.iterrows():\n # the row comes in (left, top, width, height) format.\n x, y, w, h = tuple(row)\n # we turn it into (left, top, left+width, top+height) to get the actual box.\n actual_box = [x, y, x + w, y + h]\n actual_boxes.append(actual_box)\n\n # normalize the bounding boxes.\n boxes = []\n for box in actual_boxes:\n boxes.append(self.normalize_box(box, width, height))\n\n assert len(words) == len(boxes)\n\n results[\"words\"] = words\n results[BBOX] = boxes\n return results\n\n def get_ocr_features(self, doc_path, doc_image):\n \"\"\"\n Get the OCR features, and avoid running OCR multiple times.\n\n Parameters\n ----------\n doc_path\n The document path.\n doc_image\n The document image.\n\n Returns\n -------\n Extracted words, image pixel features, and bounding boxes.\n \"\"\"\n # The OCR process is time-consuming, so apply OCR on each image only once.\n if doc_path not in self.documents:\n ocr_res = self.apply_ocr(doc_image)\n # store the ocr results.\n self.documents.update({doc_path: ocr_res})\n else:\n # reuse the ocr results.\n ocr_res = self.documents[doc_path]\n\n words = ocr_res[\"words\"]\n normalized_word_boxes = ocr_res[BBOX]\n\n return words, normalized_word_boxes\n\n def process_one_sample(\n self,\n document_features: Dict[str, Union[NDArray, list]],\n data_types: Dict[str, Union[NDArray, list]],\n is_training: bool,\n image_mode: Optional[str] = \"RGB\",\n ):\n \"\"\"\n Read documents, process them, and stack them. One sample has one document image.\n\n Parameters\n ----------\n document_features\n One sample has one document image column in a pd.DataFrame.\n data_types\n Data type of all columns.\n is_training\n Whether to process document images in the training mode.\n image_mode\n A string which defines the type and depth of a pixel in the image.\n For example, RGB, RGBA, CMYK, and etc.\n\n Returns\n -------\n A dictionary containing one sample's document and its features.\n \"\"\"\n ret = {}\n for per_col_name, per_col_image_features in document_features.items():\n try:\n # Process PDF documents.\n if data_types[per_col_name] == DOCUMENT_PDF:\n from pdf2image import convert_from_path\n\n # Convert PDF to PIL images.\n doc_images = convert_from_path(per_col_image_features[0])\n if doc_images:\n doc_image = doc_images[0].convert(image_mode)\n words, normalized_word_boxes = self.get_ocr_features(per_col_image_features[0], doc_image)\n else:\n raise ValueError(\"Failed to convert PDF to images.\")\n else:\n # Process document image.\n with PIL.Image.open(per_col_image_features[0]) as doc_image:\n doc_image = doc_image.convert(image_mode)\n words, normalized_word_boxes = self.get_ocr_features(per_col_image_features[0], doc_image)\n except ImportError as e:\n raise e\n except Exception as e:\n if self.missing_value_strategy.lower() == \"zero\":\n logger.debug(f\"Using a zero image due to '{e}'\")\n doc_image = PIL.Image.new(image_mode, (self.size, self.size), color=0)\n doc_image = doc_image.convert(image_mode)\n words = \"\" # empty words\n normalized_word_boxes = [self.pad_token_box]\n else:\n raise e\n\n if is_training:\n doc_image = self.train_processor(doc_image)\n else:\n doc_image = self.val_processor(doc_image)\n\n if self.is_text_only_flag:\n # Padding of token_boxes up the bounding boxes to the sequence length.\n token_boxes = []\n all_tokens = []\n for word, box in zip(words, normalized_word_boxes):\n # Tokenize bounding box separately.\n word_tokens = self.tokenizer.tokenize(word)\n all_tokens.append(word_tokens)\n token_boxes.extend([box] * len(word_tokens))\n\n encoding = self.tokenizer(\n \" \".join(words), padding=\"max_length\", truncation=True, return_token_type_ids=True\n )\n\n # Truncation of token_boxes\n special_tokens_count = 2 # one cls token and one sep token\n if len(token_boxes) > self.text_max_len - special_tokens_count:\n token_boxes = token_boxes[: (self.text_max_len - special_tokens_count)]\n # add bounding boxes of cls + sep tokens\n token_boxes = [self.pad_token_box] + token_boxes + [self.sep_token_box]\n\n padding_length = self.text_max_len - sum(encoding.attention_mask)\n token_boxes += [self.pad_token_box] * padding_length\n\n else:\n if not isinstance(words, list):\n words = [words]\n encoding = self.tokenizer(\n words,\n boxes=normalized_word_boxes,\n padding=\"max_length\",\n truncation=True,\n return_token_type_ids=True,\n )\n token_boxes = encoding.bbox\n\n ret.update(\n {\n self.text_token_ids_key: np.array(encoding.input_ids, dtype=np.int32),\n self.text_attention_mask_key: encoding.attention_mask,\n self.text_bbox_key: np.array(token_boxes, dtype=np.int32),\n self.text_segment_ids_key: np.array(encoding.token_type_ids, dtype=np.int32),\n }\n )\n if not self.is_text_only_flag:\n ret.update({self.document_pixel_value_key: doc_image})\n\n return ret\n\n def save_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Save the text tokenizer and record its relative paths, e.g, hf_text.\n\n Parameters\n ----------\n path\n The root path of saving.\n\n \"\"\"\n save_path = os.path.join(path, self.prefix)\n self.tokenizer.save_pretrained(save_path)\n self.tokenizer = self.prefix\n\n def load_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Load saved text tokenizers. If text/ner processors already have tokenizers,\n then do nothing.\n\n Parameters\n ----------\n path\n The root path of loading.\n\n Returns\n -------\n A list of text/ner processors with tokenizers loaded.\n \"\"\"\n if isinstance(self.tokenizer, str):\n load_path = os.path.join(path, self.tokenizer)\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=load_path,\n )\n\n def __call__(\n self,\n all_features: Dict[str, Union[NDArray, list]],\n data_types: Dict[str, Union[NDArray, list]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Extract one sample's multimodal data.\n\n Parameters\n ----------\n all\n All the raw input data.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's features and/or labels.\n \"\"\"\n\n ret = self.process_one_sample(all_features, data_types, is_training)\n\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_image.py",
"content": "import ast\nimport copy\nimport logging\nimport random\nimport warnings\nfrom io import BytesIO\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport numpy as np\nimport PIL\nimport torch\nfrom omegaconf import ListConfig\nfrom PIL import ImageFile\nfrom torch import nn\nfrom torchvision import transforms\n\nfrom .randaug import RandAugment\nfrom .trivial_augmenter import TrivialAugment\n\ntry:\n from torchvision.transforms import InterpolationMode\n\n BICUBIC = InterpolationMode.BICUBIC\n NEAREST = InterpolationMode.NEAREST\nexcept ImportError:\n BICUBIC = PIL.Image.BICUBIC\n NEAREST = PIL.Image.NEAREST\n\nfrom ..constants import COLUMN, IMAGE, IMAGE_BASE64_STR, IMAGE_BYTEARRAY, IMAGE_VALID_NUM\nfrom .collator import PadCollator, StackCollator\n\nlogger = logging.getLogger(__name__)\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nclass ImageProcessor:\n \"\"\"\n Prepare image data for the model specified by \"prefix\". For multiple models requiring image data,\n we need to create a ImageProcessor for each related model so that they will have independent input.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n train_transforms: Union[List[str], Callable, List[Callable]],\n val_transforms: Union[List[str], Callable, List[Callable]],\n max_image_num_per_column: Optional[int] = 1,\n missing_value_strategy: Optional[str] = \"zero\",\n requires_column_info: Optional[bool] = False,\n dropout: Optional[float] = 0,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n train_transforms\n A list of image transforms used in training. Note that the transform order matters.\n val_transforms\n A list of image transforms used in validation/test/prediction. Note that the transform order matters.\n max_image_num_per_column\n The maximum number of images one sample can have.\n missing_value_strategy\n How to deal with a missing image. We now support:\n - skip\n Skip this sample\n -zero\n Use an image with zero pixels.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n logger.debug(f\"initializing image processor for model {model.prefix}\")\n self.train_transforms = train_transforms\n self.val_transforms = val_transforms\n logger.debug(f\"image training transforms: {self.train_transforms}\")\n logger.debug(f\"image validation transforms: {self.val_transforms}\")\n\n self.prefix = model.prefix\n self.missing_value_strategy = missing_value_strategy\n self.requires_column_info = requires_column_info\n assert 0 <= dropout <= 1\n if dropout > 0:\n logger.debug(f\"image dropout probability: {dropout}\")\n self.dropout = dropout\n self.size = model.image_size\n self.mean = model.image_mean\n self.std = model.image_std\n\n self.normalization = transforms.Normalize(self.mean, self.std)\n if max_image_num_per_column <= 0:\n logger.debug(f\"max_image_num_per_column {max_image_num_per_column} is reset to 1\")\n max_image_num_per_column = 1\n self.max_image_num_per_column = max_image_num_per_column\n logger.debug(f\"max_image_num_per_column: {max_image_num_per_column}\")\n\n self.train_processor = self.construct_image_processor(\n image_transforms=self.train_transforms, size=self.size, normalization=self.normalization\n )\n self.val_processor = self.construct_image_processor(\n image_transforms=self.val_transforms, size=self.size, normalization=self.normalization\n )\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n def collate_fn(self, image_column_names: Optional[List] = None, per_gpu_batch_size: Optional[int] = None) -> Dict:\n \"\"\"\n Collate images into a batch. Here it pads images since the image number may\n vary from sample to sample. Samples with fewer images will be padded zeros.\n The valid image numbers of samples will be stacked into a vector.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for image data.\n \"\"\"\n fn = {}\n if self.requires_column_info:\n assert image_column_names, \"Empty image column names.\"\n for col_name in image_column_names:\n fn[f\"{self.image_column_prefix}_{col_name}\"] = StackCollator()\n\n fn.update(\n {\n self.image_key: PadCollator(pad_val=0),\n self.image_valid_num_key: StackCollator(),\n }\n )\n\n return fn\n\n def process_one_sample(\n self,\n images: Dict[str, Union[List[str], List[bytearray]]],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n image_mode: Optional[str] = \"RGB\",\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_image_num_per_column is the available image number.\n\n Parameters\n ----------\n images\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n sub_dtypes\n What modality each column belongs to.\n is_training\n Whether to process images in the training mode.\n image_mode\n A string which defines the type and depth of a pixel in the image.\n For example, RGB, RGBA, CMYK, and etc.\n\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n valid_images = []\n zero_images = []\n ret = {}\n column_start = 0\n\n for per_col_name, per_col_image_raw in images.items():\n for img_raw in per_col_image_raw[: self.max_image_num_per_column]:\n if is_training and self.dropout > 0 and random.uniform(0, 1) <= self.dropout:\n img = PIL.Image.new(image_mode, (self.size, self.size), color=0)\n is_zero_img = True\n else:\n with warnings.catch_warnings():\n warnings.filterwarnings(\n \"ignore\",\n message=(\n \"Palette images with Transparency expressed in bytes should be converted to RGBA images\"\n ),\n )\n is_zero_img = False\n try:\n if sub_dtypes.get(per_col_name) in [IMAGE_BYTEARRAY, IMAGE_BASE64_STR]:\n img_raw = BytesIO(img_raw)\n\n with PIL.Image.open(img_raw) as img:\n img = img.convert(image_mode)\n except Exception as e:\n if self.missing_value_strategy.lower() == \"zero\":\n logger.debug(f\"Using a zero image due to '{e}'\")\n img = PIL.Image.new(image_mode, (self.size, self.size), color=0)\n is_zero_img = True\n else:\n raise e\n if is_training:\n img = self.train_processor(img)\n else:\n img = self.val_processor(img)\n\n if is_zero_img:\n zero_images.append(img)\n else:\n valid_images.append(img)\n\n if self.requires_column_info:\n # only count the valid images since they are put ahead of the zero images in the below returning\n ret[f\"{self.image_column_prefix}_{per_col_name}\"] = np.array(\n [column_start, len(valid_images)], dtype=np.int64\n )\n column_start = len(valid_images)\n\n ret.update(\n {\n self.image_key: torch.tensor([])\n if len(valid_images + zero_images) == 0\n else torch.stack(valid_images + zero_images, dim=0),\n self.image_valid_num_key: len(valid_images),\n }\n )\n return ret\n\n @staticmethod\n def get_image_transform_funcs(transform_types: Union[List[str], ListConfig, List[Callable]], size: int):\n \"\"\"\n Parse a list of transform strings into callable objects.\n\n Parameters\n ----------\n transform_types\n A list of transforms, which can be strings or callable objects.\n size\n Image size.\n\n Returns\n -------\n A list of transform objects.\n \"\"\"\n image_transforms = []\n\n if not transform_types:\n return image_transforms\n\n if isinstance(transform_types, ListConfig):\n transform_types = list(transform_types)\n elif not isinstance(transform_types, list):\n transform_types = [transform_types]\n\n if all([isinstance(trans_type, str) for trans_type in transform_types]):\n pass\n elif all([isinstance(trans_type, Callable) for trans_type in transform_types]):\n return copy.copy(transform_types)\n else:\n raise ValueError(\n f\"transform_types {transform_types} contain neither all strings nor all callable objects.\"\n )\n\n for trans_type in transform_types:\n args = None\n kargs = None\n if \"(\" in trans_type:\n trans_mode = trans_type[0 : trans_type.find(\"(\")]\n if \"{\" in trans_type:\n kargs = ast.literal_eval(trans_type[trans_type.find(\"{\") : trans_type.rfind(\")\")])\n else:\n args = ast.literal_eval(trans_type[trans_type.find(\"(\") :])\n else:\n trans_mode = trans_type\n\n if trans_mode == \"resize_to_square\":\n image_transforms.append(transforms.Resize((size, size), interpolation=BICUBIC))\n elif trans_mode == \"resize_gt_to_square\":\n image_transforms.append(transforms.Resize((size, size), interpolation=NEAREST))\n elif trans_mode == \"resize_shorter_side\":\n image_transforms.append(transforms.Resize(size, interpolation=BICUBIC))\n elif trans_mode == \"center_crop\":\n image_transforms.append(transforms.CenterCrop(size))\n elif trans_mode == \"random_resize_crop\":\n image_transforms.append(transforms.RandomResizedCrop(size))\n elif trans_mode == \"random_horizontal_flip\":\n image_transforms.append(transforms.RandomHorizontalFlip())\n elif trans_mode == \"random_vertical_flip\":\n image_transforms.append(transforms.RandomVerticalFlip())\n elif trans_mode == \"color_jitter\":\n if kargs is not None:\n image_transforms.append(transforms.ColorJitter(**kargs))\n elif args is not None:\n image_transforms.append(transforms.ColorJitter(*args))\n else:\n image_transforms.append(transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1))\n elif trans_mode == \"affine\":\n if kargs is not None:\n image_transforms.append(transforms.RandomAffine(**kargs))\n elif args is not None:\n image_transforms.append(transforms.RandomAffine(*args))\n else:\n image_transforms.append(\n transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1))\n )\n elif trans_mode == \"randaug\":\n if kargs is not None:\n image_transforms.append(RandAugment(**kargs))\n elif args is not None:\n image_transforms.append(RandAugment(*args))\n else:\n image_transforms.append(RandAugment(2, 9))\n elif trans_mode == \"trivial_augment\":\n image_transforms.append(TrivialAugment(IMAGE, 30))\n else:\n raise ValueError(f\"unknown transform type: {trans_mode}\")\n\n return image_transforms\n\n def construct_image_processor(\n self,\n image_transforms: Union[List[Callable], List[str]],\n size: int,\n normalization,\n ) -> transforms.Compose:\n \"\"\"\n Build up an image processor from the provided list of transform types.\n\n Parameters\n ----------\n image_transforms\n A list of image transform types.\n size\n Image size.\n normalization\n A transforms.Normalize object. When the image is ground truth image, 'normalization=None' should be specified.\n\n Returns\n -------\n A transforms.Compose object.\n \"\"\"\n image_transforms = self.get_image_transform_funcs(transform_types=image_transforms, size=size)\n if not any([isinstance(trans, transforms.ToTensor) for trans in image_transforms]):\n image_transforms.append(transforms.ToTensor())\n if (\n not any([isinstance(trans, transforms.Normalize) for trans in image_transforms])\n and normalization is not None\n ):\n image_transforms.append(normalization)\n return transforms.Compose(image_transforms)\n\n def __call__(\n self,\n images: Dict[str, List[str]],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Obtain one sample's images and customized them for a specific model.\n\n Parameters\n ----------\n images\n Images of one sample.\n sub_dtypes\n The sub data types of all image columns.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's processed images and their number.\n \"\"\"\n images = {k: [v] if isinstance(v, str) else v for k, v in images.items()}\n\n return self.process_one_sample(images=images, sub_dtypes=sub_dtypes, is_training=is_training)\n\n def __getstate__(self):\n odict = self.__dict__.copy() # get attribute dictionary\n del odict[\"train_processor\"] # remove augmenter to support pickle\n return odict\n\n def __setstate__(self, state):\n self.__dict__ = state\n self.train_processor = self.construct_image_processor(\n image_transforms=self.train_transforms,\n size=self.size,\n normalization=self.normalization,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_label.py",
"content": "import logging\nfrom typing import Any, Dict, List, Optional, Union\n\nfrom torch import nn\n\nfrom ..constants import LABEL, MMDET_IMAGE\nfrom .collator import ListCollator, StackCollator\n\nlogger = logging.getLogger(__name__)\n\n\nclass LabelProcessor:\n \"\"\"\n Prepare ground-truth labels for the model specified by \"prefix\".\n For multiple models, we need to create a LabelProcessor for each model so that\n each model will have independent labels.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n \"\"\"\n logger.debug(f\"initializing label processor for model {model.prefix}\")\n self.prefix = model.prefix\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def collate_fn(self, label_column_names: Optional[List] = None, per_gpu_batch_size: Optional[int] = None) -> Dict:\n \"\"\"\n Collate individual labels into a batch. Here it stacks labels.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for labels.\n \"\"\"\n if self.prefix == MMDET_IMAGE:\n fn = {self.label_key: ListCollator()}\n else:\n fn = {self.label_key: StackCollator()}\n return fn\n\n def process_one_sample(\n self,\n labels: Dict[str, Union[int, float, list]],\n ) -> Dict:\n \"\"\"\n Process one sample's labels. Here it only picks the first label.\n New rules can be added if necessary.\n\n Parameters\n ----------\n labels\n One sample may have multiple labels.\n Returns\n -------\n A dictionary containing one sample's label.\n \"\"\"\n\n return {\n self.label_key: labels[next(iter(labels))], # get the first key's value\n }\n\n def __call__(\n self,\n labels: Dict[str, Union[int, float]],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n load_only: bool = False, # TODO: refactor mmdet_image and remove this\n ) -> Dict:\n \"\"\"\n Extract one sample's labels and customize them for a specific model.\n\n Parameters\n ----------\n labels\n Labels of one sample.\n sub_dtypes\n The sub data types of all label columns.\n is_training\n Whether to do processing in the training mode. This unused flag is for the API compatibility.\n load_only\n Whether to only load the data. Other processing steps may happen in dataset.__getitem__.\n\n Returns\n -------\n A dictionary containing one sample's processed label.\n \"\"\"\n return self.process_one_sample(labels)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_mmlab/__init__.py",
"content": "from .process_mmdet import MMDetProcessor\nfrom .process_mmocr import MMOcrProcessor\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_mmlab/process_mmdet.py",
"content": "import logging\nfrom typing import Dict, List, Optional, Union\n\nimport numpy as np\nimport PIL\nfrom PIL import ImageFile\nfrom torch import nn\n\nfrom ..infer_types import is_rois_input\nfrom .process_mmlab_base import MMLabProcessor\n\ntry:\n from mmcv.transforms import Compose\nexcept ImportError as e:\n pass\n\nlogger = logging.getLogger(__name__)\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nclass MMDetProcessor(MMLabProcessor):\n \"\"\"\n Prepare rois data for mmdetection models.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n max_img_num_per_col: Optional[int] = 1,\n missing_value_strategy: Optional[str] = \"zero\",\n requires_column_info: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model using this data processor.\n max_img_num_per_col\n The maximum number of images one sample can have.\n missing_value_strategy\n How to deal with a missing image. We now support:\n - skip\n Skip this sample\n -zero\n Use an image with zero pixels.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n from ...utils import CollateMMDet, check_if_packages_installed\n\n check_if_packages_installed(package_names=[\"mmcv\", \"mmengine\", \"mmdet\"])\n\n super().__init__(\n model=model,\n collate_func=CollateMMDet,\n max_img_num_per_col=max_img_num_per_col,\n missing_value_strategy=missing_value_strategy,\n requires_column_info=requires_column_info,\n )\n\n # for yolox we separate loading pipeline to support multi_image_mix_dataset\n if \"loading_pipeline\" in self.cfg.keys():\n self.load_processor = Compose(self.cfg.loading_pipeline)\n\n def prepare_one_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ):\n mm_data = dict(img_prefix=None, bbox_fields=[])\n\n for per_col_name, per_col_content in image_paths.items():\n if is_rois_input(per_col_content):\n rois = np.array(per_col_content)\n # https://github.com/open-mmlab/mmdetection/blob/ecac3a77becc63f23d9f6980b2a36f86acd00a8a/mmdet/datasets/transforms/loading.py#L155\n mm_data[\"instances\"] = []\n for roi in rois:\n mm_data[\"instances\"].append(\n {\n \"bbox\": roi[:4],\n \"bbox_label\": roi[4],\n \"ignore_flag\": 0,\n }\n )\n else:\n with PIL.Image.open(per_col_content[0]) as img:\n # mm_data[\"img_info\"] = dict(filename=per_col_content[0], height=img.height, width=img.width)\n mm_data[\"img_path\"] = per_col_content[0]\n if self.requires_column_info:\n pass # TODO\n\n mm_data[\"img_id\"] = 0 # TODO: use a non trivial image id for TTA (test time augmentation)\n\n return mm_data\n\n def load_one_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.\n\n Parameters\n ----------\n image_paths\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n mm_data = self.prepare_one_sample(image_paths=image_paths, is_training=is_training)\n ret = {self.image_key: self.load_processor(mm_data)}\n return ret\n\n def process_one_loaded_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.\n\n Parameters\n ----------\n image_paths\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n assert is_training\n\n image_paths.update({self.image_key: self.train_processor(image_paths[self.image_key])})\n\n return image_paths\n\n def process_one_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.\n Parameters\n ----------\n image_paths\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n is_training\n Whether to process images in the training mode.\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n mm_data = self.prepare_one_sample(image_paths=image_paths, is_training=is_training)\n ret = {self.image_key: self.train_processor(mm_data) if is_training else self.val_processor(mm_data)}\n return ret\n\n def __call__(\n self,\n images: Dict[str, List[str]],\n data_types: Dict[str, Union[int, float, list]],\n is_training: bool,\n load_only: bool = False,\n ) -> Dict:\n \"\"\"\n Obtain one sample's images and customized them for a specific model.\n\n Parameters\n ----------\n images\n Images of one sample.\n data_types\n Data types of all columns.\n is_training\n Whether to process images in the training mode.\n load_only\n Whether to only load the data. Other processing steps may happen in dataset.__getitem__.\n\n Returns\n -------\n A dictionary containing one sample's processed images and their number.\n \"\"\"\n images = {k: [v] if isinstance(v, str) else v for k, v in images.items()}\n if load_only:\n return self.load_one_sample(images, is_training)\n else:\n return self.process_one_sample(images, is_training)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_mmlab/process_mmlab_base.py",
"content": "import logging\nimport warnings\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport numpy as np\nimport PIL\nfrom PIL import ImageFile\nfrom torch import nn\n\nfrom ...constants import COLUMN, IMAGE, IMAGE_VALID_NUM, MMDET_IMAGE\nfrom ..collator import StackCollator\nfrom ..infer_types import is_rois_input\n\ntry:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n import mmcv\n from mmcv.transforms import Compose\nexcept ImportError as e:\n mmcv = None\n\ntry:\n import mmdet\n from mmdet.datasets.transforms import ImageToTensor\nexcept ImportError as e:\n mmdet = None\n\ntry:\n import mmocr\nexcept ImportError:\n mmocr = None\n\n\nlogger = logging.getLogger(__name__)\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nclass MMLabProcessor:\n \"\"\"\n The base class to prepare data for mmlab models specified by \"prefix\".\n Child class shall provide its specific collate function in __init__.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n collate_func: Callable,\n max_img_num_per_col: Optional[int] = 1,\n missing_value_strategy: Optional[str] = \"zero\",\n requires_column_info: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model using this data processor.\n collate_func\n The collate function to use for this processor\n max_img_num_per_col\n The maximum number of images one sample can have.\n missing_value_strategy\n How to deal with a missing image. We now support:\n - skip\n Skip this sample\n -zero\n Use an image with zero pixels.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n from ...utils import check_if_packages_installed\n\n check_if_packages_installed(package_names=[\"mmcv\"])\n\n self.prefix = model.prefix\n self.missing_value_strategy = missing_value_strategy\n self.requires_column_info = requires_column_info\n self.collate_func = collate_func\n\n self.max_img_num_per_col = max_img_num_per_col\n if max_img_num_per_col <= 0:\n logger.debug(f\"max_img_num_per_col {max_img_num_per_col} is reset to 1\")\n max_img_num_per_col = 1\n self.max_img_num_per_col = max_img_num_per_col\n logger.debug(f\"max_img_num_per_col: {max_img_num_per_col}\")\n\n if self.prefix.lower().startswith(MMDET_IMAGE):\n check_if_packages_installed(package_names=[\"mmdet\"])\n else:\n assert mmocr is not None, \"Please install MMOCR by: pip install mmocr.\"\n self.cfg = model.model.cfg\n self.val_processor = Compose(self.cfg.test_pipeline)\n self.train_processor = Compose(self.cfg.train_pipeline)\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n def collate_fn(self, image_column_names: Optional[List] = None, per_gpu_batch_size: Optional[int] = None) -> Dict:\n \"\"\"\n Collate images into a batch. Here it pads images since the image number may\n vary from sample to sample. Samples with less images will be padded zeros.\n The valid image numbers of samples will be stacked into a vector.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for image data.\n \"\"\"\n\n fn = {}\n if self.requires_column_info:\n assert image_column_names, \"Empty image column names.\"\n for col_name in image_column_names:\n fn[f\"{self.image_column_prefix}_{col_name}\"] = StackCollator()\n\n fn.update(\n {\n self.image_key: self.collate_func(samples_per_gpu=per_gpu_batch_size),\n }\n )\n\n return fn\n\n def process_one_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.\n\n Parameters\n ----------\n image_paths\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n mm_data = dict(img_prefix=None, bbox_fields=[], mask_fields=[])\n ret = {}\n\n for per_col_name, per_col_content in image_paths.items():\n if is_rois_input(per_col_content):\n rois = np.array(per_col_content)\n # TODO: add gt masks\n mm_data[\"ann_info\"] = dict(bboxes=rois[:, :4], labels=rois[:, 4], masks=[])\n else:\n with PIL.Image.open(per_col_content[0]) as img:\n mm_data[\"img_info\"] = dict(filename=per_col_content[0], height=img.height, width=img.width)\n if self.requires_column_info:\n pass # TODO\n\n ret.update({self.image_key: self.train_processor(mm_data) if is_training else self.val_processor(mm_data)})\n\n return ret\n\n def __call__(\n self,\n images: Dict[str, List[str]],\n data_types: Dict[str, Union[int, float, list]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Obtain one sample's images and customized them for a specific model.\n\n Parameters\n ----------\n images\n Images of one sample.\n data_types\n Data type of all columns.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's processed images and their number.\n \"\"\"\n images = {k: [v] if isinstance(v, str) else v for k, v in images.items()}\n\n return self.process_one_sample(images, is_training)\n\n def __getstate__(self):\n odict = self.__dict__.copy() # get attribute dictionary\n return odict\n\n def __setstate__(self, state):\n self.__dict__ = state\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_mmlab/process_mmocr.py",
"content": "import logging\nfrom typing import Dict, List, Optional\n\nimport numpy as np\nimport PIL\nfrom PIL import ImageFile\nfrom torch import nn\n\nfrom ..infer_types import is_rois_input\nfrom .process_mmlab_base import MMLabProcessor\n\nlogger = logging.getLogger(__name__)\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nclass MMOcrProcessor(MMLabProcessor):\n \"\"\"\n Prepare data for mmocr models.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n max_img_num_per_col: Optional[int] = 1,\n missing_value_strategy: Optional[str] = \"zero\",\n requires_column_info: bool = False,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model using this data processor.\n max_img_num_per_col\n The maximum number of images one sample can have.\n missing_value_strategy\n How to deal with a missing image. We now support:\n - skip\n Skip this sample\n -zero\n Use an image with zero pixels.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n from ...utils import CollateMMOcr\n\n super().__init__(\n model=model,\n collate_func=CollateMMOcr,\n max_img_num_per_col=max_img_num_per_col,\n missing_value_strategy=missing_value_strategy,\n requires_column_info=requires_column_info,\n )\n\n def process_one_sample(\n self,\n image_paths: Dict[str, List[str]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them. One sample can have multiple images,\n resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.\n\n Parameters\n ----------\n image_paths\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's images and their number.\n \"\"\"\n # TODO: modify for MMOCR\n mm_data = dict(img_prefix=None, bbox_fields=[])\n ret = {}\n\n for per_col_name, per_col_content in image_paths.items():\n if is_rois_input(per_col_content):\n rois = np.array(per_col_content)\n mm_data[\"ann_info\"] = dict(bboxes=rois[:, :4], labels=rois[:, 4])\n else:\n with PIL.Image.open(per_col_content[0]) as img:\n mm_data[\"img_info\"] = dict(filename=per_col_content[0], height=img.height, width=img.width)\n if self.requires_column_info:\n pass # TODO\n\n ret.update({self.image_key: self.train_processor(mm_data) if is_training else self.val_processor(mm_data)})\n\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_ner.py",
"content": "import logging\nimport os\nimport re\nimport warnings\nfrom typing import Any, Dict, List, Optional, Union\n\nimport numpy as np\nfrom numpy.typing import NDArray\nfrom omegaconf import DictConfig\nfrom tokenizers import pre_tokenizers\nfrom torch import nn\n\nfrom ..constants import NER_ANNOTATION, NER_TEXT, TEXT, TEXT_NER\nfrom ..models.utils import get_pretrained_tokenizer\nfrom .collator import PadCollator, StackCollator\n\nlogger = logging.getLogger(__name__)\n\n\nclass NerProcessor:\n \"\"\"\n Prepare NER data for the model specified by \"prefix\".\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n max_len: Optional[int] = None,\n entity_map: Optional[DictConfig] = None,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The NER model.\n max_len\n The max length of the tokenizer.\n entity_map\n The map between tags and tag indexes. e.g., {\"PER\":2, \"LOC\":3}.\n \"\"\"\n self.prefix = model.prefix\n self.text_token_ids_key = model.text_token_ids_key\n self.text_valid_length_key = model.text_valid_length_key\n self.text_segment_ids_key = model.text_segment_ids_key\n self.text_token_word_mapping_key = model.text_token_word_mapping_key\n self.text_word_offsets_key = model.text_word_offsets_key\n self.label_key = model.label_key\n\n self.tokenizer = None\n self.tokenizer_name = model.tokenizer_name\n self.max_len = max_len\n self.entity_map = entity_map\n\n # Disable tokenizer parallelism\n os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n if self.prefix == NER_TEXT:\n self.tokenizer = model.tokenizer\n\n if max_len is None or max_len <= 0:\n self.max_len = self.tokenizer.model_max_length\n else:\n if max_len < self.tokenizer.model_max_length:\n warnings.warn(\n f\"provided max length: {max_len} \"\n f\"is smaller than {model.checkpoint_name}'s default: {self.tokenizer.model_max_length}\"\n )\n self.max_len = min(max_len, self.tokenizer.model_max_length)\n\n self.tokenizer.model_max_length = self.max_len\n\n def collate_fn(self, text_column_names: Optional[List] = None) -> Dict:\n \"\"\"\n Collate multimodal features into a batch.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for multimodal data.\n \"\"\"\n fn = {}\n if self.prefix == NER_TEXT:\n fn.update(\n {\n self.text_token_ids_key: PadCollator(pad_val=self.tokenizer.pad_token_id),\n self.text_valid_length_key: StackCollator(),\n self.text_segment_ids_key: PadCollator(pad_val=0),\n self.text_token_word_mapping_key: PadCollator(pad_val=0),\n self.text_word_offsets_key: PadCollator(pad_val=0),\n self.label_key: StackCollator(),\n }\n )\n return fn\n\n def process_ner(\n self,\n all_features: Dict[str, Union[int, float, list]],\n data_types: Dict[str, Union[int, float, list]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Process one NER sample's label and text features.\n New rules can be added if necessary.\n\n Parameters\n ----------\n all_features\n All features including text and ner annotations.\n data_types\n Data type of all columns.\n\n Returns\n -------\n A dictionary containing one NER sample's features and/or labels.\n \"\"\"\n ret = {}\n # overwrite model_max_length for standalone checkpoints if it's not specified.\n if self.max_len is not None and self.tokenizer.model_max_length > 10**6:\n self.tokenizer.model_max_length = self.max_len\n text_column, annotation_column = None, None\n for column_name, column_modality in data_types.items():\n if column_modality.startswith((TEXT_NER, TEXT)):\n text_column = column_name\n if column_modality == NER_ANNOTATION:\n annotation_column = column_name\n\n ner_text = all_features[text_column]\n if is_training or annotation_column is not None:\n ner_annotation = all_features[annotation_column]\n label, col_tokens, token_to_word_mappings, word_offsets = self.process_ner_annotations(\n ner_annotation, ner_text, self.entity_map, self.tokenizer\n )\n ret.update({self.label_key: label})\n else:\n col_tokens, token_to_word_mappings, word_offsets = self.tokenize_ner_text(ner_text, self.tokenizer)\n ret.update({self.label_key: np.array([], dtype=np.int32)})\n\n ret.update(\n {\n self.text_token_ids_key: np.array(col_tokens.input_ids, dtype=np.int32),\n self.text_valid_length_key: sum(col_tokens.attention_mask),\n self.text_segment_ids_key: np.array(col_tokens.token_type_ids, dtype=np.int32),\n self.text_token_word_mapping_key: token_to_word_mappings,\n self.text_word_offsets_key: word_offsets,\n }\n )\n\n return ret\n\n @classmethod\n def process_ner_annotations(cls, ner_annotations, ner_text, entity_map, tokenizer, is_eval=False):\n \"\"\"\n Generate token-level/word-level labels with given text and NER annotations.\n\n Parameters\n ----------\n ner_annotations\n The NER annotations.\n ner_text\n The corresponding raw text.\n entity_map\n The map between tags and tag indexes. e.g., {\"PER\":2, \"LOC\":3}.\n tokenizer\n The tokenizer to be used.\n is_eval\n Whether it is for evaluation or not, default: False\n\n Returns\n -------\n Token-level/word-level labels and text features.\n \"\"\"\n col_tokens, token_to_word_mappings, word_offsets = cls.tokenize_ner_text(ner_text, tokenizer)\n num_words = len(set(token_to_word_mappings)) - 1\n word_label = [1] * num_words\n # TODO: Potentially optimize word label generation via binary search\n b_prefix = \"B-\"\n i_prefix = \"I-\"\n for annot in ner_annotations:\n custom_offset = annot[0]\n custom_label = annot[1]\n is_start_word = True\n for idx, word_offset in enumerate(word_offsets[:num_words, :]):\n # support multiple words in an annotated offset range.\n # Allow partial overlapping between custom annotations and pretokenized words.\n if (word_offset[0] < custom_offset[1]) and (custom_offset[0] < word_offset[1]):\n if not (\n re.match(b_prefix, custom_label, re.IGNORECASE)\n or re.match(i_prefix, custom_label, re.IGNORECASE)\n ):\n if is_start_word and b_prefix + custom_label in entity_map:\n word_label[idx] = entity_map[b_prefix + custom_label]\n is_start_word = False\n elif i_prefix + custom_label in entity_map:\n word_label[idx] = entity_map[i_prefix + custom_label]\n else:\n if custom_label in entity_map:\n word_label[idx] = entity_map[custom_label]\n\n token_label = [0] * len(col_tokens.input_ids)\n temp = set()\n counter = 0\n for idx, token_to_word in enumerate(token_to_word_mappings):\n if token_to_word != -1 and token_to_word not in temp:\n temp.add(token_to_word)\n token_label[idx] = word_label[counter]\n counter += 1\n if not is_eval:\n label = token_label # return token-level labels for training\n else:\n label = word_label # return word-level labels for evaluation\n\n return label, col_tokens, token_to_word_mappings, word_offsets\n\n @classmethod\n def tokenize_ner_text(cls, text, tokenizer):\n \"\"\"\n Tokenization process for the NER task. It will be used for the token-level label generation\n and the input text tokenization.\n\n Parameters\n ----------\n text\n The raw text data.\n tokenizer\n The tokenizer to be used.\n\n Returns\n -------\n The output of tokenizer and word offsets.\n \"\"\"\n # pre-tokenization is required for NER token-level label generation.\n words_with_offsets = pre_tokenizers.BertPreTokenizer().pre_tokenize_str(text)\n words_with_offsets = (\n cls.is_space_counted(words_with_offsets) if len(words_with_offsets) > 1 else words_with_offsets\n )\n words = [word for word, offset in words_with_offsets]\n word_offsets = np.array([[offset[0], offset[1]] for word, offset in words_with_offsets], dtype=np.int32)\n col_tokens = tokenizer(\n words,\n is_split_into_words=True,\n return_offsets_mapping=True,\n padding=\"max_length\",\n truncation=True,\n max_length=tokenizer.model_max_length,\n return_token_type_ids=True,\n )\n offset_mapping = np.array(col_tokens.offset_mapping, dtype=np.int32)\n if len(words_with_offsets) > 1:\n if offset_mapping.shape[0] > len(words):\n word_offsets = np.pad(word_offsets, ((0, offset_mapping.shape[0] - len(words)), (0, 0)), \"constant\")\n # token to word mappings: it will tell us which token belongs to which word.\n token_to_word_mappings = [i if i != None else -1 for i in col_tokens.word_ids()]\n if len(set(token_to_word_mappings)) != len(words) + 1:\n warnings.warn(f\"The token to word mappings are incorrect!\")\n else:\n # If pre_tokenizer does not give word offsets, use word_ids and offset_mappings instead.\n word_offsets = np.append(offset_mapping[1:], [[0, 0]], axis=0)\n word_idx = np.arange(len(col_tokens.word_ids()) - col_tokens.word_ids().count(None))\n token_to_word_mappings = [\n val + word_idx[idx - 1] if val != None else -1 for idx, val in enumerate(col_tokens.word_ids())\n ]\n\n return col_tokens, token_to_word_mappings, word_offsets\n\n @staticmethod\n def is_space_counted(words_with_offsets):\n \"\"\"\n Some tokenizers will count space into words for example.\n Given text: 'hello world', normal bert will output: [('hello', (0, 5)), ('world', (6, 11))]\n while some checkpoint will output: [('âhello', (0, 5)), ('âworld', (5, 11))]\n This will lead to inconsistency issue during labelling, details can be found here:\n https://github.com/huggingface/transformers/issues/18111\n\n This function will check whether space is counted or not and realign the offset.\n \"\"\"\n offset0, offset1 = [], []\n for word, offset in words_with_offsets:\n offset0.append(offset[0])\n offset1.append(offset[1])\n\n realign = []\n if offset0[1:] == offset1[:-1]: # space are counted\n realign = [words_with_offsets[0]]\n for word, offset in words_with_offsets[1:]:\n if word.startswith(\"â\"): # it is \"Lower One Eighth Block\" (U+2581) rather than lower line (U+005F).\n realign.append((word, (offset[0] + 1, offset[1])))\n else:\n realign.append((word, offset))\n\n if realign:\n return realign\n else:\n return words_with_offsets\n\n def save_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Save the text tokenizer and record its relative paths, e.g, hf_text.\n\n Parameters\n ----------\n path\n The root path of saving.\n\n \"\"\"\n save_path = os.path.join(path, self.prefix)\n self.tokenizer.save_pretrained(save_path)\n self.tokenizer = self.prefix\n\n def load_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Load saved text tokenizers. If text/ner processors already have tokenizers,\n then do nothing.\n\n Parameters\n ----------\n path\n The root path of loading.\n\n Returns\n -------\n A list of text/ner processors with tokenizers loaded.\n \"\"\"\n if isinstance(self.tokenizer, str):\n load_path = os.path.join(path, self.tokenizer)\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=load_path,\n )\n\n def __call__(\n self,\n all_features: Dict[str, Union[NDArray, list]],\n data_types: Dict[str, Union[NDArray, list]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Extract one sample's multimodal data.\n\n Parameters\n ----------\n all\n All the raw input data.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's features and/or labels.\n \"\"\"\n ret = {}\n if self.prefix == NER_TEXT:\n ret = self.process_ner(all_features, data_types, is_training)\n\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_numerical.py",
"content": "import logging\nimport random\nfrom typing import Any, Dict, List, Optional, Union\n\nimport numpy as np\nfrom torch import nn\n\nfrom ..constants import COLUMN, NUMERICAL\nfrom .collator import StackCollator\n\nlogger = logging.getLogger(__name__)\n\n\nclass NumericalProcessor:\n \"\"\"\n Prepare numerical data for the model specified by \"prefix\".\n For multiple models requiring numerical data, we need to create a NumericalProcessor\n for each related model so that they will have independent input.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n merge: Optional[str] = \"concat\",\n requires_column_info: bool = False,\n dropout: Optional[float] = 0,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n merge\n How to merge numerical features from multiple columns in a multimodal pd.DataFrame.\n Currently, it only supports one choice:\n - concat\n Concatenate the numerical features.\n requires_column_info\n Whether to require feature column information in dataloader.\n \"\"\"\n logger.debug(f\"initializing numerical processor for model {model.prefix}\")\n self.prefix = model.prefix\n self.merge = merge\n self.requires_column_info = requires_column_info\n self.numerical_fill_values = model.numerical_fill_values\n self.dropout = dropout\n assert 0 <= self.dropout <= 1\n if self.dropout > 0:\n logger.debug(f\"numerical value dropout probability: {self.dropout}\")\n logger.debug(f\"dropped values will be replaced by {self.numerical_fill_values}\")\n\n @property\n def numerical_key(self):\n return f\"{self.prefix}_{NUMERICAL}\"\n\n @property\n def numerical_column_prefix(self):\n return f\"{self.numerical_key}_{COLUMN}\"\n\n def collate_fn(self, numerical_column_names: List) -> Dict:\n \"\"\"\n Collate individual samples into a batch. Here it stacks numerical features.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for numerical features.\n \"\"\"\n fn = {}\n if self.requires_column_info:\n assert numerical_column_names, \"Empty numerical column names.\"\n for col_name in numerical_column_names:\n fn[f\"{self.numerical_column_prefix}_{col_name}\"] = StackCollator()\n\n fn[self.numerical_key] = StackCollator()\n\n return fn\n\n def process_one_sample(\n self,\n numerical_features: Dict[str, float],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Process one sample's numerical features.\n Here it converts numerical features to a NumPy array.\n\n Parameters\n ----------\n numerical_features\n Numerical features of one sample.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing the processed numerical features.\n \"\"\"\n ret = {}\n if self.requires_column_info:\n # TODO: consider moving this for loop into __init__() since each sample has the same information.\n for i, col_name in enumerate(numerical_features.keys()):\n ret[f\"{self.numerical_column_prefix}_{col_name}\"] = i\n\n if is_training and self.dropout > 0:\n numerical_features_copy = dict()\n for k, v in numerical_features.items():\n if random.uniform(0, 1) <= self.dropout:\n numerical_features_copy[k] = self.numerical_fill_values[k]\n else:\n numerical_features_copy[k] = v\n numerical_features = numerical_features_copy\n\n if self.merge == \"concat\":\n ret[self.numerical_key] = np.array(list(numerical_features.values()), dtype=np.float32)\n else:\n raise ValueError(f\"Unknown merging type: {self.merge}\")\n\n return ret\n\n def __call__(\n self,\n numerical_features: Dict[str, float],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Extract one sample's numerical features and customize it for a specific model.\n\n Parameters\n ----------\n numerical_features\n Numerical features of one sample.\n sub_dtypes\n The sub data types of all numerical columns.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's processed numerical features.\n \"\"\"\n return self.process_one_sample(\n numerical_features=numerical_features,\n is_training=is_training,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_semantic_seg_img.py",
"content": "import logging\nimport random\nfrom typing import Dict, List, Optional, Union\n\nimport numpy as np\nimport PIL\nimport torch\nfrom PIL import Image, ImageFile\nfrom torch import nn\nfrom torchvision import transforms\n\nfrom ..constants import (\n CLASS_LABEL,\n COLUMN,\n IMAGE,\n IMAGE_VALID_NUM,\n LABEL,\n MASK_LABEL,\n SEMANTIC_SEGMENTATION_GT,\n SEMANTIC_SEGMENTATION_IMG,\n)\nfrom .collator import ListCollator, PadCollator\nfrom .process_image import ImageProcessor\n\nlogger = logging.getLogger(__name__)\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nclass SemanticSegImageProcessor(ImageProcessor):\n \"\"\"\n Prepare image data for the model specified by \"prefix\". For multiple models requiring image data,\n we need to create a ImageProcessor for each related model so that they will have independent input.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n img_transforms: List[str],\n gt_transforms: List[str],\n train_transforms: Optional[List[str]] = None,\n val_transforms: Optional[List[str]] = None,\n max_img_num_per_col: Optional[int] = 1,\n missing_value_strategy: Optional[str] = \"skip\",\n requires_column_info: bool = False,\n ignore_label: int = 255,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n img_transforms\n A list of image transforms for image.\n gt_transforms\n A list of image transforms for ground truth image.\n train_transforms\n A list of image transforms used in training for data augmentation. Note that the transform order matters.\n val_transforms\n A list of image transforms used in validation/test/prediction. Note that the transform order matters.\n max_img_num_per_col\n The maximum number of images one sample can have.\n missing_value_strategy\n How to deal with a missing image. We now support:\n - skip\n Skip this sample\n requires_column_info\n Whether to require feature column information in dataloader.\n ignore_label\n Specifies a target value that is ignored and does not contribute to the training loss and metric calculation.\n \"\"\"\n\n self.img_transforms, self.gt_transforms = img_transforms, gt_transforms\n\n self.prefix = model.prefix\n self.missing_value_strategy = missing_value_strategy\n self.requires_column_info = requires_column_info\n\n self.size = model.image_size\n self.mean = model.image_mean\n self.std = model.image_std\n self.normalization = transforms.Normalize(self.mean, self.std)\n self.num_classes = model.num_classes\n self.ignore_label = ignore_label\n\n self.max_img_num_per_col = max_img_num_per_col\n if max_img_num_per_col <= 0:\n logger.debug(f\"max_img_num_per_col {max_img_num_per_col} is reset to 1\")\n max_img_num_per_col = 1\n self.max_img_num_per_col = max_img_num_per_col\n logger.debug(f\"max_img_num_per_col: {max_img_num_per_col}\")\n\n self.img_processor = self.construct_image_processor(\n image_transforms=self.img_transforms, size=self.size, normalization=self.normalization\n )\n self.gt_processor = self.construct_image_processor(\n image_transforms=self.gt_transforms, size=self.size, normalization=None\n )\n self.train_transforms = self.get_train_transforms(train_transforms)\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def mask_label_key(self):\n return f\"{self.prefix}_{MASK_LABEL}\"\n\n @property\n def class_label_key(self):\n return f\"{self.prefix}_{CLASS_LABEL}\"\n\n def collate_fn(self, image_column_names: Optional[List] = None, per_gpu_batch_size: Optional[int] = None) -> Dict:\n \"\"\"\n Collate images into a batch. Here it pads images since the image number may\n vary from sample to sample. Samples with less images will be padded zeros.\n The valid image numbers of samples will be stacked into a vector.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for image data.\n \"\"\"\n fn = {}\n if self.requires_column_info:\n return NotImplementedError(\n f\"requires_column_info={self.requires_column_info} not implemented for semantic segmentation tasks.\"\n )\n fn.update(\n {\n self.image_key: PadCollator(pad_val=0),\n self.label_key: PadCollator(pad_val=0),\n }\n )\n\n if self.num_classes > 1:\n fn.update(\n {\n self.mask_label_key: ListCollator(),\n self.class_label_key: ListCollator(),\n }\n )\n\n return fn\n\n def process_one_sample(\n self,\n image_features: Dict[str, Union[List[str], List[bytearray]]],\n data_types: Dict[str, List[str]],\n is_training: bool,\n image_mode: Optional[str] = \"RGB\",\n ) -> Dict:\n \"\"\"\n Read images, process them, and stack them for semantic segmentation.\n\n Parameters\n ----------\n image_features\n One sample may have multiple image columns in a pd.DataFrame and multiple images\n inside each image column.\n data_types\n Data type of all columns.\n is_training\n Whether to process images in the training mode.\n image_mode\n A string which defines the type and depth of a pixel in the image.\n For example, RGB, RGBA, CMYK, and etc.\n\n Returns\n -------\n A dictionary containing one sample's image, the valid number and the ground truth image label.\n For multi-class semantic segmentation, the dictionary also includes information of per-category binary masks derived from the ground truth image. This logic follows the data processing of mask-based semantic segmentation.\n \"\"\"\n images = []\n gts = []\n if self.num_classes > 1:\n gt_masks_per_category = []\n gt_classes = []\n\n ret = {}\n annotation_column = None\n for column_name, column_modality in data_types.items():\n if column_modality == SEMANTIC_SEGMENTATION_IMG:\n image_column = column_name\n if column_modality == SEMANTIC_SEGMENTATION_GT:\n annotation_column = column_name\n\n per_col_image_features = image_features[image_column]\n if is_training or annotation_column is not None:\n per_col_gt_features = image_features[annotation_column]\n\n for idx, img_feature in enumerate(per_col_image_features[: self.max_img_num_per_col]):\n try:\n with PIL.Image.open(img_feature) as img:\n img = img.convert(image_mode)\n except Exception as e:\n continue\n if annotation_column:\n gt_feature = per_col_gt_features[idx]\n\n with PIL.Image.open(gt_feature) as gt:\n gt = gt.convert(gt.mode)\n if self.num_classes == 1:\n gt = gt.convert(\"L\")\n if self.num_classes > 1:\n gt = np.array(\n gt\n ).astype(\n \"float32\"\n ) # There may be issues with 'transforms.ToTensor()' without this line because 'transforms.ToTensor()' converts 'unit8' to values between 0 and 1.\n gt = Image.fromarray(gt)\n if is_training:\n if random.random() < 0.5:\n img = self.train_transforms(img)\n gt = self.train_transforms(gt)\n img = self.img_processor(img)\n gt = self.gt_processor(gt)\n else:\n img = self.img_processor(img)\n if annotation_column is not None:\n gt = self.gt_processor(gt)\n\n images.append(img)\n if is_training or annotation_column is not None:\n gts.append(gt)\n\n if self.num_classes > 1:\n # Prepare per-category binary masks\n per_gt_masks_per_category, per_gt_classes = self.prepare_per_category_binary_masks(gt)\n gt_masks_per_category.append(per_gt_masks_per_category)\n gt_classes.append(per_gt_classes)\n\n ret.update(\n {\n self.image_key: images[0] if len(images) != 0 else torch.tensor([]),\n self.label_key: gts[0] if len(gts) != 0 else torch.tensor([]),\n }\n )\n if self.num_classes > 1:\n ret.update(\n {\n self.mask_label_key: gt_masks_per_category[0] if len(gt_masks_per_category) != 0 else [],\n self.class_label_key: gt_classes[0] if len(gt_classes) != 0 else [],\n }\n )\n return ret\n\n def prepare_per_category_binary_masks(self, gt):\n gt = gt[0]\n classes = torch.unique(gt)\n # remove ignored region\n gt_classes = classes[classes != self.ignore_label].to(torch.int64)\n\n masks = []\n for class_id in classes:\n masks.append(gt == class_id)\n\n if len(masks) == 0:\n # Some image does not have annotation (all ignored)\n gt_masks_per_category = torch.zeros((0, gt.shape[-2], gt.shape[-1]))\n else:\n gt_masks_per_category = torch.stack(masks)\n return gt_masks_per_category, gt_classes\n\n def __call__(\n self,\n images: Dict[str, List[str]],\n data_types: Dict[str, Union[int, float, list]],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Obtain one sample's images and customized them for a specific model.\n\n Parameters\n ----------\n images\n Images of one sample.\n data_types\n Data type of all columns.\n is_training\n Whether to process images in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's processed images and their number.\n \"\"\"\n images = {k: [v] if isinstance(v, str) else v for k, v in images.items()}\n return self.process_one_sample(images, data_types, is_training)\n\n def get_train_transforms(self, train_transforms):\n train_trans = []\n for trans_mode in train_transforms:\n if trans_mode == \"random_horizontal_flip\":\n train_trans.append(transforms.RandomHorizontalFlip(1.0))\n return transforms.Compose(train_trans)\n\n def __getstate__(self):\n odict = self.__dict__.copy() # get attribute dictionary\n del odict[\"img_processor\"]\n del odict[\"gt_processor\"]\n\n return odict\n\n def __setstate__(self, state):\n self.__dict__ = state\n self.img_processor = self.construct_image_processor(\n image_transforms=self.img_transforms, size=self.size, normalization=self.normalization\n )\n self.gt_processor = self.construct_image_processor(\n image_transforms=self.gt_transforms, size=self.size, normalization=None\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/process_text.py",
"content": "import ast\nimport codecs\nimport logging\nimport os\nimport random\nimport warnings\nfrom copy import deepcopy\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport numpy as np\nfrom numpy.typing import NDArray\nfrom omegaconf import DictConfig\nfrom text_unidecode import unidecode\nfrom torch import nn\n\nfrom ..constants import CHOICES_IDS, COLUMN, TEXT, TEXT_SEGMENT_IDS, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH\nfrom ..models.utils import get_pretrained_tokenizer\nfrom .collator import PadCollator, StackCollator\nfrom .template_engine import TemplateEngine\nfrom .trivial_augmenter import TrivialAugment\n\nlogger = logging.getLogger(__name__)\n\n# Disable tokenizer parallelism\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n\nclass TextProcessor:\n \"\"\"\n Prepare text data for the model specified by \"prefix\". For multiple models requiring text data,\n we need to create a TextProcessor for each related model so that they will have independent input.\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n insert_sep: Optional[bool] = True,\n stochastic_chunk: Optional[bool] = False,\n requires_column_info: bool = False,\n text_detection_length: Optional[int] = None,\n text_trivial_aug_maxscale: Optional[float] = 0.0,\n train_augment_types: Optional[List[str]] = None,\n template_config: Optional[DictConfig] = None,\n normalize_text: Optional[bool] = False,\n dropout: Optional[float] = 0,\n ):\n \"\"\"\n Parameters\n ----------\n model\n The model for which this processor would be created.\n insert_sep\n Whether to insert SEP tokens.\n stochastic_chunk\n Whether to use stochastic chunking, which will randomly slice each individual text.\n requires_column_info\n Whether to require feature column information in dataloader.\n text_detection_length\n A naive way to detect text column versus tabular column that were treated as text\n text_trivial_aug_maxscale\n Used in trivial augment as the maximum scale that can be random generated\n A value of 0 means turn off trivial augment\n https://arxiv.org/pdf/2103.10158.pdf\n train_augment_types\n All possible augmentation operations\n normalize_text\n Whether to normalize text to resolve encoding problems.\n Examples of normalized texts can be found at\n https://github.com/autogluon/autogluon/tree/master/examples/automm/kaggle_feedback_prize#15-a-few-examples-of-normalized-texts\n \"\"\"\n logger.debug(f\"initializing text processor for model {model.prefix}\")\n self.prefix = model.prefix\n self.requires_column_info = requires_column_info\n self.tokenizer_name = model.tokenizer_name\n # model should have a tokenizer\n self.tokenizer = model.tokenizer\n if hasattr(self.tokenizer, \"deprecation_warnings\"):\n # Disable the warning \"Token indices sequence length is longer than the specified maximum sequence...\"\n # See https://github.com/huggingface/transformers/blob/6ac77534bfe97c00e0127bb4fc846ae0faf1c9c5/src/transformers/tokenization_utils_base.py#L3362\n self.tokenizer.deprecation_warnings[\"sequence-length-is-longer-than-the-specified-maximum\"] = True\n\n self.cls_token_id, self.sep_token_id, self.eos_token_id = self.get_special_tokens(tokenizer=self.tokenizer)\n self.max_len = model.max_text_len\n self.insert_sep = insert_sep\n self.eos_only = self.cls_token_id == self.sep_token_id == self.eos_token_id\n self.text_segment_num = model.text_segment_num\n\n self.stochastic_chunk = stochastic_chunk\n self.normalize_text = normalize_text\n assert 0 <= dropout <= 1\n if dropout > 0:\n logger.debug(f\"text dropout probability: {dropout}\")\n self.dropout = dropout\n\n # construct augmentor\n self.train_augment_types = train_augment_types\n self.text_detection_length = text_detection_length\n self.text_trivial_aug_maxscale = text_trivial_aug_maxscale\n self.train_augmenter = self.construct_text_augmenter(self.text_trivial_aug_maxscale, self.train_augment_types)\n self.template_config = template_config\n if self.template_config.turn_on:\n self.template_engine = TemplateEngine(self.template_config)\n else:\n self.template_engine = None\n\n if self.normalize_text:\n self.register_encoding_decoding_error_handlers()\n\n @property\n def text_token_ids_key(self):\n return f\"{self.prefix}_{TEXT_TOKEN_IDS}\"\n\n @property\n def text_segment_ids_key(self):\n return f\"{self.prefix}_{TEXT_SEGMENT_IDS}\"\n\n @property\n def choices_ids_key(self):\n return f\"{self.prefix}_{CHOICES_IDS}\"\n\n @property\n def text_valid_length_key(self):\n return f\"{self.prefix}_{TEXT_VALID_LENGTH}\"\n\n @property\n def text_column_prefix(self):\n return f\"{self.text_token_ids_key}_{COLUMN}\"\n\n def collate_fn(self, text_column_names: Optional[List] = None) -> Dict:\n \"\"\"\n Collate text features into a batch.\n This function will be used when creating Pytorch DataLoader.\n\n Returns\n -------\n A dictionary containing one model's collator function for text data.\n \"\"\"\n fn = {}\n if self.requires_column_info:\n assert text_column_names, \"Empty text column names.\"\n for col_name in text_column_names:\n fn[f\"{self.text_column_prefix}_{col_name}\"] = StackCollator()\n\n fn.update(\n {\n self.text_token_ids_key: PadCollator(pad_val=self.tokenizer.pad_token_id),\n self.text_valid_length_key: StackCollator(),\n self.text_segment_ids_key: PadCollator(pad_val=0),\n self.choices_ids_key: PadCollator(pad_val=0),\n }\n )\n\n return fn\n\n def build_one_token_sequence(\n self,\n text_tokens: Dict[str, NDArray],\n ) -> Dict:\n \"\"\"\n Construct one token sequence based on multiple token sequences coming from different\n text columns in a multimodal pd.DataFrame. The token sequence length and the text segment\n id are upper bounded by \"self.max_len\" and \"self.text_segment_num\".\n\n Parameters\n ----------\n text_tokens\n One sample's text token sequences from different text columns in a multimodal pd.DataFrame.\n\n Returns\n -------\n A dictionary containing one sample's text tokens, valid length, and segment ids.\n \"\"\"\n if self.insert_sep:\n max_length = self.max_len - (len(text_tokens) + 1)\n else:\n max_length = self.max_len - 2\n if self.eos_only:\n # For EOS-only, the tokens will be combined as\n # [Field1 Tokens] [EOS] [Field2 Tokens] [EOS] [Field3 Tokens] [EOS]\n # Otherwise, the tokens will be combined as\n # [CLS] [Field1 Tokens] [SEP] [Field2 Tokens] [SEP] [Field3 Tokens] [EOS]\n max_length += 1\n trimmed_lengths = self.get_trimmed_lengths(\n [len(txt_token) for txt_token in text_tokens.values()],\n max_length,\n do_merge=True,\n )\n seg = 0\n if self.eos_only:\n # There is no cls token in the EOS-only mode\n token_ids = []\n else:\n token_ids = [self.cls_token_id]\n\n choices_ids = []\n segment_ids = [seg]\n ret = {}\n\n for (col_name, txt_token), trim_length in zip(text_tokens.items(), trimmed_lengths):\n if col_name == CHOICES_IDS:\n choices_ids = txt_token\n continue\n segment_start = len(token_ids)\n if self.stochastic_chunk:\n start_ptr = np.random.randint(0, len(txt_token) - trim_length + 1)\n else:\n start_ptr = 0\n token_ids.extend(txt_token[start_ptr : (start_ptr + trim_length)].tolist())\n segment_ids.extend([seg] * trim_length)\n if self.requires_column_info:\n # np.int64 corresponds to torch.LongTensor\n col_token_idxs = np.array([segment_start, segment_start + trim_length], dtype=np.int64)\n ret[f\"{self.text_column_prefix}_{col_name}\"] = col_token_idxs\n if self.insert_sep:\n token_ids.append(self.sep_token_id)\n segment_ids.append(seg)\n seg = (seg + 1) % self.text_segment_num\n\n if token_ids[-1] != self.eos_token_id:\n token_ids.append(self.eos_token_id)\n segment_ids.append(seg)\n\n ret.update(\n {\n self.text_token_ids_key: np.array(token_ids, dtype=np.int32),\n self.text_valid_length_key: len(token_ids),\n self.text_segment_ids_key: np.array(segment_ids, dtype=np.int32),\n self.choices_ids_key: np.array(choices_ids, dtype=np.int32),\n }\n )\n\n return ret\n\n def build_one_token_sequence_from_text(\n self,\n text: Dict[str, str],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Tokenize a sample's text data and build one token sequence. One sample may have\n multiple text columns in a multimodal pd.DataFrame.\n\n Parameters\n ----------\n text\n The raw text data of one sample.\n\n is_training\n Flag to apply augmentation only to training.\n\n Returns\n -------\n A dictionary containing one sample's text tokens, valid length, and segment ids.\n \"\"\"\n # tokenize text\n tokens = {}\n warnings.filterwarnings(\"ignore\", \"Token indices sequence length is longer than.*result in indexing errors\")\n\n if self.template_config.turn_on:\n template, applied_template = self.template_engine.sample_and_apply_template(text)\n if template:\n answer_choices = template.get_answer_choices_list(text)\n text = {}\n text[TEXT_TOKEN_IDS] = applied_template[0]\n text[CHOICES_IDS] = answer_choices\n\n for col_name, col_text in text.items():\n if is_training:\n if self.dropout > 0 and random.uniform(0, 1) <= self.dropout:\n col_text = \"\"\n elif self.train_augmenter is not None:\n # naive way to detect categorical/numerical text:\n if len(col_text.split(\" \")) >= self.text_detection_length:\n col_text = self.train_augmenter(col_text)\n # After text augmentation, \"col_text\" may become a list. An error will be raised when calling \"tokenizer.encode\".\n if type(col_text) == list and len(col_text) == 1:\n col_text = col_text[0]\n\n if col_name == CHOICES_IDS:\n answer_ids = self.tokenizer(\n col_text,\n padding=\"max_length\",\n max_length=self.template_engine.get_max_choice_length(self.tokenizer),\n )[\"input_ids\"]\n tokens[col_name] = answer_ids\n continue\n col_tokens = self.tokenizer.encode(\n col_text,\n add_special_tokens=False,\n truncation=False,\n )\n tokens[col_name] = np.array(col_tokens, dtype=np.int32)\n # build token sequence\n return self.build_one_token_sequence(tokens)\n\n @staticmethod\n def get_special_tokens(tokenizer):\n \"\"\"\n Extract the cls, sep, and eos token ids from a huggingface tokenizer. In most cases,\n we can use the attributes \"cls_token_id\" and \"sep_token_id\". But for CLIP, we\n need to use \"bos_token_id\" and \"eos_token_id\".\n\n Parameters\n ----------\n tokenizer\n A huggingface tokenizer instance.\n\n Returns\n -------\n The cls, sep, and eos token ids.\n \"\"\"\n cls_id, sep_id, eos_id = tokenizer.cls_token_id, tokenizer.sep_token_id, tokenizer.sep_token_id\n if cls_id is None or sep_id is None:\n # CLIP uses eos_token's feature as the pooled output.\n # See https://github.com/huggingface/transformers/blob/v4.14.1/src/transformers/models/clip/modeling_clip.py#L657\n cls_id, sep_id, eos_id = tokenizer.bos_token_id, tokenizer.bos_token_id, tokenizer.eos_token_id\n\n if cls_id is None and sep_id is None:\n # Special treatment for T5 (EOS-only).\n cls_id = sep_id = eos_id\n\n if cls_id is None or sep_id is None or eos_id is None:\n raise ValueError(f\"tokenizer class: {tokenizer.__class__.__name__} has no valid cls, sep, and eos ids.\")\n return cls_id, sep_id, eos_id\n\n @staticmethod\n def get_trimmed_lengths(\n lengths: List[int],\n max_length: int,\n do_merge: bool = False,\n ) -> List:\n \"\"\"\n Get the trimmed lengths of multiple text token sequences. It will make sure that\n the trimmed length is smaller than or equal to the max_length.\n - do_merge is True\n Make sure that sum(trimmed_lengths) <= max_length.\n The strategy is always trying to trim the longer lengths.\n - do_merge is False\n Make sure that all(trimmed_lengths <= max_length).\n\n Parameters\n ----------\n lengths\n The original lengths of each token sequence.\n max_length\n When do_merge is True,\n We set the max_length constraint on the total length.\n When do_merge is False,\n We set the max_length constraint on individual sequences.\n do_merge\n Whether these sentences will be merged.\n\n Returns\n -------\n trimmed_lengths\n The trimmed lengths of each individual text field.\n \"\"\"\n lengths = np.array(lengths)\n if do_merge:\n total_length = sum(lengths)\n if total_length <= max_length:\n return list(lengths)\n trimmed_lengths = np.zeros_like(lengths)\n while sum(trimmed_lengths) != max_length:\n remainder = max_length - sum(trimmed_lengths)\n budgets = lengths - trimmed_lengths\n nonzero_idx = (budgets > 0).nonzero()[0]\n nonzero_budgets = budgets[nonzero_idx]\n if remainder < len(nonzero_idx):\n for i in range(remainder):\n trimmed_lengths[nonzero_idx[i]] += 1\n else:\n increment = min(min(nonzero_budgets), remainder // len(nonzero_idx))\n trimmed_lengths[nonzero_idx] += increment\n return list(trimmed_lengths)\n else:\n return list(np.minimum(lengths, max_length))\n\n @staticmethod\n def construct_text_augmenter(\n augment_maxscale: float,\n augment_types: List[str],\n ) -> Optional[TrivialAugment]:\n \"\"\"\n Build up a text augmentor from the provided list of augmentation types\n\n Parameters\n ----------\n augment_maxscale:\n maximum scale for text augmentation\n augment_types\n A list of text augment types.\n\n Returns\n -------\n A trivial augment instance.\n \"\"\"\n if augment_maxscale == 0.0 or augment_maxscale is None:\n return None\n\n if augment_types is None or len(augment_types) == 0:\n return TrivialAugment(TEXT, max_strength=augment_maxscale)\n else:\n auglist = []\n for aug_type in augment_types:\n if \"(\" in aug_type:\n trans_mode = aug_type[0 : aug_type.find(\"(\")]\n args = ast.literal_eval(aug_type[aug_type.find(\"(\") :])\n else:\n trans_mode = aug_type\n args = None\n\n auglist.append((trans_mode, args))\n\n return TrivialAugment(TEXT, augment_maxscale, auglist)\n\n def __call__(\n self,\n text: Dict[str, str],\n sub_dtypes: Dict[str, str],\n is_training: bool,\n ) -> Dict:\n \"\"\"\n Extract one sample's text data, tokenize them, and build one token sequence.\n\n Parameters\n ----------\n text\n Text of one sample.\n sub_dtypes\n The sub data types of all text columns.\n is_training\n Whether to do processing in the training mode.\n\n Returns\n -------\n A dictionary containing one sample's text tokens, valid length, and segment ids.\n \"\"\"\n if self.normalize_text:\n text = {col_name: self.normalize_txt(col_text) for col_name, col_text in text.items()}\n\n return self.build_one_token_sequence_from_text(text=text, is_training=is_training)\n\n def __deepcopy__(self, memo):\n cls = self.__class__\n result = cls.__new__(cls)\n memo[id(self)] = result\n\n for k, v in self.__dict__.items():\n if k != \"train_augmenter\":\n setattr(result, k, deepcopy(v, memo))\n # manual reconstruct augmenter\n result.train_augmenter = self.construct_text_augmenter(\n result.text_trivial_aug_maxscale, result.train_augment_types\n )\n return result\n\n def __getstate__(self):\n odict = self.__dict__.copy() # get attribute dictionary\n del odict[\"train_augmenter\"] # remove textaugmenter to support pickle\n return odict\n\n def __setstate__(self, state):\n self.__dict__ = state\n self.train_augmenter = self.construct_text_augmenter(\n state[\"text_trivial_aug_maxscale\"], state[\"train_augment_types\"]\n )\n\n def save_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Save the text tokenizer and record its relative paths, e.g, hf_text.\n\n Parameters\n ----------\n path\n The root path of saving.\n\n \"\"\"\n save_path = os.path.join(path, self.prefix)\n self.tokenizer.save_pretrained(save_path)\n self.tokenizer = self.prefix\n\n def load_tokenizer(\n self,\n path: str,\n ):\n \"\"\"\n Load saved text tokenizers. If text/ner processors already have tokenizers,\n then do nothing.\n\n Parameters\n ----------\n path\n The root path of loading.\n\n Returns\n -------\n A list of text/ner processors with tokenizers loaded.\n \"\"\"\n if isinstance(self.tokenizer, str):\n load_path = os.path.join(path, self.tokenizer)\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=load_path,\n )\n\n @staticmethod\n def normalize_txt(text: str) -> str:\n \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n\n text = (\n text.encode(\"raw_unicode_escape\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n )\n text = unidecode(text)\n return text\n\n @staticmethod\n def register_encoding_decoding_error_handlers() -> None:\n \"\"\"Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\"\"\"\n\n def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n def replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n codecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\n codecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/randaug.py",
"content": "# code in this file is adapted from rpmcruz/autoaugment\n# https://github.com/rpmcruz/autoaugment/blob/master/transformations.py\nimport random\n\nimport numpy as np\nimport PIL\nimport PIL.ImageDraw\nimport PIL.ImageEnhance\nimport PIL.ImageOps\nimport torch\nfrom PIL import Image\n\n\ndef ShearX(img, v): # [-0.3, 0.3]\n assert -0.3 <= v <= 0.3\n if random.random() > 0.5:\n v = -v\n return img.transform(img.size, PIL.Image.AFFINE, (1, v, 0, 0, 1, 0))\n\n\ndef ShearY(img, v): # [-0.3, 0.3]\n assert -0.3 <= v <= 0.3\n if random.random() > 0.5:\n v = -v\n return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, v, 1, 0))\n\n\ndef TranslateX(img, v): # [-150, 150] => percentage: [-0.45, 0.45]\n assert -0.45 <= v <= 0.45\n if random.random() > 0.5:\n v = -v\n v = v * img.size[0]\n return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))\n\n\ndef TranslateXabs(img, v): # [-150, 150] => percentage: [-0.45, 0.45]\n assert 0 <= v\n if random.random() > 0.5:\n v = -v\n return img.transform(img.size, PIL.Image.AFFINE, (1, 0, v, 0, 1, 0))\n\n\ndef TranslateY(img, v): # [-150, 150] => percentage: [-0.45, 0.45]\n assert -0.45 <= v <= 0.45\n if random.random() > 0.5:\n v = -v\n v = v * img.size[1]\n return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))\n\n\ndef TranslateYabs(img, v): # [-150, 150] => percentage: [-0.45, 0.45]\n assert 0 <= v\n if random.random() > 0.5:\n v = -v\n return img.transform(img.size, PIL.Image.AFFINE, (1, 0, 0, 0, 1, v))\n\n\ndef Rotate(img, v): # [-30, 30]\n assert -30 <= v <= 30\n if random.random() > 0.5:\n v = -v\n return img.rotate(v)\n\n\ndef AutoContrast(img, _):\n return PIL.ImageOps.autocontrast(img)\n\n\ndef Invert(img, _):\n return PIL.ImageOps.invert(img)\n\n\ndef Equalize(img, _):\n return PIL.ImageOps.equalize(img)\n\n\ndef Flip(img, _): # not from the paper\n return PIL.ImageOps.mirror(img)\n\n\ndef Solarize(img, v): # [0, 256]\n assert 0 <= v <= 256\n return PIL.ImageOps.solarize(img, v)\n\n\ndef SolarizeAdd(img, addition=0, threshold=128):\n img_np = np.array(img).astype(int)\n img_np = img_np + addition\n img_np = np.clip(img_np, 0, 255)\n img_np = img_np.astype(np.uint8)\n img = Image.fromarray(img_np)\n return PIL.ImageOps.solarize(img, threshold)\n\n\ndef Posterize(img, v): # [4, 8]\n v = int(v)\n v = max(1, v)\n return PIL.ImageOps.posterize(img, v)\n\n\ndef Contrast(img, v): # [0.1,1.9]\n assert 0.1 <= v <= 1.9\n return PIL.ImageEnhance.Contrast(img).enhance(v)\n\n\ndef Color(img, v): # [0.1,1.9]\n assert 0.1 <= v <= 1.9\n return PIL.ImageEnhance.Color(img).enhance(v)\n\n\ndef Brightness(img, v): # [0.1,1.9]\n assert 0.1 <= v <= 1.9\n return PIL.ImageEnhance.Brightness(img).enhance(v)\n\n\ndef Sharpness(img, v): # [0.1,1.9]\n assert 0.1 <= v <= 1.9\n return PIL.ImageEnhance.Sharpness(img).enhance(v)\n\n\ndef Cutout(img, v): # [0, 60] => percentage: [0, 0.2]\n assert 0.0 <= v <= 0.2\n if v <= 0.0:\n return img\n\n v = v * img.size[0]\n return CutoutAbs(img, v)\n\n\ndef CutoutAbs(img, v): # [0, 60] => percentage: [0, 0.2]\n # assert 0 <= v <= 20\n if v < 0:\n return img\n w, h = img.size\n x0 = np.random.uniform(w)\n y0 = np.random.uniform(h)\n\n x0 = int(max(0, x0 - v / 2.0))\n y0 = int(max(0, y0 - v / 2.0))\n x1 = min(w, x0 + v)\n y1 = min(h, y0 + v)\n\n xy = (x0, y0, x1, y1)\n color = (125, 123, 114)\n # color = (0, 0, 0)\n img = img.copy()\n PIL.ImageDraw.Draw(img).rectangle(xy, color)\n return img\n\n\ndef SamplePairing(imgs): # [0, 0.4]\n def f(img1, v):\n i = np.random.choice(len(imgs))\n img2 = PIL.Image.fromarray(imgs[i])\n return PIL.Image.blend(img1, img2, v)\n\n return f\n\n\ndef Identity(img, v):\n return img\n\n\ndef augment_list(): # 16 operations and their ranges\n # https://github.com/google-research/uda/blob/master/image/randaugment/policies.py#L57\n # l = [\n # (Identity, 0., 1.0),\n # (ShearX, 0., 0.3), # 0\n # (ShearY, 0., 0.3), # 1\n # (TranslateX, 0., 0.33), # 2\n # (TranslateY, 0., 0.33), # 3\n # (Rotate, 0, 30), # 4\n # (AutoContrast, 0, 1), # 5\n # (Invert, 0, 1), # 6\n # (Equalize, 0, 1), # 7\n # (Solarize, 0, 110), # 8\n # (Posterize, 4, 8), # 9\n # # (Contrast, 0.1, 1.9), # 10\n # (Color, 0.1, 1.9), # 11\n # (Brightness, 0.1, 1.9), # 12\n # (Sharpness, 0.1, 1.9), # 13\n # # (Cutout, 0, 0.2), # 14\n # # (SamplePairing(imgs), 0, 0.4), # 15\n # ]\n\n # https://github.com/tensorflow/tpu/blob/8462d083dd89489a79e3200bcc8d4063bf362186/models/official/efficientnet/autoaugment.py#L505\n l = [\n (AutoContrast, 0, 1),\n (Equalize, 0, 1),\n # (Invert, 0, 1),\n (Rotate, 0, 30),\n (Posterize, 0, 4),\n (Solarize, 0, 256),\n (SolarizeAdd, 0, 110),\n (Color, 0.1, 1.9),\n (Contrast, 0.1, 1.9),\n (Brightness, 0.1, 1.9),\n (Sharpness, 0.1, 1.9),\n (ShearX, 0.0, 0.3),\n (ShearY, 0.0, 0.3),\n # (CutoutAbs, 0, 40),\n (TranslateXabs, 0.0, 100),\n (TranslateYabs, 0.0, 100),\n ]\n\n return l\n\n\nclass Lighting(object):\n \"\"\"Lighting noise(AlexNet - style PCA - based noise)\"\"\"\n\n def __init__(self, alphastd, eigval, eigvec):\n self.alphastd = alphastd\n self.eigval = torch.Tensor(eigval)\n self.eigvec = torch.Tensor(eigvec)\n\n def __call__(self, img):\n if self.alphastd == 0:\n return img\n\n alpha = img.new().resize_(3).normal_(0, self.alphastd)\n rgb = (\n self.eigvec.type_as(img)\n .clone()\n .mul(alpha.view(1, 3).expand(3, 3))\n .mul(self.eigval.view(1, 3).expand(3, 3))\n .sum(1)\n .squeeze()\n )\n\n return img.add(rgb.view(3, 1, 1).expand_as(img))\n\n\nclass CutoutDefault(object):\n \"\"\"\n Reference : https://github.com/quark0/darts/blob/master/cnn/utils.py\n \"\"\"\n\n def __init__(self, length):\n self.length = length\n\n def __call__(self, img):\n h, w = img.size(1), img.size(2)\n mask = np.ones((h, w), np.float32)\n y = np.random.randint(h)\n x = np.random.randint(w)\n\n y1 = np.clip(y - self.length // 2, 0, h)\n y2 = np.clip(y + self.length // 2, 0, h)\n x1 = np.clip(x - self.length // 2, 0, w)\n x2 = np.clip(x + self.length // 2, 0, w)\n\n mask[y1:y2, x1:x2] = 0.0\n mask = torch.from_numpy(mask)\n mask = mask.expand_as(img)\n img *= mask\n return img\n\n\nclass RandAugment:\n def __init__(self, n, m):\n self.n = n\n self.m = m # [0, 30]\n self.augment_list = augment_list()\n\n def __call__(self, img):\n ops = random.choices(self.augment_list, k=self.n)\n for op, minval, maxval in ops:\n val = (float(self.m) / 30) * float(maxval - minval) + minval\n img = op(img, val)\n\n return img\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/template_engine.py",
"content": "import logging\n\nimport numpy as np\nfrom omegaconf import DictConfig, OmegaConf\n\nfrom .templates import DatasetTemplates, Template, TemplateCollection\n\nlogger = logging.getLogger(__name__)\n\n\nclass TemplateEngine:\n \"\"\"\n Class to manage the selection and use of templates.\n \"\"\"\n\n def __init__(self, template_config: DictConfig):\n \"\"\"\n Initialize the TemplateEngine using preset templates from existing datasets or custom templates specified in config config.data.templates, if specified.\n\n Parameters\n ---------------\n template_config\n The templates configuration specified in config.data.templates.\n \"\"\"\n self.templates = []\n self.template_config = template_config\n collection = TemplateCollection()\n self.all_datasets = collection.keys\n self.preset_templates = self.template_config.preset_templates\n self.custom_templates = self.template_config.custom_templates\n self.num_templates = self.template_config.num_templates\n self.template_length = self.template_config.template_length\n\n if self.preset_templates:\n assert len(self.preset_templates) == 2, (\n f\"Preset templates has the wrong format. Needs to be [DATASET, SUBSET].\"\n )\n dataset_templates = DatasetTemplates(self.preset_templates[0], self.preset_templates[1])\n current_templates = list(dataset_templates.templates.values())\n self.templates += current_templates[: self.num_templates]\n\n if self.custom_templates:\n for key, value in self.custom_templates.items():\n if len(self.templates) >= self.num_templates:\n logger.warning(\n f\"Ignored custom template '{value.template}' as template engine already has {self.num_templates} templates.\"\n )\n break\n template = Template(key, value.template, \"custom\", answer_choices=value.answer_choices)\n self.templates.append(template)\n\n def has_templates(self):\n return len(self.templates) > 0\n\n def get_templates(self):\n return self.templates\n\n def get_max_choice_length(self, tokenizer):\n text = {}\n max_length = 0\n for template in self.templates:\n answer_choices = template.get_answer_choices_list(text)\n for choice in answer_choices:\n answer_ids = tokenizer(\n choice,\n )[\"input_ids\"]\n curr_length = len(answer_ids)\n if curr_length > max_length:\n max_length = curr_length\n\n return max_length\n\n def sample_and_apply_template(self, example: dict):\n \"\"\"\n Randomly sample a template from the collection of available templates and apply it to the sample.\n If collection of templates is empty return original sample.\n\n Parameters\n ---------------\n example\n A data sample, i.e. a dictionary of text columns.\n\n Returns\n ------------------\n A tuple consisting of the selected tuple and the sample after the template has been applied to it.\n \"\"\"\n if not self.templates:\n return [None, example]\n template = np.random.choice(self.templates)\n return [template, template.apply(example, truncation_length=self.template_length)]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/templates.py",
"content": "\"\"\"\nBased on: https://github.com/bigscience-workshop/promptsource\nApache-2.0 license\n\"\"\"\n\nimport logging\nimport os\nimport random\nimport re\nimport uuid\nfrom collections import Counter, defaultdict\nfrom shutil import rmtree\nfrom typing import Dict, List, Optional, Tuple\n\nimport pandas as pd\nimport pkg_resources\nimport yaml\nfrom jinja2 import BaseLoader, Environment, meta\n\nlogger = logging.getLogger(__name__)\n\n# Local path to the folder containing the templates\nTEMPLATES_FOLDER_PATH = pkg_resources.resource_filename(__name__, \"templates\")\n\nenv = Environment(loader=BaseLoader, autoescape=True)\n\n# Allow the python function zip()\nenv.globals.update(zip=zip)\n\n# These are users whose datasets should be included in the results returned by\n# filter_english_datasets (regardless of their metadata)\nINCLUDED_USERS = {\"Zaid\", \"craffel\"}\n\n# These are the metrics with which templates can be tagged\nMETRICS = {\n \"BLEU\",\n \"ROUGE\",\n \"Squad\",\n \"Trivia QA\",\n \"Accuracy\",\n \"Pearson Correlation\",\n \"Spearman Correlation\",\n \"MultiRC\",\n \"AUC\",\n \"COQA F1\",\n \"Edit Distance\",\n \"Mean Reciprocal Rank\",\n \"Other\",\n}\n\n# These are the languages with which templates can be tagged. Keys are ISO 639-1\n# tags, which are the actual tags we use. Values are English names shown in the\n# UI for convenience.\nLANGUAGES = {\n \"ab\": \"Abkhazian\",\n \"aa\": \"Afar\",\n \"af\": \"Afrikaans\",\n \"ak\": \"Akan\",\n \"sq\": \"Albanian\",\n \"am\": \"Amharic\",\n \"ar\": \"Arabic\",\n \"an\": \"Aragonese\",\n \"hy\": \"Armenian\",\n \"as\": \"Assamese\",\n \"av\": \"Avaric\",\n \"ae\": \"Avestan\",\n \"ay\": \"Aymara\",\n \"az\": \"Azerbaijani\",\n \"bm\": \"Bambara\",\n \"ba\": \"Bashkir\",\n \"eu\": \"Basque\",\n \"be\": \"Belarusian\",\n \"bn\": \"Bengali\",\n \"bi\": \"Bislama\",\n \"bs\": \"Bosnian\",\n \"br\": \"Breton\",\n \"bg\": \"Bulgarian\",\n \"my\": \"Burmese\",\n \"ca\": \"Catalan, Valencian\",\n \"ch\": \"Chamorro\",\n \"ce\": \"Chechen\",\n \"ny\": \"Chichewa, Chewa, Nyanja\",\n \"zh\": \"Chinese\",\n \"cu\": \"Church Slavic, Old Slavonic, Church Slavonic, Old Bulgarian, Old Church Slavonic\",\n \"cv\": \"Chuvash\",\n \"kw\": \"Cornish\",\n \"co\": \"Corsican\",\n \"cr\": \"Cree\",\n \"hr\": \"Croatian\",\n \"cs\": \"Czech\",\n \"da\": \"Danish\",\n \"dv\": \"Divehi, Dhivehi, Maldivian\",\n \"nl\": \"Dutch, Flemish\",\n \"dz\": \"Dzongkha\",\n \"en\": \"English\",\n \"eo\": \"Esperanto\",\n \"et\": \"Estonian\",\n \"ee\": \"Ewe\",\n \"fo\": \"Faroese\",\n \"fj\": \"Fijian\",\n \"fi\": \"Finnish\",\n \"fr\": \"French\",\n \"fy\": \"Western Frisian\",\n \"ff\": \"Fulah\",\n \"gd\": \"Gaelic, Scottish Gaelic\",\n \"gl\": \"Galician\",\n \"lg\": \"Ganda\",\n \"ka\": \"Georgian\",\n \"de\": \"German\",\n \"el\": \"Greek, Modern (1453â)\",\n \"kl\": \"Kalaallisut, Greenlandic\",\n \"gn\": \"Guarani\",\n \"gu\": \"Gujarati\",\n \"ht\": \"Haitian, Haitian Creole\",\n \"ha\": \"Hausa\",\n \"he\": \"Hebrew\",\n \"hz\": \"Herero\",\n \"hi\": \"Hindi\",\n \"ho\": \"Hiri Motu\",\n \"hu\": \"Hungarian\",\n \"is\": \"Icelandic\",\n \"io\": \"Ido\",\n \"ig\": \"Igbo\",\n \"id\": \"Indonesian\",\n \"ia\": \"Interlingua (International Auxiliary Language Association)\",\n \"ie\": \"Interlingue, Occidental\",\n \"iu\": \"Inuktitut\",\n \"ik\": \"Inupiaq\",\n \"ga\": \"Irish\",\n \"it\": \"Italian\",\n \"ja\": \"Japanese\",\n \"jv\": \"Javanese\",\n \"kn\": \"Kannada\",\n \"kr\": \"Kanuri\",\n \"ks\": \"Kashmiri\",\n \"kk\": \"Kazakh\",\n \"km\": \"Central Khmer\",\n \"ki\": \"Kikuyu, Gikuyu\",\n \"rw\": \"Kinyarwanda\",\n \"ky\": \"Kirghiz, Kyrgyz\",\n \"kv\": \"Komi\",\n \"kg\": \"Kongo\",\n \"ko\": \"Korean\",\n \"kj\": \"Kuanyama, Kwanyama\",\n \"ku\": \"Kurdish\",\n \"lo\": \"Lao\",\n \"la\": \"Latin\",\n \"lv\": \"Latvian\",\n \"li\": \"Limburgan, Limburger, Limburgish\",\n \"ln\": \"Lingala\",\n \"lt\": \"Lithuanian\",\n \"lu\": \"Luba-Katanga\",\n \"lb\": \"Luxembourgish, Letzeburgesch\",\n \"mk\": \"Macedonian\",\n \"mg\": \"Malagasy\",\n \"ms\": \"Malay\",\n \"ml\": \"Malayalam\",\n \"mt\": \"Maltese\",\n \"gv\": \"Manx\",\n \"mi\": \"Maori\",\n \"mr\": \"Marathi\",\n \"mh\": \"Marshallese\",\n \"mn\": \"Mongolian\",\n \"na\": \"Nauru\",\n \"nv\": \"Navajo, Navaho\",\n \"nd\": \"North Ndebele\",\n \"nr\": \"South Ndebele\",\n \"ng\": \"Ndonga\",\n \"ne\": \"Nepali\",\n \"no\": \"Norwegian\",\n \"nb\": \"Norwegian BokmÃĨl\",\n \"nn\": \"Norwegian Nynorsk\",\n \"ii\": \"Sichuan Yi, Nuosu\",\n \"oc\": \"Occitan\",\n \"oj\": \"Ojibwa\",\n \"or\": \"Oriya\",\n \"om\": \"Oromo\",\n \"os\": \"Ossetian, Ossetic\",\n \"pi\": \"Pali\",\n \"ps\": \"Pashto, Pushto\",\n \"fa\": \"Persian\",\n \"pl\": \"Polish\",\n \"pt\": \"Portuguese\",\n \"pa\": \"Punjabi, Panjabi\",\n \"qu\": \"Quechua\",\n \"ro\": \"Romanian, Moldavian, Moldovan\",\n \"rm\": \"Romansh\",\n \"rn\": \"Rundi\",\n \"ru\": \"Russian\",\n \"se\": \"Northern Sami\",\n \"sm\": \"Samoan\",\n \"sg\": \"Sango\",\n \"sa\": \"Sanskrit\",\n \"sc\": \"Sardinian\",\n \"sr\": \"Serbian\",\n \"sn\": \"Shona\",\n \"sd\": \"Sindhi\",\n \"si\": \"Sinhala, Sinhalese\",\n \"sk\": \"Slovak\",\n \"sl\": \"Slovenian\",\n \"so\": \"Somali\",\n \"st\": \"Southern Sotho\",\n \"es\": \"Spanish, Castilian\",\n \"su\": \"Sundanese\",\n \"sw\": \"Swahili\",\n \"ss\": \"Swati\",\n \"sv\": \"Swedish\",\n \"tl\": \"Tagalog\",\n \"ty\": \"Tahitian\",\n \"tg\": \"Tajik\",\n \"ta\": \"Tamil\",\n \"tt\": \"Tatar\",\n \"te\": \"Telugu\",\n \"th\": \"Thai\",\n \"bo\": \"Tibetan\",\n \"ti\": \"Tigrinya\",\n \"to\": \"Tonga (Tonga Islands)\",\n \"ts\": \"Tsonga\",\n \"tn\": \"Tswana\",\n \"tr\": \"Turkish\",\n \"tk\": \"Turkmen\",\n \"tw\": \"Twi\",\n \"ug\": \"Uighur, Uyghur\",\n \"uk\": \"Ukrainian\",\n \"ur\": \"Urdu\",\n \"uz\": \"Uzbek\",\n \"ve\": \"Venda\",\n \"vi\": \"Vietnamese\",\n \"vo\": \"VolapÃŧk\",\n \"wa\": \"Walloon\",\n \"cy\": \"Welsh\",\n \"wo\": \"Wolof\",\n \"xh\": \"Xhosa\",\n \"yi\": \"Yiddish\",\n \"yo\": \"Yoruba\",\n \"za\": \"Zhuang, Chuang\",\n \"zu\": \"Zulu\",\n}\n\n\ndef download_sourceprompt_templates():\n import zipfile\n\n from autogluon.multimodal.utils import download\n\n from ..constants import SOURCEPROMPT_SHA1, SOURCEPROMPT_URL\n\n temporary_zip_file = pkg_resources.resource_filename(__name__, \"templates.zip\")\n temporary_zip_file = download(url=SOURCEPROMPT_URL, path=temporary_zip_file, sha1_hash=SOURCEPROMPT_SHA1)\n with zipfile.ZipFile(temporary_zip_file, \"r\") as zip_ref:\n zip_ref.extractall(os.path.join(TEMPLATES_FOLDER_PATH, \"..\"))\n os.remove(temporary_zip_file)\n\n\ndef fetching_templates_if_not_exist():\n if os.path.exists(TEMPLATES_FOLDER_PATH):\n return True\n else:\n download_sourceprompt_templates()\n\n\ndef highlight(input):\n return \"\" + input + \"\"\n\n\ndef choice(choices):\n return random.choice(choices)\n\n\ndef most_frequent(items):\n \"\"\"Returns the set of items which appear most frequently in the input\"\"\"\n if not items:\n return\n item_counts = Counter(items).most_common()\n max_freq = item_counts[0][1]\n most_frequent_items = [c[0] for c in item_counts if c[1] == max_freq]\n return most_frequent_items\n\n\nenv.filters[\"highlight\"] = highlight\nenv.filters[\"choice\"] = choice\nenv.filters[\"most_frequent\"] = most_frequent\n\n\nclass Template(yaml.YAMLObject):\n \"\"\"\n A prompt template.\n \"\"\"\n\n yaml_tag = \"!Template\"\n yaml_loader = yaml.SafeLoader\n\n def __init__(self, name, jinja, reference, metadata=None, answer_choices=None):\n \"\"\"\n Creates a prompt template.\n\n A prompt template is expressed in Jinja. It is rendered using an example\n from the corresponding Hugging Face datasets library (a dictionary). The\n separator ||| should appear once to divide the template into prompt and\n output. Generally, the prompt should provide information on the desired\n behavior, e.g., text passage and instructions, and the output should be\n a desired response.\n\n :param name: unique name (per dataset) for template\n :param jinja: template expressed in Jinja\n :param reference: string describing author or paper reference for template\n :param metadata: a Metadata object with template annotations\n :param answer_choices: Jinja expression for answer choices. Should produce\n a ||| delimited string of choices that enumerates\n the possible completions for templates that should\n be evaluated as ranked completions. If None, then\n the template is open-ended. This list is accessible\n from within Jinja as the variable `answer_choices`.\n \"\"\"\n self.id = str(uuid.uuid4())\n self.name = name\n self.jinja = jinja\n self.reference = reference\n self.metadata = metadata if metadata is not None else Template.Metadata()\n self.answer_choices = answer_choices\n\n def get_id(self):\n \"\"\"\n Returns the id of the template\n\n :return: unique id for template\n \"\"\"\n return self.id\n\n def get_template_fields(self):\n \"\"\"\n Returns all template fields.\n \"\"\"\n return [x.strip() for x in re.findall(r\"\\{\\{(.+?)\\}\\}\", str(self.jinja))]\n\n def get_name(self):\n \"\"\"\n Returns the name of the template\n\n :return: unique (per dataset) name for template\n \"\"\"\n return self.name\n\n def get_reference(self):\n \"\"\"\n Returns the bibliographic reference (or author) for the template\n\n :return: reference as a string\n \"\"\"\n return self.reference\n\n def get_answer_choices_expr(self):\n \"\"\"\n Returns a Jinja expression for computing the answer choices from an example.\n\n :return: String, or None if no answer choices\n \"\"\"\n return self.answer_choices\n\n def get_answer_choices_list(self, example):\n \"\"\"\n Returns a list of answer choices for a given example\n\n :return: list of strings, or None if get_answer_choices_expr is None\n \"\"\"\n jinja = self.get_answer_choices_expr()\n if jinja is None:\n return None\n\n rtemplate = env.from_string(jinja)\n protected_example = self._escape_pipe(example)\n rendered_choices = rtemplate.render(**protected_example)\n return [self._unescape_pipe(answer_choice.strip()) for answer_choice in rendered_choices.split(\"|||\")]\n\n def get_fixed_answer_choices_list(self):\n \"\"\"\n Returns a list of answer choices that is static across examples, if possible\n :return: list of strings, or None if no static list exists\n \"\"\"\n jinja = self.get_answer_choices_expr()\n if jinja is None:\n return None\n\n parse = env.parse(jinja)\n variables = meta.find_undeclared_variables(parse)\n if len(variables) == 0:\n rtemplate = env.from_string(jinja)\n rendered_choices = rtemplate.render()\n return [answer_choice.strip() for answer_choice in rendered_choices.split(\"|||\")]\n else:\n return None\n\n def apply(self, example, truncate=True, truncation_length=2048, highlight_variables=False):\n \"\"\"\n Creates a prompt by applying this template to an example\n\n :param example: the dataset example to create a prompt for\n :param truncate: if True, example fields will be truncated to truncation_length chars\n :param highlight_variables: highlight the added variables\n :return: tuple of 2 strings, for prompt and output\n \"\"\"\n jinja = self.jinja\n\n # Truncates the prompt if needed\n if truncate:\n trunc_command = (\n f\" | string | truncate({truncation_length}) }}}}\" # Escaping curly braces requires doubling them\n )\n jinja = jinja.replace(\"}}\", trunc_command)\n\n # Highlights text that was substituted for variables, if requested\n if highlight_variables:\n jinja = jinja.replace(\"}}\", \" | highlight }}\")\n rtemplate = env.from_string(jinja)\n\n protected_example = self._escape_pipe(example)\n\n # Adds in answer_choices variable\n if \"answer_choices\" in protected_example:\n raise ValueError(\"Example contains the restricted key 'answer_choices'.\")\n\n protected_example[\"answer_choices\"] = self.get_answer_choices_list(example)\n\n # Renders the Jinja template\n rendered_example = rtemplate.render(**protected_example)\n\n # Splits on the separator, and then replaces back any occurrences of the\n # separator in the original example\n return [self._unescape_pipe(part).strip() for part in rendered_example.split(\"|||\")]\n\n pipe_protector = \"3ed2dface8203c4c9dfb1a5dc58e41e0\"\n\n @classmethod\n def _escape_pipe(cls, example):\n # Replaces any occurrences of the \"|||\" separator in the example, which\n # which will be replaced back after splitting\n protected_example = {\n key: value.replace(\"|||\", cls.pipe_protector) if isinstance(value, str) else value\n for key, value in example.items()\n }\n return protected_example\n\n @classmethod\n def _unescape_pipe(cls, string):\n # replaces back any occurrences of the separator in a string\n return string.replace(cls.pipe_protector, \"|||\")\n\n class Metadata(yaml.YAMLObject):\n \"\"\"\n Metadata for a prompt template.\n \"\"\"\n\n yaml_tag = \"!TemplateMetadata\"\n yaml_loader = yaml.SafeLoader\n\n def __init__(\n self,\n original_task: Optional[bool] = None,\n choices_in_prompt: Optional[bool] = None,\n metrics: Optional[List[str]] = None,\n languages: Optional[List[str]] = None,\n ):\n \"\"\"\n Initializes template metadata.\n\n In the following, trivial choices are defined as Yes/No, True/False,\n etc. and nontrivial choices are other types of choices denoted in\n the answer_choices field.\n\n :param original_task: If True, this prompt asks a model to perform the original task designed for\n this dataset.\n :param choices_in_prompt: If True, the answer choices are included in the templates such that models\n see those choices in the input. Only applicable to classification tasks.\n :param metrics: List of strings denoting metrics to use for evaluation\n :param metrics: List of strings denoting languages used in the prompt (not the associated dataset!)\n \"\"\"\n self.original_task = original_task\n self.choices_in_prompt = choices_in_prompt\n self.metrics = metrics\n self.languages = languages\n\n\nclass TemplateCollection:\n \"\"\"\n This helper class wraps the DatasetTemplates class\n - Initialized the DatasetTemplates for all existing template folder\n - Give access to each DatasetTemplates\n - Provides aggregated counts over all DatasetTemplates\n \"\"\"\n\n def __init__(self):\n # Dict of all the DatasetTemplates, key is the tuple (dataset_name, subset_name)\n fetching_templates_if_not_exist()\n self.datasets_templates: Dict[(str, Optional[str]), DatasetTemplates] = self._collect_datasets()\n\n @property\n def keys(self):\n return list(self.datasets_templates.keys())\n\n def __len__(self) -> int:\n return len(self.datasets_templates)\n\n def remove(self, dataset_name: str, subset_name: Optional[str] = None) -> None:\n del self.datasets_templates[dataset_name, subset_name]\n\n def _collect_datasets(self) -> Dict[Tuple[str, str], \"DatasetTemplates\"]:\n \"\"\"\n Initialize a DatasetTemplates object for each templates.yaml detected in the templates folder\n\n Returns: a dict with key=(dataset_name, subset_name)\n \"\"\"\n dataset_folders = os.listdir(TEMPLATES_FOLDER_PATH)\n dataset_folders = [folder for folder in dataset_folders if not folder.startswith(\".\")]\n\n output = {} # format is {(dataset_name, subset_name): DatasetsTemplates}\n for dataset in dataset_folders:\n if dataset in INCLUDED_USERS:\n for filename in os.listdir(os.path.join(TEMPLATES_FOLDER_PATH, dataset)):\n output = {**output, **self._collect_dataset(dataset + \"/\" + filename)}\n else:\n output = {**output, **self._collect_dataset(dataset)}\n return output\n\n def _collect_dataset(self, dataset):\n output = {} # format is {(dataset_name, subset_name): DatasetsTemplates}\n for filename in os.listdir(os.path.join(TEMPLATES_FOLDER_PATH, dataset)):\n if filename.endswith(\".yaml\"):\n # If there is no sub-folder, there is no subset for this dataset\n output[(dataset, None)] = DatasetTemplates(dataset)\n else:\n # This is a subfolder, and its name corresponds to the subset name\n output[(dataset, filename)] = DatasetTemplates(dataset_name=dataset, subset_name=filename)\n return output\n\n def get_dataset(self, dataset_name: str, subset_name: Optional[str] = None) -> \"DatasetTemplates\":\n \"\"\"\n Return the DatasetTemplates object corresponding to the dataset name\n\n :param dataset_name: name of the dataset to get\n :param subset_name: name of the subset\n \"\"\"\n # if the dataset does not exist, we add it\n if dataset_name not in self.keys:\n self.datasets_templates[(dataset_name, subset_name)] = DatasetTemplates(dataset_name, subset_name)\n\n return self.datasets_templates[(dataset_name, subset_name)]\n\n def get_templates_count(self) -> Dict:\n \"\"\"\n Return the overall number count over all datasets\n\n NB: we don't breakdown datasets into subsets for the count, i.e subsets count are included\n into the dataset count\n \"\"\"\n\n count_dict = defaultdict(int)\n for k, v in self.datasets_templates.items():\n # Subsets count towards dataset count\n count_dict[k[0]] += len(v)\n # converting to regular dict\n return dict(count_dict)\n\n\nclass DatasetTemplates:\n \"\"\"\n Class that wraps all templates for a specific dataset/subset and implements all the helper\n functions necessary to read/write to the yaml file\n \"\"\"\n\n TEMPLATES_KEY = \"templates\"\n DATASET_KEY = \"dataset\"\n SUBSET_KEY = \"subset\"\n TEMPLATE_FILENAME = \"templates.yaml\"\n\n def __init__(self, dataset_name: str, subset_name: str = None):\n self.dataset_name: str = dataset_name\n self.subset_name: str = subset_name\n # dictionary is keyed by template name.\n self.templates: Dict = self.read_from_file()\n\n # Mapping from template name to template id\n self.name_to_id_mapping = {}\n self.sync_mapping()\n\n def sync_mapping(self) -> None:\n \"\"\"\n Re-compute the name_to_id_mapping to ensure it is in sync with self.templates\n \"\"\"\n self.name_to_id_mapping = {template.name: template.id for template in self.templates.values()}\n\n @property\n def all_template_names(self) -> List[str]:\n \"\"\"\n Sorted list of all templates names for this dataset\n \"\"\"\n return sorted([template.name for template in self.templates.values()])\n\n def get_template_fields(self):\n \"\"\"\n Returns all template fields.\n \"\"\"\n return (\n self.templates[self.name_to_id_mapping[self.all_template_names[0]]].get_template_fields()\n if len(self.all_template_names) > 0\n else []\n )\n\n @property\n def folder_path(self) -> str:\n if self.subset_name:\n return os.path.join(TEMPLATES_FOLDER_PATH, self.dataset_name, self.subset_name)\n else:\n return os.path.join(TEMPLATES_FOLDER_PATH, self.dataset_name)\n\n @property\n def yaml_path(self) -> str:\n return os.path.join(self.folder_path, self.TEMPLATE_FILENAME)\n\n def format_for_dump(self) -> Dict:\n \"\"\"\n Create a formatted dictionary for the class attributes\n \"\"\"\n formatted_dict = {self.DATASET_KEY: self.dataset_name, self.TEMPLATES_KEY: self.templates}\n if self.subset_name:\n formatted_dict[self.SUBSET_KEY] = self.subset_name\n return formatted_dict\n\n def read_from_file(self) -> Dict:\n \"\"\"\n Reads a file containing a prompt collection.\n \"\"\"\n\n if not os.path.exists(self.yaml_path):\n dataset_name = f\"{self.dataset_name} {self.subset_name}\" if self.subset_name else self.dataset_name\n logging.warning(\n f\"Tried instantiating `DatasetTemplates` for {dataset_name}, but no prompts found. \"\n \"Please ignore this warning if you are creating new prompts for this dataset.\"\n )\n return {}\n yaml_dict = yaml.safe_load(open(self.yaml_path, \"r\"))\n return yaml_dict[self.TEMPLATES_KEY]\n\n def write_to_file(self) -> None:\n \"\"\"\n Writes to a file with the current prompt collection.\n \"\"\"\n # Sync the mapping\n self.sync_mapping()\n\n # We only create the folder if a template is written\n if not os.path.exists(self.folder_path):\n os.makedirs(self.folder_path)\n yaml.dump(self.format_for_dump(), open(self.yaml_path, \"w\"))\n\n def add_template(self, template: \"Template\") -> None:\n \"\"\"\n Adds a new template for the dataset\n\n :param template: template\n \"\"\"\n self.templates[template.get_id()] = template\n\n self.write_to_file()\n\n def remove_template(self, template_name: str) -> None:\n \"\"\"\n Deletes a template\n\n :param template_name: name of template to remove\n \"\"\"\n\n # Even if we have an ID, we want to check for duplicate names\n if template_name not in self.all_template_names:\n raise ValueError(f\"No template with name {template_name} for dataset {self.dataset_name} exists.\")\n\n del self.templates[self.name_to_id_mapping[template_name]]\n\n if len(self.templates) == 0:\n # There is no remaining template, we can remove the entire folder\n self.delete_folder()\n else:\n # We just update the file\n self.write_to_file()\n\n def update_template(\n self,\n current_template_name: str,\n new_template_name: str,\n jinja: str,\n reference: str,\n metadata: Template.Metadata,\n answer_choices: str,\n ) -> None:\n \"\"\"\n Updates a pre-existing template and writes changes\n\n :param current_template_name: current name of the template stored in self.templates\n :param new_template_name: new name for the template\n :param jinja: new jinja entry\n :param reference: new reference entry\n :param metadata: a Metadata object with template annotations\n :param answer_choices: new answer_choices string\n \"\"\"\n template_id = self.name_to_id_mapping[current_template_name]\n self.templates[template_id].name = new_template_name\n self.templates[template_id].jinja = jinja\n self.templates[template_id].reference = reference\n self.templates[template_id].metadata = metadata\n self.templates[template_id].answer_choices = answer_choices\n\n self.write_to_file()\n\n def delete_folder(self) -> None:\n \"\"\"\n Delete the folder corresponding to self.folder_path\n \"\"\"\n self.sync_mapping()\n\n rmtree(self.folder_path)\n\n # If it is a subset, we have to check whether to remove the dataset folder\n if self.subset_name:\n # have to check for other folders\n base_dataset_folder = os.path.join(TEMPLATES_FOLDER_PATH, self.dataset_name)\n if len(os.listdir(base_dataset_folder)) == 0:\n rmtree(base_dataset_folder)\n\n def __getitem__(self, template_key: str) -> \"Template\":\n return self.templates[self.name_to_id_mapping[template_key]]\n\n def __len__(self) -> int:\n return len(self.templates)\n\n\ndef get_templates_data_frame():\n \"\"\"\n Gathers all template information into a Pandas DataFrame.\n\n :return: Pandas DataFrame\n \"\"\"\n data = {\n \"id\": [],\n \"dataset\": [],\n \"subset\": [],\n \"name\": [],\n \"reference\": [],\n \"original_task\": [],\n \"choices_in_prompt\": [],\n \"metrics\": [],\n \"languages\": [],\n \"answer_choices\": [],\n \"jinja\": [],\n }\n\n template_collection = TemplateCollection()\n\n for key in template_collection.keys:\n templates = template_collection.get_dataset(key[0], key[1])\n for template_name in templates.all_template_names:\n template = templates[template_name]\n data[\"id\"].append(template.get_id())\n data[\"dataset\"].append(key[0])\n data[\"subset\"].append(key[1])\n data[\"name\"].append(template.get_name())\n data[\"reference\"].append(template.get_reference())\n data[\"original_task\"].append(template.metadata.original_task)\n data[\"choices_in_prompt\"].append(template.metadata.choices_in_prompt)\n data[\"metrics\"].append(template.metadata.metrics)\n data[\"languages\"].append(template.metadata.languages)\n data[\"answer_choices\"].append(template.get_answer_choices_expr())\n data[\"jinja\"].append(template.jinja)\n\n return pd.DataFrame(data)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/trivial_augmenter.py",
"content": "\"\"\"\nThis file implements TrivialAugment.(https://arxiv.org/abs/2103.10158) We extend it for multi-modality setting.\n\nCode is partially adapted from its official implementation https://github.com/automl/trivialaugment\n\"\"\"\n\nimport logging\nimport random\n\nimport nltk\nfrom PIL import Image, ImageEnhance, ImageOps\n\nfrom ..constants import IMAGE, TEXT\n\nlogger = logging.getLogger(__name__)\n\n# Global flag to ensure NLTK resources are downloaded only once\n_nltk_downloaded = False\n\n\ndef scale_parameter(level, maxval, type):\n \"\"\"\n Helper function to scale `val` between 0 and maxval .\n\n Parameters\n ----------\n level: Level of the operation.\n maxval: Maximum value that the operation can have.\n type: return float or int\n\n Returns\n -------\n An adjust scale\n \"\"\"\n if type == \"float\":\n return float(level) * maxval\n elif type == \"int\":\n return int(level * maxval)\n\n\nclass TransformT(object):\n \"\"\"\n Each instance of this class represents a specific transform.\n \"\"\"\n\n def __init__(self, name, xform_fn):\n \"\"\"\n Parameters\n ----------\n name: name of the operation\n xform_fn: augmentation operation function\n \"\"\"\n self.name = name\n self.xform = xform_fn\n\n def __repr__(self):\n return \"<\" + self.name + \">\"\n\n def augment(self, level, data):\n return self.xform(data, level)\n\n\nidentity = TransformT(\"identity\", lambda data, level: data)\n\n\nauto_contrast = TransformT(\"AutoContrast\", lambda pil_img, level: ImageOps.autocontrast(pil_img))\n\n\nequalize = TransformT(\"Equalize\", lambda pil_img, level: ImageOps.equalize(pil_img))\n\n\ndef _rotate_impl(pil_img, level):\n \"\"\"\n Rotates `pil_img` from -30 to 30 degrees depending on `level`.\n \"\"\"\n max = 30\n degrees = scale_parameter(level, max, \"int\")\n if random.random() > 0.5:\n degrees = -degrees\n return pil_img.rotate(degrees)\n\n\nrotate = TransformT(\"Rotate\", _rotate_impl)\n\n\ndef _solarize_impl(pil_img, level):\n \"\"\"\n Applies PIL Solarize to `pil_img` with strength level.\n \"\"\"\n max = 256\n level = scale_parameter(level, max, \"int\")\n return ImageOps.solarize(pil_img, max - level)\n\n\nsolarize = TransformT(\"Solarize\", _solarize_impl)\n\n\ndef _posterize_impl(pil_img, level):\n \"\"\"\n Applies PIL Posterize to `pil_img` with strength level.\n \"\"\"\n max = 4\n min = 0\n level = scale_parameter(level, max - min, \"int\")\n return ImageOps.posterize(pil_img, max - level)\n\n\nposterize = TransformT(\"Posterize\", _posterize_impl)\n\n\ndef _enhancer_impl(enhancer):\n \"\"\"\n Sets level to be between 0.1 and 1.8 for ImageEnhance transforms of PIL.\n \"\"\"\n min = 0.1\n max = 1.8\n\n def impl(pil_img, level):\n v = scale_parameter(level, max - min, \"float\") + min # going to 0 just destroys it\n return enhancer(pil_img).enhance(v)\n\n return impl\n\n\ncolor = TransformT(\"Color\", _enhancer_impl(ImageEnhance.Color))\n\ncontrast = TransformT(\"Contrast\", _enhancer_impl(ImageEnhance.Contrast))\n\nbrightness = TransformT(\"Brightness\", _enhancer_impl(ImageEnhance.Brightness))\n\nsharpness = TransformT(\"Sharpness\", _enhancer_impl(ImageEnhance.Sharpness))\n\n\ndef _shear_x_impl(pil_img, level):\n \"\"\"\n Shears the image along the horizontal axis with `level` magnitude.\n \"\"\"\n max = 0.3\n level = scale_parameter(level, max, \"float\")\n if random.random() > 0.5:\n level = -level\n return pil_img.transform(pil_img.size, Image.AFFINE, (1, level, 0, 0, 1, 0))\n\n\nshear_x = TransformT(\"ShearX\", _shear_x_impl)\n\n\ndef _shear_y_impl(pil_img, level):\n \"\"\"\n Shear the image along the vertical axis with `level` magnitude.\n \"\"\"\n max = 0.3\n level = scale_parameter(level, max, \"float\")\n if random.random() > 0.5:\n level = -level\n return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, 0, level, 1, 0))\n\n\nshear_y = TransformT(\"ShearY\", _shear_y_impl)\n\n\ndef _translate_x_impl(pil_img, level):\n \"\"\"\n Translate the image in the horizontal direction by `level` number of pixels.\n \"\"\"\n max = 10\n level = scale_parameter(level, max, \"int\")\n if random.random() > 0.5:\n level = -level\n return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, level, 0, 1, 0))\n\n\ntranslate_x = TransformT(\"TranslateX\", _translate_x_impl)\n\n\ndef _translate_y_impl(pil_img, level):\n \"\"\"\n Translate the image in the vertical direction by `level` number of pixels.\n \"\"\"\n max = 10\n level = scale_parameter(level, max, \"int\")\n if random.random() > 0.5:\n level = -level\n return pil_img.transform(pil_img.size, Image.AFFINE, (1, 0, 0, 0, 1, level))\n\n\ntranslate_y = TransformT(\"TranslateY\", _translate_y_impl)\n\n\ndef set_image_augmentation_space():\n image_all_transform = [\n identity,\n auto_contrast,\n equalize,\n rotate, # extra coin-flip\n solarize,\n color, # enhancer\n posterize,\n contrast, # enhancer\n brightness, # enhancer\n sharpness, # enhancer\n shear_x, # extra coin-flip\n shear_y, # extra coin-flip\n translate_x, # extra coin-flip\n translate_y, # extra coin-flip\n ]\n return image_all_transform\n\n\ndef download_nltk():\n \"\"\"\n Download required NLTK resources with singleton pattern to prevent multiple downloads.\n\n This function handles NLTK 3.9+ changes where resource names changed and\n the quiet=True parameter behavior was affected. Uses a global flag to ensure\n downloads happen only once even when TrivialAugment is instantiated multiple times.\n \"\"\"\n global _nltk_downloaded\n if _nltk_downloaded:\n return\n\n # Try to download required NLTK data with proper error handling for NLTK 3.9+\n # Include both old and new resource names for compatibility\n resources_to_download = [\n (\"taggers/averaged_perceptron_tagger_eng\", \"averaged_perceptron_tagger_eng\"),\n (\"taggers/averaged_perceptron_tagger\", \"averaged_perceptron_tagger\"),\n (\"corpora/wordnet\", \"wordnet\"),\n (\"corpora/omw-1.4\", \"omw-1.4\"),\n ]\n\n for resource_path, download_name in resources_to_download:\n try:\n nltk.data.find(resource_path)\n except LookupError:\n nltk.download(download_name, quiet=True)\n _nltk_downloaded = True\n\n\ndef set_text_augmentation_space(space):\n if space == None:\n text_all_transform = [\n \"identity\",\n \"syn_replacement\",\n \"random_delete\",\n \"random_swap\",\n \"insert_punc\",\n ]\n else:\n text_all_transform = [\"identity\"]\n text_all_transform += space\n\n return text_all_transform\n\n\nclass TrivialAugment:\n \"\"\"\n Implementation for TrivialAugment (https://arxiv.org/abs/2103.10158)\n Random sample one operation from all_transform\n Random a strength between [0, max_strength]\n \"\"\"\n\n def __init__(self, datatype, max_strength, space=None) -> None:\n \"\"\"\n Parameters\n ----------\n datatype\n Modality type, currently support \"text\" and \"img\"\n max_strength\n Max strength for augmentation operation.\n space\n Use to set augmentation space if specified in config. Text only for now.\n \"\"\"\n self.max_strength = max_strength\n self.data_type = datatype\n if datatype == IMAGE:\n self.all_transform = set_image_augmentation_space()\n elif datatype == TEXT:\n download_nltk()\n self.all_transform = set_text_augmentation_space(space)\n else:\n raise NotImplementedError\n logger.debug(f\"{self.data_type} augmentation space {self.all_transform}\")\n\n def __call__(self, data):\n if self.data_type == IMAGE:\n return self.augment_image(data)\n elif self.data_type == TEXT:\n return self.augment_text(data)\n\n def augment_image(self, data):\n op = random.choice(self.all_transform)\n scale = float(random.randint(0, self.max_strength) / self.max_strength)\n return op.augment(scale, data)\n\n def augment_text(self, data):\n op = random.choice(self.all_transform)\n\n # use specified operation magnitude if available\n if isinstance(op, tuple):\n op, scale = op\n else:\n scale = random.uniform(0, self.max_strength)\n\n if op == \"identity\":\n return data\n\n # lazy import of nlpaug due to the speed issue. See more in https://github.com/autogluon/autogluon/issues/2706\n import nlpaug.augmenter.word as naw\n\n from .nlpaug import InsertPunctuation\n\n if op == \"syn_replacement\":\n op = naw.SynonymAug(aug_src=\"wordnet\", aug_p=scale, aug_max=None)\n elif op == \"random_swap\":\n op = naw.RandomWordAug(action=\"swap\", aug_p=scale, aug_max=None)\n elif op == \"random_delete\":\n op = naw.RandomWordAug(action=\"delete\", aug_p=scale, aug_max=None)\n elif op == \"insert_punc\":\n op = InsertPunctuation() # scale will be randomized inside function\n else:\n raise NotImplementedError\n return op.augment(data)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/data/utils.py",
"content": "import logging\nimport warnings\nfrom typing import Callable, Dict, Iterable, List, Optional, Tuple, Union\n\nimport pandas as pd\nfrom omegaconf import DictConfig, OmegaConf\nfrom torch import nn\n\nfrom autogluon.core.utils import default_holdout_frac, generate_train_test_split_combined\nfrom autogluon.core.utils.loaders import load_pd\n\nfrom ..constants import (\n BINARY,\n CATEGORICAL,\n DEFAULT_SHOT,\n DOCUMENT,\n FEW_SHOT,\n IDENTIFIER,\n IMAGE,\n IMAGE_PATH,\n LABEL,\n MMDET_IMAGE,\n MMLAB_MODELS,\n MULTICLASS,\n NER_ANNOTATION,\n NER_TEXT,\n NUMERICAL,\n REGRESSION,\n ROIS,\n SAM,\n SEMANTIC_SEGMENTATION_IMG,\n TEXT,\n TEXT_NER,\n)\nfrom .collator import DictCollator\nfrom .infer_types import is_image_column\nfrom .label_encoder import NerLabelEncoder\nfrom .mixup import MixupModule\nfrom .preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom .process_categorical import CategoricalProcessor\nfrom .process_document import DocumentProcessor\nfrom .process_image import ImageProcessor\nfrom .process_label import LabelProcessor\nfrom .process_mmlab import MMDetProcessor\nfrom .process_ner import NerProcessor\nfrom .process_numerical import NumericalProcessor\nfrom .process_semantic_seg_img import SemanticSegImageProcessor\nfrom .process_text import TextProcessor\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_collate_fn(\n df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],\n data_processors: Union[Dict, List[Dict]],\n per_gpu_batch_size: Optional[int] = None,\n):\n \"\"\"\n Collect collator functions for each modality input of every model.\n These collator functions are wrapped by the \"Dict\" collator function,\n which can then be used by the Pytorch DataLoader.\n\n Parameters\n ----------\n df_preprocessor\n One or a list of dataframe preprocessors.\n data_processors\n One or a list of data processor dicts.\n per_gpu_batch_size\n Mini-batch size for each GPU.\n\n Returns\n -------\n A \"Dict\" collator wrapping other collators.\n \"\"\"\n if isinstance(df_preprocessor, MultiModalFeaturePreprocessor):\n df_preprocessor = [df_preprocessor]\n if isinstance(data_processors, dict):\n data_processors = [data_processors]\n\n collate_fn = {}\n for per_preprocessor, per_data_processors_group in zip(df_preprocessor, data_processors):\n for per_modality in per_data_processors_group:\n per_modality_column_names = per_preprocessor.get_column_names(modality=per_modality)\n if per_modality_column_names:\n for per_model_processor in per_data_processors_group[per_modality]:\n if per_model_processor.prefix.lower().startswith(MMLAB_MODELS):\n collate_fn.update(\n per_model_processor.collate_fn(\n per_modality_column_names, per_gpu_batch_size=per_gpu_batch_size\n )\n )\n else:\n collate_fn.update(per_model_processor.collate_fn(per_modality_column_names))\n return DictCollator(collate_fn)\n\n\ndef apply_df_preprocessor(\n data: pd.DataFrame,\n df_preprocessor: MultiModalFeaturePreprocessor,\n modalities: Iterable,\n):\n \"\"\"\n Preprocess one dataframe with one df_preprocessor.\n\n Parameters\n ----------\n data\n A pandas dataframe.\n df_preprocessor\n One dataframe preprocessor object.\n modalities\n A list of data modalities to preprocess.\n\n Returns\n -------\n modality_features\n Preprocessed features of given modalities.\n modality_types\n Minor modality types of each major modality.\n sample_num\n Number of samples.\n \"\"\"\n lengths = []\n modality_features = {}\n modality_types = {}\n for per_modality in modalities:\n per_modality_features, per_modality_types = getattr(df_preprocessor, f\"transform_{per_modality}\")(data)\n modality_features[per_modality] = per_modality_features\n modality_types[per_modality] = per_modality_types\n if per_modality_features:\n lengths.append(len(per_modality_features[next(iter(per_modality_features))]))\n assert len(set(lengths)) == 1 # make sure each modality has the same sample num\n sample_num = lengths[0]\n\n return modality_features, modality_types, sample_num\n\n\ndef apply_data_processor(\n per_sample_features: Dict,\n data_processors: Dict,\n data_types: Dict,\n is_training: bool,\n load_only=False,\n):\n \"\"\"\n Process one sample's features.\n\n Parameters\n ----------\n per_sample_features\n Features of one sample.\n data_processors\n A dict of data processors.\n data_types\n Data types of all columns.\n is_training\n Whether is training.\n load_only\n Whether to only load the data. Other processing steps may happen in dataset.__getitem__.\n\n Returns\n -------\n The processed features of one sample.\n \"\"\"\n sample_features = {}\n for per_modality, per_modality_processors in data_processors.items():\n for per_model_processor in per_modality_processors:\n if per_modality in per_sample_features and per_sample_features[per_modality]:\n sample_features.update(\n per_model_processor(\n per_sample_features[per_modality],\n data_types[per_modality],\n is_training=is_training,\n load_only=load_only,\n )\n if per_model_processor.prefix.lower().startswith(MMDET_IMAGE)\n else per_model_processor(\n per_sample_features[per_modality],\n data_types[per_modality],\n is_training=is_training,\n )\n )\n\n return sample_features\n\n\ndef get_per_sample_features(\n modality_features: Dict, modality_types: Dict, idx: int, id_mappings: Optional[Dict] = None\n):\n \"\"\"\n Extract the modality features of one sample.\n\n Parameters\n ----------\n modality_features\n Modality features of all samples.\n modality_types\n Data types of all columns.\n idx\n The sample index.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n\n Returns\n -------\n One sample's modality features.\n \"\"\"\n ret = dict()\n for per_modality, per_modality_features in modality_features.items():\n if per_modality_features:\n per_modality_ret = dict()\n for per_col_name, per_col_features in per_modality_features.items():\n per_sample_features = per_col_features[idx]\n if (\n modality_types\n and modality_types[per_modality]\n and modality_types[per_modality][per_col_name].endswith(IDENTIFIER)\n ):\n per_sample_features = id_mappings[per_col_name][per_sample_features]\n\n per_modality_ret[per_col_name] = per_sample_features\n ret[per_modality] = per_modality_ret\n\n return ret\n\n\ndef default_holdout_frac(num_train_rows, hyperparameter_tune=False):\n \"\"\"Returns default holdout_frac used in fit().\n Between row count 5,000 and 25,000 keep 0.1 holdout_frac, as we want to grow validation set to a stable 2500 examples.\n \"\"\"\n if num_train_rows < 5000:\n holdout_frac = max(0.1, min(0.2, 500.0 / num_train_rows))\n else:\n holdout_frac = max(0.01, min(0.1, 2500.0 / num_train_rows))\n\n if hyperparameter_tune:\n holdout_frac = min(\n 0.2, holdout_frac * 2\n ) # We want to allocate more validation data for HPO to avoid overfitting\n\n return holdout_frac\n\n\ndef init_df_preprocessor(\n config: DictConfig,\n column_types: Dict,\n label_column: Optional[str] = None,\n train_df_x: Optional[pd.DataFrame] = None,\n train_df_y: Optional[pd.Series] = None,\n):\n \"\"\"\n Initialize the dataframe preprocessor by calling .fit().\n\n Parameters\n ----------\n config\n A DictConfig containing only the data config.\n column_types\n A dictionary that maps column names to their data types.\n For example: `column_types = {\"item_name\": \"text\", \"image\": \"image_path\",\n \"product_description\": \"text\", \"height\": \"numerical\"}`\n may be used for a table with columns: \"item_name\", \"brand\", \"product_description\", and \"height\".\n label_column\n Name of the column that contains the target variable to predict.\n train_df_x\n A pd.DataFrame containing only the feature columns.\n train_df_y\n A pd.Series object containing only the label column.\n\n Returns\n -------\n Initialized dataframe preprocessor.\n \"\"\"\n if label_column in column_types and column_types[label_column] == NER_ANNOTATION:\n label_generator = NerLabelEncoder(config)\n else:\n label_generator = None\n\n df_preprocessor = MultiModalFeaturePreprocessor(\n config=config.data,\n column_types=column_types,\n label_column=label_column,\n label_generator=label_generator,\n )\n df_preprocessor.fit(\n X=train_df_x,\n y=train_df_y,\n )\n\n return df_preprocessor\n\n\ndef get_image_transforms(model_config: DictConfig, model_name: str, advanced_hyperparameters: Dict):\n \"\"\"\n Get the image transforms of one image-related model.\n Use the transforms in advanced_hyperparameters with higher priority.\n\n Parameters\n ----------\n model_config\n Config of one model.\n model_name\n Name of one model.\n advanced_hyperparameters\n The advanced hyperparameters whose values are complex objects.\n\n Returns\n -------\n The image transforms used in training and validation.\n \"\"\"\n train_transform_key = f\"model.{model_name}.train_transforms\"\n val_transform_key = f\"model.{model_name}.val_transforms\"\n if advanced_hyperparameters and train_transform_key in advanced_hyperparameters:\n train_transforms = advanced_hyperparameters[train_transform_key]\n else:\n train_transforms = model_config.train_transforms\n train_transforms = list(train_transforms)\n\n if advanced_hyperparameters and val_transform_key in advanced_hyperparameters:\n val_transforms = advanced_hyperparameters[val_transform_key]\n else:\n val_transforms = model_config.val_transforms\n val_transforms = list(val_transforms)\n\n return train_transforms, val_transforms\n\n\ndef create_data_processor(\n data_type: str,\n config: DictConfig,\n model: nn.Module,\n advanced_hyperparameters: Optional[Dict] = None,\n):\n \"\"\"\n Create one data processor based on the data type and model.\n\n Parameters\n ----------\n data_type\n Data type.\n config\n The config may contain information required by creating a data processor.\n In future, we may move the required config information into the model.config\n to make the data processor conditioned only on the model itself.\n model\n The model.\n\n Returns\n -------\n One data processor.\n \"\"\"\n model_config = getattr(config.model, model.prefix)\n if data_type == IMAGE:\n train_transforms, val_transforms = get_image_transforms(\n model_config=model_config,\n model_name=model.prefix,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n data_processor = ImageProcessor(\n model=model,\n train_transforms=train_transforms,\n val_transforms=val_transforms,\n max_image_num_per_column=model_config.max_image_num_per_column,\n missing_value_strategy=config.data.image.missing_value_strategy,\n dropout=config.data.modality_dropout,\n )\n elif data_type == TEXT:\n data_processor = TextProcessor(\n model=model,\n insert_sep=model_config.insert_sep,\n stochastic_chunk=model_config.stochastic_chunk,\n text_detection_length=model_config.text_aug_detect_length,\n text_trivial_aug_maxscale=model_config.text_trivial_aug_maxscale,\n train_augment_types=model_config.text_train_augment_types,\n normalize_text=config.data.text.normalize_text,\n template_config=config.data.templates,\n dropout=config.data.modality_dropout,\n )\n elif data_type == CATEGORICAL:\n data_processor = CategoricalProcessor(\n model=model,\n dropout=config.data.modality_dropout,\n )\n elif data_type == NUMERICAL:\n data_processor = NumericalProcessor(\n model=model,\n merge=model_config.merge,\n dropout=config.data.modality_dropout,\n )\n elif data_type == LABEL:\n data_processor = LabelProcessor(model=model)\n elif data_type == TEXT_NER:\n data_processor = NerProcessor(\n model=model,\n max_len=model_config.max_text_len,\n entity_map=config.entity_map,\n )\n elif data_type == ROIS:\n data_processor = MMDetProcessor(\n model=model,\n max_img_num_per_col=model_config.max_img_num_per_col,\n missing_value_strategy=config.data.image.missing_value_strategy,\n )\n elif data_type == DOCUMENT:\n train_transforms, val_transforms = get_image_transforms(\n model_config=model_config,\n model_name=model.prefix,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n data_processor = DocumentProcessor(\n model=model,\n train_transforms=train_transforms,\n val_transforms=val_transforms,\n size=model_config.image_size,\n text_max_len=model_config.max_text_len,\n missing_value_strategy=config.data.document.missing_value_strategy,\n )\n elif data_type == SEMANTIC_SEGMENTATION_IMG:\n data_processor = SemanticSegImageProcessor(\n model=model,\n img_transforms=model_config.img_transforms,\n gt_transforms=model_config.gt_transforms,\n train_transforms=model_config.train_transforms,\n val_transforms=model_config.val_transforms,\n ignore_label=model_config.ignore_label,\n )\n else:\n raise ValueError(f\"unknown data type: {data_type}\")\n\n return data_processor\n\n\ndef create_fusion_data_processors(\n config: DictConfig,\n model: nn.Module,\n requires_label: Optional[bool] = True,\n requires_data: Optional[bool] = True,\n advanced_hyperparameters: Optional[Dict] = None,\n):\n \"\"\"\n Create the data processors for late-fusion models. This function creates one processor for\n each modality of each model. For example, if one model config contains BERT, ViT, and CLIP, then\n BERT would have its own text processor, ViT would have its own image processor, and CLIP would have\n its own text and image processors. This is to support training arbitrary combinations of single-modal\n and multimodal models since two models may share the same modality but have different processing. Text\n sequence length is a good example. BERT's sequence length is generally 512, while CLIP uses sequences of\n length 77.\n\n Parameters\n ----------\n config\n A DictConfig object. The model config should be accessible by \"config.model\".\n model\n The model object.\n\n Returns\n -------\n A dictionary with modalities as the keys. Each modality has a list of processors.\n Note that \"label\" is also treated as a modality for convenience.\n \"\"\"\n data_processors = {\n IMAGE: [],\n TEXT: [],\n CATEGORICAL: [],\n NUMERICAL: [],\n LABEL: [],\n ROIS: [],\n TEXT_NER: [],\n DOCUMENT: [],\n SEMANTIC_SEGMENTATION_IMG: [],\n }\n\n model_dict = {model.prefix: model}\n\n if model.prefix.lower().startswith(\"fusion\"):\n for per_model in model.model:\n model_dict[per_model.prefix] = per_model\n\n assert sorted(list(model_dict.keys())) == sorted(config.model.names)\n\n for per_name, per_model in model_dict.items():\n model_config = getattr(config.model, per_model.prefix)\n if model_config.data_types is not None:\n data_types = model_config.data_types.copy()\n else:\n data_types = None\n\n if per_name == NER_TEXT:\n # create a multimodal processor for NER.\n data_processors[TEXT_NER].append(\n create_data_processor(\n data_type=TEXT_NER,\n config=config,\n model=per_model,\n )\n )\n requires_label = False\n if data_types is not None and TEXT_NER in data_types:\n data_types.remove(TEXT_NER)\n elif per_name.lower().startswith(MMLAB_MODELS):\n # create a multimodal processor for NER.\n data_processors[ROIS].append(\n create_data_processor(\n data_type=ROIS,\n config=config,\n model=per_model,\n )\n )\n if data_types is not None and IMAGE in data_types:\n data_types.remove(IMAGE)\n elif per_name == SAM:\n data_processors[SEMANTIC_SEGMENTATION_IMG].append(\n create_data_processor(\n data_type=SEMANTIC_SEGMENTATION_IMG,\n config=config,\n model=per_model,\n )\n )\n if data_types is not None and SEMANTIC_SEGMENTATION_IMG in data_types:\n data_types.remove(SEMANTIC_SEGMENTATION_IMG)\n requires_label = False\n\n if requires_label:\n # each model has its own label processor\n label_processor = create_data_processor(\n data_type=LABEL,\n config=config,\n model=per_model,\n )\n data_processors[LABEL].append(label_processor)\n\n if requires_data and data_types:\n for data_type in data_types:\n per_data_processor = create_data_processor(\n data_type=data_type,\n model=per_model,\n config=config,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n data_processors[data_type].append(per_data_processor)\n\n # Only keep the modalities with non-empty processors.\n data_processors = {k: v for k, v in data_processors.items() if len(v) > 0}\n\n if TEXT_NER in data_processors and LABEL in data_processors:\n # LabelProcessor is not needed for NER tasks as annotations are handled in NerProcessor.\n data_processors.pop(LABEL)\n return data_processors\n\n\ndef turn_on_off_feature_column_info(\n data_processors: Dict,\n flag: bool,\n):\n \"\"\"\n Turn on or off returning feature column information in data processors.\n Since feature column information is not always required in training models,\n we optionally turn this flag on or off.\n\n Parameters\n ----------\n data_processors\n The data processors.\n flag\n True/False\n \"\"\"\n for per_modality_processors in data_processors.values():\n for per_model_processor in per_modality_processors:\n # label processor doesn't have requires_column_info.\n if hasattr(per_model_processor, \"requires_column_info\"):\n per_model_processor.requires_column_info = flag\n\n\ndef get_mixup(\n model_config: DictConfig,\n mixup_config: DictConfig,\n num_classes: int,\n):\n \"\"\"\n Get the mixup state for loss function choice.\n Now the mixup can only support image data.\n And the problem type can not support Regression.\n Parameters\n ----------\n model_config\n The model configs to find image model for the necessity of mixup.\n mixup_config\n The mixup configs for mixup and cutmix.\n num_classes\n The number of classes in the task. Class <= 1 will cause faults.\n\n Returns\n -------\n The mixup is on or off.\n \"\"\"\n model_active = False\n names = model_config.names\n if isinstance(names, str):\n names = [names]\n for model_name in names:\n permodel_config = getattr(model_config, model_name)\n if hasattr(permodel_config.data_types, IMAGE):\n model_active = True\n break\n\n mixup_active = False\n if mixup_config is not None and mixup_config.turn_on:\n mixup_active = (\n mixup_config.mixup_alpha > 0 or mixup_config.cutmix_alpha > 0.0 or mixup_config.cutmix_minmax is not None\n )\n\n mixup_state = model_active & mixup_active & ((num_classes is not None) and (num_classes > 1))\n mixup_fn = None\n if mixup_state:\n mixup_args = dict(\n mixup_alpha=mixup_config.mixup_alpha,\n cutmix_alpha=mixup_config.cutmix_alpha,\n cutmix_minmax=mixup_config.cutmix_minmax,\n prob=mixup_config.prob,\n switch_prob=mixup_config.switch_prob,\n mode=mixup_config.mode,\n label_smoothing=mixup_config.label_smoothing,\n num_classes=num_classes,\n )\n mixup_fn = MixupModule(**mixup_args)\n return mixup_state, mixup_fn\n\n\ndef data_to_df(\n data: Union[pd.DataFrame, Dict, List],\n required_columns: Optional[List] = None,\n all_columns: Optional[List] = None,\n header: Optional[str] = None,\n):\n \"\"\"\n Convert the input data to a dataframe.\n\n Parameters\n ----------\n data\n Input data provided by users during prediction/evaluation.\n required_columns\n Required columns.\n all_columns\n All the possible columns got from training data. The column order is preserved.\n header\n Provided header to create a dataframe.\n\n Returns\n -------\n A dataframe with required columns.\n \"\"\"\n has_header = True\n if isinstance(data, pd.DataFrame):\n pass\n elif isinstance(data, dict):\n data = pd.DataFrame(data)\n elif isinstance(data, list):\n assert len(data) > 0, f\"Expected data to have length > 0, but got {data} of len {len(data)}\"\n if header is None:\n has_header = False\n data = pd.DataFrame(data)\n else:\n data = pd.DataFrame({header: data})\n elif isinstance(data, str):\n df = pd.DataFrame([data])\n col_name = list(df.columns)[0]\n if is_image_column(df[col_name], col_name=col_name, image_type=IMAGE_PATH):\n has_header = False\n data = df\n else:\n data = load_pd.load(data)\n else:\n raise NotImplementedError(\n f\"The format of data is not understood. \"\n f'We have type(data)=\"{type(data)}\", but a pd.DataFrame was required.'\n )\n\n if required_columns and all_columns:\n detected_columns = data.columns.values.tolist()\n missing_columns = []\n for per_col in required_columns:\n if per_col not in detected_columns:\n missing_columns.append(per_col)\n\n if len(missing_columns) > 0:\n # assume no column names are provided and users organize data in the same column order of training data.\n if len(detected_columns) == len(all_columns):\n if has_header:\n warnings.warn(\n f\"Replacing detected dataframe columns `{detected_columns}` with columns \"\n f\"`{all_columns}` from training data.\"\n \"Double check the correspondences between them to avoid unexpected behaviors.\",\n UserWarning,\n )\n data.rename(dict(zip(detected_columns, required_columns)), axis=1, inplace=True)\n else:\n raise ValueError(\n f\"Dataframe columns `{detected_columns}` are detected, but columns `{missing_columns}` are missing. \"\n f\"Please double check your input data to provide all the \"\n f\"required columns `{required_columns}`.\"\n )\n\n return data\n\n\ndef infer_scarcity_mode_by_data_size(df_train: pd.DataFrame, scarcity_threshold: int = 50):\n \"\"\"\n Infer based on the number of training sample the data scarsity. Select mode accordingly from [DEFAULT_SHOT, FEW_SHOT, ZERO_SHOT].\n\n Parameters\n ---------------\n df_train\n Training dataframe\n scarcity_threshold\n Threshold number of samples when to select FEW_SHOT mode\n\n Returns\n --------\n Mode in [DEFAULT_SHOT, FEW_SHOT, ZERO_SHOT]\n \"\"\"\n row_num = len(df_train)\n if row_num < scarcity_threshold:\n return FEW_SHOT\n else:\n return DEFAULT_SHOT\n\n\ndef infer_dtypes_by_model_names(model_config: DictConfig):\n \"\"\"\n Get data types according to model types.\n\n Parameters\n ----------\n model_config\n Model config from `config.model`.\n\n Returns\n -------\n The data types allowed by models and the default fallback data type.\n \"\"\"\n allowable_dtypes = []\n fallback_dtype = None\n for per_model in model_config.names:\n per_model_dtypes = OmegaConf.select(model_config, f\"{per_model}.data_types\")\n if per_model_dtypes:\n allowable_dtypes.extend(per_model_dtypes)\n\n allowable_dtypes = set(allowable_dtypes)\n if allowable_dtypes == {IMAGE, TEXT}:\n fallback_dtype = TEXT\n elif len(allowable_dtypes) == 1:\n fallback_dtype = list(allowable_dtypes)[0]\n\n return allowable_dtypes, fallback_dtype\n\n\ndef split_train_tuning_data(\n data: pd.DataFrame,\n holdout_frac: float = None,\n problem_type: str = None,\n label_column: str = None,\n random_state: int = 0,\n) -> (pd.DataFrame, pd.DataFrame):\n \"\"\"\n Splits `data` into `train_data` and `tuning_data`.\n If the problem_type is one of ['binary', 'multiclass']:\n The split will be done with stratification on the label column.\n Will guarantee at least 1 sample of every class in `data` will be present in `train_data`.\n If only 1 sample of a class exists, it will always be put in `train_data` and not `tuning_data`.\n\n Parameters\n ----------\n data : pd.DataFrame\n The data to be split\n holdout_frac : float, default = None\n The ratio of data to use as validation.\n If 0.2, 20% of the data will be used for validation, and 80% for training.\n If None, the ratio is automatically determined,\n ranging from 0.2 for small row count to 0.01 for large row count.\n random_state : int, default = 0\n The random state to use when splitting the data, to make the splitting process deterministic.\n If None, a random value is used.\n\n Returns\n -------\n Tuple of (train_data, tuning_data) of the split `data`\n \"\"\"\n if holdout_frac is None:\n holdout_frac = default_holdout_frac(num_train_rows=len(data), hyperparameter_tune=False)\n\n # TODO: Hack since the recognized problem types are only binary, multiclass, and regression\n # Problem types used for purpose of stratification, so regression = no stratification\n if problem_type in [BINARY, MULTICLASS]:\n problem_type_for_split = problem_type\n else:\n problem_type_for_split = REGRESSION\n\n train_data, tuning_data = generate_train_test_split_combined(\n data=data,\n label=label_column,\n test_size=holdout_frac,\n problem_type=problem_type_for_split,\n random_state=random_state,\n )\n return train_data, tuning_data\n\n\ndef get_detected_data_types(column_types: Dict):\n \"\"\"\n Extract data types from column types.\n\n Parameters\n ----------\n column_types\n A dataframe's column types.\n\n Returns\n -------\n A list of detected data types.\n \"\"\"\n data_types = []\n for col_type in column_types.values():\n if col_type.startswith(IMAGE) and IMAGE not in data_types:\n data_types.append(IMAGE)\n elif col_type.startswith(TEXT_NER) and TEXT_NER not in data_types:\n data_types.append(TEXT_NER)\n elif col_type.startswith(TEXT) and TEXT not in data_types:\n data_types.append(TEXT)\n elif col_type.startswith(DOCUMENT) and DOCUMENT not in data_types:\n data_types.append(DOCUMENT)\n elif col_type.startswith(NUMERICAL) and NUMERICAL not in data_types:\n data_types.append(NUMERICAL)\n elif col_type.startswith(CATEGORICAL) and CATEGORICAL not in data_types:\n data_types.append(CATEGORICAL)\n elif col_type.startswith(ROIS) and ROIS not in data_types:\n data_types.append(ROIS)\n\n return data_types\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/__init__.py",
"content": "from .base import BaseLearner\nfrom .ensemble import EnsembleLearner\nfrom .few_shot_svm import FewShotSVMLearner\nfrom .matching import MatchingLearner\nfrom .ner import NERLearner\nfrom .object_detection import ObjectDetectionLearner\nfrom .semantic_segmentation import SemanticSegmentationLearner\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/base.py",
"content": "from __future__ import annotations\n\nimport copy\nimport json\nimport logging\nimport operator\nimport os\nimport pickle\nimport shutil\nimport sys\nimport time\nimport warnings\nfrom datetime import timedelta\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport numpy as np\nimport pandas as pd\nimport torch\nimport yaml\nfrom lightning.pytorch.strategies import DeepSpeedStrategy\nfrom omegaconf import DictConfig, OmegaConf\nfrom packaging import version\nfrom torch import nn\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_ray\nfrom autogluon.core.metrics import Scorer, get_metric\nfrom autogluon.core.utils.loaders import load_pd\n\nfrom .. import version as ag_version\nfrom ..constants import (\n BEST,\n BEST_K_MODELS_FILE,\n BINARY,\n COLUMN_FEATURES,\n DDP,\n DEEPSPEED_MIN_PL_VERSION,\n DEEPSPEED_MODULE,\n DEEPSPEED_OFFLOADING,\n DISTILLER,\n DOCUMENT,\n FEATURE_EXTRACTION,\n FEATURES,\n FEW_SHOT,\n FEW_SHOT_CLASSIFICATION,\n GREEDY_SOUP,\n IMAGE_BASE64_STR,\n IMAGE_BYTEARRAY,\n IMAGE_PATH,\n LABEL,\n LAST_CHECKPOINT,\n LOGITS,\n MASKS,\n MAX,\n MIN,\n MODEL_CHECKPOINT,\n MULTICLASS,\n NER,\n RAY_TUNE_CHECKPOINT,\n REGRESSION,\n TEXT,\n TEXT_NER,\n TORCH_COMPILE_MIN_VERSION,\n UNIFORM_SOUP,\n Y_PRED,\n Y_PRED_PROB,\n Y_TRUE,\n ZERO_SHOT_IMAGE_CLASSIFICATION,\n)\nfrom ..data import (\n BaseDataModule,\n MultiModalFeaturePreprocessor,\n create_fusion_data_processors,\n data_to_df,\n get_mixup,\n infer_column_types,\n infer_dtypes_by_model_names,\n infer_output_shape,\n infer_problem_type,\n infer_scarcity_mode_by_data_size,\n init_df_preprocessor,\n is_image_column,\n split_train_tuning_data,\n turn_on_off_feature_column_info,\n)\nfrom ..models import (\n create_fusion_model,\n get_model_postprocess_fn,\n is_lazy_weight_tensor,\n list_timm_models,\n select_model,\n)\nfrom ..optim import (\n compute_score,\n get_aug_loss_func,\n get_loss_func,\n get_minmax_mode,\n get_norm_layer_param_names,\n get_peft_param_names,\n get_stopping_threshold,\n get_torchmetric,\n infer_metrics,\n)\nfrom ..optim.lit_distiller import DistillerLitModule\nfrom ..optim.lit_module import LitModule\nfrom ..utils import (\n AutoMMModelCheckpoint,\n AutoMMModelCheckpointIO,\n DDPPredictionWriter,\n DistillationMixin,\n ExportMixin,\n LogFilter,\n RealtimeMixin,\n apply_log_filter,\n average_checkpoints,\n compute_inference_batch_size,\n compute_num_gpus,\n extract_from_output,\n filter_hyperparameters,\n get_config,\n get_dir_ckpt_paths,\n get_gpu_message,\n get_load_ckpt_paths,\n get_local_pretrained_config_paths,\n hyperparameter_tune,\n infer_precision,\n infer_problem_type_by_eval_metric,\n is_interactive_env,\n is_interactive_strategy,\n logits_to_prob,\n on_fit_end_message,\n on_fit_per_run_start_message,\n on_fit_start_message,\n run_ddp_only_once,\n save_pretrained_model_configs,\n setup_save_path,\n split_hyperparameters,\n tensor_to_ndarray,\n update_config_by_rules,\n update_hyperparameters,\n update_tabular_config_by_resources,\n)\nfrom ..utils.problem_types import PROBLEM_TYPES_REG\n\npl_logger = logging.getLogger(\"lightning\")\npl_logger.propagate = False # https://github.com/Lightning-AI/lightning/issues/4621\nlogger = logging.getLogger(__name__)\n\n\nclass BaseLearner(ExportMixin, DistillationMixin, RealtimeMixin):\n \"\"\" \"\"\"\n\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = None,\n presets: Optional[str] = None,\n eval_metric: Optional[Union[str, Scorer]] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = None,\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n label\n Name of the column that contains the target variable to predict.\n problem_type\n Type of the prediction problem. We support standard problems like\n\n - 'binary': Binary classification\n - 'multiclass': Multi-class classification\n - 'regression': Regression\n - 'classification': Classification problems include 'binary' and 'multiclass' classification.\n\n In addition, we support advanced problems such as\n\n - 'object_detection': Object detection\n - 'ner' or 'named_entity_recognition': Named entity extraction\n - 'text_similarity': Text-text similarity problem\n - 'image_similarity': Image-image similarity problem\n - 'image_text_similarity': Text-image similarity problem\n - 'feature_extraction': Extracting feature (only support inference)\n - 'zero_shot_image_classification': Zero-shot image classification (only support inference)\n - 'few_shot_classification': Few-shot classification for image or text data.\n\n For certain problem types, the default behavior is to load a pretrained model based on\n the presets / hyperparameters and the learner will support zero-shot inference\n (running inference without .fit()). This includes the following\n problem types:\n\n - 'object_detection'\n - 'text_similarity'\n - 'image_similarity'\n - 'image_text_similarity'\n - 'feature_extraction'\n - 'zero_shot_image_classification'\n - 'few_shot_classification'\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n eval_metric\n Evaluation metric name. If `eval_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'accuracy' for multiclass classification, `roc_auc` for binary classification, and 'root_mean_squared_error' for regression.\n hyperparameters\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n hyperparameters = \"model.hf_text.checkpoint_name=google/electra-small-discriminator model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n hyperparameters = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\", \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n path\n Path to directory where models and intermediate outputs should be saved.\n If unspecified, a time-stamped folder called \"AutogluonAutoMM/ag-[TIMESTAMP]\"\n will be created in the working directory to store all models.\n Note: To call `fit()` twice and save all results of each fit,\n you must specify different `path` locations or don't specify `path` at all.\n Otherwise files from first `fit()` will be overwritten by second `fit()`.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50\n (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels)\n warn_if_exist\n Whether to raise warning if the specified path already exists.\n enable_progress_bar\n Whether to show progress bar. It will be True by default and will also be\n disabled if the environment variable os.environ[\"AUTOMM_DISABLE_PROGRESS_BAR\"] is set.\n pretrained\n Whether to init model with pretrained weights. If False, it creates a model with random initialization.\n validation_metric\n Validation metric name. If `validation_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'accuracy' for multiclass classification, `roc_auc` for binary classification, and 'root_mean_squared_error' for regression.\n \"\"\"\n self._eval_metric_name = None\n self._eval_metric_func = None\n if isinstance(eval_metric, str):\n self._eval_metric_name = eval_metric.lower()\n self.set_eval_metric_func()\n elif isinstance(eval_metric, Scorer):\n self._eval_metric_name = eval_metric.name\n self._eval_metric_func = eval_metric\n\n self._problem_type = infer_problem_type_by_eval_metric(\n eval_metric_name=self._eval_metric_name,\n problem_type=problem_type,\n )\n\n self._label_column = label\n self._presets = presets.lower() if presets else None\n self._validation_metric_name = validation_metric.lower() if validation_metric else None\n self._minmax_mode = None\n self._output_shape = None\n self._ckpt_path = None\n self._pretrained_path = None\n self._pretrained = pretrained\n self._config = None\n self._df_preprocessor = None\n self._column_types = None\n self._data_processors = None\n self._model_postprocess_fn = None\n self._model = None\n self._resume = False\n self._verbosity = verbosity\n self._warn_if_exist = warn_if_exist\n self._enable_progress_bar = enable_progress_bar if enable_progress_bar is not None else True\n self._fit_called = False # Flag whether is continuous training.\n self._save_path = path\n self._hyperparameters = hyperparameters\n self._advanced_hyperparameters = None\n self._hyperparameter_tune_kwargs = None\n self._train_data = None\n self._tuning_data = None\n self._is_hpo = False\n self._teacher_learner = None\n self._fit_args = None\n # Summary statistics used in fit summary. TODO: wrap it in a class.\n self._total_train_time = None\n self._best_score = None\n\n self._log_filters = [\n \".*does not have many workers.* in the `DataLoader` init to improve performance.*\",\n \"Checkpoint directory .* exists and is not empty.\",\n ]\n\n @property\n def path(self):\n return self._save_path\n\n @property\n def label(self):\n return self._label_column\n\n @property\n def problem_type(self):\n return self._problem_type\n\n @property\n def problem_property(self):\n if self.problem_type is None:\n return None\n else:\n return PROBLEM_TYPES_REG.get(self.problem_type)\n\n @property\n def column_types(self):\n return self._column_types\n\n @property\n def eval_metric(self):\n return self._eval_metric_name\n\n @property\n def validation_metric(self):\n return self._validation_metric_name\n\n @property\n def total_parameters(self) -> int:\n return sum(p.numel() if not is_lazy_weight_tensor(p) else 0 for p in self._model.parameters())\n\n @property\n def trainable_parameters(self) -> int:\n return sum(\n p.numel() if not is_lazy_weight_tensor(p) else 0 for p in self._model.parameters() if p.requires_grad\n )\n\n @property\n def model_size(self) -> float:\n \"\"\"\n Returns the model size in Megabyte.\n \"\"\"\n model_size = sum(\n p.numel() * p.element_size() if not is_lazy_weight_tensor(p) else 0 for p in self._model.parameters()\n )\n return model_size * 1e-6 # convert to megabytes\n\n def set_eval_metric_func(self):\n from .matching import MatchingLearner\n from .ner import NERLearner\n from .object_detection import ObjectDetectionLearner\n from .semantic_segmentation import SemanticSegmentationLearner\n\n if (\n not isinstance(self, (NERLearner, SemanticSegmentationLearner, MatchingLearner, ObjectDetectionLearner))\n and self._eval_metric_func is None\n ):\n self._eval_metric_func = get_metric(self._eval_metric_name)\n\n def ensure_fit_ready(self):\n if self._problem_type and not self.problem_property.support_fit:\n raise RuntimeError(\n f\"The problem_type='{self._problem_type}' does not support `learner.fit()`. \"\n f\"You may try to use `learner.predict()` or `learner.evaluate()`.\"\n )\n\n def infer_problem_type(self, train_data: pd.DataFrame):\n if self._fit_called:\n return\n if self._label_column:\n if isinstance(train_data, str):\n train_data = load_pd.load(train_data)\n self._problem_type = infer_problem_type(\n y_train_data=train_data[self._label_column],\n provided_problem_type=self._problem_type,\n )\n\n def setup_save_path(self, save_path: str):\n self._save_path = setup_save_path(\n resume=self._resume,\n old_save_path=self._save_path,\n proposed_save_path=save_path,\n raise_if_exist=True,\n warn_if_exist=False,\n fit_called=self._fit_called,\n )\n\n def infer_column_types(\n self, column_types: Optional[Dict] = None, data: Optional[pd.DataFrame] = None, is_train=True\n ):\n if is_train and self._fit_called:\n return\n elif is_train:\n self._column_types = infer_column_types(\n data=self._train_data,\n valid_data=self._tuning_data,\n label_columns=self._label_column,\n provided_column_types=column_types,\n problem_type=self._problem_type, # used to update the corresponding column type\n )\n logger.debug(f\"column_types: {self._column_types}\")\n logger.debug(f\"image columns: {[k for k, v in self._column_types.items() if v == 'image_path']}\")\n elif column_types is None:\n allowable_dtypes, fallback_dtype = infer_dtypes_by_model_names(model_config=self._config.model)\n return infer_column_types(\n data=data,\n label_columns=self._label_column,\n problem_type=self._problem_type,\n allowable_column_types=allowable_dtypes,\n fallback_column_type=fallback_dtype,\n )\n else:\n return column_types\n\n def infer_output_shape(self):\n if self._fit_called:\n return\n self._output_shape = infer_output_shape(\n label_column=self._label_column,\n data=self._train_data,\n problem_type=self._problem_type,\n )\n\n def prepare_train_tuning_data(\n self,\n train_data: Union[pd.DataFrame, str],\n tuning_data: Optional[Union[pd.DataFrame, str]],\n holdout_frac: Optional[float],\n seed: Optional[int],\n ):\n if isinstance(train_data, str):\n train_data = load_pd.load(train_data)\n if isinstance(tuning_data, str):\n tuning_data = load_pd.load(tuning_data)\n\n if tuning_data is None:\n train_data, tuning_data = split_train_tuning_data(\n data=train_data,\n holdout_frac=holdout_frac,\n problem_type=self._problem_type,\n label_column=self._label_column,\n random_state=seed,\n )\n\n self._train_data = train_data\n self._tuning_data = tuning_data\n\n def detect_data_scarcity_mode(self):\n # Determine data scarcity mode, i.e. a few-shot scenario\n scarcity_mode = infer_scarcity_mode_by_data_size(\n df_train=self._train_data, scarcity_threshold=50\n ) # Add as separate hyperparameter somewhere?\n if scarcity_mode == FEW_SHOT and (\n not self._presets or FEW_SHOT not in self._presets\n ): # TODO: check for data type\n logger.warning(\n f\"Detected data scarcity. Consider running using the problem_type '{FEW_SHOT_CLASSIFICATION}' for better performance.\"\n )\n\n def update_attributes(\n self,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n model_postprocess_fn: Optional[Callable] = None,\n best_score: Optional[float] = None,\n **kwargs,\n ):\n if config:\n self._config = config\n if df_preprocessor:\n self._df_preprocessor = df_preprocessor\n if data_processors:\n self._data_processors = data_processors\n if model:\n self._model = model\n if model_postprocess_fn:\n self._model_postprocess_fn = model_postprocess_fn\n if best_score:\n self._best_score = best_score\n\n def infer_validation_metric(self, is_matching: Optional[bool] = False):\n if self._fit_called:\n return\n self._validation_metric_name, self._eval_metric_name = infer_metrics(\n problem_type=self._problem_type,\n eval_metric=self._eval_metric_name if self._eval_metric_func is None else self._eval_metric_func,\n validation_metric_name=self._validation_metric_name,\n is_matching=is_matching,\n )\n self.set_eval_metric_func()\n self._minmax_mode = get_minmax_mode(self._validation_metric_name)\n logger.debug(f\"validation_metric_name: {self._validation_metric_name}\")\n logger.debug(f\"minmax_mode: {self._minmax_mode}\")\n\n def update_hyperparameters(self, hyperparameters: Dict, hyperparameter_tune_kwargs: Dict):\n problem_type = self._pipeline if hasattr(self, \"_pipeline\") else self._problem_type # matching uses pipeline\n if self._hyperparameters and hyperparameters:\n self._hyperparameters.update(hyperparameters)\n elif hyperparameters:\n self._hyperparameters = hyperparameters\n\n if self._hyperparameter_tune_kwargs and hyperparameter_tune_kwargs:\n self._hyperparameter_tune_kwargs.update(hyperparameter_tune_kwargs)\n elif hyperparameter_tune_kwargs:\n self._hyperparameter_tune_kwargs = hyperparameter_tune_kwargs\n\n self._hyperparameters, self._hyperparameter_tune_kwargs = update_hyperparameters(\n problem_type=problem_type,\n presets=self._presets,\n provided_hyperparameters=self._hyperparameters,\n provided_hyperparameter_tune_kwargs=self._hyperparameter_tune_kwargs,\n )\n # split out the hyperparameters whose values are complex objects\n self._hyperparameters, self._advanced_hyperparameters = split_hyperparameters(self._hyperparameters)\n self._is_hpo = True if self._hyperparameter_tune_kwargs else False\n if self._is_hpo:\n try_import_ray()\n self._hyperparameters = filter_hyperparameters(\n hyperparameters=self._hyperparameters,\n column_types=self._column_types,\n config=self._config,\n fit_called=self._fit_called,\n )\n\n def fit_sanity_check(self):\n assert not self._resume or not self._is_hpo, \"You can not resume training with HPO.\"\n if self._is_hpo and hasattr(self, \"_teacher_learner\") and self._teacher_learner is not None:\n assert isinstance(self._teacher_learner, str), (\n \"HPO with distillation only supports passing a path to the learner.\"\n )\n\n def prepare_fit_args(\n self,\n time_limit: int,\n seed: int,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n if time_limit is not None:\n time_limit = timedelta(seconds=time_limit)\n self._fit_args = dict(\n max_time=time_limit,\n save_path=self._save_path, # In HPO mode, this would be overwritten by per trial path.\n ckpt_path=None if self._is_hpo else self._ckpt_path,\n resume=False if self._is_hpo else self._resume,\n enable_progress_bar=False if self._is_hpo else self._enable_progress_bar,\n seed=seed,\n hyperparameters=self._hyperparameters, # In HPO mode, this would be overwritten by sampled hyperparameters.\n advanced_hyperparameters=self._advanced_hyperparameters,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n if self._fit_called: # continuous training\n continuous_train_args = dict(\n config=self._config,\n df_preprocessor=self._df_preprocessor,\n data_processors=self._data_processors,\n model=self._model,\n )\n self._fit_args.update(continuous_train_args)\n\n def execute_fit(self):\n if self._is_hpo:\n self._fit_args[\"learner\"] = self\n hyperparameter_tune(\n hyperparameter_tune_kwargs=self._hyperparameter_tune_kwargs,\n resources=dict(num_gpus=ResourceManager.get_gpu_count_torch()), # TODO: allow customizing GPUs\n **self._fit_args,\n )\n return dict()\n else:\n attributes = self.fit_per_run(**self._fit_args)\n self.update_attributes(**attributes) # only update attributes for non-HPO mode\n return attributes\n\n def on_fit_start(\n self,\n presets: Optional[str] = None,\n teacher_learner: Optional[Union[str, BaseLearner]] = None,\n ):\n self.ensure_fit_ready()\n if presets:\n if self._fit_called:\n warnings.warn(\"Ignoring the provided `presets` as fit() was called before.\", UserWarning)\n else:\n self._presets = presets\n if teacher_learner:\n self._teacher_learner = teacher_learner\n\n logger.info(on_fit_start_message(path=self._save_path))\n training_start = time.time()\n return training_start\n\n def on_fit_end(\n self,\n training_start: float,\n strategy: Optional[str] = None,\n strict_loading: Optional[bool] = True,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n self._fit_called = True\n if not self._is_hpo:\n # top_k_average is called inside hyperparameter_tune() when building the final predictor.\n self.top_k_average(\n save_path=self._save_path,\n top_k_average_method=self._config.optim.top_k_average_method,\n strategy=strategy,\n strict_loading=strict_loading,\n # Not strict loading if using parameter-efficient finetuning\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n\n training_end = time.time()\n self._total_train_time = training_end - training_start\n # TODO(?) We should have a separate \"_post_training_event()\" for logging messages.\n logger.info(on_fit_end_message(self._save_path))\n\n def fit(\n self,\n train_data: Union[pd.DataFrame, str],\n presets: Optional[str] = None,\n tuning_data: Optional[Union[pd.DataFrame, str]] = None,\n time_limit: Optional[int] = None,\n save_path: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n column_types: Optional[Dict] = None,\n holdout_frac: Optional[float] = None,\n teacher_learner: Union[str, BaseLearner] = None,\n seed: Optional[int] = 0,\n standalone: Optional[bool] = True,\n hyperparameter_tune_kwargs: Optional[Dict] = None,\n clean_ckpts: Optional[bool] = True,\n **kwargs,\n ):\n self.setup_save_path(save_path=save_path)\n training_start = self.on_fit_start(presets=presets, teacher_learner=teacher_learner)\n self.infer_problem_type(train_data=train_data)\n self.prepare_train_tuning_data(\n train_data=train_data,\n tuning_data=tuning_data,\n holdout_frac=holdout_frac,\n seed=seed,\n )\n self.infer_column_types(column_types=column_types)\n self.infer_output_shape()\n self.infer_validation_metric()\n self.update_hyperparameters(\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n self.fit_sanity_check()\n self.prepare_fit_args(\n time_limit=time_limit,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n fit_returns = self.execute_fit()\n self.on_fit_end(\n training_start=training_start,\n strategy=fit_returns.get(\"strategy\", None),\n strict_loading=fit_returns.get(\"strict_loading\", True),\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n\n return self\n\n def init_pretrained(self):\n # split out the hyperparameters whose values are complex objects\n hyperparameters, advanced_hyperparameters = split_hyperparameters(self._hyperparameters)\n if self._config is None:\n self._config = get_config(\n problem_type=self._problem_type,\n presets=self._presets,\n overrides=hyperparameters,\n )\n if self._model is None:\n assert len(self._config.model.names) == 1, (\n f\"Zero shot mode only supports using one model, but detects multiple models {self._config.model.names}\"\n )\n self._model = create_fusion_model(\n config=self._config,\n pretrained=self._pretrained,\n num_classes=self._output_shape,\n )\n if self._data_processors is None:\n self._data_processors = create_fusion_data_processors(\n config=self._config,\n model=self._model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n self._config = self.update_strategy_by_env(config=self._config)\n\n def get_config_per_run(self, config, hyperparameters):\n config = get_config(\n problem_type=self._problem_type,\n presets=self._presets,\n config=config,\n overrides=hyperparameters, # don't use self._hyperparameters due to HPO.\n extra=[DISTILLER] if self._teacher_learner is not None else None,\n )\n config = update_config_by_rules(\n problem_type=self._problem_type,\n config=config,\n )\n config = self.update_strategy_by_env(config=config)\n return config\n\n def get_df_preprocessor_per_run(\n self,\n df_preprocessor,\n data=None,\n config=None,\n column_types=None,\n is_train=True,\n ):\n if df_preprocessor is None:\n if is_train:\n df_preprocessor = init_df_preprocessor(\n config=config,\n column_types=self._column_types,\n label_column=self._label_column,\n train_df_x=self._train_data.drop(columns=self._label_column),\n train_df_y=self._train_data[self._label_column],\n )\n else:\n df_preprocessor = init_df_preprocessor(\n config=self._config,\n column_types=column_types,\n label_column=self._label_column,\n train_df_x=data,\n train_df_y=data[self._label_column] if self._label_column in data else None,\n )\n\n return df_preprocessor\n\n @staticmethod\n def update_config_by_data_per_run(config, df_preprocessor):\n # Avoid passing tabular data with many columns to MultiHeadAttention.\n # If models have additive_attention=\"auto\", we enable it automatically for large tables.\n config = update_tabular_config_by_resources(\n config,\n num_numerical_columns=len(df_preprocessor.numerical_feature_names),\n num_categorical_columns=len(df_preprocessor.categorical_num_categories),\n )\n config = select_model(config=config, df_preprocessor=df_preprocessor)\n return config\n\n def get_model_per_run(self, model, config, df_preprocessor):\n if model is None:\n model = create_fusion_model(\n config=config,\n num_classes=self._output_shape,\n num_numerical_columns=len(df_preprocessor.numerical_feature_names),\n num_categories=df_preprocessor.categorical_num_categories,\n numerical_fill_values=df_preprocessor.numerical_fill_values,\n )\n return model\n\n @staticmethod\n def compile_model_per_run(config, model):\n if config.env.compile.turn_on:\n assert version.parse(torch.__version__) >= version.parse(TORCH_COMPILE_MIN_VERSION), (\n f\"torch.compile requires torch version >= {TORCH_COMPILE_MIN_VERSION}, \"\n f\"but torch version {torch.__version__} is detected.\"\n )\n logger.debug(\"Using torch.compile() in compiling the model.\")\n model = torch.compile(\n model,\n mode=config.env.compile.mode,\n dynamic=config.env.compile.dynamic,\n backend=config.env.compile.backend,\n )\n return model\n\n @staticmethod\n def get_peft_param_names_per_run(model, config):\n peft_param_names = None\n peft = config.optim.peft\n if peft:\n norm_param_names = get_norm_layer_param_names(model)\n peft_param_names = get_peft_param_names(\n norm_param_names,\n peft=peft,\n )\n return peft_param_names\n\n def get_data_processors_per_run(\n self,\n data_processors,\n config=None,\n model=None,\n advanced_hyperparameters=None,\n requires_label=None,\n is_train=True,\n ):\n if is_train:\n if data_processors is None:\n data_processors = create_fusion_data_processors(\n config=config,\n model=model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n data_processors_count = {k: len(v) for k, v in data_processors.items()}\n logger.debug(f\"data_processors_count: {data_processors_count}\")\n else:\n # For each inference call, decouple the used data processors from the learner's attribute\n data_processors = copy.copy(data_processors)\n # For prediction data with no labels provided.\n if not requires_label:\n data_processors.pop(LABEL, None)\n\n return data_processors\n\n def get_validation_metric_per_run(self):\n validation_metric, custom_metric_func = get_torchmetric(\n metric_name=self._validation_metric_name,\n num_classes=self._output_shape,\n problem_type=self._problem_type,\n )\n return validation_metric, custom_metric_func\n\n def get_mixup_func_per_run(self, config):\n mixup_active, mixup_func = get_mixup(\n model_config=config.model,\n mixup_config=config.data.mixup,\n num_classes=self._output_shape,\n )\n if mixup_active and (config.env.per_gpu_batch_size == 1 or config.env.per_gpu_batch_size % 2 == 1):\n warnings.warn(\n \"The mixup is done on the batch.The per_gpu_batch_size should be >1 and even for reasonable operation\",\n UserWarning,\n )\n return mixup_active, mixup_func\n\n def get_loss_func_per_run(self, config, mixup_active=None):\n loss_func = get_loss_func(\n problem_type=self._problem_type,\n mixup_active=mixup_active,\n loss_func_name=config.optim.loss_func,\n config=config.optim,\n )\n aug_loss_func = get_aug_loss_func(\n config=config.optim,\n problem_type=self._problem_type,\n )\n return loss_func, aug_loss_func\n\n def get_model_postprocess_fn_per_run(self, loss_func):\n model_postprocess_fn = get_model_postprocess_fn(\n problem_type=self._problem_type,\n loss_func=loss_func,\n )\n return model_postprocess_fn\n\n def get_datamodule_per_run(\n self,\n df_preprocessor,\n data_processors,\n per_gpu_batch_size,\n num_workers,\n predict_data=None,\n is_train=True,\n ):\n if is_train and self._teacher_learner is not None:\n df_preprocessor = [df_preprocessor, self._teacher_learner._df_preprocessor]\n data_processors = [data_processors, self._teacher_learner._data_processors]\n datamodule_kwargs = dict(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=per_gpu_batch_size,\n num_workers=num_workers,\n )\n if is_train:\n datamodule_kwargs.update(dict(train_data=self._train_data, validate_data=self._tuning_data))\n else:\n datamodule_kwargs.update(dict(predict_data=predict_data))\n\n datamodule = BaseDataModule(**datamodule_kwargs)\n return datamodule\n\n def get_optim_kwargs_per_run(\n self,\n config,\n validation_metric,\n custom_metric_func,\n loss_func,\n aug_loss_func,\n mixup_func,\n grad_steps,\n ):\n return dict(\n optim_type=config.optim.optim_type,\n lr_choice=config.optim.lr_choice,\n lr_schedule=config.optim.lr_schedule,\n lr=config.optim.lr,\n lr_decay=config.optim.lr_decay,\n end_lr=config.optim.end_lr,\n lr_mult=config.optim.lr_mult,\n weight_decay=config.optim.weight_decay,\n warmup_steps=config.optim.warmup_steps,\n track_grad_norm=config.optim.track_grad_norm,\n validation_metric=validation_metric,\n validation_metric_name=self._validation_metric_name,\n custom_metric_func=custom_metric_func,\n loss_func=loss_func,\n mixup_fn=mixup_func,\n peft=config.optim.peft,\n mixup_off_epoch=config.data.mixup.turn_off_epoch,\n skip_final_val=config.optim.skip_final_val,\n cross_modal_align=config.optim.cross_modal_align,\n cross_modal_align_weight=config.optim.cross_modal_align_weight,\n automatic_optimization=config.optim.automatic_optimization,\n accumulate_grad_batches=grad_steps,\n gradient_clip_val=config.optim.gradient_clip_val,\n gradient_clip_algorithm=config.optim.gradient_clip_algorithm,\n use_aug_optim=config.optim.lemda.turn_on,\n aug_loss_func=aug_loss_func,\n aug_lr=config.optim.lemda.lr,\n aug_weight_decay=config.optim.lemda.weight_decay,\n aug_optim_type=config.optim.lemda.optim_type,\n )\n\n def get_litmodule_per_run(\n self,\n model=None,\n model_postprocess_fn=None,\n peft_param_names=None,\n optim_kwargs=dict(),\n distillation_kwargs=dict(),\n is_train=True,\n ):\n if is_train:\n if self._teacher_learner is not None:\n return DistillerLitModule(\n student_model=model,\n teacher_model=self._teacher_learner._model,\n **optim_kwargs,\n **distillation_kwargs,\n )\n else:\n return LitModule(\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n trainable_param_names=peft_param_names,\n **optim_kwargs,\n )\n else:\n return LitModule(\n model=self._model,\n model_postprocess_fn=self._model_postprocess_fn,\n **optim_kwargs,\n )\n\n def get_callbacks_per_run(self, save_path=None, config=None, litmodule=None, pred_writer=None, is_train=True):\n if not is_train:\n if pred_writer is not None:\n callbacks = [pred_writer]\n else:\n callbacks = []\n return callbacks\n\n checkpoint_callback = AutoMMModelCheckpoint(\n dirpath=save_path,\n save_top_k=config.optim.top_k,\n verbose=True,\n monitor=litmodule.validation_metric_name,\n mode=self._minmax_mode,\n save_last=True,\n )\n early_stopping_callback = pl.callbacks.EarlyStopping(\n monitor=litmodule.validation_metric_name,\n patience=config.optim.patience,\n mode=self._minmax_mode,\n stopping_threshold=get_stopping_threshold(self._validation_metric_name),\n )\n lr_callback = pl.callbacks.LearningRateMonitor(logging_interval=\"step\")\n model_summary = pl.callbacks.ModelSummary(max_depth=1)\n callbacks = [\n checkpoint_callback,\n early_stopping_callback,\n lr_callback,\n model_summary,\n ]\n if self._is_hpo:\n from ..utils.hpo import get_ray_tune_ckpt_callback\n\n TuneReportCheckpointCallback = get_ray_tune_ckpt_callback()\n tune_report_callback = TuneReportCheckpointCallback(\n {f\"{litmodule.validation_metric_name}\": f\"{litmodule.validation_metric_name}\"},\n filename=RAY_TUNE_CHECKPOINT,\n )\n callbacks = [\n tune_report_callback,\n early_stopping_callback,\n lr_callback,\n model_summary,\n ]\n\n return callbacks\n\n @staticmethod\n def get_plugins_per_run(model, peft_param_names=None):\n custom_checkpoint_plugin = AutoMMModelCheckpointIO(\n trainable_param_names=peft_param_names,\n model_name_to_id=model.name_to_id,\n )\n return [custom_checkpoint_plugin]\n\n @staticmethod\n def get_tb_logger(save_path):\n return pl.loggers.TensorBoardLogger(\n save_dir=save_path,\n name=\"\",\n version=\"\",\n )\n\n @staticmethod\n def log_gpu_info(num_gpus, config):\n logger.info(\n get_gpu_message(\n detected_num_gpus=ResourceManager.get_gpu_count_torch(),\n used_num_gpus=num_gpus,\n strategy=config.env.strategy,\n )\n )\n\n @staticmethod\n def get_grad_steps(num_gpus, config):\n if num_gpus == 0: # CPU only training\n grad_steps = max(\n config.env.batch_size // (config.env.per_gpu_batch_size * config.env.num_nodes),\n 1,\n )\n else:\n grad_steps = max(\n config.env.batch_size // (config.env.per_gpu_batch_size * num_gpus * config.env.num_nodes),\n 1,\n )\n return grad_steps\n\n @staticmethod\n def get_strategy_per_run(num_gpus, config):\n if (\n config.env.strategy == DEEPSPEED_OFFLOADING and num_gpus == 1 and DEEPSPEED_MODULE not in sys.modules\n ): # Offloading currently only tested for single GPU\n assert version.parse(pl.__version__) >= version.parse(DEEPSPEED_MIN_PL_VERSION), (\n f\"For DeepSpeed Offloading to work reliably you need at least lightning version {DEEPSPEED_MIN_PL_VERSION}, however, found {pl.__version__}. Please update your lightning version.\"\n )\n from ..optim.deepspeed import CustomDeepSpeedStrategy\n\n strategy = CustomDeepSpeedStrategy(\n stage=3,\n offload_optimizer=True,\n offload_parameters=False,\n allgather_bucket_size=config.env.deepspeed_allgather_size,\n reduce_bucket_size=config.env.deepspeed_allreduce_size,\n )\n elif num_gpus <= 1:\n strategy = \"auto\"\n else:\n strategy = config.env.strategy\n return strategy\n\n def update_strategy_and_num_gpus_for_hpo(self, strategy, num_gpus):\n if self._is_hpo:\n strategy = \"auto\"\n num_gpus = min(num_gpus, 1) # Currently only support one trial using one gpu.\n return strategy, num_gpus\n\n @staticmethod\n def get_precision_per_run(num_gpus: int, precision: Union[str, int], cpu_only_warning: Optional[bool] = True):\n return infer_precision(num_gpus=num_gpus, precision=precision, cpu_only_warning=cpu_only_warning)\n\n def get_num_gpus_and_strategy_per_run(\n self,\n config: Optional[DictConfig] = None,\n predict_data: Optional[pd.DataFrame] = None,\n is_train: Optional[bool] = True,\n ):\n if is_train:\n data = self._train_data\n else:\n data = predict_data\n config = self._config\n\n num_gpus = compute_num_gpus(\n config_num_gpus=config.env.num_gpus,\n accelerator=config.env.accelerator,\n )\n num_gpus = self.update_num_gpus_by_data_size(num_gpus=num_gpus, data=data)\n strategy = self.get_strategy_per_run(num_gpus=num_gpus, config=config)\n strategy, num_gpus = self.update_strategy_and_num_gpus_for_hpo(strategy=strategy, num_gpus=num_gpus)\n num_gpus, strategy = run_ddp_only_once(num_gpus=num_gpus, strategy=strategy)\n\n if is_train:\n self.log_gpu_info(num_gpus=num_gpus, config=config)\n\n return num_gpus, strategy\n\n @staticmethod\n def post_update_config_per_run(config, num_gpus, precision, strategy):\n config.env.num_gpus = num_gpus\n config.env.precision = precision\n # for deepspeed offloading, the strategy becomes is a customized strategy object instead of a string,\n # but config still needs a string.\n config.env.strategy = strategy if not config.env.strategy == DEEPSPEED_OFFLOADING else DEEPSPEED_OFFLOADING\n return config\n\n def init_trainer_per_run(\n self,\n num_gpus,\n precision,\n strategy,\n callbacks,\n max_time=None,\n config=None,\n tb_logger=None,\n grad_steps=None,\n plugins=None,\n enable_progress_bar=None,\n barebones=False,\n is_train=True,\n ):\n if is_train:\n trainer_kwargs = dict(\n accelerator=\"gpu\" if num_gpus > 0 else config.env.accelerator,\n devices=num_gpus if num_gpus > 0 else \"auto\",\n num_nodes=config.env.num_nodes,\n precision=precision,\n strategy=strategy if strategy else \"auto\",\n benchmark=False,\n deterministic=config.env.deterministic,\n max_epochs=config.optim.max_epochs,\n max_steps=config.optim.max_steps,\n max_time=max_time,\n callbacks=callbacks,\n logger=tb_logger,\n log_every_n_steps=config.optim.log_every_n_steps,\n enable_progress_bar=enable_progress_bar,\n fast_dev_run=config.env.fast_dev_run,\n val_check_interval=config.optim.val_check_interval,\n check_val_every_n_epoch=config.optim.check_val_every_n_epoch,\n plugins=plugins,\n )\n if config.optim.automatic_optimization:\n trainer_kwargs.update(\n dict(\n gradient_clip_val=config.optim.gradient_clip_val,\n gradient_clip_algorithm=config.optim.gradient_clip_algorithm,\n accumulate_grad_batches=grad_steps,\n )\n )\n blacklist_msgs = [\"already configured with model summary\"]\n log_filter = LogFilter(blacklist_msgs)\n with apply_log_filter(log_filter):\n trainer = pl.Trainer(**trainer_kwargs)\n else:\n blacklist_msgs = []\n if self._verbosity <= 3: # turn off logging in prediction\n blacklist_msgs.append(\"Automatic Mixed Precision\")\n blacklist_msgs.append(\"GPU available\")\n blacklist_msgs.append(\"TPU available\")\n blacklist_msgs.append(\"IPU available\")\n blacklist_msgs.append(\"HPU available\")\n blacklist_msgs.append(\"select gpus\")\n blacklist_msgs.append(\"Trainer(barebones=True)\")\n log_filter = LogFilter(blacklist_msgs)\n\n with apply_log_filter(log_filter):\n trainer = pl.Trainer(\n accelerator=\"gpu\" if num_gpus > 0 else self._config.env.accelerator,\n devices=num_gpus if num_gpus > 0 else \"auto\",\n num_nodes=self._config.env.num_nodes,\n precision=precision,\n strategy=strategy,\n benchmark=False,\n enable_progress_bar=False if barebones else self._enable_progress_bar,\n deterministic=self._config.env.deterministic,\n max_epochs=-1, # Add max_epochs to disable warning\n logger=False,\n callbacks=callbacks,\n barebones=barebones,\n )\n\n return trainer\n\n def run_trainer(\n self,\n trainer,\n litmodule,\n datamodule,\n ckpt_path=None,\n resume=None,\n pred_writer=None,\n is_train=True,\n ):\n with warnings.catch_warnings():\n for filter in self._log_filters:\n warnings.filterwarnings(\"ignore\", filter)\n if is_train:\n trainer.fit(\n litmodule,\n datamodule=datamodule,\n ckpt_path=ckpt_path if resume else None, # this is to resume training that was broken accidentally\n )\n else:\n blacklist_msgs = []\n if self._verbosity <= 3: # turn off logging in prediction\n blacklist_msgs.append(\"LOCAL_RANK\")\n blacklist_msgs.append(\"Trainer(barebones=True)\")\n log_filter = LogFilter(blacklist_msgs)\n with apply_log_filter(log_filter):\n outputs = trainer.predict(\n litmodule,\n datamodule=datamodule,\n return_predictions=pred_writer is None,\n )\n return outputs\n\n def on_fit_per_run_start(self, seed, save_path):\n # TODO(?) We should have a separate \"_pre_training_event()\" for logging messages.\n logger.info(on_fit_per_run_start_message(save_path, self._validation_metric_name))\n pl.seed_everything(seed, workers=True)\n\n def on_fit_per_run_end(\n self,\n trainer: pl.Trainer,\n config: DictConfig,\n model: nn.Module,\n df_preprocessor: MultiModalFeaturePreprocessor,\n data_processors: Dict,\n save_path: str,\n standalone: bool,\n ):\n self.clean_trainer_processes(trainer=trainer, is_train=True)\n self.save(\n path=save_path,\n standalone=standalone,\n config=config,\n model=model,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=True, # fit is called on one run.\n save_model=False, # The final model will be saved in top_k_average\n )\n\n def fit_per_run(\n self,\n max_time: timedelta,\n save_path: str,\n ckpt_path: str,\n resume: bool,\n enable_progress_bar: bool,\n seed: int,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n advanced_hyperparameters: Optional[Dict] = None,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n standalone: bool = True,\n clean_ckpts: bool = True,\n ):\n self.on_fit_per_run_start(seed=seed, save_path=save_path)\n config = self.get_config_per_run(config=config, hyperparameters=hyperparameters)\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=df_preprocessor,\n config=config,\n )\n config = self.update_config_by_data_per_run(config=config, df_preprocessor=df_preprocessor)\n model = self.get_model_per_run(model=model, config=config, df_preprocessor=df_preprocessor)\n model = self.compile_model_per_run(config=config, model=model)\n peft_param_names = self.get_peft_param_names_per_run(model=model, config=config)\n data_processors = self.get_data_processors_per_run(\n data_processors=data_processors,\n config=config,\n model=model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n validation_metric, custom_metric_func = self.get_validation_metric_per_run()\n mixup_active, mixup_func = self.get_mixup_func_per_run(config=config)\n loss_func, aug_loss_func = self.get_loss_func_per_run(config=config, mixup_active=mixup_active)\n model_postprocess_fn = self.get_model_postprocess_fn_per_run(loss_func=loss_func)\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(config=config)\n precision = self.get_precision_per_run(num_gpus=num_gpus, precision=config.env.precision)\n grad_steps = self.get_grad_steps(num_gpus=num_gpus, config=config)\n\n if max_time == timedelta(seconds=0):\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n strict_loading=not peft_param_names,\n )\n # setup distillation in each fit_per_run call to support distillation + HPO\n distillation_kwargs = self.setup_distillation(\n model=model,\n loss_func=loss_func,\n config=config,\n data_processors=data_processors,\n )\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=config.env.per_gpu_batch_size,\n num_workers=config.env.num_workers,\n )\n optim_kwargs = self.get_optim_kwargs_per_run(\n config=config,\n validation_metric=validation_metric,\n custom_metric_func=custom_metric_func,\n loss_func=loss_func,\n aug_loss_func=aug_loss_func,\n mixup_func=mixup_func,\n grad_steps=grad_steps,\n )\n litmodule = self.get_litmodule_per_run(\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n peft_param_names=peft_param_names,\n optim_kwargs=optim_kwargs,\n distillation_kwargs=distillation_kwargs,\n )\n callbacks = self.get_callbacks_per_run(save_path=save_path, config=config, litmodule=litmodule)\n plugins = self.get_plugins_per_run(model=model, peft_param_names=peft_param_names)\n tb_logger = self.get_tb_logger(save_path=save_path)\n config = self.post_update_config_per_run(\n config=config,\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n )\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n config=config,\n precision=precision,\n strategy=strategy,\n max_time=max_time,\n callbacks=callbacks,\n tb_logger=tb_logger,\n grad_steps=grad_steps,\n plugins=plugins,\n enable_progress_bar=enable_progress_bar,\n )\n\n self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n ckpt_path=ckpt_path,\n resume=resume,\n )\n self.on_fit_per_run_end(\n save_path=save_path,\n standalone=standalone,\n trainer=trainer,\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n )\n\n best_score = (\n trainer.callback_metrics[f\"val_{self._validation_metric_name}\"].item()\n if f\"val_{self._validation_metric_name}\" in trainer.callback_metrics\n else self._best_score\n ) # https://github.com/autogluon/autogluon/issues/4428\n\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n best_score=best_score,\n strategy=strategy,\n strict_loading=not peft_param_names,\n )\n\n def top_k_average(\n self,\n save_path,\n top_k_average_method,\n strategy=None,\n last_ckpt_path=None,\n strict_loading=True,\n standalone=True,\n clean_ckpts=True,\n ):\n eval_metric = self._eval_metric_name if self._eval_metric_func is None else self._eval_metric_func\n minmax_mode = get_minmax_mode(eval_metric)\n best_k_models_yaml_path = os.path.join(save_path, BEST_K_MODELS_FILE)\n if os.path.exists(best_k_models_yaml_path):\n with open(best_k_models_yaml_path, \"r\") as f:\n best_k_models = yaml.safe_load(f)\n else:\n # In some cases, the training ends up too early (e.g., due to time_limit) so that there is\n # no saved best_k model checkpoints. In that scenario, we won't perform any model averaging.\n best_k_models = None\n if last_ckpt_path is None:\n last_ckpt_path = os.path.join(save_path, LAST_CHECKPOINT)\n\n if self._teacher_learner is not None:\n prefix = \"student_model.\"\n else:\n prefix = \"model.\"\n\n if best_k_models:\n if top_k_average_method == UNIFORM_SOUP:\n logger.info(f\"Start to fuse {len(best_k_models)} checkpoints via the uniform soup algorithm.\")\n ingredients = top_k_model_paths = list(best_k_models.keys())\n else:\n top_k_model_paths = [\n v[0]\n for v in sorted(\n list(best_k_models.items()),\n key=lambda ele: ele[1],\n reverse=(minmax_mode == MAX),\n )\n ]\n if top_k_average_method == GREEDY_SOUP:\n # Select the ingredients based on the methods proposed in paper\n # \"Model soups: averaging weights of multiple fine-tuned models improves accuracy without\n # increasing inference time\", https://arxiv.org/pdf/2203.05482.pdf\n monitor_op = {MIN: operator.le, MAX: operator.ge}[minmax_mode]\n ingredients = [top_k_model_paths[0]]\n if len(top_k_model_paths) > 1:\n logger.info(\n f\"Start to fuse {len(top_k_model_paths)} checkpoints via the greedy soup algorithm.\"\n )\n\n self._load_state_dict(\n path=top_k_model_paths[0],\n prefix=prefix,\n strict=strict_loading,\n )\n best_score = self.evaluate(self._tuning_data, metrics=[eval_metric])\n best_score = next(iter(best_score.values()))\n for i in range(1, len(top_k_model_paths)):\n cand_avg_state_dict = average_checkpoints(\n checkpoint_paths=ingredients + [top_k_model_paths[i]],\n )\n self._load_state_dict(\n state_dict=cand_avg_state_dict,\n prefix=prefix,\n strict=strict_loading,\n )\n cand_score = self.evaluate(self._tuning_data, metrics=[eval_metric])\n cand_score = next(iter(cand_score.values()))\n if monitor_op(cand_score, best_score):\n # Add new ingredient\n ingredients.append(top_k_model_paths[i])\n best_score = cand_score\n elif top_k_average_method == BEST:\n ingredients = [top_k_model_paths[0]]\n else:\n raise ValueError(\n f\"The key for 'optim.top_k_average_method' is not supported. \"\n f\"We only support '{GREEDY_SOUP}', '{UNIFORM_SOUP}' and '{BEST}'. \"\n f\"The provided value is '{top_k_average_method}'.\"\n )\n else:\n # best_k_models is empty so we will manually save a checkpoint from the trainer\n # and use it as the main ingredients\n ingredients = [last_ckpt_path]\n top_k_model_paths = []\n # no checkpoints are available, do nothing\n if not os.path.isfile(last_ckpt_path):\n return\n\n # Average all the ingredients\n avg_state_dict = average_checkpoints(\n checkpoint_paths=ingredients,\n )\n self._load_state_dict(\n state_dict=avg_state_dict,\n prefix=prefix,\n strict=strict_loading,\n )\n\n if self._teacher_learner is not None:\n avg_state_dict = self._replace_model_name_prefix(\n state_dict=avg_state_dict,\n old_prefix=\"student_model\",\n new_prefix=\"model\",\n )\n\n if not standalone:\n checkpoint = {\"state_dict\": avg_state_dict}\n else:\n if isinstance(strategy, DeepSpeedStrategy):\n checkpoint = {\n \"state_dict\": {\n name.partition(\"module.\")[2]: param\n for name, param in strategy.model._zero3_consolidated_16bit_state_dict().items()\n }\n }\n else:\n checkpoint = {\n \"state_dict\": {\"model.\" + name: param for name, param in self._model.state_dict().items()}\n }\n\n torch.save(checkpoint, os.path.join(save_path, MODEL_CHECKPOINT)) # nosec B614\n\n if clean_ckpts:\n # clean old checkpoints + the intermediate files stored\n for per_path in top_k_model_paths:\n if os.path.isfile(per_path):\n os.remove(per_path)\n # remove the yaml file after cleaning the checkpoints\n if os.path.isfile(best_k_models_yaml_path):\n os.remove(best_k_models_yaml_path)\n # clean the last checkpoint\n if os.path.isfile(last_ckpt_path):\n os.remove(last_ckpt_path)\n\n def prepare_deepspeed_offloading(self, strategy):\n # TODO: Using optimiation_kwargs for inference is confusing and bad design. Remove as soon as fixed in lightning.\n if self._config.env.strategy == DEEPSPEED_OFFLOADING and DEEPSPEED_MODULE not in sys.modules:\n # Need to initialize DeepSpeed and optimizer as currently required in lightning's integration of deepspeed.\n from ..optim.deepspeed import CustomDeepSpeedStrategy\n\n strategy = CustomDeepSpeedStrategy(\n stage=3,\n offload_optimizer=True,\n offload_parameters=False,\n allgather_bucket_size=self._config.env.deepspeed_allgather_size,\n reduce_bucket_size=self._config.env.deepspeed_allreduce_size,\n )\n\n optim_kwargs = dict(\n optim_type=self._config.optim.optim_type,\n lr_choice=self._config.optim.lr_choice,\n lr_schedule=self._config.optim.lr_schedule,\n lr=self._config.optim.lr,\n lr_decay=self._config.optim.lr_decay,\n end_lr=self._config.optim.end_lr,\n lr_mult=self._config.optim.lr_mult,\n weight_decay=self._config.optim.weight_decay,\n warmup_steps=self._config.optim.warmup_steps,\n )\n else:\n optim_kwargs = {}\n\n return strategy, optim_kwargs\n\n def get_pred_writer(self, strategy):\n pred_writer = None\n if isinstance(strategy, str) and DDP in strategy:\n pred_writer = DDPPredictionWriter(output_dir=self._save_path, write_interval=\"epoch\", strategy=strategy)\n return pred_writer\n\n @staticmethod\n def collect_predictions(outputs, trainer, pred_writer, num_gpus):\n if pred_writer is not None:\n if trainer.global_rank == 0:\n outputs = pred_writer.collect_all_gpu_results(num_gpus=num_gpus)\n\n return outputs\n\n @staticmethod\n def clean_trainer_processes(trainer, is_train=True):\n if is_train:\n msg = \"Training finished,\"\n else:\n msg = \"Prediction finished,\"\n if trainer.global_rank != 0:\n sys.exit(f\"{msg} exit the process with global_rank={trainer.global_rank}...\")\n\n def update_image_column_types(self, data):\n column_types = self._column_types\n column_types_copy = copy.deepcopy(column_types)\n for col_name, col_type in column_types.items():\n if col_type in [IMAGE_BYTEARRAY, IMAGE_PATH, IMAGE_BASE64_STR]:\n if is_image_column(data=data[col_name], col_name=col_name, image_type=IMAGE_PATH):\n image_type = IMAGE_PATH\n elif is_image_column(\n data=data[col_name],\n col_name=col_name,\n image_type=IMAGE_BYTEARRAY,\n ):\n image_type = IMAGE_BYTEARRAY\n elif is_image_column(data=data[col_name], col_name=col_name, image_type=IMAGE_BASE64_STR):\n image_type = IMAGE_BASE64_STR\n else:\n image_type = col_type\n if col_type != image_type:\n column_types_copy[col_name] = image_type\n return column_types_copy\n\n def data_to_df(self, data):\n if self._fit_called:\n column_names = list(self._column_types.keys())\n # remove label column since it's not required in inference.\n column_names.remove(self._label_column)\n data = data_to_df(\n data=data,\n required_columns=self._df_preprocessor.required_feature_names,\n all_columns=column_names,\n )\n else:\n data = data_to_df(data=data)\n\n return data\n\n @staticmethod\n def update_realtime_for_interactive_env(realtime: bool, num_gpus: int, barebones: bool, strategy: str):\n # TODO: support realtime inference for notebook with multi-gpus\n # realtime can initialize CUDA, which can cause failures when calling fit again in the interactive env.\n if is_interactive_strategy(strategy) and realtime:\n realtime = False\n num_gpus = min(1, num_gpus)\n barebones = True\n\n return realtime, num_gpus, barebones\n\n @staticmethod\n def update_num_gpus_by_data_size(\n num_gpus: int,\n data: pd.DataFrame,\n ):\n data_size = len(data)\n if data_size < num_gpus:\n num_gpus = data_size\n return num_gpus\n\n def realtime_predict(\n self,\n data: pd.DataFrame,\n df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],\n data_processors: Union[Dict, List[Dict]],\n num_gpus: int,\n precision: Union[int, str],\n ) -> List[Dict]:\n \"\"\"\n Perform realtime inference.\n\n Parameters\n ----------\n data\n A dataframe.\n df_preprocessor\n Dataframe preprocessors.\n data_processors\n Data processors.\n num_gpus\n Number of GPUs.\n precision\n The precision used in inference.\n\n Returns\n -------\n A list of output dicts.\n \"\"\"\n\n batch = self.process_batch(\n data=data,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n )\n output = self.predict_batch(\n batch=batch,\n model=self._model,\n precision=precision,\n num_gpus=num_gpus,\n )\n return [output]\n\n def on_predict_per_run_start(self, data: Union[str, pd.DataFrame]):\n data = self.data_to_df(data=data)\n return data\n\n def get_predict_batch_size_per_run(self, num_gpus: int, strategy: str):\n return compute_inference_batch_size(\n per_gpu_batch_size=self._config.env.per_gpu_batch_size,\n inference_batch_size_ratio=self._config.env.inference_batch_size_ratio,\n num_gpus=num_gpus,\n strategy=strategy,\n )\n\n def on_predict_per_run_end(self, trainer):\n self.clean_trainer_processes(trainer=trainer, is_train=False)\n\n def predict_per_run(\n self,\n data: Union[pd.DataFrame, dict, list],\n realtime: Optional[bool],\n requires_label: Optional[bool] = False,\n barebones: Optional[bool] = False,\n ) -> List[Dict]:\n \"\"\"\n Perform inference for learner.\n\n Parameters\n ----------\n data\n The data for inference.\n realtime\n Whether use realtime inference.\n requires_label\n Whether uses label during inference.\n barebones\n Whether to run in âbarebones modeâ, where all lightning's features that may impact raw speed are disabled.\n\n Returns\n -------\n A list of output dicts.\n \"\"\"\n data = self.on_predict_per_run_start(data=data)\n column_types = self.infer_column_types(\n column_types=self._column_types,\n data=data,\n is_train=False,\n )\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=self._df_preprocessor,\n data=data,\n column_types=column_types,\n is_train=False,\n )\n if self._fit_called:\n df_preprocessor._column_types = self.update_image_column_types(data=data)\n data_processors = self.get_data_processors_per_run(\n data_processors=self._data_processors,\n requires_label=requires_label,\n is_train=False,\n )\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(\n predict_data=data,\n is_train=False,\n )\n precision = self.get_precision_per_run(\n num_gpus=num_gpus,\n precision=self._config.env.precision,\n cpu_only_warning=False,\n )\n batch_size = self.get_predict_batch_size_per_run(num_gpus=num_gpus, strategy=strategy)\n realtime = self.use_realtime(\n realtime=realtime,\n data=data,\n data_processors=data_processors,\n batch_size=batch_size,\n )\n realtime, num_gpus, barebones = self.update_realtime_for_interactive_env(\n realtime=realtime,\n num_gpus=num_gpus,\n barebones=barebones,\n strategy=strategy,\n )\n\n if realtime:\n outputs = self.realtime_predict(\n data=data,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n num_gpus=num_gpus,\n precision=precision,\n )\n return outputs\n\n strategy, optim_kwargs = self.prepare_deepspeed_offloading(strategy=strategy)\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=batch_size,\n num_workers=self._config.env.num_workers_inference,\n predict_data=data,\n is_train=False,\n )\n pred_writer = self.get_pred_writer(strategy=strategy)\n callbacks = self.get_callbacks_per_run(pred_writer=pred_writer, is_train=False)\n # TODO: remove optim_kwargs from inference\n litmodule = self.get_litmodule_per_run(optim_kwargs=optim_kwargs, is_train=False)\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n callbacks=callbacks,\n barebones=barebones,\n is_train=False,\n )\n outputs = self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n pred_writer=pred_writer,\n is_train=False,\n )\n outputs = self.collect_predictions(\n outputs=outputs,\n trainer=trainer,\n pred_writer=pred_writer,\n num_gpus=num_gpus,\n )\n self.on_predict_per_run_end(trainer=trainer)\n\n return outputs\n\n def ensure_predict_ready(self):\n if not self._fit_called:\n if not self._problem_type or not self.problem_property.support_zero_shot:\n raise RuntimeError(\n f\"problem_type='{self._problem_type}' does not support running inference directly. \"\n f\"You need to call `learner.fit()`, or load a learner first before \"\n f\"running `learner.predict()`, `learner.evaluate()` or `learner.extract_embedding()`.\"\n )\n else:\n self.init_pretrained()\n\n def on_predict_start(self):\n self.ensure_predict_ready()\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n self.on_predict_start()\n ret_type = LOGITS\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=True,\n )\n logits = extract_from_output(ret_type=ret_type, outputs=outputs)\n metric_data = {}\n if self._problem_type in [BINARY, MULTICLASS]:\n y_pred_prob = logits_to_prob(logits)\n metric_data[Y_PRED_PROB] = y_pred_prob\n\n y_pred = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n inverse_categorical=False,\n )\n y_pred_inv = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n inverse_categorical=True,\n )\n y_true = self._df_preprocessor.transform_label_for_metric(df=data)\n metric_data.update(\n {\n Y_PRED: y_pred,\n Y_TRUE: y_true,\n }\n )\n if metrics is None:\n if self._eval_metric_func:\n metrics = [self._eval_metric_func]\n else:\n metrics = [self._eval_metric_name]\n if isinstance(metrics, str) or isinstance(metrics, Scorer):\n metrics = [metrics]\n\n results = {}\n for per_metric in metrics:\n score = compute_score(\n metric_data=metric_data,\n metric=per_metric.lower() if isinstance(per_metric, str) else per_metric,\n )\n per_metric_name = per_metric if isinstance(per_metric, str) else per_metric.name\n results[per_metric_name] = score\n\n if return_pred:\n return results, self._as_pandas(data=data, to_be_converted=y_pred_inv)\n else:\n return results\n\n def _match_queries_and_candidates(\n self,\n query_data: Union[pd.DataFrame, dict, list],\n candidate_data: Union[pd.DataFrame, dict, list],\n return_prob: Optional[bool] = False,\n ):\n query_embeddings = self.extract_embedding(query_data, as_tensor=True)\n assert len(query_embeddings) == 1, (\n f\"Multiple embedding types `{query_embeddings.keys()}` exist in query data. Please reduce them to one type.\"\n )\n query_embeddings = list(query_embeddings.values())[0]\n\n candidate_embeddings = self.extract_embedding(candidate_data, as_tensor=True)\n assert len(candidate_embeddings) == 1, (\n f\"Multiple embedding types `{candidate_embeddings.keys()}` exist in candidate data. Please reduce them to one type.\"\n )\n candidate_embeddings = list(candidate_embeddings.values())[0]\n\n if return_prob:\n ret = (100.0 * query_embeddings @ candidate_embeddings.T).float().softmax(dim=-1)\n else:\n ret = (query_embeddings @ candidate_embeddings.T).argmax(dim=-1)\n\n ret = tensor_to_ndarray(ret)\n\n return ret\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n candidate_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n candidate_data\n The candidate data from which to search the query data's matches.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n \"\"\"\n self.on_predict_start()\n ret_type = LOGITS\n if candidate_data:\n pred = self._match_queries_and_candidates(\n query_data=data,\n candidate_data=candidate_data,\n return_prob=False,\n )\n else:\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n logits = extract_from_output(outputs=outputs, ret_type=ret_type)\n\n if self._df_preprocessor:\n pred = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n )\n else:\n if isinstance(logits, (torch.Tensor, np.ndarray)) and logits.ndim == 2:\n pred = logits.argmax(axis=1)\n else:\n pred = logits\n\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n pred = self._as_pandas(data=data, to_be_converted=pred)\n\n return pred\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n candidate_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict probabilities class probabilities rather than class labels.\n This is only for the classification. Calling it for regression will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n candidate_data\n The candidate data from which to search the query data's matches.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n as_multiclass\n Whether to return the probability of all labels or\n just return the probability of the positive class for binary classification problems.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n When as_multiclass is True, the output will always have shape (#samples, #classes).\n Otherwise, the output will have shape (#samples,)\n \"\"\"\n self.on_predict_start()\n assert self._problem_type not in [\n REGRESSION,\n ], f\"Problem {self._problem_type} has no probability output.\"\n\n if candidate_data:\n prob = self._match_queries_and_candidates(\n query_data=data,\n candidate_data=candidate_data,\n return_prob=True,\n )\n else:\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n logits = extract_from_output(outputs=outputs, ret_type=LOGITS)\n prob = logits_to_prob(logits)\n\n if not as_multiclass:\n if self._problem_type == BINARY:\n prob = prob[:, 1]\n\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n prob = self._as_pandas(data=data, to_be_converted=prob)\n\n return prob\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n return_masks: Optional[bool] = False,\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Extract features for each sample, i.e., one row in the provided dataframe `data`.\n\n Parameters\n ----------\n data\n The data to extract embeddings for. Should contain same column names as training dataset and\n follow same format (except for the `label` column).\n return_masks\n If true, returns a mask dictionary, whose keys are the same as those in the features dictionary.\n If a sample has empty input in feature column `image_0`, the sample will has mask 0 under key `image_0`.\n as_tensor\n Whether to return a Pytorch tensor.\n as_pandas\n Whether to return the output as a pandas DataFrame (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of embeddings, corresponding to each row in the given data.\n It will have shape (#samples, D) where the embedding dimension D is determined\n by the neural network's architecture.\n \"\"\"\n self.on_predict_start()\n turn_on_off_feature_column_info(\n data_processors=self._data_processors,\n flag=True,\n )\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n if self._problem_type in [FEATURE_EXTRACTION, ZERO_SHOT_IMAGE_CLASSIFICATION]:\n features = extract_from_output(outputs=outputs, ret_type=COLUMN_FEATURES, as_ndarray=as_tensor is False)\n if return_masks:\n masks = extract_from_output(outputs=outputs, ret_type=MASKS, as_ndarray=as_tensor is False)\n else:\n features = extract_from_output(outputs=outputs, ret_type=FEATURES, as_ndarray=as_tensor is False)\n\n if as_pandas:\n features = pd.DataFrame(features, index=data.index)\n if return_masks:\n masks = pd.DataFrame(masks, index=data.index)\n\n if return_masks:\n return features, masks\n else:\n return features\n\n def _as_pandas(\n self,\n data: Union[pd.DataFrame, dict, list],\n to_be_converted: Union[np.ndarray, dict],\n ):\n if isinstance(data, pd.DataFrame):\n index = data.index\n else:\n index = None\n if isinstance(to_be_converted, list) or (\n isinstance(to_be_converted, np.ndarray) and to_be_converted.ndim == 1\n ):\n return pd.Series(to_be_converted, index=index, name=self._label_column)\n else:\n return pd.DataFrame(to_be_converted, index=index, columns=self.class_labels)\n\n def _load_state_dict(\n self,\n state_dict: dict = None,\n path: str = None,\n prefix: str = \"model.\",\n strict: bool = True,\n ):\n if state_dict is None:\n if os.path.isdir(path + \"-dir\"): # deepspeed save checkpoints into a directory\n from lightning.pytorch.utilities.deepspeed import convert_zero_checkpoint_to_fp32_state_dict\n\n convert_zero_checkpoint_to_fp32_state_dict(path + \"-dir\", path)\n shutil.rmtree(path + \"-dir\")\n state_dict = torch.load(path, map_location=torch.device(\"cpu\"), weights_only=False)[\"state_dict\"] # nosec B614\n else:\n state_dict = torch.load(path, map_location=torch.device(\"cpu\"), weights_only=False)[\"state_dict\"] # nosec B614\n state_dict = {k.partition(prefix)[2]: v for k, v in state_dict.items() if k.startswith(prefix)}\n\n # Some buffers like `position_ids` are registered as persistent=False since transformers 4.31.0\n # Refer to https://github.com/huggingface/transformers/pull/24505/files\n buffer_names = [k for k, v in self._model.named_buffers()]\n buffer_names_to_filter = [k for k in buffer_names if k not in self._model.state_dict().keys()]\n state_dict = {k: v for k, v in state_dict.items() if k not in buffer_names_to_filter}\n\n load_result = self._model.load_state_dict(state_dict, strict=strict)\n assert len(load_result.unexpected_keys) == 0, (\n f\"Load model failed, unexpected keys {load_result.unexpected_keys.__str__()}\"\n )\n\n @staticmethod\n def _replace_model_name_prefix(\n state_dict: dict,\n old_prefix: str,\n new_prefix: str,\n ):\n start_idx = len(old_prefix)\n state_dict_processed = {\n new_prefix + k[start_idx:]: v for k, v in state_dict.items() if k.startswith(old_prefix)\n }\n return state_dict_processed\n\n def save(\n self,\n path: str,\n standalone: Optional[bool] = True,\n config: Optional[DictConfig] = None,\n model: Optional[nn.Module] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n fit_called: Optional[bool] = None,\n save_model: Optional[bool] = True,\n ):\n \"\"\"\n Save this learner to file in directory specified by `path`.\n\n Parameters\n ----------\n path\n The directory to save this learner.\n standalone\n Whether to save the downloaded model for offline deployment.\n When standalone = True, save the transformers.CLIPModel and transformers.AutoModel to os.path.join(path,model_name),\n and reset the associate model.model_name.checkpoint_name start with `local://` in config.yaml.\n When standalone = False, the saved artifact may require an online environment to process in load().\n \"\"\"\n config = config if config else self._config\n config = copy.deepcopy(config)\n model = model if model else self._model\n if standalone and not config.optim.peft:\n config = save_pretrained_model_configs(model=model, config=config, path=path)\n os.makedirs(path, exist_ok=True)\n OmegaConf.save(config=config, f=os.path.join(path, \"config.yaml\"))\n\n df_preprocessor = df_preprocessor if df_preprocessor else self._df_preprocessor\n with open(os.path.join(path, \"df_preprocessor.pkl\"), \"wb\") as fp:\n pickle.dump(df_preprocessor, fp)\n\n data_processors = data_processors if data_processors else self._data_processors\n data_processors = copy.deepcopy(data_processors)\n\n # Save text tokenizers before saving data processors\n for modality in [TEXT, TEXT_NER, NER, DOCUMENT]:\n if modality in data_processors:\n for per_processor in data_processors[modality]:\n per_processor.save_tokenizer(path)\n\n # Clear the documents cache dictionary before saving.\n for modality in [DOCUMENT]:\n if modality in data_processors:\n for p in data_processors[modality]:\n p.documents.clear()\n\n with open(os.path.join(path, \"data_processors.pkl\"), \"wb\") as fp:\n pickle.dump(data_processors, fp)\n\n with open(os.path.join(path, \"eval_metric.pkl\"), \"wb\") as fp:\n pickle.dump(self._eval_metric_func, fp)\n\n with open(os.path.join(path, f\"assets.json\"), \"w\") as fp:\n json.dump(\n {\n \"learner_class\": self.__class__.__name__,\n \"column_types\": self._column_types,\n \"label_column\": self._label_column,\n \"problem_type\": self._problem_type,\n \"presets\": self._presets,\n \"eval_metric_name\": self._eval_metric_name,\n \"validation_metric_name\": self._validation_metric_name,\n \"minmax_mode\": self._minmax_mode,\n \"output_shape\": self._output_shape,\n \"save_path\": path,\n \"pretrained\": self._pretrained,\n \"pretrained_path\": self._pretrained_path,\n \"fit_called\": fit_called if fit_called is not None else self._fit_called,\n \"best_score\": self._best_score,\n \"total_train_time\": self._total_train_time,\n \"version\": ag_version.__version__,\n },\n fp,\n ensure_ascii=True,\n )\n\n if save_model:\n checkpoint = {\"state_dict\": {\"model.\" + name: param for name, param in model.state_dict().items()}}\n torch.save(checkpoint, os.path.join(os.path.abspath(path), MODEL_CHECKPOINT)) # nosec B614\n\n @staticmethod\n def _load_metadata(\n learner: BaseLearner,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n path = os.path.abspath(os.path.expanduser(path))\n assert os.path.isdir(path), f\"'{path}' must be an existing directory.\"\n config = OmegaConf.load(os.path.join(path, \"config.yaml\"))\n\n config = get_local_pretrained_config_paths(\n config=config, path=path\n ) # check the config to load offline pretrained model configs\n\n with open(os.path.join(path, \"assets.json\"), \"r\") as fp:\n assets = json.load(fp)\n\n with open(os.path.join(path, \"df_preprocessor.pkl\"), \"rb\") as fp:\n df_preprocessor = pickle.load(fp) # nosec B301\n try:\n with open(os.path.join(path, \"data_processors.pkl\"), \"rb\") as fp:\n data_processors = pickle.load(fp) # nosec B301\n # Load text tokenizers after loading data processors.\n for modality in [TEXT, TEXT_NER, NER, DOCUMENT]:\n if modality in data_processors:\n for per_processor in data_processors[modality]:\n per_processor.load_tokenizer(path)\n\n # Only keep the modalities with non-empty processors.\n data_processors = {k: v for k, v in data_processors.items() if len(v) > 0}\n except: # reconstruct the data processor in case something went wrong.\n data_processors = None\n\n learner._label_column = assets[\"label_column\"]\n learner._problem_type = assets[\"problem_type\"]\n learner._presets = assets[\"presets\"]\n learner._best_score = assets[\"best_score\"]\n learner._total_train_time = assets[\"total_train_time\"]\n learner._eval_metric_name = assets[\"eval_metric_name\"]\n with open(os.path.join(path, \"eval_metric.pkl\"), \"rb\") as fp:\n learner._eval_metric_func = pickle.load(fp) # nosec B301\n learner._verbosity = verbosity\n learner._resume = resume\n learner._save_path = path # in case the original exp dir is copied to somewhere else\n learner._pretrained_path = path\n learner._pretrained = assets[\"pretrained\"]\n learner._fit_called = assets[\"fit_called\"]\n learner._config = config\n learner._output_shape = assets[\"output_shape\"]\n if \"classes\" in assets:\n learner._classes = assets[\"classes\"]\n learner._column_types = assets[\"column_types\"]\n learner._validation_metric_name = assets[\"validation_metric_name\"]\n learner._df_preprocessor = df_preprocessor\n learner._data_processors = data_processors\n learner._minmax_mode = assets[\"minmax_mode\"]\n\n return learner\n\n @classmethod\n def load(\n cls,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n \"\"\"\n Load a learner object from a directory specified by `path`. The to-be-loaded learner\n can be completely or partially trained by .fit(). If a previous training has completed,\n it will load the checkpoint `model.ckpt`. Otherwise if a previous training accidentally\n collapses in the middle, it can load the `last.ckpt` checkpoint by setting `resume=True`.\n It also supports loading one specific checkpoint given its path.\n\n Parameters\n ----------\n path\n The directory to load the learner object.\n resume\n Whether to resume training from `last.ckpt`. This is useful when a training was accidentally\n broken during the middle and we want to resume the training from the last saved checkpoint.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n\n Returns\n -------\n The loaded learner object.\n \"\"\"\n dir_path, ckpt_path = get_dir_ckpt_paths(path=path)\n\n assert os.path.isdir(dir_path), f\"'{dir_path}' must be an existing directory.\"\n learner = cls(label=\"dummy_label\")\n learner = cls._load_metadata(learner=learner, path=dir_path, resume=resume, verbosity=verbosity)\n peft = learner._config.optim.peft\n learner._model = create_fusion_model(\n config=learner._config,\n num_classes=learner._output_shape,\n classes=learner._classes if hasattr(learner, \"_classes\") else None,\n num_numerical_columns=len(learner._df_preprocessor.numerical_feature_names),\n num_categories=learner._df_preprocessor.categorical_num_categories,\n pretrained=False if not peft else True, # set \"pretrain=False\" to prevent downloading online models\n )\n if learner._data_processors is None:\n learner._data_processors = create_fusion_data_processors(\n config=learner._config,\n model=learner._model,\n )\n load_path, ckpt_path = get_load_ckpt_paths(\n ckpt_path=ckpt_path,\n dir_path=dir_path,\n resume=resume,\n )\n learner._load_state_dict(\n path=load_path,\n strict=not peft,\n )\n learner._ckpt_path = ckpt_path\n loss_func = get_loss_func(\n problem_type=learner._problem_type,\n mixup_active=False,\n loss_func_name=learner._config.optim.loss_func,\n config=learner._config.optim,\n num_classes=learner._output_shape,\n )\n model_postprocess_fn = get_model_postprocess_fn(\n problem_type=learner._problem_type,\n loss_func=loss_func,\n )\n learner._model_postprocess_fn = model_postprocess_fn\n learner._config = learner.update_strategy_by_env(learner._config)\n\n return learner\n\n @property\n def class_labels(self):\n \"\"\"\n The original name of the class labels.\n For example, the tabular data may contain classes equal to\n \"entailment\", \"contradiction\", \"neutral\". Internally, these will be converted to\n 0, 1, 2, ...\n This function returns the original names of these raw labels.\n\n Returns\n -------\n List that contain the class names. It will be None if it's not a classification problem.\n \"\"\"\n if self._problem_type == MULTICLASS or self._problem_type == BINARY:\n return self._df_preprocessor.label_generator.classes_\n else:\n warnings.warn(\"Accessing class names for a non-classification problem. Return None.\")\n return None\n\n @property\n def positive_class(self):\n \"\"\"\n Name of the class label that will be mapped to 1.\n This is only meaningful for binary classification problems.\n\n It is useful for computing metrics such as F1 which require a positive and negative class.\n You may refer to https://en.wikipedia.org/wiki/F-score for more details.\n In binary classification, :class:`BaseLearner.predict_proba(as_multiclass=False)`\n returns the estimated probability that each row belongs to the positive class.\n Will print a warning and return None if called when `learner.problem_type != 'binary'`.\n\n Returns\n -------\n The positive class name in binary classification or None if the problem is not binary classification.\n \"\"\"\n if self._problem_type != BINARY:\n logger.warning(\n f\"Warning: Attempted to retrieve positive class label in a non-binary problem. \"\n f\"Positive class labels only exist in binary classification. \"\n f\"Returning None instead. The problem type is '{self._problem_type}'\"\n f\" but positive_class only exists for '{BINARY}'.\"\n )\n return None\n else:\n return self.class_labels[1]\n\n def fit_summary(self, verbosity=0, show_plot=False):\n \"\"\"\n Output summary of information about models produced during `fit()`.\n\n Parameters\n ----------\n verbosity : int, default = 2\n Verbosity levels range from 0 to 4 and control how much information is printed.\n verbosity = 0 for no output printing.\n TODO: Higher levels correspond to more detailed print statements\n show_plot : bool, default = False\n If True, shows the model summary plot in browser when verbosity > 1.\n\n Returns\n -------\n Dict containing various detailed information.\n We do not recommend directly printing this dict as it may be very large.\n \"\"\"\n if self._total_train_time is None:\n logging.info(\"There is no `best_score` or `total_train_time`. Have you called `predictor.fit()`?\")\n else:\n logging.info(\n f\"Here's the model summary:\"\n f\"\"\n f\"The model achieved score '{self._best_score}' on the validation metric\"\n f\" '{self._validation_metric_name}'. \"\n f\"The total training time is {timedelta(seconds=self._total_train_time)}\"\n )\n results = {\n f\"val_{self._validation_metric_name}\": self._best_score,\n \"training_time\": self._total_train_time,\n }\n return results\n\n def list_supported_models(self, pretrained=True):\n \"\"\"\n List supported models for each problem_type to let users know\n options of checkpoint name to choose during fit().\n\n Parameters\n ----------\n pretrained : bool, default = True\n If True, only return the models with pretrained weights.\n If False, return all the models as long as there is model definition.\n\n Returns\n -------\n a list of model names\n \"\"\"\n if self.problem_property and self.problem_property.is_classification:\n # FIXME (Need to list the supported models for each modality)\n return list_timm_models(pretrained=pretrained)\n else:\n raise ValueError(f\"list_supported_models() is not available for problem type: {self._problem_type}\")\n\n @staticmethod\n def update_strategy_by_env(config):\n \"\"\"\n Set strategy to ddp_fork or ddp_notebook if an iterative env is detected.\n \"\"\"\n assert config is not None\n if is_interactive_env() and not is_interactive_strategy(config.env.strategy):\n strs = list(config.env.strategy.partition(\"_find_unused_parameters\"))\n strs[0] = \"ddp_fork\"\n config.env.strategy = \"\".join(strs)\n\n return config\n\n def set_num_gpus(self, num_gpus):\n assert isinstance(num_gpus, int)\n self._config.env.num_gpus = num_gpus\n\n def get_num_gpus(self):\n try:\n return self._config.env.num_gpus\n except:\n return None\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/ensemble.py",
"content": "import copy\nimport json\nimport logging\nimport os\nimport pathlib\nimport pickle\nimport pprint\nimport time\nimport warnings\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.core.metrics import Scorer\nfrom autogluon.core.models.greedy_ensemble.ensemble_selection import EnsembleSelection\n\nfrom .. import version as ag_version\nfrom ..constants import BINARY, LOGITS, MULTICLASS, REGRESSION, TEST, VAL, Y_PRED, Y_TRUE\nfrom ..optim import compute_score\nfrom ..utils import (\n extract_from_output,\n get_dir_ckpt_paths,\n logits_to_prob,\n on_fit_end_message,\n update_ensemble_hyperparameters,\n)\nfrom .base import BaseLearner\n\nlogger = logging.getLogger(__name__)\n\n\nclass EnsembleLearner(BaseLearner):\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = None,\n presets: Optional[str] = \"high_quality\",\n eval_metric: Optional[Union[str, Scorer]] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n ensemble_size: Optional[int] = 2,\n ensemble_mode: Optional[str] = \"one_shot\",\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n label\n Name of the column that contains the target variable to predict.\n problem_type\n Type of the prediction problem. We support standard problems like\n\n - 'binary': Binary classification\n - 'multiclass': Multi-class classification\n - 'regression': Regression\n - 'classification': Classification problems include 'binary' and 'multiclass' classification.\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n eval_metric\n Evaluation metric name. If `eval_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'accuracy' for multiclass classification, `roc_auc` for binary classification, and 'root_mean_squared_error' for regression.\n hyperparameters\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n hyperparameters = \"model.hf_text.checkpoint_name=google/electra-small-discriminator model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n hyperparameters = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\", \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n path\n Path to directory where models and intermediate outputs should be saved.\n If unspecified, a time-stamped folder called \"AutogluonAutoMM/ag-[TIMESTAMP]\"\n will be created in the working directory to store all models.\n Note: To call `fit()` twice and save all results of each fit,\n you must specify different `path` locations or don't specify `path` at all.\n Otherwise files from first `fit()` will be overwritten by second `fit()`.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50\n (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels)\n warn_if_exist\n Whether to raise warning if the specified path already exists.\n enable_progress_bar\n Whether to show progress bar. It will be True by default and will also be\n disabled if the environment variable os.environ[\"AUTOMM_DISABLE_PROGRESS_BAR\"] is set.\n ensemble_size\n A multiple of number of models in the ensembling pool (Default 2). The actual ensemble size = ensemble_size * the model number\n ensemble_mode\n The mode of conducting ensembling:\n - `one_shot`: the classic ensemble selection\n - `sequential`: iteratively calling the classic ensemble selection with each time growing the model zoo by the best next model.\n \"\"\"\n super().__init__(\n label=label,\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n )\n self._ensemble_size = int(ensemble_size)\n assert ensemble_mode in [\"sequential\", \"one_shot\"]\n self._ensemble_mode = ensemble_mode\n self._weighted_ensemble = None\n self._selected_learners = None\n self._all_learners = None\n self._selected_indices = []\n self._relative_path = False\n\n return\n\n def get_learner_path(self, learner_path: str):\n if self._relative_path:\n learner_path = os.path.join(self._save_path, learner_path)\n return learner_path\n\n def get_learner_name(self, learner):\n if isinstance(learner, str):\n if self._relative_path:\n learner_name = learner\n else:\n learner_name = pathlib.PurePath(learner).name\n else:\n learner_name = pathlib.PurePath(learner.path).name\n\n return learner_name\n\n def predict_all_for_ensembling(\n self,\n learners: List[Union[str, BaseLearner]],\n data: Union[pd.DataFrame, str],\n mode: str,\n requires_label: Optional[bool] = False,\n save: Optional[bool] = False,\n ):\n assert mode in [VAL, TEST]\n predictions = []\n labels = None\n i = 0\n for per_learner in learners:\n i += 1\n logger.info(f\"\\npredicting with learner {i}: {per_learner}\\n\")\n if isinstance(per_learner, str):\n per_learner_path = self.get_learner_path(per_learner)\n else:\n per_learner_path = per_learner.path\n\n pred_file_path = os.path.join(per_learner_path, f\"{mode}_predictions.npy\")\n if os.path.isfile(pred_file_path):\n logger.info(f\"{mode}_predictions.npy exists. loading it...\")\n y_pred = np.load(pred_file_path)\n else:\n if isinstance(per_learner, str):\n per_learner = BaseLearner.load(path=per_learner_path)\n if not self._problem_type:\n self._problem_type = per_learner.problem_type\n else:\n assert self._problem_type == per_learner.problem_type\n outputs = per_learner.predict_per_run(\n data=data,\n realtime=False,\n requires_label=False,\n )\n y_pred = extract_from_output(outputs=outputs, ret_type=LOGITS)\n\n if self._problem_type == REGRESSION:\n y_pred = per_learner._df_preprocessor.transform_prediction(y_pred=y_pred)\n if self._problem_type in [BINARY, MULTICLASS]:\n y_pred = logits_to_prob(y_pred)\n if self._problem_type == BINARY:\n y_pred = y_pred[:, 1]\n\n if save:\n np.save(pred_file_path, y_pred)\n\n if requires_label:\n label_file_path = os.path.join(per_learner_path, f\"{mode}_labels.npy\")\n if os.path.isfile(label_file_path):\n logger.info(f\"{mode}_labels.npy exists. loading it...\")\n y_true = np.load(label_file_path)\n else:\n if isinstance(per_learner, str):\n per_learner = BaseLearner.load(path=per_learner_path)\n y_true = per_learner._df_preprocessor.transform_label_for_metric(df=data)\n\n if save:\n np.save(label_file_path, y_true)\n\n if labels is None:\n labels = y_true\n else:\n assert np.array_equal(y_true, labels)\n\n predictions.append(y_pred)\n\n if requires_label:\n return predictions, labels\n else:\n return predictions\n\n @staticmethod\n def verify_predictions_labels(predictions, learners, labels=None):\n if labels is not None:\n assert isinstance(labels, np.ndarray)\n assert isinstance(predictions, list) and all(isinstance(ele, np.ndarray) for ele in predictions)\n assert len(learners) == len(predictions), (\n f\"len(learners) {len(learners)} doesn't match len(predictions) {len(predictions)}\"\n )\n\n def fit_per_ensemble(\n self,\n predictions: List[np.ndarray],\n labels: np.ndarray,\n ):\n weighted_ensemble = EnsembleSelection(\n ensemble_size=self._ensemble_size * len(predictions),\n problem_type=self._problem_type,\n metric=self._eval_metric_func,\n )\n weighted_ensemble.fit(predictions=predictions, labels=labels)\n\n return weighted_ensemble\n\n def select_next_best(self, left_learner_indices, selected_learner_indices, predictions, labels):\n best_regret = None\n best_weighted_ensemble = None\n best_next_index = None\n for i in left_learner_indices:\n tmp_learner_indices = selected_learner_indices + [i]\n tmp_predictions = [predictions[j] for j in tmp_learner_indices]\n tmp_weighted_ensemble = self.fit_per_ensemble(\n predictions=tmp_predictions,\n labels=labels,\n )\n if best_regret is None or tmp_weighted_ensemble.train_score_ < best_regret:\n best_regret = tmp_weighted_ensemble.train_score_\n best_weighted_ensemble = tmp_weighted_ensemble\n best_next_index = i\n\n return best_regret, best_next_index, best_weighted_ensemble\n\n def sequential_ensemble(\n self,\n predictions: List[np.ndarray],\n labels: np.ndarray,\n ):\n selected_learner_indices = []\n all_learner_indices = list(range(len(predictions)))\n best_regret = None\n best_weighted_ensemble = None\n best_selected_learner_indices = None\n while len(selected_learner_indices) < len(all_learner_indices):\n left_learner_indices = [i for i in all_learner_indices if i not in selected_learner_indices]\n assert sorted(all_learner_indices) == sorted(selected_learner_indices + left_learner_indices)\n logger.debug(f\"\\nleft_learner_indices: {left_learner_indices}\")\n if not left_learner_indices:\n break\n logger.debug(f\"selected_learner_indices: {selected_learner_indices}\")\n tmp_reget, next_index, tmp_weighted_ensemble = self.select_next_best(\n left_learner_indices=left_learner_indices,\n selected_learner_indices=selected_learner_indices,\n predictions=predictions,\n labels=labels,\n )\n selected_learner_indices.append(next_index)\n if best_regret is None or tmp_reget < best_regret:\n best_regret = tmp_reget\n best_weighted_ensemble = tmp_weighted_ensemble\n best_selected_learner_indices = copy.deepcopy(selected_learner_indices)\n logger.debug(f\"\\nbest score: {self._eval_metric_func._optimum - best_regret}\")\n logger.debug(f\"best_selected_learner_indices: {best_selected_learner_indices}\")\n logger.debug(f\"best_ensemble_weights: {best_weighted_ensemble.weights_}\")\n\n return best_weighted_ensemble, best_selected_learner_indices\n\n def update_hyperparameters(self, hyperparameters: Dict):\n if self._hyperparameters and hyperparameters:\n self._hyperparameters.update(hyperparameters)\n elif hyperparameters:\n self._hyperparameters = hyperparameters\n\n self._hyperparameters = update_ensemble_hyperparameters(\n presets=self._presets,\n provided_hyperparameters=self._hyperparameters,\n )\n # filter out meta-transformer if no local checkpoint path is provided\n if \"early_fusion\" in self._hyperparameters:\n if self._hyperparameters[\"early_fusion\"][\"model.meta_transformer.checkpoint_path\"] == \"null\":\n self._hyperparameters.pop(\"early_fusion\")\n message = (\n \"`early_fusion` will not be used in ensembling because `early_fusion` relies on MetaTransformer, \"\n \"but no local MetaTransformer model checkpoint is provided. To use `early_fusion`, \"\n \"download its model checkpoints from https://github.com/invictus717/MetaTransformer to local \"\n \"and set the checkpoint path as follows:\\n\"\n \"```python\\n\"\n \"hyperparameters = {\\n\"\n ' \"early_fusion\": {\\n'\n ' \"model.meta_transformer.checkpoint_path\": args.meta_transformer_ckpt_path,\\n'\n \" }\\n\"\n \"}\\n\"\n \"```\\n\"\n \"Note that presets `high_quality` (default) and `medium_quality` need the base model, while preset \"\n \"`best_quality` requires the large model. Make sure to download the right MetaTransformer version. \"\n \"We recommend using the download links under tag `åŊå
ä¸čŊŊæē` because the corresponding \"\n \"downloaded models are not compressed and can be loaded directly.\\n\"\n )\n\n logger.warning(message)\n\n def fit_all(\n self,\n train_data,\n tuning_data,\n hyperparameters,\n column_types,\n holdout_frac,\n time_limit,\n seed,\n standalone,\n clean_ckpts,\n ):\n self._relative_path = True\n self.update_hyperparameters(hyperparameters=hyperparameters)\n\n learners = []\n assert len(self._hyperparameters) > 1, (\n f\"Ensembling requires training more than 1 learners, but got {len(self._hyperparameters)} sets of hyperparameters.\"\n )\n logger.info(\n f\"Will ensemble {len(self._hyperparameters)} models with the following configs:\\n {pprint.pformat(self._hyperparameters)}\"\n )\n for per_name, per_hparams in self._hyperparameters.items():\n per_learner_path = os.path.join(self._save_path, per_name)\n if not os.path.isdir(per_learner_path):\n logger.info(f\"\\nfitting learner {per_name}\")\n logger.debug(f\"hyperparameters: {per_hparams}\")\n per_learner = BaseLearner(\n label=self._label_column,\n problem_type=self._problem_type,\n presets=self._presets,\n eval_metric=self._eval_metric_func,\n hyperparameters=per_hparams,\n path=per_learner_path,\n verbosity=self._verbosity,\n warn_if_exist=self._warn_if_exist,\n enable_progress_bar=self._enable_progress_bar,\n pretrained=self._pretrained,\n validation_metric=self._validation_metric_name,\n )\n per_learner.fit(\n train_data=train_data,\n tuning_data=tuning_data,\n time_limit=time_limit,\n column_types=column_types,\n holdout_frac=holdout_frac,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n learners.append(per_name)\n\n return learners\n\n def on_fit_end(\n self,\n training_start: float,\n **kwargs,\n ):\n self._fit_called = True\n training_end = time.time()\n self._total_train_time = training_end - training_start\n logger.info(on_fit_end_message(self._save_path))\n\n def update_attributes_by_first_learner(self, learners: List):\n # load df preprocessor from the first learner\n if isinstance(learners[0], str):\n first_learner_path = self.get_learner_path(learners[0])\n dir_path, ckpt_path = get_dir_ckpt_paths(path=first_learner_path)\n assert os.path.isdir(dir_path), f\"'{dir_path}' must be an existing directory.\"\n first_learner = BaseLearner(label=\"dummy_label\")\n first_learner = BaseLearner._load_metadata(learner=first_learner, path=dir_path)\n else:\n first_learner = learners[0]\n\n self._df_preprocessor = first_learner._df_preprocessor\n self._eval_metric_func = first_learner._eval_metric_func\n self._eval_metric_name = first_learner._eval_metric_name\n self._problem_type = first_learner._problem_type\n\n def fit_ensemble(\n self,\n predictions: Optional[List[np.ndarray]] = None,\n labels: Optional[np.ndarray] = None,\n learners: Optional[List[Union[str, BaseLearner]]] = None,\n train_data: Optional[Union[pd.DataFrame, str]] = None,\n tuning_data: Optional[Union[pd.DataFrame, str]] = None,\n holdout_frac: Optional[float] = None,\n seed: Optional[int] = 0,\n ):\n if not predictions or labels is None:\n self.prepare_train_tuning_data(\n train_data=train_data,\n tuning_data=tuning_data,\n holdout_frac=holdout_frac,\n seed=seed,\n )\n predictions, labels = self.predict_all_for_ensembling(\n learners=learners,\n data=self._tuning_data,\n mode=VAL,\n requires_label=True,\n save=True,\n )\n\n self.verify_predictions_labels(\n predictions=predictions,\n labels=labels,\n learners=learners,\n )\n\n if self._ensemble_mode == \"sequential\":\n weighted_ensemble, selected_learner_indices = self.sequential_ensemble(\n predictions=predictions,\n labels=labels,\n )\n elif self._ensemble_mode == \"one_shot\":\n weighted_ensemble = self.fit_per_ensemble(\n predictions=predictions,\n labels=labels,\n )\n selected_learner_indices = list(range(len(learners)))\n else:\n raise ValueError(f\"Unsupported ensemble_mode: {self._ensemble_mode}\")\n\n predictions = [predictions[j] for j in selected_learner_indices]\n predictions = weighted_ensemble.predict_proba(predictions)\n\n # for regression, the transform_prediction() is already called in predict_all()\n if self._eval_metric_func.needs_pred and self._problem_type != REGRESSION:\n predictions = self._df_preprocessor.transform_prediction(\n y_pred=predictions,\n inverse_categorical=False,\n )\n metric_data = {\n Y_PRED: predictions,\n Y_TRUE: labels,\n }\n score = compute_score(\n metric_data=metric_data,\n metric=self._eval_metric_func,\n )\n\n logger.debug(f\"\\nEnsembling score on validation data: {score}\")\n\n return weighted_ensemble, selected_learner_indices\n\n def fit(\n self,\n train_data: Union[pd.DataFrame, str],\n presets: Optional[str] = None,\n tuning_data: Optional[Union[pd.DataFrame, str]] = None,\n time_limit: Optional[int] = None,\n save_path: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n column_types: Optional[Dict] = None,\n holdout_frac: Optional[float] = None,\n teacher_learner: Union[str, BaseLearner] = None,\n seed: Optional[int] = 0,\n standalone: Optional[bool] = True,\n hyperparameter_tune_kwargs: Optional[Dict] = None,\n clean_ckpts: Optional[bool] = True,\n learners: Optional[List[Union[str, BaseLearner]]] = None,\n predictions: Optional[List[np.ndarray]] = None,\n labels: Optional[np.ndarray] = None,\n **kwargs,\n ):\n self.setup_save_path(save_path=save_path)\n training_start = self.on_fit_start(presets=presets)\n if learners is None:\n learners = self.fit_all(\n train_data=train_data,\n tuning_data=tuning_data,\n hyperparameters=hyperparameters,\n column_types=column_types,\n holdout_frac=holdout_frac,\n time_limit=time_limit,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n assert len(learners) > 1, f\"Ensembling requires more than 1 learners, but got {len(learners)}.\"\n\n self.update_attributes_by_first_learner(learners=learners)\n weighted_ensemble, selected_learner_indices = self.fit_ensemble(\n predictions=predictions,\n labels=labels,\n learners=learners,\n train_data=train_data,\n tuning_data=tuning_data,\n holdout_frac=holdout_frac,\n seed=seed,\n )\n\n assert len(selected_learner_indices) == len(weighted_ensemble.weights_)\n self._weighted_ensemble = weighted_ensemble\n self._selected_learners = [learners[i] for i in selected_learner_indices]\n self._all_learners = learners\n self._selected_indices = selected_learner_indices\n\n self.on_fit_end(training_start=training_start)\n self.save(path=self._save_path)\n\n return self\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n predictions: Optional[List[np.ndarray]] = None,\n as_pandas: Optional[bool] = None,\n **kwargs,\n ):\n self.on_predict_start()\n if not predictions:\n predictions = self.predict_all_for_ensembling(\n learners=self._selected_learners,\n data=data,\n mode=TEST,\n requires_label=False,\n save=False,\n )\n else:\n predictions = [predictions[i] for i in self._selected_indices]\n\n self.verify_predictions_labels(\n predictions=predictions,\n learners=self._selected_learners,\n )\n pred = self._weighted_ensemble.predict_proba(predictions)\n # for regression, the transform_prediction() is already called in predict_all()\n if self._problem_type in [BINARY, MULTICLASS]:\n pred = self._df_preprocessor.transform_prediction(\n y_pred=pred,\n inverse_categorical=True,\n )\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n pred = self._as_pandas(data=data, to_be_converted=pred)\n\n return pred\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n predictions: Optional[List[np.ndarray]] = None,\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n **kwargs,\n ):\n self.on_predict_start()\n assert self._problem_type not in [\n REGRESSION,\n ], f\"Problem {self._problem_type} has no probability output.\"\n\n if not predictions:\n predictions = self.predict_all_for_ensembling(\n learners=self._selected_learners,\n data=data,\n mode=TEST,\n requires_label=False,\n save=False,\n )\n else:\n predictions = [predictions[i] for i in self._selected_indices]\n\n self.verify_predictions_labels(\n predictions=predictions,\n learners=self._selected_learners,\n )\n prob = self._weighted_ensemble.predict_proba(predictions)\n if as_multiclass and self._problem_type == BINARY:\n prob = np.column_stack((1 - prob, prob))\n\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n prob = self._as_pandas(data=data, to_be_converted=prob)\n\n return prob\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n predictions: Optional[List[np.ndarray]] = None,\n labels: Optional[np.ndarray] = None,\n save_all: Optional[bool] = True,\n **kwargs,\n ):\n self.on_predict_start()\n if not predictions or labels is None:\n if save_all:\n learners = self._all_learners\n else:\n learners = self._selected_learners\n predictions, labels = self.predict_all_for_ensembling(\n learners=learners,\n data=data,\n mode=TEST,\n requires_label=True,\n save=True,\n )\n if save_all:\n predictions = [predictions[i] for i in self._selected_indices]\n else:\n predictions = [predictions[i] for i in self._selected_indices]\n\n self.verify_predictions_labels(\n predictions=predictions,\n labels=labels,\n learners=self._selected_learners,\n )\n all_scores = dict()\n for per_predictions, per_learner in zip(predictions, self._selected_learners):\n if not isinstance(per_learner, str):\n per_learner = per_learner.path\n metric_data = {\n Y_PRED: per_predictions,\n Y_TRUE: labels,\n }\n all_scores[per_learner] = compute_score(\n metric_data=metric_data,\n metric=self._eval_metric_func,\n )\n\n predictions = self._weighted_ensemble.predict_proba(predictions)\n # for regression, the transform_prediction() is already called in predict_all()\n if self._eval_metric_func.needs_pred and self._problem_type != REGRESSION:\n predictions = self._df_preprocessor.transform_prediction(\n y_pred=predictions,\n inverse_categorical=False,\n )\n metric_data = {\n Y_PRED: predictions,\n Y_TRUE: labels,\n }\n all_scores[\"ensemble\"] = compute_score(\n metric_data=metric_data,\n metric=self._eval_metric_func,\n )\n\n return all_scores\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n return_masks: Optional[bool] = False,\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n raise ValueError(f\"EnsembleLearner doesn't support extracting embedding yet.\")\n\n def save(\n self,\n path: str,\n **kwargs,\n ):\n selected_learner_names = [self.get_learner_name(per_learner) for per_learner in self._selected_learners]\n all_learner_names = [self.get_learner_name(per_learner) for per_learner in self._all_learners]\n\n os.makedirs(path, exist_ok=True)\n with open(os.path.join(path, f\"assets.json\"), \"w\") as fp:\n json.dump(\n {\n \"learner_class\": self.__class__.__name__,\n \"ensemble_size\": self._ensemble_size,\n \"ensemble_mode\": self._ensemble_mode,\n \"selected_learners\": selected_learner_names,\n \"all_learners\": all_learner_names,\n \"selected_indices\": self._selected_indices,\n \"ensemble_weights\": self._weighted_ensemble.weights_.tolist(),\n \"save_path\": path,\n \"relative_path\": True,\n \"fit_called\": self._fit_called,\n \"version\": ag_version.__version__,\n \"hyperparameters\": self._hyperparameters,\n },\n fp,\n ensure_ascii=True,\n )\n\n with open(os.path.join(path, \"ensemble.pkl\"), \"wb\") as fp:\n pickle.dump(self._weighted_ensemble, fp)\n\n # save each learner\n for per_learner in self._all_learners:\n per_learner_name = self.get_learner_name(per_learner)\n if isinstance(per_learner, str):\n per_learner_path = self.get_learner_path(per_learner)\n per_learner = BaseLearner.load(per_learner_path)\n\n per_learner_save_path = os.path.join(path, per_learner_name)\n per_learner.save(per_learner_save_path)\n\n return\n\n @classmethod\n def load(\n cls,\n path: str,\n **kwargs,\n ):\n dir_path, ckpt_path = get_dir_ckpt_paths(path=path)\n assert os.path.isdir(dir_path), f\"'{dir_path}' must be an existing directory.\"\n with open(os.path.join(dir_path, \"assets.json\"), \"r\") as fp:\n assets = json.load(fp)\n\n learner = cls(\n hyperparameters=assets[\"hyperparameters\"],\n )\n learner._ensemble_size = assets[\"ensemble_size\"]\n learner._ensemble_mode = assets[\"ensemble_mode\"]\n learner._selected_learners = assets[\"selected_learners\"]\n learner._all_learners = assets[\"all_learners\"]\n learner._selected_indices = assets[\"selected_indices\"]\n learner._save_path = path # in case the original exp dir is copied to somewhere else\n learner._relative_path = assets[\"relative_path\"]\n learner._fit_called = assets[\"fit_called\"]\n\n with open(os.path.join(path, \"ensemble.pkl\"), \"rb\") as fp:\n learner._weighted_ensemble = pickle.load(fp) # nosec B301\n\n learner.update_attributes_by_first_learner(learners=learner._selected_learners)\n\n return learner\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/few_shot_svm.py",
"content": "import logging\nimport os\nimport pickle\nfrom datetime import timedelta\nfrom typing import Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom omegaconf import DictConfig\nfrom sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import SVC\nfrom torch import nn\n\nfrom autogluon.core.metrics import Scorer\nfrom autogluon.core.utils.loaders import load_pd\n\nfrom ..constants import CLIP, COLUMN_FEATURES, HF_TEXT, TIMM_IMAGE, Y_PRED, Y_TRUE\nfrom ..data import BaseDataModule, MultiModalFeaturePreprocessor, data_to_df, turn_on_off_feature_column_info\nfrom ..models import select_model\nfrom ..optim import compute_score\nfrom ..utils import LogFilter, apply_log_filter, extract_from_output, get_available_devices, logits_to_prob\nfrom .base import BaseLearner\n\nlogger = logging.getLogger(__name__)\n\n\nclass FewShotSVMLearner(BaseLearner):\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = None,\n hyperparameters: Optional[dict] = None,\n presets: Optional[str] = None,\n eval_metric: Optional[str] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n label\n Name of the column that contains the target variable to predict.\n hyperparameters\n This is to override some default configurations.\n example:\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-mpnet-base-v2\",\n \"model.hf_text.pooling_mode\": \"mean\",\n \"env.per_gpu_batch_size\": 32,\n \"env.inference_batch_size_ratio\": 4,\n }\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n eval_metric\n Evaluation metric name.\n path\n Path to directory where models and intermediate outputs should be saved.\n If unspecified, a time-stamped folder called \"AutogluonAutoMM/ag-[TIMESTAMP]\"\n will be created in the working directory to store all models.\n Note: To call `fit()` twice and save all results of each fit,\n you must specify different `path` locations or don't specify `path` at all.\n Otherwise files from first `fit()` will be overwritten by second `fit()`.\n problem_type\n The problem type specified by user. Currently the SVM predictor only supports classification types\n \"\"\"\n super().__init__(\n label=label,\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n )\n\n self._svm = None\n\n def update_attributes(\n self,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n svm: Optional[Pipeline] = None,\n **kwargs,\n ):\n super().update_attributes(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n )\n if svm:\n self._svm = svm\n\n def prepare_train_tuning_data(\n self,\n train_data: Union[pd.DataFrame, str],\n tuning_data: Optional[Union[pd.DataFrame, str]],\n holdout_frac: Optional[float],\n seed: Optional[int],\n ):\n if isinstance(train_data, str):\n train_data = load_pd.load(train_data)\n if isinstance(tuning_data, str):\n tuning_data = load_pd.load(tuning_data)\n\n self._train_data = train_data\n self._tuning_data = tuning_data # TODO: use tuning_data in few shot learning?\n\n def infer_problem_type(self, train_data: pd.DataFrame):\n return # problem type should be provided in the learner initialization.\n\n def infer_output_shape(self):\n return # learner doesn't need output shape since svm handles it.\n\n def prepare_fit_args(\n self,\n time_limit: int,\n seed: int,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n super().prepare_fit_args(\n time_limit=time_limit,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n self._fit_args.pop(\"ckpt_path\", None)\n self._fit_args.pop(\"resume\", None)\n self._fit_args.pop(\"clean_ckpts\", None)\n if self._fit_called:\n self._fit_args.update(dict(svm=self._svm))\n\n def fit_sanity_check(self):\n feature_column_types = {k: v for k, v in self._column_types.items() if k != self._label_column}\n unique_dtypes = set(feature_column_types.values())\n assert len(unique_dtypes) == 1, (\n f\"Few shot SVM learner allows single modality data for now, but detected modalities {unique_dtypes}.\"\n )\n\n @staticmethod\n def get_svm_per_run(svm: Pipeline):\n if svm is None:\n svm = make_pipeline(StandardScaler(), SVC(gamma=\"auto\"))\n return svm\n\n def on_fit_per_run_end(\n self,\n trainer: pl.Trainer,\n config: DictConfig,\n model: nn.Module,\n svm: Pipeline,\n df_preprocessor: MultiModalFeaturePreprocessor,\n data_processors: Dict,\n save_path: str,\n standalone: bool,\n ):\n self.clean_trainer_processes(trainer=trainer, is_train=True)\n self.save(\n path=save_path,\n standalone=standalone,\n config=config,\n model=model,\n svm=svm,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=True, # fit is called on one run.\n save_model=True, # need to save the model now because there will be no top k averaging.\n )\n\n def get_datamodule_per_run(\n self,\n df_preprocessor,\n data_processors,\n per_gpu_batch_size,\n num_workers,\n predict_data=None,\n is_train=True,\n ):\n datamodule_kwargs = dict(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=per_gpu_batch_size,\n num_workers=num_workers,\n )\n if is_train:\n datamodule_kwargs.update(dict(predict_data=self._train_data))\n else:\n datamodule_kwargs.update(dict(predict_data=predict_data))\n\n datamodule = BaseDataModule(**datamodule_kwargs)\n return datamodule\n\n @staticmethod\n def update_config_by_data_per_run(config, df_preprocessor):\n if df_preprocessor.text_feature_names and CLIP in config.model.names:\n config.model.names.remove(CLIP) # for text only data, remove clip model and use hf_text\n if df_preprocessor.image_feature_names and TIMM_IMAGE in config.model.names and CLIP in config.model.names:\n config.model.names.remove(\n CLIP\n ) # if users add timm_image in hyperparameters, then remove clip and use timm_image\n config = select_model(config=config, df_preprocessor=df_preprocessor, strict=False)\n return config\n\n def init_trainer_per_run(\n self,\n num_gpus,\n precision,\n strategy,\n callbacks,\n max_time=None,\n config=None,\n enable_progress_bar=None,\n barebones=False,\n is_train=True,\n ):\n if not is_train:\n config = self._config\n enable_progress_bar = self._enable_progress_bar\n\n blacklist_msgs = []\n if self._verbosity <= 3: # turn off logging in prediction\n blacklist_msgs.append(\"Automatic Mixed Precision\")\n blacklist_msgs.append(\"GPU available\")\n blacklist_msgs.append(\"TPU available\")\n blacklist_msgs.append(\"IPU available\")\n blacklist_msgs.append(\"HPU available\")\n blacklist_msgs.append(\"select gpus\")\n blacklist_msgs.append(\"Trainer(barebones=True)\")\n log_filter = LogFilter(blacklist_msgs)\n\n with apply_log_filter(log_filter):\n trainer = pl.Trainer(\n accelerator=\"gpu\" if num_gpus > 0 else \"auto\",\n devices=get_available_devices(num_gpus, config.env.auto_select_gpus),\n num_nodes=config.env.num_nodes,\n precision=precision,\n strategy=strategy,\n benchmark=False,\n enable_progress_bar=False if barebones else enable_progress_bar,\n deterministic=config.env.deterministic,\n max_epochs=-1, # Add max_epochs to disable warning\n logger=False,\n callbacks=callbacks,\n barebones=barebones,\n )\n\n return trainer\n\n def fit_per_run(\n self,\n max_time: timedelta,\n save_path: str,\n enable_progress_bar: bool,\n seed: int,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n advanced_hyperparameters: Optional[Dict] = None,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n svm: Optional[Pipeline] = None,\n standalone: bool = True,\n ):\n self.on_fit_per_run_start(seed=seed, save_path=save_path)\n config = self.get_config_per_run(config=config, hyperparameters=hyperparameters)\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=df_preprocessor,\n config=config,\n )\n config = self.update_config_by_data_per_run(config=config, df_preprocessor=df_preprocessor)\n model = self.get_model_per_run(model=model, config=config, df_preprocessor=df_preprocessor)\n model = self.compile_model_per_run(config=config, model=model)\n data_processors = self.get_data_processors_per_run(\n data_processors=data_processors,\n config=config,\n model=model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n turn_on_off_feature_column_info(\n data_processors=data_processors,\n flag=True,\n )\n svm = self.get_svm_per_run(svm=svm)\n if max_time == timedelta(seconds=0):\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n svm=svm,\n )\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=config.env.per_gpu_batch_size,\n num_workers=config.env.num_workers,\n )\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(config=config)\n precision = self.get_precision_per_run(num_gpus=num_gpus, precision=config.env.precision)\n config = self.post_update_config_per_run(\n config=config,\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n )\n pred_writer = self.get_pred_writer(strategy=strategy)\n callbacks = self.get_callbacks_per_run(pred_writer=pred_writer, is_train=False)\n litmodule = self.get_litmodule_per_run(model=model)\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n config=config,\n precision=precision,\n strategy=strategy,\n max_time=max_time,\n callbacks=callbacks,\n enable_progress_bar=enable_progress_bar,\n )\n outputs = self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n pred_writer=pred_writer,\n is_train=False, # only use a pretrained model to extract embeddings\n )\n outputs = self.collect_predictions(\n outputs=outputs,\n trainer=trainer,\n pred_writer=pred_writer,\n num_gpus=num_gpus,\n )\n self.clean_trainer_processes(trainer=trainer, is_train=False)\n features = extract_from_output(outputs=outputs, ret_type=COLUMN_FEATURES, as_ndarray=True)\n features = self.aggregate_column_features(\n features=features,\n column_features_pooling_mode=config.data.column_features_pooling_mode,\n )\n # no need to call df_preprocessor.transform_label_for_metric since the sklearn pipeline encodes the label automatically\n labels = np.array(self._train_data[self._label_column])\n svm.fit(features, labels)\n self.on_fit_per_run_end(\n save_path=save_path,\n standalone=standalone,\n trainer=trainer,\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n svm=svm,\n )\n\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n svm=svm,\n )\n\n def aggregate_column_features(\n self,\n features: Union[Dict, np.ndarray],\n column_features_pooling_mode: Optional[str] = None,\n is_train: Optional[bool] = True,\n ):\n if not is_train:\n column_features_pooling_mode = self._config.data.column_features_pooling_mode\n\n if isinstance(features, np.ndarray):\n return features\n elif isinstance(features, dict):\n assert len(features) != 0, f\"column features are empty.\"\n if len(features) == 1:\n return next(iter(features.values()))\n if column_features_pooling_mode == \"concat\":\n return np.concatenate(list(features.values()), axis=1)\n elif column_features_pooling_mode == \"mean\":\n return np.mean(list(features.values()), axis=0)\n else:\n raise ValueError(f\"Unsupported column_features_pooling_mode: {column_features_pooling_mode}.\")\n else:\n raise ValueError(f\"Unsupported features type: {type(features)} in aggregating column features.\")\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n \"\"\"\n self.on_predict_start()\n features = self.extract_embedding(data=data, realtime=realtime)\n pred = self._svm.predict(features)\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n pred = self._as_pandas(data=data, to_be_converted=pred)\n return pred\n\n def predict_proba(\n self,\n data,\n as_pandas: Optional[bool] = False,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict probabilities class probabilities rather than class labels.\n This is only for the classification tasks. Calling it for a regression task will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n as_multiclass\n Whether to return the probability of all labels or\n just return the probability of the positive class for binary classification problems.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n When as_multiclass is True, the output will always have shape (#samples, #classes).\n Otherwise, the output will have shape (#samples,)\n \"\"\"\n self.on_predict_start()\n features = self.extract_embedding(data, realtime=realtime)\n logits = self._svm.decision_function(features)\n prob = logits_to_prob(logits)\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n prob = self._as_pandas(data=data, to_be_converted=prob)\n return prob\n\n def extract_embedding(\n self,\n data: pd.DataFrame,\n realtime: Optional[bool] = False,\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Extract features for each sample, i.e., one row in the provided dataframe `data`.\n\n Parameters\n ----------\n data\n The data to extract embeddings for. Should contain same column names as training dataset and\n follow same format (except for the `label` column).\n as_tensor\n Whether to return a Pytorch tensor.\n as_pandas\n Whether to return the output as a pandas DataFrame (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of embeddings, corresponding to each row in the given data.\n It will have shape (#samples, D) where the embedding dimension D is determined\n by the neural network's architecture.\n \"\"\"\n self.on_predict_start()\n turn_on_off_feature_column_info(\n data_processors=self._data_processors,\n flag=True,\n )\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n features = extract_from_output(outputs=outputs, ret_type=COLUMN_FEATURES, as_ndarray=as_tensor is False)\n features = self.aggregate_column_features(features=features, is_train=False)\n return features\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n Or a str, that is a path of the annotation file for detection.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n self.on_predict_start()\n data = data_to_df(data=data)\n features = self.extract_embedding(data)\n pred = self._svm.predict(features)\n assert self._label_column in data.columns, (\n f\"Label {self._label_column} is not in the data. Cannot perform evaluation without ground truth labels.\"\n )\n y_true = np.array(data[self._label_column])\n metric_data = {Y_PRED: pred, Y_TRUE: y_true}\n if metrics is None:\n if self._eval_metric_func:\n metrics = [self._eval_metric_func]\n else:\n metrics = [self._eval_metric_name]\n if isinstance(metrics, str) or isinstance(metrics, Scorer):\n metrics = [metrics]\n\n results = {}\n for per_metric in metrics:\n score = compute_score(\n metric_data=metric_data,\n metric=per_metric.lower() if isinstance(per_metric, str) else per_metric,\n )\n per_metric_name = per_metric if isinstance(per_metric, str) else per_metric.name\n results[per_metric_name] = score\n\n if return_pred:\n return results, self._as_pandas(data=data, to_be_converted=pred)\n return results\n\n @classmethod\n def load(\n cls,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n predictor = super().load(path=path, resume=resume, verbosity=verbosity)\n with open(os.path.join(path, \"svm.pkl\"), \"rb\") as fp:\n params = pickle.load(fp) # nosec B301\n svm = make_pipeline(StandardScaler(), SVC(gamma=\"auto\"))\n svm.set_params(**params)\n predictor._svm = svm\n\n return predictor\n\n def save(\n self,\n path: str,\n standalone: Optional[bool] = True,\n config: Optional[DictConfig] = None,\n model: Optional[nn.Module] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n svm: Optional[Pipeline] = None,\n fit_called: Optional[bool] = None,\n save_model: Optional[bool] = True,\n ):\n super().save(\n path=path,\n standalone=standalone,\n config=config,\n model=model,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=fit_called,\n save_model=save_model,\n )\n svm = svm if svm else self._svm\n with open(os.path.join(path, \"svm.pkl\"), \"wb\") as fp:\n pickle.dump(svm.get_params(), fp)\n\n def fit_summary(self, verbosity=0, show_plot=False):\n if self._total_train_time is None:\n logging.info(\"There is no `best_score` or `total_train_time`. Have you called `predictor.fit()`?\")\n else:\n logging.info(\n f\"Here's the model summary:The total training time is {timedelta(seconds=self._total_train_time)}\"\n )\n results = {\n \"training_time\": self._total_train_time,\n }\n return results\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/matching.py",
"content": "from __future__ import annotations\n\nimport copy\nimport json\nimport logging\nimport operator\nimport os\nimport pickle\nimport time\nimport warnings\nfrom datetime import timedelta\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport numpy as np\nimport pandas as pd\nimport torch\nimport yaml\nfrom omegaconf import DictConfig, OmegaConf\nfrom torch import nn\n\nfrom autogluon.common.utils.log_utils import set_logger_verbosity\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom .. import version as ag_version\nfrom ..constants import (\n BEST,\n BEST_K_MODELS_FILE,\n BINARY,\n FEATURES,\n GREEDY_SOUP,\n IMAGE_TEXT_SIMILARITY,\n LABEL,\n LAST_CHECKPOINT,\n MAX,\n MIN,\n MODEL_CHECKPOINT,\n MULTICLASS,\n PAIR,\n PROBABILITY,\n QUERY,\n RESPONSE,\n TEXT,\n UNIFORM_SOUP,\n Y_PRED,\n Y_PRED_PROB,\n Y_TRUE,\n)\nfrom ..data import (\n BaseDataModule,\n MultiModalFeaturePreprocessor,\n create_fusion_data_processors,\n data_to_df,\n infer_column_types,\n infer_dtypes_by_model_names,\n init_df_preprocessor,\n)\nfrom ..models import is_lazy_weight_tensor, select_model\nfrom ..optim import (\n MatcherLitModule,\n compute_ranking_score,\n compute_score,\n get_matcher_loss_func,\n get_matcher_miner_func,\n get_torchmetric,\n)\nfrom ..utils import (\n average_checkpoints,\n compute_semantic_similarity,\n convert_data_for_ranking,\n create_siamese_model,\n customize_model_names,\n extract_from_output,\n get_config,\n get_dir_ckpt_paths,\n get_load_ckpt_paths,\n get_local_pretrained_config_paths,\n hyperparameter_tune,\n matcher_presets,\n on_fit_end_message,\n save_pretrained_model_configs,\n split_hyperparameters,\n update_config_by_rules,\n)\nfrom ..utils.problem_types import PROBLEM_TYPES_REG\nfrom .base import BaseLearner\n\npl_logger = logging.getLogger(\"lightning\")\npl_logger.propagate = False # https://github.com/Lightning-AI/lightning/issues/4621\nlogger = logging.getLogger(__name__)\n\n\nclass MatchingLearner(BaseLearner):\n \"\"\"\n MatchingLearner is a framework to learn/extract embeddings for multimodal data including image, text, and tabular.\n These embeddings can be used e.g. with cosine-similarity to find items with similar semantic meanings.\n This can be useful for computing the semantic similarity of two items, semantic search, paraphrase mining, etc.\n \"\"\"\n\n def __init__(\n self,\n query: Optional[Union[str, List[str]]] = None,\n response: Optional[Union[str, List[str]]] = None,\n label: Optional[str] = None,\n match_label: Optional[Union[int, str]] = None,\n problem_type: Optional[str] = None,\n presets: Optional[str] = None,\n eval_metric: Optional[str] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 3,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = None,\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n query\n Column names of query data.\n response\n Column names of response data. If no label column is provided,\n query and response columns form positive pairs.\n label\n Name of the label column.\n match_label\n The label class that indicates the pair is counted as \"match\".\n This is used when the problem_type is one of the matching problem types, and when the labels are binary.\n For example, the label column can contain [\"match\", \"not match\"]. And match_label can be \"match\".\n It is similar as the \"pos_label\" in F1-score: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score\n Internally, we will set match_label to self.class_labels[1] by default.\n problem_type\n Type of matching problem if the label column is available.\n This could be binary, multiclass, or regression\n if the label column contains binary, multiclass, or numeric labels.\n If `problem_type = None`, the prediction problem type is inferred\n based on the label-values in provided dataset.\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n eval_metric\n Evaluation metric name. If `eval_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'roc_auc' for binary classification and 'spearmanr' for multiclass classification and regression.\n path\n Path to directory where models and intermediate outputs should be saved.\n If unspecified, a time-stamped folder called \"AutogluonAutoMM/ag-[TIMESTAMP]\"\n will be created in the working directory to store all models.\n Note: To call `fit()` twice and save all results of each fit,\n you must specify different `path` locations or don't specify `path` at all.\n Otherwise files from first `fit()` will be overwritten by second `fit()`.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50\n (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels)\n warn_if_exist\n Whether to raise warning if the specified path already exists.\n enable_progress_bar\n Whether to show progress bar. It will be True by default and will also be\n disabled if the environment variable os.environ[\"AUTOMM_DISABLE_PROGRESS_BAR\"] is set.\n pretrained\n Whether to init model with pretrained weights. If False, it creates a model with random initialization.\n validation_metric\n Validation metric name. If `validation_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'roc_auc' for binary classification and 'spearmanr' for multiclass classification and regression.\n \"\"\"\n if eval_metric is not None and not isinstance(eval_metric, str):\n eval_metric = eval_metric.name\n\n if isinstance(query, str):\n query = [query]\n if query:\n assert all(isinstance(q, str) for q in query)\n\n if isinstance(response, str):\n response = [response]\n if response:\n assert all(isinstance(r, str) for r in response)\n\n self._query = query\n self._response = response\n self._data_format = PAIR # TODO: Support Triplet\n self._match_label = match_label\n self._label_column = label\n self._problem_type = None # always infer problem type for matching.\n self._pipeline = problem_type.lower() if problem_type is not None else None\n self._presets = presets.lower() if presets else None\n self._eval_metric_name = eval_metric.lower() if eval_metric else None\n self._eval_metric_func = None\n self._validation_metric_name = validation_metric.lower() if validation_metric else None\n self._minmax_mode = None\n self._hyperparameters = hyperparameters\n self._advanced_hyperparameters = None\n self._hyperparameter_tune_kwargs = None\n self._output_shape = None\n self._save_path = path\n self._ckpt_path = None\n self._pretrained_path = None\n self._pretrained = pretrained\n self._config = None\n self._query_config = None\n self._response_config = None\n self._query_df_preprocessor = None\n self._response_df_preprocessor = None\n self._label_df_preprocessor = None\n self._column_types = None\n self._query_processors = None\n self._response_processors = None\n self._label_processors = None\n self._query_model = None\n self._response_model = None\n self._is_hpo = False\n self._resume = False\n self._fit_called = False\n self._train_data = None\n self._tuning_data = None\n self._verbosity = verbosity\n self._warn_if_exist = warn_if_exist\n self._enable_progress_bar = enable_progress_bar if enable_progress_bar is not None else True\n self._fit_args = None\n self._total_train_time = None\n self._best_score = None\n\n self._log_filters = [\n \".*does not have many workers.* in the `DataLoader` init to improve performance.*\",\n \"Checkpoint directory .* exists and is not empty.\",\n ]\n\n @property\n def query(self):\n return self._query\n\n @property\n def response(self):\n return self._response\n\n @property\n def match_label(self):\n return self._match_label\n\n @property\n def path(self):\n return self._save_path\n\n @property\n def label(self):\n return self._label_column\n\n @property\n def problem_type(self):\n if self._pipeline and self._problem_type:\n return f\"{self._pipeline}_{self._problem_type}\"\n elif self._pipeline:\n return self._pipeline\n else:\n return self._problem_type\n\n @property\n def problem_property(self):\n return PROBLEM_TYPES_REG.get(self._pipeline)\n\n @property\n def column_types(self):\n return self._column_types\n\n @property\n def eval_metric(self):\n return self._eval_metric_name\n\n @property\n def validation_metric(self):\n return self._validation_metric_name\n\n @property\n def total_parameters(self) -> int:\n return sum(p.numel() if not is_lazy_weight_tensor(p) else 0 for p in self._query_model.parameters())\n\n @property\n def trainable_parameters(self) -> int:\n return sum(\n p.numel() if not is_lazy_weight_tensor(p) else 0 for p in self._query_model.parameters() if p.requires_grad\n )\n\n @property\n def model_size(self) -> float:\n \"\"\"\n Returns the model size in Megabyte.\n \"\"\"\n model_size = sum(\n p.numel() * p.element_size() if not is_lazy_weight_tensor(p) else 0 for p in self._query_model.parameters()\n )\n return model_size * 1e-6 # convert to megabytes\n\n # This func is required by the abstract trainer of TabularPredictor.\n def set_verbosity(self, verbosity: int):\n \"\"\"\n Set the verbosity level of the log.\n\n Parameters\n ----------\n verbosity\n The verbosity level\n\n \"\"\"\n self._verbosity = verbosity\n set_logger_verbosity(verbosity, logger=logger)\n\n def _init_pretrained(self):\n if self._config is None:\n # split out the hyperparameters whose values are complex objects\n hyperparameters, advanced_hyperparameters = split_hyperparameters(self._hyperparameters)\n self._config = get_config(\n problem_type=self._pipeline,\n presets=self._presets,\n overrides=hyperparameters,\n extra=[\"matcher\"],\n )\n else:\n advanced_hyperparameters = None\n\n assert len(self._config.model.names) == 1, (\n f\"Zero shot mode only supports using one model, but detects multiple models {self._config.model.names}\"\n )\n\n if self._query_config is None:\n self._query_config = copy.deepcopy(self._config)\n # customize config model names to make them consistent with model prefixes.\n self._query_config.model, query_advanced_hyperparameters = customize_model_names(\n config=self._query_config.model,\n customized_names=[f\"{n}_{QUERY}\" for n in self._query_config.model.names],\n advanced_hyperparameters=advanced_hyperparameters,\n )\n else:\n query_advanced_hyperparameters = (None,)\n\n if self._response_config is None:\n self._response_config = copy.deepcopy(self._config)\n # customize config model names to make them consistent with model prefixes.\n self._response_config.model, response_advanced_hyperparameters = customize_model_names(\n config=self._response_config.model,\n customized_names=[f\"{n}_{RESPONSE}\" for n in self._response_config.model.names],\n advanced_hyperparameters=advanced_hyperparameters,\n )\n else:\n response_advanced_hyperparameters = None\n\n if self._query_model is None or self._response_model is None:\n self._query_model, self._response_model = create_siamese_model(\n query_config=self._query_config,\n response_config=self._response_config,\n pretrained=self._pretrained,\n )\n\n self._query_processors, self._response_processors, self._label_processors = self._get_matcher_data_processors(\n query_model=self._query_model,\n query_config=self._query_config,\n response_model=self._response_model,\n response_config=self._response_config,\n query_advanced_hyperparameters=query_advanced_hyperparameters,\n response_advanced_hyperparameters=response_advanced_hyperparameters,\n )\n\n def _ensure_inference_ready(self):\n if not self._fit_called:\n self._init_pretrained()\n\n def prepare_fit_args(\n self,\n time_limit: int,\n seed: int,\n id_mappings: Dict,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n if time_limit is not None:\n time_limit = timedelta(seconds=time_limit)\n self._fit_args = dict(\n id_mappings=id_mappings,\n max_time=time_limit,\n save_path=self._save_path, # In HPO mode, this would be overwritten by per trial path.\n ckpt_path=None if self._is_hpo else self._ckpt_path,\n resume=False if self._is_hpo else self._resume,\n enable_progress_bar=False if self._is_hpo else self._enable_progress_bar,\n seed=seed,\n hyperparameters=self._hyperparameters, # In HPO mode, this would be overwritten by sampled hyperparameters.\n advanced_hyperparameters=self._advanced_hyperparameters,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n if self._fit_called: # continuous training\n continuous_train_args = dict(\n config=self._config,\n ) # TODO: add more continuous training arguments\n self._fit_args.update(continuous_train_args)\n\n def update_attributes(\n self,\n config: Dict,\n query_config: DictConfig,\n response_config: DictConfig,\n query_model: nn.Module,\n response_model: nn.Module,\n query_df_preprocessor: MultiModalFeaturePreprocessor,\n response_df_preprocessor: MultiModalFeaturePreprocessor,\n label_df_preprocessor: MultiModalFeaturePreprocessor,\n query_processors: Dict,\n response_processors: Dict,\n label_processors: Dict,\n best_score: Optional[float] = None,\n ):\n self._config = config\n self._query_config = query_config\n self._response_config = response_config\n self._query_model = query_model\n self._response_model = response_model\n self._query_df_preprocessor = query_df_preprocessor\n self._response_df_preprocessor = response_df_preprocessor\n self._label_df_preprocessor = label_df_preprocessor\n self._query_processors = query_processors\n self._response_processors = response_processors\n self._label_processors = label_processors\n if best_score:\n self._best_score = best_score\n\n def execute_fit(self):\n if self._is_hpo:\n self._fit_args[\"learner\"] = self\n hyperparameter_tune(\n hyperparameter_tune_kwargs=self._hyperparameter_tune_kwargs,\n resources=dict(num_gpus=ResourceManager.get_gpu_count_torch()), # TODO: allow customizing GPUs\n is_matching=True,\n **self._fit_args,\n )\n return dict()\n else:\n attributes = self.fit_per_run(**self._fit_args)\n self.update_attributes(**attributes) # only update attributes for non-HPO mode\n return attributes\n\n def on_fit_end(\n self,\n training_start: float,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n self._fit_called = True\n if not self._is_hpo:\n # top_k_average is called inside hyperparameter_tune() when building the final predictor.\n self.top_k_average(\n save_path=self._save_path,\n top_k_average_method=self._config.optim.top_k_average_method,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n\n training_end = time.time()\n self._total_train_time = training_end - training_start\n # TODO(?) We should have a separate \"_post_training_event()\" for logging messages.\n logger.info(on_fit_end_message(self._save_path))\n\n def fit(\n self,\n train_data: pd.DataFrame,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n presets: Optional[str] = None,\n tuning_data: Optional[pd.DataFrame] = None,\n time_limit: Optional[int] = None,\n save_path: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n column_types: Optional[dict] = None,\n holdout_frac: Optional[float] = None,\n hyperparameter_tune_kwargs: Optional[dict] = None,\n seed: Optional[int] = 123,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n **kwargs,\n ):\n \"\"\"\n Fit MatchingLearner. Train the model to learn embeddings to simultaneously maximize and minimize\n the semantic similarities of positive and negative pairs.\n The data may contain image, text, numeric, or categorical features.\n\n Parameters\n ----------\n train_data\n A dataframe, containing the query data, response data, and their relevance scores. For example,\n | query_col1 | query_col2 | response_col1 | response_col2 | relevance_score |\n |-------------|------------|---------------|---------------|-----------------|\n | .... | .... | .... | ... | ... |\n | .... | .... | .... | ... | ... |\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response identifiers instead of their contents.\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n tuning_data\n A dataframe containing validation data, which should have the same columns as the train_data.\n If `tuning_data = None`, `fit()` will automatically\n hold out some random validation examples from `train_data`.\n time_limit\n How long `fit()` should run for (wall clock time in seconds).\n If not specified, `fit()` will run until the model has completed training.\n save_path\n Path to directory where models and intermediate outputs should be saved.\n hyperparameters\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n hyperparameters = \"model.hf_text.checkpoint_name=google/electra-small-discriminator model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n hyperparameters = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\", \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n column_types\n A dictionary that maps column names to their data types.\n For example: `column_types = {\"item_name\": \"text\", \"image\": \"image_path\",\n \"product_description\": \"text\", \"height\": \"numerical\"}`\n may be used for a table with columns: \"item_name\", \"brand\", \"product_description\", and \"height\".\n If None, column_types will be automatically inferred from the data.\n The current supported types are:\n - \"image_path\": each row in this column is one image path.\n - \"text\": each row in this column contains text (sentence, paragraph, etc.).\n - \"numerical\": each row in this column contains a number.\n - \"categorical\": each row in this column belongs to one of K categories.\n holdout_frac\n Fraction of train_data to holdout as tuning_data for optimizing hyper-parameters or\n early stopping (ignored unless `tuning_data = None`).\n Default value (if None) is selected based on the number of rows in the training data\n and whether hyper-parameter-tuning is utilized.\n seed\n The random seed to use for this training run.\n\n Returns\n -------\n An \"MatchingLearner\" object (itself).\n \"\"\"\n self.setup_save_path(save_path=save_path)\n training_start = self.on_fit_start(presets=presets)\n self.infer_problem_type(train_data=train_data)\n self.prepare_train_tuning_data(\n train_data=train_data,\n tuning_data=tuning_data,\n holdout_frac=holdout_frac,\n seed=seed,\n )\n self.infer_column_types(column_types=column_types)\n self.infer_output_shape()\n self.infer_validation_metric(is_matching=True)\n self.update_hyperparameters(\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n self.fit_sanity_check()\n self.prepare_fit_args(\n time_limit=time_limit,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n id_mappings=id_mappings,\n )\n self.execute_fit()\n self.on_fit_end(\n training_start=training_start,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n return self\n\n def _get_matcher_df_preprocessor(\n self,\n data: pd.DataFrame,\n column_types: Dict,\n query_config: Optional[DictConfig] = None,\n response_config: Optional[DictConfig] = None,\n query_columns: Optional[List] = None,\n response_columns: Optional[List] = None,\n ):\n if query_columns is None:\n query_df_preprocessor = None\n elif self._query_df_preprocessor is None and all(v is not None for v in [query_columns, query_config]):\n query_df_preprocessor = init_df_preprocessor(\n config=query_config,\n column_types={k: column_types[k] for k in query_columns},\n train_df_x=data[query_columns],\n )\n else: # continuing training\n query_df_preprocessor = self._query_df_preprocessor\n\n if response_columns is None:\n response_df_preprocessor = None\n elif self._response_df_preprocessor is None and all(\n v is not None for v in [response_columns, response_config]\n ):\n response_df_preprocessor = init_df_preprocessor(\n config=response_config,\n column_types={k: column_types[k] for k in response_columns},\n train_df_x=data[response_columns],\n )\n else: # continuing training\n response_df_preprocessor = self._response_df_preprocessor\n\n if self._label_column is None:\n label_df_preprocessor = None\n elif (\n self._label_df_preprocessor is None and response_config is not None and self._label_column in column_types\n ):\n label_df_preprocessor = init_df_preprocessor(\n config=response_config,\n column_types={self._label_column: column_types[self._label_column]},\n label_column=self._label_column,\n train_df_y=data[self._label_column],\n )\n else: # continuing training\n label_df_preprocessor = self._label_df_preprocessor\n\n return query_df_preprocessor, response_df_preprocessor, label_df_preprocessor\n\n def _get_matcher_data_processors(\n self,\n query_model: Optional[nn.Module] = None,\n query_config: Optional[DictConfig] = None,\n response_model: Optional[nn.Module] = None,\n response_config: Optional[DictConfig] = None,\n query_advanced_hyperparameters: Optional[Dict] = None,\n response_advanced_hyperparameters: Optional[Dict] = None,\n ):\n if query_model is None:\n query_processors = None\n elif self._query_processors is None and all(v is not None for v in [query_model, query_config]):\n query_processors = create_fusion_data_processors(\n model=query_model,\n config=query_config,\n requires_label=False,\n requires_data=True,\n advanced_hyperparameters=query_advanced_hyperparameters,\n )\n else: # continuing training\n query_processors = self._query_processors\n\n if response_model is None:\n response_processors = None\n elif self._response_processors is None and all(v is not None for v in [response_model, response_config]):\n response_processors = create_fusion_data_processors(\n model=response_model,\n config=response_config,\n requires_label=False,\n requires_data=True,\n advanced_hyperparameters=response_advanced_hyperparameters,\n )\n else: # continuing training\n response_processors = self._response_processors\n\n # only need labels for the response model\n if response_model is None:\n label_processors = None\n elif self._label_processors is None and all(\n v is not None for v in [self._label_column, response_model, response_config]\n ):\n label_processors = create_fusion_data_processors(\n model=response_model,\n config=response_config,\n requires_label=True,\n requires_data=False,\n )\n else: # continuing training\n label_processors = self._label_processors\n\n return query_processors, response_processors, label_processors\n\n def get_config_per_run(self, config, hyperparameters):\n config = get_config(\n problem_type=self._pipeline,\n presets=self._presets,\n config=config,\n overrides=hyperparameters, # don't use self._hyperparameters due to HPO.\n extra=[\"matcher\"],\n )\n config = update_config_by_rules(\n problem_type=self._pipeline,\n config=config,\n )\n config = self.update_strategy_by_env(config=config)\n return config\n\n def on_fit_per_run_end(\n self,\n trainer: pl.Trainer,\n config: DictConfig,\n query_config: DictConfig,\n response_config: DictConfig,\n query_model: nn.Module,\n response_model: nn.Module,\n query_df_preprocessor: MultiModalFeaturePreprocessor,\n response_df_preprocessor: MultiModalFeaturePreprocessor,\n label_df_preprocessor: MultiModalFeaturePreprocessor,\n query_processors: Dict,\n response_processors: Dict,\n label_processors: Dict,\n save_path: str,\n standalone: bool,\n ):\n self.clean_trainer_processes(trainer=trainer, is_train=True)\n self.save(\n path=save_path,\n standalone=standalone,\n config=config,\n query_config=query_config,\n response_config=response_config,\n query_model=query_model,\n response_model=response_model,\n query_df_preprocessor=query_df_preprocessor,\n response_df_preprocessor=response_df_preprocessor,\n label_df_preprocessor=label_df_preprocessor,\n query_processors=query_processors,\n response_processors=response_processors,\n label_processors=label_processors,\n fit_called=True, # fit is called on one run.\n save_model=False, # The final model will be saved in top_k_average\n )\n\n def fit_per_run(\n self,\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]],\n max_time: timedelta,\n save_path: str,\n ckpt_path: str,\n resume: bool,\n enable_progress_bar: bool,\n seed: int,\n config: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n advanced_hyperparameters: Optional[Dict] = None,\n standalone: bool = True,\n clean_ckpts: bool = True,\n ):\n self.on_fit_per_run_start(seed=seed, save_path=save_path)\n config = self.get_config_per_run(config=config, hyperparameters=hyperparameters)\n\n if self._query_config is None:\n query_config = copy.deepcopy(config)\n # customize config model names to make them consistent with model prefixes.\n query_config.model, query_advanced_hyperparameters = customize_model_names(\n config=query_config.model,\n customized_names=[f\"{n}_{QUERY}\" for n in query_config.model.names],\n advanced_hyperparameters=advanced_hyperparameters,\n )\n else:\n query_config = self._query_config\n query_advanced_hyperparameters = None\n\n if self._response_config is None:\n response_config = copy.deepcopy(config)\n # customize config model names to make them consistent with model prefixes.\n response_config.model, response_advanced_hyperparameters = customize_model_names(\n config=response_config.model,\n customized_names=[f\"{n}_{RESPONSE}\" for n in response_config.model.names],\n advanced_hyperparameters=advanced_hyperparameters,\n )\n else:\n response_config = self._response_config\n response_advanced_hyperparameters = None\n\n query_df_preprocessor, response_df_preprocessor, label_df_preprocessor = self._get_matcher_df_preprocessor(\n data=self._train_data,\n column_types=self._column_types,\n query_config=query_config,\n response_config=response_config,\n query_columns=self._query,\n response_columns=self._response,\n )\n\n query_config = select_model(config=query_config, df_preprocessor=query_df_preprocessor, strict=False)\n response_config = select_model(config=response_config, df_preprocessor=response_df_preprocessor, strict=False)\n\n if self._query_model is None or self._response_model is None:\n query_model, response_model = create_siamese_model(\n query_config=query_config,\n response_config=response_config,\n pretrained=self._pretrained,\n )\n else: # continuing training\n query_model = self._query_model\n response_model = self._response_model\n\n query_processors, response_processors, label_processors = self._get_matcher_data_processors(\n query_model=query_model,\n query_config=query_config,\n response_model=response_model,\n response_config=response_config,\n query_advanced_hyperparameters=query_advanced_hyperparameters,\n response_advanced_hyperparameters=response_advanced_hyperparameters,\n )\n\n query_processors_count = {k: len(v) for k, v in query_processors.items()}\n logger.debug(f\"query_processors_count: {query_processors_count}\")\n response_processors_count = {k: len(v) for k, v in response_processors.items()}\n logger.debug(f\"response_processors_count: {response_processors_count}\")\n if label_processors:\n label_processors_count = {k: len(v) for k, v in label_processors.items()}\n logger.debug(f\"label_processors_count: {label_processors_count}\")\n\n validation_metric, custom_metric_func = get_torchmetric(\n metric_name=self._validation_metric_name,\n num_classes=self._output_shape,\n is_matching=self._pipeline in matcher_presets.list_keys(),\n problem_type=self._problem_type,\n )\n logger.debug(f\"validation_metric_name: {self._validation_metric_name}\")\n logger.debug(f\"validation_metric: {validation_metric}\")\n logger.debug(f\"custom_metric_func: {custom_metric_func}\")\n\n loss_func = get_matcher_loss_func(\n data_format=self._data_format,\n problem_type=self._problem_type,\n loss_type=config.matcher.loss.type,\n pos_margin=config.matcher.loss.pos_margin,\n neg_margin=config.matcher.loss.neg_margin,\n distance_type=config.matcher.distance.type,\n )\n\n miner_func = None\n if self._problem_type == BINARY:\n miner_func = get_matcher_miner_func(\n miner_type=config.matcher.miner.type,\n pos_margin=config.matcher.miner.pos_margin,\n neg_margin=config.matcher.miner.neg_margin,\n distance_type=config.matcher.distance.type,\n )\n\n if max_time == timedelta(seconds=0):\n return dict(\n config=config,\n query_config=query_config,\n response_config=response_config,\n query_model=query_model,\n response_model=response_model,\n query_df_preprocessor=query_df_preprocessor,\n response_df_preprocessor=response_df_preprocessor,\n label_df_preprocessor=label_df_preprocessor,\n query_processors=query_processors,\n response_processors=response_processors,\n label_processors=label_processors,\n )\n\n df_preprocessors = [query_df_preprocessor, response_df_preprocessor, label_df_preprocessor]\n data_processors = [query_processors, response_processors, label_processors]\n df_preprocessors = [item for item in df_preprocessors if item is not None]\n data_processors = [item for item in data_processors if item is not None]\n assert len(df_preprocessors) == len(data_processors)\n\n datamodule = BaseDataModule(\n df_preprocessor=df_preprocessors,\n data_processors=data_processors,\n per_gpu_batch_size=config.env.per_gpu_batch_size,\n num_workers=config.env.num_workers,\n train_data=self._train_data,\n validate_data=self._tuning_data,\n id_mappings=id_mappings,\n )\n optim_kwargs = dict(\n optim_type=config.optim.optim_type,\n lr_choice=config.optim.lr_choice,\n lr_schedule=config.optim.lr_schedule,\n lr=config.optim.lr,\n lr_decay=config.optim.lr_decay,\n end_lr=config.optim.end_lr,\n lr_mult=config.optim.lr_mult,\n weight_decay=config.optim.weight_decay,\n warmup_steps=config.optim.warmup_steps,\n track_grad_norm=config.optim.track_grad_norm,\n )\n metrics_kwargs = dict(\n validation_metric=validation_metric,\n validation_metric_name=self._validation_metric_name,\n custom_metric_func=custom_metric_func,\n )\n\n if self._match_label is not None:\n match_label = label_df_preprocessor.label_generator.transform([self._match_label]).item()\n else:\n match_label = None\n\n litmodule = MatcherLitModule(\n query_model=query_model,\n response_model=response_model,\n match_label=match_label,\n loss_func=loss_func,\n miner_func=miner_func,\n **metrics_kwargs,\n **optim_kwargs,\n )\n callbacks = self.get_callbacks_per_run(save_path=save_path, config=config, litmodule=litmodule)\n tb_logger = self.get_tb_logger(save_path=save_path)\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(config=config)\n precision = self.get_precision_per_run(num_gpus=num_gpus, precision=config.env.precision)\n grad_steps = self.get_grad_steps(num_gpus=num_gpus, config=config)\n config = self.post_update_config_per_run(\n config=config,\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n )\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n config=config,\n precision=precision,\n strategy=strategy,\n max_time=max_time,\n callbacks=callbacks,\n tb_logger=tb_logger,\n grad_steps=grad_steps,\n enable_progress_bar=enable_progress_bar,\n )\n self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n ckpt_path=ckpt_path,\n resume=resume,\n )\n\n self.on_fit_per_run_end(\n trainer=trainer,\n standalone=standalone,\n save_path=save_path,\n config=config,\n query_config=query_config,\n response_config=response_config,\n query_model=query_model,\n response_model=response_model,\n query_df_preprocessor=query_df_preprocessor,\n response_df_preprocessor=response_df_preprocessor,\n label_df_preprocessor=label_df_preprocessor,\n query_processors=query_processors,\n response_processors=response_processors,\n label_processors=label_processors,\n )\n\n return dict(\n config=config,\n query_config=query_config,\n response_config=response_config,\n query_model=query_model,\n response_model=response_model,\n query_df_preprocessor=query_df_preprocessor,\n response_df_preprocessor=response_df_preprocessor,\n label_df_preprocessor=label_df_preprocessor,\n query_processors=query_processors,\n response_processors=response_processors,\n label_processors=label_processors,\n best_score=trainer.callback_metrics[f\"val_{self._validation_metric_name}\"].item(),\n )\n\n def top_k_average(\n self,\n save_path: str,\n top_k_average_method: str,\n last_ckpt_path: Optional[str] = None,\n standalone: Optional[bool] = True,\n clean_ckpts: Optional[bool] = True,\n ):\n best_k_models_yaml_path = os.path.join(save_path, BEST_K_MODELS_FILE)\n if os.path.exists(best_k_models_yaml_path):\n with open(best_k_models_yaml_path, \"r\") as f:\n best_k_models = yaml.safe_load(f)\n else:\n # In some cases, the training ends up too early (e.g., due to time_limit) so that there is\n # no saved best_k model checkpoints. In that scenario, we won't perform any model averaging.\n best_k_models = None\n if last_ckpt_path is None:\n last_ckpt_path = os.path.join(save_path, LAST_CHECKPOINT)\n\n if best_k_models:\n if top_k_average_method == UNIFORM_SOUP:\n logger.info(f\"Start to fuse {len(best_k_models)} checkpoints via the uniform soup algorithm.\")\n ingredients = top_k_model_paths = list(best_k_models.keys())\n else:\n top_k_model_paths = [\n v[0]\n for v in sorted(\n list(best_k_models.items()),\n key=lambda ele: ele[1],\n reverse=(self._minmax_mode == MAX),\n )\n ]\n if top_k_average_method == GREEDY_SOUP:\n # Select the ingredients based on the methods proposed in paper\n # \"Model soups: averaging weights of multiple fine-tuned models improves accuracy without\n # increasing inference time\", https://arxiv.org/pdf/2203.05482.pdf\n monitor_op = {MIN: operator.le, MAX: operator.ge}[self._minmax_mode]\n\n logger.info(f\"Start to fuse {len(top_k_model_paths)} checkpoints via the greedy soup algorithm.\")\n\n ingredients = [top_k_model_paths[0]]\n self._load_state_dict(path=top_k_model_paths[0])\n\n if self._pipeline == IMAGE_TEXT_SIMILARITY:\n best_score = self._evaluate_symmetric_ranking(self._tuning_data)\n else:\n best_score = self.evaluate(self._tuning_data, metrics=[self._validation_metric_name])[\n self._validation_metric_name\n ]\n for i in range(1, len(top_k_model_paths)):\n cand_avg_state_dict = average_checkpoints(\n checkpoint_paths=ingredients + [top_k_model_paths[i]],\n )\n self._load_state_dict(state_dict=cand_avg_state_dict)\n if self._pipeline == IMAGE_TEXT_SIMILARITY:\n cand_score = self._evaluate_symmetric_ranking(self._tuning_data)\n else:\n cand_score = self.evaluate(self._tuning_data, metrics=[self._validation_metric_name])[\n self._validation_metric_name\n ]\n if monitor_op(cand_score, best_score):\n # Add new ingredient\n ingredients.append(top_k_model_paths[i])\n best_score = cand_score\n elif top_k_average_method == BEST:\n ingredients = [top_k_model_paths[0]]\n else:\n raise ValueError(\n f\"The key for 'optim.top_k_average_method' is not supported. \"\n f\"We only support '{GREEDY_SOUP}', '{UNIFORM_SOUP}' and '{BEST}'. \"\n f\"The provided value is '{top_k_average_method}'.\"\n )\n else:\n # best_k_models is empty so we will manually save a checkpoint from the trainer\n # and use it as the main ingredients\n ingredients = [last_ckpt_path]\n top_k_model_paths = []\n # no checkpoints are available, do nothing\n if not os.path.isfile(last_ckpt_path):\n return\n\n # Average all the ingredients\n avg_state_dict = average_checkpoints(\n checkpoint_paths=ingredients,\n )\n self._load_state_dict(state_dict=avg_state_dict)\n\n self._query_model, self._response_model = create_siamese_model(\n query_config=self._query_config,\n response_config=self._response_config,\n query_model=self._query_model,\n response_model=self._response_model,\n )\n\n task = MatcherLitModule(\n query_model=self._query_model,\n response_model=self._response_model,\n )\n\n checkpoint = {\"state_dict\": task.state_dict()}\n torch.save(checkpoint, os.path.join(save_path, MODEL_CHECKPOINT)) # nosec B614\n\n if clean_ckpts:\n # clean old checkpoints + the intermediate files stored\n for per_path in top_k_model_paths:\n if os.path.isfile(per_path):\n os.remove(per_path)\n # remove the yaml file after cleaning the checkpoints\n if os.path.isfile(best_k_models_yaml_path):\n os.remove(best_k_models_yaml_path)\n # clean the last checkpoint\n if os.path.isfile(last_ckpt_path):\n os.remove(last_ckpt_path)\n\n def _on_predict_start(\n self,\n data: Union[pd.DataFrame, dict, list],\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]],\n requires_label: bool,\n signature: Optional[str] = None,\n ):\n assert signature in [QUERY, RESPONSE, None]\n\n data = data_to_df(data=data, header=signature)\n\n if self._column_types is None:\n if signature is None or signature == QUERY:\n allowable_query_dtypes, fallback_query_dtype = infer_dtypes_by_model_names(\n model_config=self._query_config.model\n )\n if signature is None or signature == RESPONSE:\n allowable_response_dtypes, fallback_response_dtype = infer_dtypes_by_model_names(\n model_config=self._response_config.model\n )\n if signature == QUERY:\n allowable_dtypes = allowable_query_dtypes\n fallback_dtype = fallback_query_dtype\n elif signature == RESPONSE:\n allowable_dtypes = allowable_response_dtypes\n fallback_dtype = fallback_response_dtype\n else:\n # TODO: consider that query and response have different modalities.\n assert sorted(allowable_query_dtypes) == sorted(allowable_response_dtypes)\n assert fallback_query_dtype == fallback_response_dtype\n allowable_dtypes = allowable_query_dtypes\n fallback_dtype = fallback_query_dtype\n\n column_types = infer_column_types(\n data=data,\n label_columns=self._label_column,\n problem_type=self._problem_type,\n allowable_column_types=allowable_dtypes,\n fallback_column_type=fallback_dtype,\n id_mappings=id_mappings,\n )\n else: # called .fit() or .load()\n column_types = self._column_types\n\n query_config = None\n query_model = None\n response_config = None\n response_model = None\n query_columns = None\n response_columns = None\n\n if signature == QUERY:\n query_config = self._query_config\n query_model = self._query_model\n query_columns = self._query if self._query else list(data.columns)\n if isinstance(query_columns, tuple):\n query_columns = query_columns[0]\n elif signature == RESPONSE:\n response_config = self._response_config\n response_model = self._response_model\n response_columns = self._response if self._response else list(data.columns)\n if isinstance(response_columns, tuple):\n response_columns = response_columns[0]\n else:\n query_config = self._query_config\n query_model = self._query_model\n response_config = self._response_config\n response_model = self._response_model\n assert self._query and self._response\n query_columns = self._query\n response_columns = self._response\n\n logger.debug(f\"signature: {signature}\")\n logger.debug(f\"column_types: {column_types}\")\n logger.debug(f\"query_columns: {query_columns}\")\n logger.debug(f\"response_columns: {response_columns}\")\n query_df_preprocessor, response_df_preprocessor, label_df_preprocessor = self._get_matcher_df_preprocessor(\n data=data,\n column_types=column_types,\n query_config=query_config,\n response_config=response_config,\n query_columns=query_columns,\n response_columns=response_columns,\n )\n\n query_processors, response_processors, label_processors = self._get_matcher_data_processors(\n query_model=query_model,\n query_config=query_config,\n response_model=response_model,\n response_config=response_config,\n )\n\n logger.debug(f\"query_processors: {query_processors}\")\n logger.debug(f\"response_processors: {response_processors}\")\n logger.debug(f\"label_processors: {label_processors}\")\n\n # For prediction data with no labels provided.\n df_preprocessors = [query_df_preprocessor, response_df_preprocessor]\n data_processors = [query_processors, response_processors]\n if requires_label:\n df_preprocessors.append(label_df_preprocessor)\n data_processors.append(label_processors)\n\n df_preprocessors = [item for item in df_preprocessors if item is not None]\n data_processors = [item for item in data_processors if item is not None]\n\n if self._match_label is not None:\n match_label = label_df_preprocessor.label_generator.transform([self._match_label]).item()\n else:\n match_label = None\n\n return data, df_preprocessors, data_processors, match_label\n\n def _default_predict(\n self,\n data: Union[pd.DataFrame, Dict, List],\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]],\n df_preprocessor: List[MultiModalFeaturePreprocessor],\n data_processors: List[Dict],\n num_gpus: int,\n precision: Union[int, str],\n batch_size: int,\n strategy: str,\n match_label: int,\n signature: Optional[str] = None,\n barebones: Optional[bool] = False,\n ) -> List[Dict]:\n datamodule = BaseDataModule(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=batch_size,\n num_workers=self._config.env.num_workers_inference,\n predict_data=data,\n id_mappings=id_mappings,\n )\n litmodule = MatcherLitModule(\n query_model=self._query_model,\n response_model=self._response_model,\n signature=signature,\n match_label=match_label,\n )\n pred_writer = self.get_pred_writer(strategy=strategy)\n callbacks = self.get_callbacks_per_run(pred_writer=pred_writer, is_train=False)\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n callbacks=callbacks,\n barebones=barebones,\n is_train=False,\n )\n outputs = self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n pred_writer=pred_writer,\n is_train=False,\n )\n outputs = self.collect_predictions(\n outputs=outputs,\n trainer=trainer,\n pred_writer=pred_writer,\n num_gpus=num_gpus,\n )\n self.clean_trainer_processes(trainer=trainer, is_train=False)\n\n return outputs\n\n def _realtime_predict(\n self,\n data: pd.DataFrame,\n df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],\n data_processors: Union[Dict, List[Dict]],\n num_gpus: int,\n precision: Union[int, str],\n query_model: Optional[nn.Module] = None,\n response_model: Optional[nn.Module] = None,\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]] = None,\n signature: Optional[str] = None,\n match_label: Optional[int] = None,\n ) -> List[Dict]:\n \"\"\"\n Perform realtime inference.\n\n Parameters\n ----------\n data\n A dataframe.\n df_preprocessor\n Dataframe preprocessors.\n data_processors\n Data processors.\n num_gpus\n Number of GPUs.\n precision\n The precision used in inference.\n query_model\n Matcher's query model.\n response_model\n Matcher's response model.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n signature\n query or response.\n match_label\n 0 or 1.\n\n Returns\n -------\n A list of output dicts.\n \"\"\"\n\n batch = self.process_batch(\n data=data,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n id_mappings=id_mappings,\n )\n output = self.predict_matcher_batch(\n batch=batch,\n query_model=query_model,\n response_model=response_model,\n signature=signature,\n match_label=match_label,\n precision=precision,\n num_gpus=num_gpus,\n )\n\n return [output]\n\n def predict_per_run(\n self,\n data: Union[pd.DataFrame, dict, list],\n realtime: Optional[bool],\n requires_label: bool,\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]] = None,\n signature: Optional[str] = None,\n barebones: Optional[bool] = False,\n ) -> List[Dict]:\n \"\"\"\n Perform inference for predictor or matcher.\n\n Parameters\n ----------\n data\n The data for inference.\n realtime\n Whether use realtime inference.\n requires_label\n Whether uses label during inference.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n signature\n query or response.\n barebones\n Whether to run in âbarebones modeâ, where all lightning's features that may impact raw speed are disabled.\n\n Returns\n -------\n A list of output dicts.\n \"\"\"\n data, df_preprocessor, data_processors, match_label = self._on_predict_start(\n data=data,\n id_mappings=id_mappings,\n requires_label=requires_label,\n signature=signature,\n )\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(\n predict_data=data,\n is_train=False,\n )\n precision = self.get_precision_per_run(\n num_gpus=num_gpus,\n precision=self._config.env.precision,\n cpu_only_warning=False,\n )\n batch_size = self.get_predict_batch_size_per_run(\n num_gpus=num_gpus,\n strategy=strategy,\n )\n realtime = self.use_realtime(\n realtime=realtime,\n data=data,\n data_processors=data_processors,\n batch_size=batch_size,\n )\n realtime, num_gpus, barebones = self.update_realtime_for_interactive_env(\n realtime=realtime,\n num_gpus=num_gpus,\n barebones=barebones,\n strategy=strategy,\n )\n if realtime:\n outputs = self._realtime_predict(\n query_model=self._query_model,\n response_model=self._response_model,\n data=data,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n num_gpus=num_gpus,\n precision=precision,\n id_mappings=id_mappings,\n signature=signature,\n match_label=match_label,\n )\n else:\n outputs = self._default_predict(\n data=data,\n id_mappings=id_mappings,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n num_gpus=num_gpus,\n precision=precision,\n batch_size=batch_size,\n strategy=strategy,\n match_label=match_label,\n signature=signature,\n )\n\n return outputs\n\n def _evaluate_symmetric_ranking(self, data):\n data_with_label, query_data, response_data, label_column = convert_data_for_ranking(\n data=data,\n query_column=self._query[0],\n response_column=self._response[0],\n )\n logger.debug(f\"first _evaluate_ranking...\\n\")\n score_1 = self._evaluate_ranking(\n qr_relevance=data_with_label,\n query_data=query_data,\n response_data=response_data,\n label_column=label_column,\n cutoffs=[1, 5, 10],\n )\n data_with_label, query_data, response_data, label_column = convert_data_for_ranking(\n data=data,\n query_column=self._response[0],\n response_column=self._query[0],\n )\n logger.debug(f\"second _evaluate_ranking...\\n\")\n score_2 = self._evaluate_ranking(\n qr_relevance=data_with_label,\n query_data=query_data,\n response_data=response_data,\n label_column=label_column,\n cutoffs=[1, 5, 10],\n )\n\n return sum(score_1.values()) + sum(score_2.values())\n\n def _evaluate_ranking(\n self,\n qr_relevance: Union[pd.DataFrame, dict, list],\n query_data: Union[pd.DataFrame, dict, list],\n response_data: Union[pd.DataFrame, dict, list],\n label_column: str,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n metrics: Optional[Union[str, List[str]]] = None,\n chunk_size: Optional[int] = 1024,\n similarity_type: Optional[str] = \"cosine\",\n cutoffs: Optional[List[int]] = [1, 5, 10],\n realtime: Optional[bool] = False,\n ):\n query_column = query_data.columns[0]\n response_column = response_data.columns[0]\n\n qr_relevance = data_to_df(data=qr_relevance)\n assert query_column in qr_relevance.columns\n assert response_column in qr_relevance.columns\n\n if metrics is None:\n metrics = [self._eval_metric_name]\n if isinstance(metrics, str):\n metrics = [metrics]\n\n rank_labels = {}\n for i, per_row in qr_relevance.iterrows():\n rank_labels.setdefault(per_row[query_column], {})[per_row[response_column]] = int(per_row[label_column])\n\n rank_results = dict()\n query_embeddings = self.extract_embedding(\n query_data, id_mappings=id_mappings, as_tensor=True, realtime=realtime\n )\n num_chunks = max(1, len(response_data) // chunk_size)\n top_k = max(cutoffs)\n for response_chunk in np.array_split(response_data, num_chunks):\n response_embeddings = self.extract_embedding(\n response_chunk, id_mappings=id_mappings, as_tensor=True, realtime=realtime\n )\n similarity_scores = compute_semantic_similarity(\n a=query_embeddings, b=response_embeddings, similarity_type=similarity_type\n )\n similarity_scores[torch.isnan(similarity_scores)] = -1\n top_k_scores, top_k_indices = torch.topk(\n similarity_scores,\n k=min(top_k + 1, len(similarity_scores[1])),\n dim=1,\n largest=True,\n sorted=False,\n )\n top_k_indices = top_k_indices.cpu().tolist()\n top_k_scores = top_k_scores.cpu().tolist()\n for i in range(len(query_data)):\n query_idx = query_data.iloc[i][query_column]\n for sub_response_idx, score in zip(top_k_indices[i], top_k_scores[i]):\n response_idx = response_chunk.iloc[int(sub_response_idx)][response_column]\n rank_results.setdefault(query_idx, {})[response_idx] = score\n\n results = compute_ranking_score(results=rank_results, qrel_dict=rank_labels, metrics=metrics, cutoffs=cutoffs)\n\n return results\n\n def _evaluate_matching(\n self,\n data: Union[pd.DataFrame, dict, list],\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n ):\n outputs = self.predict_per_run(\n data=data,\n id_mappings=id_mappings,\n realtime=realtime,\n requires_label=True,\n )\n prob = extract_from_output(ret_type=PROBABILITY, outputs=outputs)\n\n metric_data = {Y_PRED_PROB: prob}\n\n y_pred = self._label_df_preprocessor.transform_prediction(\n y_pred=prob,\n inverse_categorical=False,\n )\n y_pred_inv = self._label_df_preprocessor.transform_prediction(\n y_pred=prob,\n inverse_categorical=True,\n )\n y_true = self._label_df_preprocessor.transform_label_for_metric(df=data)\n\n metric_data.update(\n {\n Y_PRED: y_pred,\n Y_TRUE: y_true,\n }\n )\n\n if metrics is None:\n metrics = [self._eval_metric_name]\n if isinstance(metrics, str):\n metrics = [metrics]\n\n results = {}\n for per_metric in metrics:\n score = compute_score(\n metric_data=metric_data,\n metric=per_metric.lower(),\n )\n results[per_metric] = score\n\n if return_pred:\n return results, self._as_pandas(data=data, to_be_converted=y_pred_inv)\n else:\n return results\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list],\n query_data: Optional[pd.DataFrame, dict, list] = None,\n response_data: Optional[pd.DataFrame, dict, list] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n chunk_size: Optional[int] = 1024,\n similarity_type: Optional[str] = \"cosine\",\n cutoffs: Optional[List[int]] = [1, 5, 10],\n label: Optional[str] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the query data, response data, and their relevance. For example,\n | query_col1 | query_col2 | response_col1 | response_col2 | relevance_score |\n |-------------|------------|---------------|---------------|-----------------|\n | | .... | .... | ... | ... |\n | | .... | .... | ... | ... |\n query_data\n Query data used for ranking.\n response_data\n Response data used for ranking.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data/query_data/response_data contain the query/response identifiers instead of their contents.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n chunk_size\n Scan the response data by chunk_size each time. Increasing the value increases the speed, but requires more memory.\n similarity_type\n Use what function (cosine/dot_prod) to score the similarity (default: cosine).\n cutoffs\n A list of cutoff values to evaluate ranking.\n label\n The label column name in data. Some tasks, e.g., image<-->text matching, have no label column in training data,\n but the label column is still required in evaluation.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n self._ensure_inference_ready()\n if all(v is not None for v in [data, query_data, response_data]):\n if isinstance(query_data, list):\n assert self._query is not None, (\n \"query_data is a list. Need a dict or dataframe, whose keys or headers should be in data's headers.\"\n )\n\n if isinstance(response_data, list):\n assert self._response is not None, (\n \"response_data is a list. Need a dict or dataframe, whose keys or headers should be in data's headers.\"\n )\n\n query_header = self._query[0] if self._query else None\n query_data = data_to_df(data=query_data, header=query_header)\n\n response_header = self._response[0] if self._response else None\n response_data = data_to_df(data=response_data, header=response_header)\n\n if label is None:\n label = self._label_column\n\n return self._evaluate_ranking(\n qr_relevance=data,\n query_data=query_data,\n response_data=response_data,\n label_column=label,\n id_mappings=id_mappings,\n metrics=metrics,\n chunk_size=chunk_size,\n similarity_type=similarity_type,\n cutoffs=cutoffs,\n realtime=realtime,\n )\n elif data is not None:\n return self._evaluate_matching(\n data=data,\n id_mappings=id_mappings,\n metrics=metrics,\n return_pred=return_pred,\n realtime=realtime,\n )\n else:\n raise ValueError(f\"Invalid input.\")\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list],\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n \"\"\"\n self._ensure_inference_ready()\n outputs = self.predict_per_run(\n data=data,\n id_mappings=id_mappings,\n realtime=realtime,\n requires_label=False,\n )\n prob = extract_from_output(outputs=outputs, ret_type=PROBABILITY)\n\n if self._label_df_preprocessor:\n pred = self._label_df_preprocessor.transform_prediction(\n y_pred=prob,\n )\n else:\n if isinstance(prob, (torch.Tensor, np.ndarray)) and prob.ndim == 2:\n pred = prob.argmax(axis=1)\n else:\n pred = prob\n\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n pred = self._as_pandas(data=data, to_be_converted=pred)\n\n return pred\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict probabilities class probabilities rather than class labels.\n This is only for the classification tasks. Calling it for a regression task will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n as_multiclass\n Whether to return the probability of all labels or\n just return the probability of the positive class for binary classification problems.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n When as_multiclass is True, the output will always have shape (#samples, #classes).\n Otherwise, the output will have shape (#samples,)\n \"\"\"\n self._ensure_inference_ready()\n outputs = self.predict_per_run(\n data=data,\n id_mappings=id_mappings,\n realtime=realtime,\n requires_label=False,\n )\n prob = extract_from_output(outputs=outputs, ret_type=PROBABILITY)\n\n if not as_multiclass:\n if self._problem_type == BINARY:\n prob = prob[:, 1]\n\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n prob = self._as_pandas(data=data, to_be_converted=prob)\n\n return prob\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n signature: Optional[str] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Extract features for each sample, i.e., one row in the provided dataframe `data`.\n\n Parameters\n ----------\n data\n The data to extract embeddings for. Should contain same column names as training dataset and\n follow same format (except for the `label` column).\n signature\n query or response\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n as_tensor\n Whether to return a Pytorch tensor.\n as_pandas\n Whether to return the output as a pandas DataFrame (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of embeddings, corresponding to each row in the given data.\n It will have shape (#samples, D) where the embedding dimension D is determined\n by the neural network's architecture.\n \"\"\"\n self._ensure_inference_ready()\n if signature is None:\n if self._query or self._response:\n if isinstance(data, list):\n raise ValueError(\"data can't be a list. Provide a dict or a dataframe instead.\")\n else:\n data = data_to_df(data=data)\n if self._query and all(c in data.columns for c in self._query):\n signature = QUERY\n elif self._response and all(c in data.columns for c in self._response):\n signature = RESPONSE\n else:\n raise ValueError(\n f\"Both query `{self._query}` and response `{self._response}` are not within the data headers `{data.columns}`.\"\n )\n else:\n signature = QUERY\n\n outputs = self.predict_per_run(\n data=data,\n id_mappings=id_mappings,\n signature=signature,\n realtime=realtime,\n requires_label=False,\n )\n features = extract_from_output(outputs=outputs, ret_type=FEATURES, as_ndarray=as_tensor is False)\n\n if as_pandas:\n features = pd.DataFrame(features, index=data.index)\n\n return features\n\n def _as_pandas(\n self,\n data: Union[pd.DataFrame, dict, list],\n to_be_converted: np.ndarray,\n ):\n if isinstance(data, pd.DataFrame):\n index = data.index\n else:\n index = None\n if to_be_converted.ndim == 1:\n return pd.Series(to_be_converted, index=index, name=self._label_column)\n else:\n return pd.DataFrame(to_be_converted, index=index, columns=self.class_labels)\n\n def _load_state_dict(\n self,\n state_dict: dict = None,\n path: str = None,\n query_prefix: str = \"query_model.\",\n response_prefix: str = \"response_model.\",\n ):\n if state_dict is None:\n state_dict = torch.load(path, map_location=torch.device(\"cpu\"))[\"state_dict\"] # nosec B614\n query_state_dict = {\n k.partition(query_prefix)[2]: v for k, v in state_dict.items() if k.startswith(query_prefix)\n }\n self._query_model.load_state_dict(query_state_dict)\n\n response_state_dict = {\n k.partition(response_prefix)[2]: v for k, v in state_dict.items() if k.startswith(response_prefix)\n }\n self._response_model.load_state_dict(response_state_dict)\n\n @staticmethod\n def _replace_model_name_prefix(\n state_dict: dict,\n old_prefix: str,\n new_prefix: str,\n ):\n start_idx = len(old_prefix)\n state_dict_processed = {\n new_prefix + k[start_idx:]: v for k, v in state_dict.items() if k.startswith(old_prefix)\n }\n return state_dict_processed\n\n def save(\n self,\n path: str,\n standalone: Optional[bool] = True,\n config: Optional[DictConfig] = None,\n query_config: Optional[DictConfig] = None,\n response_config: Optional[DictConfig] = None,\n query_model: Optional[nn.Module] = None,\n response_model: Optional[nn.Module] = None,\n query_df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n response_df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n label_df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n query_processors: Optional[Dict] = None,\n response_processors: Optional[Dict] = None,\n label_processors: Optional[Dict] = None,\n fit_called: Optional[bool] = None,\n save_model: Optional[bool] = True,\n ):\n \"\"\"\n Save this matcher to file in directory specified by `path`.\n\n Parameters\n ----------\n path\n The directory to save this matcher.\n standalone\n Whether to save the downloaded model for offline deployment.\n When standalone = True, save the transformers.CLIPModel and transformers.AutoModel to os.path.join(path,model_name),\n and reset the associate model.model_name.checkpoint_name start with `local://` in config.yaml.\n When standalone = False, the saved artifact may require an online environment to process in load().\n \"\"\"\n config = config if config else self._config\n query_config = query_config if query_config else self._query_config\n response_config = response_config if response_config else self._response_config\n query_model = query_model if query_model else self._query_model\n response_model = response_model if response_model else self._response_model\n query_df_preprocessor = query_df_preprocessor if query_df_preprocessor else self._query_df_preprocessor\n response_df_preprocessor = (\n response_df_preprocessor if response_df_preprocessor else self._response_df_preprocessor\n )\n label_df_preprocessor = label_df_preprocessor if label_df_preprocessor else self._label_df_preprocessor\n query_processors = query_processors if query_processors else self._query_processors\n response_processors = response_processors if response_processors else self._response_processors\n label_processors = label_processors if label_processors else self._label_processors\n\n path = os.path.abspath(os.path.expanduser(path))\n query_config = copy.deepcopy(query_config)\n response_config = copy.deepcopy(response_config)\n if standalone:\n query_config = save_pretrained_model_configs(model=query_model, config=query_config, path=path)\n response_config = save_pretrained_model_configs(model=response_model, config=response_config, path=path)\n\n os.makedirs(path, exist_ok=True)\n config = {\"generic\": config, QUERY: query_config, RESPONSE: response_config}\n OmegaConf.save(config=config, f=os.path.join(path, \"config.yaml\"))\n\n df_preprocessor = {\n QUERY: query_df_preprocessor,\n RESPONSE: response_df_preprocessor,\n LABEL: label_df_preprocessor,\n }\n with open(os.path.join(path, \"df_preprocessor.pkl\"), \"wb\") as fp:\n pickle.dump(df_preprocessor, fp)\n\n # Save text tokenizers before saving data processors\n query_processors = copy.deepcopy(query_processors)\n if TEXT in query_processors:\n for per_text_processor in query_processors[TEXT]:\n per_text_processor.save_tokenizer(path)\n\n # Save text tokenizers before saving data processors\n response_processors = copy.deepcopy(response_processors)\n if TEXT in response_processors:\n for per_text_processor in response_processors[TEXT]:\n per_text_processor.save_tokenizer(path)\n\n data_processors = {\n QUERY: query_processors,\n RESPONSE: response_processors,\n LABEL: label_processors,\n }\n with open(os.path.join(path, \"data_processors.pkl\"), \"wb\") as fp:\n pickle.dump(data_processors, fp)\n\n with open(os.path.join(path, f\"assets.json\"), \"w\") as fp:\n json.dump(\n {\n \"learner_class\": self.__class__.__name__,\n \"query\": self._query,\n \"response\": self._response,\n \"match_label\": self._match_label,\n \"column_types\": self._column_types,\n \"label_column\": self._label_column,\n \"problem_type\": self._problem_type,\n \"pipeline\": self._pipeline,\n \"presets\": self._presets,\n \"eval_metric_name\": self._eval_metric_name,\n \"validation_metric_name\": self._validation_metric_name,\n \"minmax_mode\": self._minmax_mode,\n \"output_shape\": self._output_shape,\n \"save_path\": self._save_path,\n \"pretrained\": self._pretrained,\n \"pretrained_path\": self._pretrained_path,\n \"fit_called\": fit_called if fit_called is not None else self._fit_called,\n \"best_score\": self._best_score,\n \"total_train_time\": self._total_train_time,\n \"version\": ag_version.__version__,\n },\n fp,\n ensure_ascii=True,\n )\n\n if save_model:\n task = MatcherLitModule(\n query_model=self._query_model,\n response_model=self._response_model,\n )\n checkpoint = {\"state_dict\": task.state_dict()}\n torch.save(checkpoint, os.path.join(path, MODEL_CHECKPOINT)) # nosec B614\n\n @staticmethod\n def _load_metadata(\n matcher: MatchingLearner,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n path = os.path.abspath(os.path.expanduser(path))\n assert os.path.isdir(path), f\"'{path}' must be an existing directory.\"\n config = OmegaConf.load(os.path.join(path, \"config.yaml\"))\n query_config = config[QUERY]\n response_config = config[RESPONSE]\n config = config[\"generic\"]\n\n query_config = get_local_pretrained_config_paths(\n config=query_config, path=path\n ) # check the config to load offline pretrained model configs\n\n response_config = get_local_pretrained_config_paths(\n config=response_config, path=path\n ) # check the config to load offline pretrained model configs\n\n with open(os.path.join(path, \"assets.json\"), \"r\") as fp:\n assets = json.load(fp)\n\n with open(os.path.join(path, \"df_preprocessor.pkl\"), \"rb\") as fp:\n df_preprocessor = pickle.load(fp) # nosec B301\n\n query_df_preprocessor = df_preprocessor[QUERY]\n response_df_preprocessor = df_preprocessor[RESPONSE]\n label_df_preprocessor = df_preprocessor[LABEL]\n\n try:\n with open(os.path.join(path, \"data_processors.pkl\"), \"rb\") as fp:\n data_processors = pickle.load(fp) # nosec B301\n\n query_processors = data_processors[QUERY]\n response_processors = data_processors[RESPONSE]\n label_processors = data_processors[LABEL]\n\n # Load text tokenizers after loading data processors.\n if TEXT in query_processors:\n for per_text_processor in query_processors[TEXT]:\n per_text_processor.load_tokenizer(path)\n\n # Only keep the modalities with non-empty processors.\n query_processors = {k: v for k, v in query_processors.items() if len(v) > 0}\n\n # Load text tokenizers after loading data processors.\n if TEXT in response_processors:\n for per_text_processor in response_processors[TEXT]:\n per_text_processor.load_tokenizer(path)\n\n # Only keep the modalities with non-empty processors.\n response_processors = {k: v for k, v in response_processors.items() if len(v) > 0}\n except: # reconstruct the data processor in case something went wrong.\n query_processors = None\n response_processors = None\n label_processors = None\n\n matcher._query = assets[\"query\"]\n matcher._response = assets[\"response\"]\n matcher._match_label = assets[\"match_label\"]\n matcher._label_column = assets[\"label_column\"]\n matcher._problem_type = assets[\"problem_type\"]\n matcher._pipeline = assets[\"pipeline\"]\n matcher._presets = assets[\"presets\"]\n matcher._eval_metric_name = assets[\"eval_metric_name\"]\n matcher._verbosity = verbosity\n matcher._resume = resume\n matcher._save_path = path # in case the original exp dir is copied to somewhere else\n matcher._pretrained_path = path\n matcher._pretrained = assets[\"pretrained\"]\n matcher._fit_called = assets[\"fit_called\"]\n matcher._config = config\n matcher._query_config = query_config\n matcher._response_config = response_config\n matcher._output_shape = assets[\"output_shape\"]\n matcher._column_types = assets[\"column_types\"]\n matcher._validation_metric_name = assets[\"validation_metric_name\"]\n matcher._query_df_preprocessor = query_df_preprocessor\n matcher._response_df_preprocessor = response_df_preprocessor\n matcher._label_df_preprocessor = label_df_preprocessor\n matcher._query_processors = query_processors\n matcher._response_processors = response_processors\n matcher._label_processors = label_processors\n matcher._minmax_mode = assets[\"minmax_mode\"]\n\n return matcher\n\n @classmethod\n def load(\n cls,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n \"\"\"\n Load a matcher object from a directory specified by `path`. The to-be-loaded matcher\n can be completely or partially trained by .fit(). If a previous training has completed,\n it will load the checkpoint `model.ckpt`. Otherwise if a previous training accidentally\n collapses in the middle, it can load the `last.ckpt` checkpoint by setting `resume=True`.\n\n Parameters\n ----------\n path\n The directory to load the matcher object.\n resume\n Whether to resume training from `last.ckpt`. This is useful when a training was accidentally\n broken during the middle and we want to resume the training from the last saved checkpoint.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n\n Returns\n -------\n The loaded matcher object.\n \"\"\"\n dir_path, ckpt_path = get_dir_ckpt_paths(path=path)\n\n assert os.path.isdir(dir_path), f\"'{dir_path}' must be an existing directory.\"\n matcher = cls(query=\"\", response=\"\")\n matcher = cls._load_metadata(matcher=matcher, path=dir_path, resume=resume, verbosity=verbosity)\n matcher._query_model, matcher._response_model = create_siamese_model(\n query_config=matcher._query_config,\n response_config=matcher._response_config,\n pretrained=False,\n )\n load_path, ckpt_path = get_load_ckpt_paths(\n ckpt_path=ckpt_path,\n dir_path=dir_path,\n resume=resume,\n )\n matcher._load_state_dict(path=load_path)\n matcher._ckpt_path = ckpt_path\n\n return matcher\n\n @property\n def class_labels(self):\n \"\"\"\n The original name of the class labels.\n For example, the tabular data may contain classes equal to\n \"entailment\", \"contradiction\", \"neutral\". Internally, these will be converted to\n 0, 1, 2, ...\n This function returns the original names of these raw labels.\n\n Returns\n -------\n List that contain the class names. It will be None if it's not a classification problem.\n \"\"\"\n if self._problem_type == MULTICLASS or self._problem_type == BINARY:\n return self._label_df_preprocessor.label_generator.classes_\n else:\n warnings.warn(\"Accessing class names for a non-classification problem. Return None.\")\n return None\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/ner.py",
"content": "import json\nimport logging\nimport os\nimport warnings\nfrom datetime import timedelta\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport pandas as pd\nfrom omegaconf import DictConfig\nfrom torch import nn\n\nfrom autogluon.core.metrics import Scorer\n\nfrom ..constants import NER, NER_RET, Y_PRED, Y_TRUE\nfrom ..data import MultiModalFeaturePreprocessor\nfrom ..models import create_fusion_model\nfrom ..optim import NerLitModule, compute_score, get_minmax_mode, get_torchmetric, infer_metrics\nfrom ..utils import extract_from_output, merge_bio_format\nfrom .base import BaseLearner\n\nlogger = logging.getLogger(__name__)\n\n\nclass NERLearner(BaseLearner):\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = NER,\n presets: Optional[str] = None,\n eval_metric: Optional[str] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = None,\n **kwargs,\n ):\n super().__init__(\n label=label,\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n pretrained=pretrained,\n validation_metric=validation_metric,\n )\n # set the convert_to_text=True assuming there are no categorical data in NER\n convert_to_text = {\"data.categorical.convert_to_text\": True}\n if self._hyperparameters:\n self._hyperparameters.update(convert_to_text)\n else:\n self._hyperparameters = convert_to_text\n\n def infer_problem_type(self, train_data: pd.DataFrame):\n return # problem type should be provided in learner initialization\n\n def infer_output_shape(self):\n return # output shape is conditioned on df_preprocessor in fit_per_run().\n\n def update_attributes(\n self,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n model_postprocess_fn: Optional[Callable] = None,\n best_score: Optional[float] = None,\n output_shape: Optional[int] = None,\n **kwargs,\n ):\n super().update_attributes(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n best_score=best_score,\n )\n if output_shape:\n self._output_shape = output_shape # since ner infers output_shape in fit_per_run(), the learners needs to update the attribute afterwards.\n\n def get_validation_metric_per_run(self, output_shape: int):\n validation_metric, custom_metric_func = get_torchmetric(\n metric_name=self._validation_metric_name,\n num_classes=output_shape,\n problem_type=self._problem_type,\n )\n return validation_metric, custom_metric_func\n\n def get_model_per_run(\n self,\n model: nn.Module,\n config: DictConfig,\n df_preprocessor: MultiModalFeaturePreprocessor,\n output_shape: int,\n ):\n if model is None:\n model = create_fusion_model(\n config=config,\n num_classes=output_shape,\n num_numerical_columns=len(df_preprocessor.numerical_feature_names),\n num_categories=df_preprocessor.categorical_num_categories,\n )\n return model\n\n def get_optim_kwargs_per_run(self, config, validation_metric, custom_metric_func, loss_func):\n return dict(\n optim_type=config.optim.optim_type,\n lr_choice=config.optim.lr_choice,\n lr_schedule=config.optim.lr_schedule,\n lr=config.optim.lr,\n lr_decay=config.optim.lr_decay,\n end_lr=config.optim.end_lr,\n lr_mult=config.optim.lr_mult,\n weight_decay=config.optim.weight_decay,\n warmup_steps=config.optim.warmup_steps,\n track_grad_norm=config.optim.track_grad_norm,\n validation_metric=validation_metric,\n validation_metric_name=self._validation_metric_name,\n custom_metric_func=custom_metric_func,\n loss_func=loss_func,\n peft=config.optim.peft,\n skip_final_val=config.optim.skip_final_val,\n )\n\n def get_litmodule_per_run(\n self,\n model: Optional[nn.Module] = None,\n peft_param_names: Optional[List[str]] = None,\n optim_kwargs: Optional[dict] = None,\n is_train=True,\n ):\n if is_train:\n return NerLitModule(\n model=model,\n trainable_param_names=peft_param_names,\n **optim_kwargs,\n )\n else:\n return NerLitModule(model=self._model)\n\n @staticmethod\n def get_output_shape_per_run(df_preprocessor):\n # ner needs to update output_shape with label_generator.\n return len(df_preprocessor.label_generator.unique_entity_groups)\n\n def on_fit_per_run_end(\n self,\n trainer: pl.Trainer,\n config: DictConfig,\n model: nn.Module,\n df_preprocessor: MultiModalFeaturePreprocessor,\n data_processors: Dict,\n save_path: str,\n standalone: bool,\n output_shape: int,\n ):\n self.clean_trainer_processes(trainer=trainer, is_train=True)\n self.save(\n path=save_path,\n standalone=standalone,\n config=config,\n model=model,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=True, # fit is called on one run.\n save_model=False, # The final model will be saved in top_k_average\n output_shape=output_shape,\n )\n\n def fit_per_run(\n self,\n max_time: timedelta,\n save_path: str,\n ckpt_path: str,\n resume: bool,\n enable_progress_bar: bool,\n seed: int,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n advanced_hyperparameters: Optional[Dict] = None,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n standalone: bool = True,\n clean_ckpts: bool = True,\n ):\n self.on_fit_per_run_start(seed=seed, save_path=save_path)\n config = self.get_config_per_run(config=config, hyperparameters=hyperparameters)\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=df_preprocessor,\n config=config,\n )\n config = self.update_config_by_data_per_run(config=config, df_preprocessor=df_preprocessor)\n output_shape = self.get_output_shape_per_run(df_preprocessor=df_preprocessor)\n model = self.get_model_per_run(\n model=model,\n config=config,\n df_preprocessor=df_preprocessor,\n output_shape=output_shape,\n )\n model = self.compile_model_per_run(config=config, model=model)\n peft_param_names = self.get_peft_param_names_per_run(model=model, config=config)\n data_processors = self.get_data_processors_per_run(\n data_processors=data_processors,\n config=config,\n model=model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n validation_metric, custom_metric_func = self.get_validation_metric_per_run(output_shape=output_shape)\n loss_func, _ = self.get_loss_func_per_run(config=config)\n if max_time == timedelta(seconds=0):\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n strict_loading=not peft_param_names,\n )\n\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=config.env.per_gpu_batch_size,\n num_workers=config.env.num_workers,\n )\n optim_kwargs = self.get_optim_kwargs_per_run(\n config=config,\n validation_metric=validation_metric,\n custom_metric_func=custom_metric_func,\n loss_func=loss_func,\n )\n litmodule = self.get_litmodule_per_run(\n model=model,\n peft_param_names=peft_param_names,\n optim_kwargs=optim_kwargs,\n )\n callbacks = self.get_callbacks_per_run(save_path=save_path, config=config, litmodule=litmodule)\n plugins = self.get_plugins_per_run(model=model, peft_param_names=peft_param_names)\n tb_logger = self.get_tb_logger(save_path=save_path)\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(config=config)\n precision = self.get_precision_per_run(num_gpus=num_gpus, precision=config.env.precision)\n grad_steps = self.get_grad_steps(num_gpus=num_gpus, config=config)\n config = self.post_update_config_per_run(\n config=config,\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n )\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n config=config,\n precision=precision,\n strategy=strategy,\n max_time=max_time,\n callbacks=callbacks,\n tb_logger=tb_logger,\n grad_steps=grad_steps,\n plugins=plugins,\n enable_progress_bar=enable_progress_bar,\n )\n\n self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n ckpt_path=ckpt_path,\n resume=resume,\n )\n self.on_fit_per_run_end(\n save_path=save_path,\n standalone=standalone,\n trainer=trainer,\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n output_shape=output_shape,\n )\n\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n best_score=trainer.callback_metrics[f\"val_{self._validation_metric_name}\"].item(),\n strategy=strategy,\n strict_loading=not peft_param_names,\n output_shape=output_shape,\n )\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n self.ensure_predict_ready()\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=True,\n )\n logits = extract_from_output(ret_type=NER_RET, outputs=outputs)\n metric_data = {}\n y_pred = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n inverse_categorical=False,\n )\n y_pred_inv = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n inverse_categorical=True,\n )\n y_true = self._df_preprocessor.transform_label_for_metric(\n df=data,\n tokenizer=self._model.tokenizer,\n )\n metric_data.update(\n {\n Y_PRED: y_pred,\n Y_TRUE: y_true,\n }\n )\n metrics_is_none = False\n if metrics is None:\n metrics_is_none = True\n if self._eval_metric_func:\n metrics = [self._eval_metric_func]\n else:\n metrics = [self._eval_metric_name]\n if isinstance(metrics, str) or isinstance(metrics, Scorer):\n metrics = [metrics]\n\n results = {}\n score = compute_score(\n metric_data=metric_data,\n metric=self._eval_metric_name.lower(),\n )\n score = {k.lower(): v for k, v in score.items()}\n if metrics_is_none:\n results = score\n else:\n for per_metric in metrics:\n if per_metric.lower() in score:\n results.update({per_metric: score[per_metric.lower()]})\n else:\n warnings.warn(f\"Warning: {per_metric} is not a supported evaluation metric!\")\n if not results:\n results = score # If the results dict is empty, return all scores.\n\n if return_pred:\n return results, self._as_pandas(data=data, to_be_converted=y_pred_inv)\n else:\n return results\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n \"\"\"\n self.ensure_predict_ready()\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n logits = extract_from_output(outputs=outputs, ret_type=NER_RET)\n if self._df_preprocessor:\n pred = self._df_preprocessor.transform_prediction(\n y_pred=logits,\n )\n else:\n pred = logits\n\n pred = merge_bio_format(data[self._df_preprocessor.ner_feature_names[0]], pred)\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n pred = self._as_pandas(data=data, to_be_converted=pred)\n\n return pred\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict probabilities class probabilities rather than class labels.\n This is only for the classification. Calling it for a regression will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n When as_multiclass is True, the output will always have shape (#samples, #classes).\n Otherwise, the output will have shape (#samples,)\n \"\"\"\n self.ensure_predict_ready()\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n ner_outputs = extract_from_output(outputs=outputs, ret_type=NER_RET)\n prob = self._df_preprocessor.transform_prediction(\n y_pred=ner_outputs,\n return_proba=True,\n )\n if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n prob = self._as_pandas(data=data, to_be_converted=prob)\n\n return prob\n\n def save(\n self,\n path: str,\n standalone: Optional[bool] = True,\n config: Optional[DictConfig] = None,\n model: Optional[nn.Module] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n fit_called: Optional[bool] = None,\n save_model: Optional[bool] = True,\n output_shape: Optional[int] = None,\n ):\n super().save(\n path=path,\n standalone=standalone,\n config=config,\n model=model,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=fit_called,\n save_model=save_model,\n )\n if output_shape: # output_shape is provided for saving artifacts at on_fit_per_run_end().\n assets_path = os.path.join(path, \"assets.json\")\n with open(assets_path, \"r\") as fp:\n assets = json.load(fp)\n assets.update(\n {\n \"output_shape\": output_shape,\n }\n )\n os.remove(assets_path)\n with open(assets_path, \"w\") as fp:\n json.dump(assets, fp, ensure_ascii=True)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/object_detection.py",
"content": "import json\nimport logging\nimport os\nfrom datetime import timedelta\nfrom typing import Dict, List, Optional, Union\n\nimport pandas as pd\nfrom omegaconf import DictConfig\nfrom torch import nn\n\nfrom ..constants import BBOX, DDP, MAP, MULTI_IMAGE_MIX_DATASET, OBJECT_DETECTION, XYWH\nfrom ..data import (\n BaseDataModule,\n MultiImageMixDataset,\n MultiModalFeaturePreprocessor,\n infer_rois_column_type,\n split_train_tuning_data,\n)\nfrom ..models import create_fusion_model\nfrom ..optim import MMDetLitModule\nfrom ..utils import (\n check_if_packages_installed,\n cocoeval,\n convert_pred_to_xywh,\n convert_result_df,\n extract_from_output,\n from_coco_or_voc,\n get_detection_classes,\n object_detection_data_to_df,\n save_result_coco_format,\n setup_save_path,\n)\nfrom .base import BaseLearner\n\nlogger = logging.getLogger(__name__)\n\n\nclass ObjectDetectionLearner(BaseLearner):\n def __init__(\n self,\n label: Optional[str] = None, # TODO: can we let users customize label?\n problem_type: Optional[str] = OBJECT_DETECTION,\n presets: Optional[str] = None,\n eval_metric: Optional[str] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n num_classes: Optional[int] = None,\n classes: Optional[list] = None,\n category_ids: Optional[list] = None,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = None,\n sample_data_path: Optional[str] = None, # TODO: can we use train/predict data instead?\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n num_classes\n Number of classes. Used in classification.\n If this is specified and is different from the pretrained model's output,\n the model's head will be changed to have output.\n classes\n All classes in this dataset.\n sample_data_path\n This is used for automatically inference num_classes, classes, or label.\n \"\"\"\n super().__init__(\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n pretrained=pretrained,\n validation_metric=validation_metric,\n )\n check_if_packages_installed(problem_type=self._problem_type)\n\n self._config = self.get_config_per_run(config=self._config, hyperparameters=hyperparameters)\n\n self._output_shape = num_classes\n self._classes = classes\n self._category_ids = category_ids\n self._sample_data_path = sample_data_path\n\n # TODO: merge object detection and open vocabulary object detection\n self._label_column = \"label\"\n if self._sample_data_path is not None:\n self._classes, self._category_ids = get_detection_classes(self._sample_data_path)\n self._output_shape = len(self._classes)\n\n # TODO: merge _detection_anno_train and detection_anno_train?\n self._detection_anno_train = None\n self.detection_anno_train = None\n\n self._log_filters += [\n \".*Creating a tensor from a list of numpy.ndarrays is extremely slow..*\",\n ]\n\n @property\n def classes(self):\n \"\"\"\n Return the classes of object detection.\n \"\"\"\n if self._model.model.CLASSES is not None and self._classes is not None:\n assert self._classes == self._model.model.CLASSES, f\"{self._classes}\\n{self._model.model.CLASSES}\"\n return self._classes if self._classes is not None else self._model.model.CLASSES\n\n @property\n def category_ids(self):\n \"\"\"\n Return the classes of object detection.\n \"\"\"\n return self._category_ids\n\n def setup_detection_train_tuning_data(self, max_num_tuning_data, seed, train_data, tuning_data):\n if isinstance(train_data, str):\n self._detection_anno_train = train_data\n train_data = from_coco_or_voc(\n train_data,\n \"train\",\n coco_root=self._config.model.mmdet_image.coco_root,\n ) # TODO: Refactor to use convert_data_to_df\n if tuning_data is not None:\n self.detection_anno_train = tuning_data\n tuning_data = from_coco_or_voc(\n tuning_data,\n \"val\",\n coco_root=self._config.model.mmdet_image.coco_root,\n ) # TODO: Refactor to use convert_data_to_df\n if max_num_tuning_data is not None:\n if len(tuning_data) > max_num_tuning_data:\n tuning_data = tuning_data.sample(\n n=max_num_tuning_data, replace=False, random_state=seed\n ).reset_index(drop=True)\n elif isinstance(train_data, pd.DataFrame):\n self._detection_anno_train = None\n # sanity check dataframe columns\n train_data = object_detection_data_to_df(\n train_data,\n coco_root=self._config.model.mmdet_image.coco_root,\n )\n if tuning_data is not None:\n self.detection_anno_train = tuning_data\n tuning_data = object_detection_data_to_df(\n tuning_data,\n coco_root=self._config.model.mmdet_image.coco_root,\n )\n if max_num_tuning_data is not None:\n if len(tuning_data) > max_num_tuning_data:\n tuning_data = tuning_data.sample(\n n=max_num_tuning_data, replace=False, random_state=seed\n ).reset_index(drop=True)\n else:\n raise TypeError(f\"Expected train_data to have type str or pd.DataFrame, but got type: {type(train_data)}\")\n return train_data, tuning_data\n\n def prepare_train_tuning_data(\n self,\n train_data: Union[pd.DataFrame, str],\n tuning_data: Optional[Union[pd.DataFrame, str]],\n holdout_frac: Optional[float],\n max_num_tuning_data: Optional[int],\n seed: Optional[int],\n ):\n # TODO: remove self from calling setup_detection_train_tuning_data()\n train_data, tuning_data = self.setup_detection_train_tuning_data(\n train_data=train_data,\n tuning_data=tuning_data,\n max_num_tuning_data=max_num_tuning_data,\n seed=seed,\n )\n\n if tuning_data is None:\n train_data, tuning_data = split_train_tuning_data(\n data=train_data,\n holdout_frac=holdout_frac,\n problem_type=self._problem_type,\n label_column=self._label_column,\n random_state=seed,\n )\n\n self._train_data = train_data\n self._tuning_data = tuning_data\n\n def infer_output_shape(self, **kwargs):\n # TODO: support inferring output during fit()?\n assert self._output_shape is not None, f\"output_shape should have been set in the learner initialization.\"\n\n def init_pretrained(self):\n super().init_pretrained()\n self._model = create_fusion_model(\n config=self._config,\n pretrained=self._pretrained,\n num_classes=self._output_shape,\n classes=self._classes,\n )\n\n def fit(\n self,\n train_data: Union[pd.DataFrame, str],\n presets: Optional[str] = None,\n tuning_data: Optional[Union[pd.DataFrame, str]] = None,\n max_num_tuning_data: Optional[int] = None,\n time_limit: Optional[int] = None,\n save_path: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n column_types: Optional[Dict] = None,\n holdout_frac: Optional[float] = None,\n seed: Optional[int] = 0,\n standalone: Optional[bool] = True,\n hyperparameter_tune_kwargs: Optional[Dict] = None,\n clean_ckpts: Optional[bool] = True,\n **kwargs,\n ):\n training_start = self.on_fit_start(presets=presets)\n self.setup_save_path(save_path=save_path)\n self.prepare_train_tuning_data(\n train_data=train_data,\n tuning_data=tuning_data,\n holdout_frac=holdout_frac,\n max_num_tuning_data=max_num_tuning_data,\n seed=seed,\n )\n self.infer_column_types(column_types=column_types)\n self.infer_validation_metric()\n self.update_hyperparameters(\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n self.fit_sanity_check()\n self.prepare_fit_args(\n time_limit=time_limit,\n seed=seed,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n fit_returns = self.execute_fit()\n self.on_fit_end(\n training_start=training_start,\n strategy=fit_returns.get(\"strategy\", None),\n strict_loading=fit_returns.get(\"strict_loading\", True),\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n\n return self\n\n def get_datamodule_per_run(\n self,\n df_preprocessor,\n data_processors,\n per_gpu_batch_size,\n num_workers,\n model_config=None,\n predict_data=None,\n is_train=True,\n ):\n datamodule_kwargs = dict(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=per_gpu_batch_size,\n num_workers=num_workers,\n )\n if is_train:\n val_use_training_mode = (self._problem_type == OBJECT_DETECTION) and (self._validation_metric_name != MAP)\n datamodule_kwargs.update(\n dict(validate_data=self._tuning_data, val_use_training_mode=val_use_training_mode)\n )\n if (\n self._problem_type == OBJECT_DETECTION\n and model_config is not None\n and MULTI_IMAGE_MIX_DATASET in model_config\n ):\n train_dataset = MultiImageMixDataset(\n data=self._train_data,\n preprocessor=[df_preprocessor],\n processors=[data_processors],\n model_config=model_config,\n id_mappings=None,\n is_training=True,\n )\n datamodule_kwargs.update(dict(train_dataset=train_dataset))\n else:\n datamodule_kwargs.update(dict(train_data=self._train_data))\n else:\n datamodule_kwargs.update(dict(predict_data=predict_data))\n\n datamodule = BaseDataModule(**datamodule_kwargs)\n return datamodule\n\n def get_strategy_per_run(self, num_gpus, config):\n if num_gpus <= 1:\n strategy = \"auto\"\n else:\n strategy = DDP\n\n return strategy\n\n def update_num_gpus_by_strategy(self, strategy, num_gpus):\n if strategy == DDP and self._fit_called:\n num_gpus = 1 # While using DDP, we can only use single gpu after fit is called\n\n return num_gpus\n\n def get_optim_kwargs_per_run(self, config, validation_metric, custom_metric_func):\n return dict(\n optim_type=config.optim.optim_type,\n lr_choice=config.optim.lr_choice,\n lr_schedule=config.optim.lr_schedule,\n lr=config.optim.lr,\n lr_decay=config.optim.lr_decay,\n end_lr=config.optim.end_lr,\n lr_mult=config.optim.lr_mult,\n weight_decay=config.optim.weight_decay,\n warmup_steps=config.optim.warmup_steps,\n track_grad_norm=config.optim.track_grad_norm,\n validation_metric=validation_metric,\n validation_metric_name=self._validation_metric_name,\n custom_metric_func=custom_metric_func,\n )\n\n def get_litmodule_per_run(\n self,\n model: Optional[nn.Module] = None,\n optim_kwargs: Optional[dict] = None,\n is_train=True,\n ):\n if self._problem_type == OBJECT_DETECTION:\n LightningModule = MMDetLitModule\n else:\n raise TypeError(f\"problem type {self._problem_type} is not supported by ObjectDetectionLearner.\")\n\n if is_train:\n return LightningModule(\n model=model,\n **optim_kwargs,\n )\n else:\n return LightningModule(model=self._model)\n\n def get_model_per_run(self, model, config):\n if model is None:\n model = create_fusion_model(\n config=config,\n num_classes=self._output_shape,\n classes=self._classes,\n )\n return model\n\n def fit_per_run(\n self,\n max_time: timedelta,\n save_path: str,\n ckpt_path: str,\n resume: bool,\n enable_progress_bar: bool,\n seed: int,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n advanced_hyperparameters: Optional[Dict] = None,\n config: Optional[Dict] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n model: Optional[nn.Module] = None,\n standalone: bool = True,\n clean_ckpts: bool = True,\n ):\n self.on_fit_per_run_start(seed=seed, save_path=save_path)\n config = self.get_config_per_run(config=config, hyperparameters=hyperparameters)\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=df_preprocessor,\n config=config,\n )\n config = self.update_config_by_data_per_run(config=config, df_preprocessor=df_preprocessor)\n model = self.get_model_per_run(model=model, config=config)\n model = self.compile_model_per_run(config=config, model=model)\n data_processors = self.get_data_processors_per_run(\n data_processors=data_processors,\n config=config,\n model=model,\n advanced_hyperparameters=advanced_hyperparameters,\n )\n validation_metric, custom_metric_func = self.get_validation_metric_per_run()\n if max_time == timedelta(seconds=0):\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n )\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=config.env.per_gpu_batch_size,\n num_workers=config.env.num_workers,\n model_config=model.config,\n )\n optim_kwargs = self.get_optim_kwargs_per_run(\n config=config,\n validation_metric=validation_metric,\n custom_metric_func=custom_metric_func,\n )\n litmodule = self.get_litmodule_per_run(\n model=model,\n optim_kwargs=optim_kwargs,\n )\n callbacks = self.get_callbacks_per_run(save_path=save_path, config=config, litmodule=litmodule)\n plugins = self.get_plugins_per_run(model=model)\n tb_logger = self.get_tb_logger(save_path=save_path)\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(config=config)\n precision = self.get_precision_per_run(num_gpus=num_gpus, precision=config.env.precision)\n grad_steps = self.get_grad_steps(num_gpus=num_gpus, config=config)\n strategy = self.get_strategy_per_run(num_gpus=num_gpus, config=config)\n config = self.post_update_config_per_run(\n config=config,\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n )\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n config=config,\n precision=precision,\n strategy=strategy,\n max_time=max_time,\n callbacks=callbacks,\n tb_logger=tb_logger,\n grad_steps=grad_steps,\n plugins=plugins,\n enable_progress_bar=enable_progress_bar,\n )\n\n self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n ckpt_path=ckpt_path,\n resume=resume,\n )\n self.on_fit_per_run_end(\n save_path=save_path,\n standalone=standalone,\n trainer=trainer,\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n )\n\n return dict(\n config=config,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n model=model,\n best_score=trainer.callback_metrics[f\"val_{self._validation_metric_name}\"].item(),\n strategy=strategy,\n )\n\n def predict_per_run(\n self,\n data: Union[pd.DataFrame, dict, list],\n realtime: Optional[bool],\n requires_label: bool,\n barebones: Optional[bool] = False,\n ) -> List[Dict]:\n \"\"\"\n Perform inference for learner.\n\n Parameters\n ----------\n data\n The data for inference.\n realtime\n Whether use realtime inference.\n requires_label\n Whether uses label during inference.\n barebones\n Whether to run in âbarebones modeâ, where all lightning's features that may impact raw speed are disabled.\n\n Returns\n -------\n A list of output dicts.\n \"\"\"\n data = self.on_predict_per_run_start(data=data)\n column_types = self.infer_column_types(\n column_types=self._column_types,\n data=data,\n is_train=False,\n )\n column_types = infer_rois_column_type(\n column_types=column_types,\n data=data,\n )\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=self._df_preprocessor,\n data=data,\n column_types=column_types,\n is_train=False,\n )\n if self._fit_called:\n df_preprocessor._column_types = self.update_image_column_types(data=data)\n data_processors = self.get_data_processors_per_run(\n data_processors=self._data_processors,\n requires_label=requires_label,\n is_train=False,\n )\n num_gpus, strategy = self.get_num_gpus_and_strategy_per_run(\n predict_data=data,\n is_train=False,\n )\n precision = self.get_precision_per_run(\n num_gpus=num_gpus,\n precision=self._config.env.precision,\n cpu_only_warning=False,\n )\n batch_size = self.get_predict_batch_size_per_run(num_gpus=num_gpus, strategy=strategy)\n realtime, num_gpus, barebones = self.update_realtime_for_interactive_env(\n realtime=realtime,\n num_gpus=num_gpus,\n barebones=barebones,\n strategy=strategy,\n )\n datamodule = self.get_datamodule_per_run(\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n per_gpu_batch_size=batch_size,\n num_workers=self._config.env.num_workers_inference,\n predict_data=data,\n is_train=False,\n )\n pred_writer = self.get_pred_writer(strategy=strategy)\n callbacks = self.get_callbacks_per_run(pred_writer=pred_writer, is_train=False)\n litmodule = self.get_litmodule_per_run(is_train=False)\n trainer = self.init_trainer_per_run(\n num_gpus=num_gpus,\n precision=precision,\n strategy=strategy,\n callbacks=callbacks,\n barebones=barebones,\n is_train=False,\n )\n outputs = self.run_trainer(\n trainer=trainer,\n litmodule=litmodule,\n datamodule=datamodule,\n pred_writer=pred_writer,\n is_train=False,\n )\n outputs = self.collect_predictions(\n outputs=outputs,\n trainer=trainer,\n pred_writer=pred_writer,\n num_gpus=num_gpus,\n )\n self.on_predict_per_run_end(trainer=trainer)\n\n # TODO: remove this by adjusting the return format of mmdet_image or lit_mmdet.\n if pred_writer is None and self._problem_type == OBJECT_DETECTION:\n outputs = [output for batch_outputs in outputs for output in batch_outputs]\n\n return outputs\n\n def evaluate_coco(\n self,\n anno_file_or_df: str,\n metrics: str,\n return_pred: Optional[bool] = False,\n eval_tool: Optional[str] = None,\n ):\n \"\"\"\n Evaluate object detection model on a test dataset in COCO format.\n\n Parameters\n ----------\n anno_file_or_df\n The annotation file in COCO format\n metrics\n Metrics used for evaluation.\n return_pred\n Whether to return the prediction result of each row.\n eval_tool\n The eval_tool for object detection. Could be \"pycocotools\" or \"torchmetrics\".\n \"\"\"\n if isinstance(anno_file_or_df, str):\n anno_file = anno_file_or_df\n data = from_coco_or_voc(\n anno_file,\n \"test\",\n coco_root=self._config.model.mmdet_image.coco_root,\n ) # TODO: maybe remove default splits hardcoding (only used in VOC)\n if os.path.isdir(anno_file):\n eval_tool = \"torchmetrics\" # we can only use torchmetrics for VOC format evaluation.\n else:\n # during validation, it will call evaluate with df as input\n anno_file = self._detection_anno_train\n data = anno_file_or_df\n\n outputs = self.predict_per_run(\n data=data,\n realtime=False,\n requires_label=True,\n ) # outputs shape: num_batch, 1([\"bbox\"]), batch_size, 2(if using mask_rcnn)/na, 80, n, 5\n\n # Cache prediction results as COCO format # TODO: refactor this\n self._save_path = setup_save_path(\n old_save_path=self._save_path,\n warn_if_exist=False,\n )\n cocoeval_cache_path = os.path.join(self._save_path, \"object_detection_result_cache.json\")\n eval_results = cocoeval(\n outputs=outputs,\n data=data,\n anno_file=anno_file,\n cache_path=cocoeval_cache_path,\n metrics=metrics,\n tool=eval_tool,\n )\n if return_pred:\n return eval_results, outputs\n else:\n return eval_results\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n eval_tool: Optional[str] = None,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n Or a str, that is a path of the annotation file for detection.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n eval_tool\n The eval_tool for object detection. Could be \"pycocotools\" or \"torchmetrics\".\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n self.ensure_predict_ready()\n\n if realtime:\n return NotImplementedError(f\"Current problem type {self._problem_type} does not support realtime predict.\")\n if isinstance(data, str):\n return self.evaluate_coco(\n anno_file_or_df=data,\n metrics=metrics,\n return_pred=return_pred,\n eval_tool=eval_tool,\n )\n else:\n data = object_detection_data_to_df(\n data,\n coco_root=self._config.model.mmdet_image.coco_root,\n )\n return self.evaluate_coco(\n anno_file_or_df=data,\n metrics=metrics,\n return_pred=return_pred,\n eval_tool=\"torchmetrics\",\n )\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n as_pandas: Optional[bool] = None,\n as_coco: Optional[bool] = True,\n realtime: Optional[bool] = False,\n save_results: Optional[bool] = None,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or Instance Data (False).\n For the definition of Instance Data in MMDetection/MMEngine, see\n https://github.com/open-mmlab/mmengine/blob/698782f9203a6bfcc0e445047fd2300796ecbf0f/mmengine/structures/instance_data.py#L34\n as_coco\n Whether to save the output as a COCO json file (True) or pandas DataFrame (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n save_results\n Whether to save the prediction results (only works for detection now)\n **kwargs\n Additional arguments including:\n - result_save_path (str, optional): Custom path to save results. If not provided,\n uses default path setup.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n \"\"\"\n self.ensure_predict_ready()\n\n if as_pandas is None:\n as_pandas = True # return pandas dataframe by default\n\n ret_type = BBOX\n\n # only supports coco/voc format for OBJECT_DETECTION\n if self._problem_type == OBJECT_DETECTION:\n data_path = data\n data_df = object_detection_data_to_df(\n data_path,\n coco_root=self._config.model.mmdet_image.coco_root,\n )\n if self._label_column not in data_df:\n self._label_column = None\n\n outputs = self.predict_per_run(\n data=data_df,\n realtime=realtime,\n requires_label=False,\n )\n pred = extract_from_output(outputs=outputs, ret_type=ret_type)\n\n self._save_path = setup_save_path(\n old_save_path=self._save_path,\n warn_if_exist=False,\n )\n result_path = os.path.join(self._save_path, \"result.txt\")\n\n pred_df = convert_result_df(\n pred=convert_pred_to_xywh(pred) if self._model.output_bbox_format == XYWH else pred,\n data=data_df,\n detection_classes=self.classes,\n result_path=result_path,\n )\n\n if save_results:\n self._save_path = setup_save_path(\n old_save_path=self._save_path,\n warn_if_exist=False,\n )\n custom_save_path = kwargs.get(\"result_save_path\")\n if custom_save_path:\n result_path = custom_save_path\n elif as_coco:\n result_path = os.path.join(self._save_path, \"result.json\")\n else:\n result_path = os.path.join(self._save_path, \"result.txt\")\n if as_coco:\n save_result_coco_format(\n data_path=data_path,\n pred=pred,\n category_ids=self.category_ids,\n result_path=result_path,\n coco_root=self._config.model.mmdet_image.coco_root,\n )\n else:\n pred_df.to_csv(result_path, index=False)\n logger.info(f\"Saved detection results {'as coco' if as_coco else 'as dataframe'} to {result_path}\")\n\n if as_pandas:\n return pred_df\n else:\n if self._model.output_bbox_format == XYWH:\n pred = convert_pred_to_xywh(pred)\n return pred\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n raise NotImplementedError(\"Object detection doesn't support calling `predict_proba` yet.\")\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n raise NotImplementedError(\"Object detection doesn't support calling `extract_embedding` yet.\")\n\n @staticmethod\n def _load_metadata(\n learner,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n learner = super()._load_metadata(learner=learner, path=path, resume=resume, verbosity=verbosity)\n learner._data_processors = None\n return learner\n\n def save(\n self,\n path: str,\n standalone: Optional[bool] = True,\n config: Optional[DictConfig] = None,\n model: Optional[nn.Module] = None,\n df_preprocessor: Optional[MultiModalFeaturePreprocessor] = None,\n data_processors: Optional[Dict] = None,\n fit_called: Optional[bool] = None,\n save_model: Optional[bool] = True,\n ):\n super().save(\n path=path,\n standalone=standalone,\n config=config,\n model=model,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n fit_called=fit_called,\n save_model=save_model,\n )\n assets_path = os.path.join(path, \"assets.json\")\n with open(assets_path, \"r\") as fp:\n assets = json.load(fp)\n assets.update(\n {\n \"classes\": self._classes,\n }\n )\n os.remove(assets_path)\n with open(assets_path, \"w\") as fp:\n json.dump(assets, fp, ensure_ascii=True)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/learners/semantic_segmentation.py",
"content": "import logging\nimport os\nfrom typing import Dict, Iterable, List, Optional, Union\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom scipy.special import softmax\n\nfrom autogluon.core.metrics import Scorer\n\nfrom ..constants import LABEL, LOGITS, SEMANTIC_MASK, SEMANTIC_SEGMENTATION_IMG\nfrom ..optim import SemanticSegmentationLitModule, get_loss_func, get_norm_layer_param_names, get_peft_param_names\nfrom ..optim.metrics.semantic_seg_metrics import Balanced_Error_Rate_Pred as Balanced_Error_Rate\nfrom ..optim.metrics.semantic_seg_metrics import Binary_IoU_Pred as Binary_IoU\nfrom ..optim.metrics.semantic_seg_metrics import COD_METRICS_NAMES_Pred as COD_METRICS_NAMES\nfrom ..optim.metrics.semantic_seg_metrics import Multiclass_IoU_Pred as Multiclass_IoU\nfrom ..utils import extract_from_output, setup_save_path\nfrom .base import BaseLearner\n\nlogger = logging.getLogger(__name__)\n\nfrom ..constants import BER, EM, FM, IOU, MAE, SEMANTIC_SEGMENTATION, SM\n\n\nclass SemanticSegmentationLearner(BaseLearner):\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = SEMANTIC_SEGMENTATION,\n presets: Optional[str] = None,\n eval_metric: Optional[Union[str, Scorer]] = \"iou\",\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n num_classes: Optional[int] = None, # TODO: can we infer this from data?\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = \"iou\",\n sample_data_path: Optional[str] = None,\n **kwargs,\n ):\n super().__init__(\n label=label,\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n pretrained=pretrained,\n validation_metric=validation_metric,\n )\n self._output_shape = num_classes\n self._sample_data_path = sample_data_path\n\n if self._sample_data_path is not None:\n infer_output_shape = self.get_semantic_segmentation_class_num(self._sample_data_path)\n if num_classes is not None:\n assert num_classes == infer_output_shape, (\n f\"The provided number of classes '{num_classes}' and the inferred class number {infer_output_shape}' from the sample data should be consistent.\"\n )\n else:\n self._output_shape = infer_output_shape\n\n def get_semantic_segmentation_class_num(self, sample_data_path):\n \"\"\"\n Get the number of classes for given data.\n\n Parameters\n ----------\n sample_data_path\n This is used for automatically inference num_classes of semantic segmentation dataset.\n Could be an image directory, image file or pd.DataFrame.\n Returns\n -------\n The number of classes.\n \"\"\"\n if isinstance(sample_data_path, str):\n if os.path.isdir(sample_data_path):\n mask_files = os.listdir(sample_data_path)\n num_classes = []\n for mask_file in mask_files:\n per_num_classes = self.get_semantic_segmentation_class_num(\n os.path.join(sample_data_path, mask_file)\n )\n num_classes.append(per_num_classes)\n return max(num_classes)\n else:\n mask = Image.open(sample_data_path)\n mode = mask.mode\n\n if mode == \"1\":\n return 1\n classes = np.unique(mask)\n if mode == \"L\" and np.array_equal(classes, np.array([0, 255])):\n return 1\n\n return max(classes).item() + 1 # include background\n\n elif isinstance(sample_data_path, pd.DataFrame):\n num_classes = []\n for idx in range(sample_data_path.shape[0]):\n row = sample_data_path.iloc[idx]\n mask_file = row[self._label_column]\n per_num_classes = self.get_semantic_segmentation_class_num(mask_file)\n num_classes.append(per_num_classes)\n return max(num_classes)\n\n def infer_output_shape(self):\n if self._output_shape is None:\n self._output_shape = self.get_semantic_segmentation_class_num(self._train_data)\n\n @staticmethod\n def get_peft_param_names_per_run(model, config):\n peft_param_names = None\n peft = config.optim.peft\n if peft:\n norm_param_names = get_norm_layer_param_names(model)\n peft_param_names = get_peft_param_names(\n norm_param_names,\n peft=peft,\n extra_params=config.optim.extra_trainable_params,\n )\n return peft_param_names\n\n def get_loss_func_per_run(self, config, mixup_active=None):\n loss_func = get_loss_func(\n problem_type=self._problem_type,\n loss_func_name=config.optim.loss_func,\n config=config.optim,\n num_classes=self._output_shape,\n )\n return loss_func, None\n\n def evaluate_semantic_segmentation(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n ):\n \"\"\"\n Evaluate semantic segmentation on a test dataset based on \"torchmetrics\".\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n Or a str, that is a path of the annotation file for detection.\n metrics\n Metrics used for evaluation.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default None).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n \"\"\"\n\n def get_metric_predict(\n metric_name: str,\n num_classes: Optional[int] = None,\n ):\n \"\"\"\n Obtain a torchmerics.Metric from its name.\n Define a customized metric function in case that torchmetrics doesn't support some metric.\n\n Parameters\n ----------\n metric_name\n Name of metric.\n num_classes\n Number of classes.\n is_matching\n Whether is matching.\n problem_type\n Type of problem, e.g., binary and multiclass.\n\n Returns\n -------\n torchmetrics.Metric\n A torchmetrics.Metric object.\n custom_metric_func\n A customized metric function.\n \"\"\"\n if metric_name == BER:\n return Balanced_Error_Rate()\n elif metric_name in [SM, EM, FM, MAE]:\n return COD_METRICS_NAMES[metric_name]\n elif metric_name == IOU:\n if num_classes == 1:\n return Binary_IoU()\n else:\n return Multiclass_IoU(num_classes=num_classes)\n else:\n raise ValueError(f\"Unknown metric {metric_name}\")\n\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n\n if self._output_shape == 1:\n logits = extract_from_output(ret_type=LOGITS, outputs=outputs, as_ndarray=False)\n else:\n logits = extract_from_output(ret_type=SEMANTIC_MASK, outputs=outputs, as_ndarray=False)\n y_pred = logits.float()\n y_true = [ele[LABEL] for ele in outputs]\n y_true = torch.cat(y_true)\n\n assert len(y_true) == len(y_pred)\n\n results = {}\n if isinstance(metrics, str):\n metrics = [metrics]\n for per_metric_name in metrics:\n per_metric = get_metric_predict(metric_name=per_metric_name.lower(), num_classes=self._output_shape)\n for y_p, y_t in zip(y_pred, y_true):\n per_metric.update(y_p.unsqueeze(0), y_t.unsqueeze(0))\n score = per_metric.compute()\n results[per_metric_name] = score.item()\n\n if return_pred:\n return results, outputs\n else:\n return results\n\n def get_litmodule_per_run(\n self,\n model=None,\n model_postprocess_fn=None,\n peft_param_names=None,\n optim_kwargs=None,\n distillation_kwargs=None,\n is_train=True,\n ):\n if is_train:\n return SemanticSegmentationLitModule(\n model=model,\n model_postprocess_fn=model_postprocess_fn,\n trainable_param_names=peft_param_names,\n **optim_kwargs,\n )\n else:\n return SemanticSegmentationLitModule(\n model=self._model,\n model_postprocess_fn=self._model_postprocess_fn,\n **optim_kwargs,\n )\n\n def on_predict_start(self, data: pd.DataFrame):\n data = self.data_to_df(data=data)\n if self._output_shape is None: # for zero-shot evaluation/prediction\n self._output_shape = self.get_semantic_segmentation_class_num(data)\n self.ensure_predict_ready()\n return data\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n metrics: Optional[Union[str, List[str]]] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Evaluate model on a test dataset.\n\n Parameters\n ----------\n data\n A dataframe, containing the same columns as the training data.\n Or a str, that is a path of the annotation file for detection.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a dataframe of prediction results.\n \"\"\"\n data = self.on_predict_start(data)\n return self.evaluate_semantic_segmentation(data, metrics, realtime)\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n realtime: Optional[bool] = False,\n save_results: Optional[bool] = None,\n **kwargs,\n ):\n \"\"\"\n Predict values for the label column of new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n save_results\n Whether to save the prediction results.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n When save_results is True, the output is a pandas dataframe containing the path of the predicted mask file for each input image.\n Otherwise, the output will have shape (#samples, height, width).\n \"\"\"\n data = self.on_predict_start(data)\n if self._output_shape == 1:\n ret_type = LOGITS\n else:\n ret_type = SEMANTIC_MASK\n\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n\n logits = self.post_process_prediction(data, outputs, ret_type)\n\n pred = []\n for logit in logits:\n logit = logit.numpy()\n if ret_type == SEMANTIC_MASK:\n pred.append(logit.argmax(axis=1))\n else:\n pred.append((logit > 0.5).squeeze(axis=1))\n\n if save_results:\n self._save_path = setup_save_path(\n old_save_path=self._save_path,\n warn_if_exist=False,\n )\n pred = self.save_segmentation_result(\n pred=pred,\n data=data,\n result_path=self._save_path,\n )\n\n return pred\n\n # if (as_pandas is None and isinstance(data, pd.DataFrame)) or as_pandas is True:\n # # TODO\n # pred = self._as_pandas(data=data, to_be_converted=pred)\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n \"\"\"\n Predict probabilities class probabilities rather than class labels.\n This is only for the classification. Calling it for regression will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n as_multiclass\n Whether to return the probability of all labels or\n just return the probability of the positive class for binary classification problems.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n The output will always have shape (#samples, #classes, height, width).\n \"\"\"\n assert (self._output_shape == 1 and as_multiclass == False) or (\n self._output_shape > 1 and as_multiclass == True\n )\n data = self.on_predict_start(data)\n\n outputs = self.predict_per_run(\n data=data,\n realtime=realtime,\n requires_label=False,\n )\n\n if as_multiclass:\n ret_type = SEMANTIC_MASK\n else:\n ret_type = LOGITS\n\n logits = self.post_process_prediction(data, outputs, ret_type)\n\n prob = []\n for logit in logits:\n logit = logit.numpy()\n if ret_type == SEMANTIC_MASK:\n prob.append(softmax(logit, axis=1))\n else:\n prob.append(logit)\n\n return prob\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n **kwargs,\n ):\n raise NotImplementedError(\"Semantic segmentation doesn't support calling `extract_embedding` yet.\")\n\n def save_segmentation_result(self, pred: Iterable, data: Union[pd.DataFrame, Dict], result_path: str):\n \"\"\"\n Saving segmentation results in pd.DataFrame format (per image)\n\n Parameters\n ----------\n pred\n List containing segmentation results for one image\n data\n Pandas data frame or dict containing the image information to be tested\n result_path\n Path to save result\n Returns\n -------\n The paths of the segmentation results as pandas DataFrame\n \"\"\"\n\n def show_mask(mask, ax):\n color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n h, w = mask.shape[-2:]\n mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n ax.imshow(mask_image)\n\n image_column_name = self.get_image_column_name(data)\n if isinstance(data, dict):\n image_names = data[image_column_name]\n else:\n image_names = data[image_column_name].to_list()\n results = []\n\n mask_path = os.path.join(result_path, \"masks\")\n txt_path = os.path.join(result_path, \"result.txt\")\n os.makedirs(mask_path, exist_ok=True)\n for image_pred, image_name in zip(pred, image_names):\n if self._output_shape == 1:\n mask = Image.fromarray(image_pred.squeeze(axis=0))\n per_mask_path = os.path.join(mask_path, os.path.basename(image_name))\n mask.save(per_mask_path)\n else:\n masks = []\n classes = np.unique(image_pred)\n for class_id in classes:\n if class_id == 0: # bg\n continue\n masks.append(image_pred == class_id)\n\n for mask in masks:\n show_mask(mask, plt.gca())\n mask_name = \"\"\n for i in os.path.basename(image_name).split(\".\")[:-1]:\n mask_name += i\n per_mask_path = os.path.join(mask_path, os.path.basename(image_name))\n plt.axis(\"off\")\n plt.savefig(per_mask_path, bbox_inches=\"tight\", dpi=300, pad_inches=0.0)\n\n results.append([image_name, per_mask_path])\n\n result_df = pd.DataFrame(results, columns=[\"image\", \"mask\"])\n result_df.to_csv(txt_path, index=False)\n return result_df\n\n def post_process_prediction(self, data, outputs, ret_type):\n \"\"\"\n Post-process segmentation results to match the size of original input images.\n\n Parameters\n ----------\n data\n Pandas data frame or dict containing the image information.\n outputs\n A list of segmentation output results.\n ret_type\n What kind of information to extract from model outputs.\n\n Returns\n -------\n A list of the post-processed segmentation results.\n \"\"\"\n logits = [ele[ret_type] for ele in outputs]\n image_column_name = self.get_image_column_name(data)\n for idx in range(data.shape[0]):\n ori_image_size = Image.open(data[image_column_name][idx]).size # width, height\n logits[idx] = F.interpolate(\n logits[idx].float(), (ori_image_size[1], ori_image_size[0]), mode=\"bilinear\", align_corners=False\n )\n return logits\n\n def get_image_column_name(self, data: pd.DataFrame):\n if self.column_types is None:\n column_names = list(data.columns)\n if self._label_column in column_names:\n column_names.remove(self._label_column)\n assert len(column_names) == 1, (\n f\"More than one image columns {column_names} exist in the data. Make sure to provide data with one image column.\"\n )\n return column_names[0]\n else:\n for k, v in self.column_types.items():\n if v == SEMANTIC_SEGMENTATION_IMG:\n return k\n return None\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/__init__.py",
"content": "from .augmenter import Augmenter\nfrom .categorical_mlp import CategoricalMLP\nfrom .clip import CLIPForImageText\nfrom .document_transformer import DocumentTransformer\nfrom .ft_transformer import FT_Transformer\nfrom .fusion import (\n AbstractMultimodalFusionModel,\n MultimodalFusionMLP,\n MultimodalFusionNER,\n MultimodalFusionTransformer,\n)\nfrom .hf_text import HFAutoModelForTextPrediction\nfrom .meta_transformer import MetaTransformer\nfrom .mmdet_image import MMDetAutoModelForObjectDetection\nfrom .mmocr_text_detection import MMOCRAutoModelForTextDetection\nfrom .mmocr_text_recognition import MMOCRAutoModelForTextRecognition\nfrom .ner_text import HFAutoModelForNER\nfrom .numerical_mlp import NumericalMLP\nfrom .sam import SAMForSemanticSegmentation\nfrom .t_few import TFewModel\nfrom .timm_image import TimmAutoModelForImagePrediction\nfrom .utils import (\n create_fusion_model,\n create_model,\n get_model_postprocess_fn,\n is_lazy_weight_tensor,\n list_timm_models,\n modify_duplicate_model_names,\n select_model,\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/adaptation_layers.py",
"content": "# Modified based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py\n\n# ------------------------------------------------------------------------------------------\n# Copyright (c) Microsoft Corporation. All rights reserved.\n# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.\n# ------------------------------------------------------------------------------------------\n\nimport math\nfrom typing import List, Optional\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.distributions.normal import Normal\n\n\ndef identity(x):\n return x\n\n\nclass LoRALayer:\n \"\"\"\n Abstract LoRA Layer.\n\n Parameters\n ----------\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n merge_weights\n Merging weights during inference to reduce latency.\n\n References\n ----------\n 1. Edward J. Hu*, Yelong Shen*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen,\n \"LoRA: Low-Rank Adaptation of Large Language Models\", 2021\n https://arxiv.org/abs/2106.09685\n 2. Code: https://github.com/microsoft/LoRA\n \"\"\"\n\n def __init__(\n self,\n r: int,\n lora_alpha: int,\n lora_dropout: float,\n merge_weights: bool,\n ):\n self.r = r\n self.lora_alpha = lora_alpha\n # Optional dropout\n if lora_dropout > 0.0:\n self.lora_dropout = nn.Dropout(p=lora_dropout)\n else:\n self.lora_dropout = identity\n # Mark the weight as unmerged\n self.merged = False\n self.merge_weights = merge_weights\n\n\nclass IA3LoRALinear(nn.Linear, LoRALayer):\n \"\"\"\n LoRA (low-rank adaptation) followed by (IA)^3 (weight rescaling) incorporated in a Linear Layer. Weights of Linear layer are set to be frozen per default.\n\n Parameters\n ----------\n in_features\n input dimension, set to the original linear layer input dimension LoRA is replacing.\n out_features\n output dimension, set to the original linear layer output dimension LoRA is replacing.\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n lora_dropout\n Dropout probability.\n fan_in_fan_out\n Set this to True if the layer to replace stores weight like (fan_in, fan_out).\n merge_weights\n Merging weights during inference to reduce latency.\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n r=8,\n lora_alpha=8,\n lora_dropout: float = 0.0,\n fan_in_fan_out=False,\n merge_weights=False,\n **kwargs,\n ):\n nn.Linear.__init__(self, in_features, out_features, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights)\n # In essence the $b$ parameter of LoRA.\n self.lora_b = nn.Parameter(torch.ones(out_features, 1))\n self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features)))\n self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r)))\n self.fan_in_fan_out = fan_in_fan_out\n self.weight.requires_grad = False\n\n self.scaling = self.lora_alpha / self.r\n self.reset_parameters()\n\n def reset_parameters(self):\n nn.Linear.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))\n nn.init.zeros_(self.lora_B)\n\n def T(self, w):\n return w.T if self.fan_in_fan_out else w\n\n def forward(self, x: torch.Tensor):\n result = F.linear(x, self.T(self.weight), bias=self.bias)\n if self.r > 0:\n result += (self.lora_dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling\n\n hidden = result * self.lora_b.flatten()\n return hidden\n\n def train(self, mode: bool = True):\n nn.Linear.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n if self.r > 0:\n self.weight.data /= self.lora_b.flatten()\n self.weight.data -= self.T(self.lora_B @ self.lora_A) * self.scaling\n self.merged = False\n\n def eval(self):\n # def T(w):\n # return w.T if self.fan_in_fan_out else w\n nn.Linear.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n if self.r > 0:\n self.weight.data += self.T(self.lora_B @ self.lora_A) * self.scaling\n self.weight.data *= self.lora_b.flatten()\n self.merged = True\n return hidden\n\n def extra_repr(self):\n return \"in_features={}, out_features={}, bias={}\".format(\n self.in_features, self.out_features, self.bias is not None\n )\n\n\nclass IA3Linear(nn.Linear, LoRALayer):\n \"\"\"\n (IA)^3 incorporated in a Linear Layer. Weights of Linear layer are set to be frozen per default.\n\n Parameters\n ----------\n in_features\n input dimension, set to the original linear layer input dimension LoRA is replacing.\n out_features\n output dimension, set to the original linear layer output dimension LoRA is replacing.\n scaling_rank\n Merging weights during inference to reduce latency.\n\n References\n ----------\n 1. Liu, Haokun and Tam, Derek and Muqeeth, Mohammed and Mohta, Jay and Huang, Tenghao and Bansal, Mohit and Raffel, Colin,\n \"Few-Shot Parameter-Efficient Fine-Tuning is Better and Cheaper than In-Context Learning\", 2022\n https://arxiv.org/pdf/2205.05638.pdf\n 2. Code: https://github.com/r-three/t-few\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n merge_weights: False,\n **kwargs,\n ):\n nn.Linear.__init__(self, in_features, out_features, **kwargs)\n LoRALayer.__init__(\n self, r=4, lora_alpha=4, lora_dropout=0.0, merge_weights=merge_weights\n ) # Default arguments, only\n # In essence the $b$ parameter of LoRA.\n self.lora_b = nn.Parameter(torch.ones(out_features, 1))\n self.weight.requires_grad = False\n\n def forward(self, x: torch.Tensor):\n hidden = F.linear(x, self.weight, self.bias)\n hidden = hidden * self.lora_b.flatten()\n return hidden\n\n def train(self, mode: bool = True):\n nn.Linear.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n if self.r > 0:\n self.weight.data /= self.lora_b.flatten()\n self.merged = False\n\n def eval(self):\n # def T(w):\n # return w.T if self.fan_in_fan_out else w\n nn.Linear.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n if self.r > 0:\n self.weight.data *= self.lora_b.flatten()\n self.merged = True\n return hidden\n\n def extra_repr(self):\n return \"in_features={}, out_features={}, bias={}\".format(\n self.in_features, self.out_features, self.bias is not None\n )\n\n\nclass LoRALinear(nn.Linear, LoRALayer):\n \"\"\"\n LoRA incorporated in Linear Layer. Weights of linear layer are set to be frozen per default.\n\n Parameters\n ----------\n in_features\n input dimension, set to the original linear layer input dimension LoRA is replacing.\n out_features\n output dimension, set to the original linear layer output dimension LoRA is replacing.\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n lora_dropout\n Dropout probability.\n fan_in_fan_out\n Set this to True if the layer to replace stores weight like (fan_in, fan_out).\n merge_weights\n Merging weights during inference to reduce latency.\n\n References\n ----------\n 1. Edward J. Hu*, Yelong Shen*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen,\n \"LoRA: Low-Rank Adaptation of Large Language Models\", 2021\n https://arxiv.org/abs/2106.09685\n 2. Code: https://github.com/microsoft/LoRA\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n r: int = 0,\n lora_alpha: int = 1,\n lora_dropout: float = 0.0,\n fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out)\n merge_weights: bool = True,\n **kwargs,\n ):\n nn.Linear.__init__(self, in_features, out_features, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights)\n\n self.fan_in_fan_out = fan_in_fan_out\n # Actual trainable parameters\n if r > 0:\n self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features)))\n self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r)))\n self.scaling = self.lora_alpha / self.r\n # Freezing the pre-trained weight matrix\n self.weight.requires_grad = False\n self.reset_parameters()\n if fan_in_fan_out:\n self.weight.data = self.weight.data.T\n\n def reset_parameters(self):\n nn.Linear.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))\n nn.init.zeros_(self.lora_B)\n\n def T(self, w):\n return w.T if self.fan_in_fan_out else w\n\n def train(self, mode: bool = True):\n nn.Linear.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n if self.r > 0:\n self.weight.data -= self.T(self.lora_B @ self.lora_A) * self.scaling\n self.merged = False\n\n def eval(self):\n # def T(w):\n # return w.T if self.fan_in_fan_out else w\n nn.Linear.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n if self.r > 0:\n self.weight.data += self.T(self.lora_B @ self.lora_A) * self.scaling\n self.merged = True\n\n def forward(self, x: torch.Tensor):\n if self.r > 0 and not self.merged:\n result = F.linear(x, self.T(self.weight), bias=self.bias)\n if self.r > 0:\n result += (self.lora_dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling\n return result\n else:\n return F.linear(x, self.T(self.weight), bias=self.bias)\n\n\nclass LoRAEmbedding(nn.Embedding, LoRALayer):\n \"\"\"\n LoRA incorporated in Embedding Layer. Weights of embedding layer are set to be frozen per default.\n\n Parameters\n ----------\n num_embeddings\n size of the dictionary of embeddings.\n embedding_dim\n the size of each embedding vector.\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n merge_weights\n Merging weights during inference to reduce latency.\n\n References\n ----------\n 1. Edward J. Hu*, Yelong Shen*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen,\n \"LoRA: Low-Rank Adaptation of Large Language Models\", 2021\n https://arxiv.org/abs/2106.09685\n 2. Code: https://github.com/microsoft/LoRA\n \"\"\"\n\n def __init__(\n self,\n num_embeddings: int,\n embedding_dim: int,\n r: int = 0,\n lora_alpha: int = 1,\n merge_weights: bool = True,\n **kwargs,\n ):\n nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=0, merge_weights=merge_weights)\n # Actual trainable parameters\n if r > 0:\n self.lora_A = nn.Parameter(self.weight.new_zeros((r, num_embeddings)))\n self.lora_B = nn.Parameter(self.weight.new_zeros((embedding_dim, r)))\n self.scaling = self.lora_alpha / self.r\n # Freezing the pre-trained weight matrix\n self.weight.requires_grad = False\n self.reset_parameters()\n\n def reset_parameters(self):\n nn.Embedding.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.zeros_(self.lora_A)\n nn.init.normal_(self.lora_B)\n\n def train(self, mode: bool = True):\n nn.Embedding.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n if self.r > 0:\n self.weight.data -= (self.lora_B @ self.lora_A).T * self.scaling\n self.merged = False\n\n def eval(self):\n nn.Linear.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n if self.r > 0:\n self.weight.data += (self.lora_B @ self.lora_A) * self.scaling\n self.merged = True\n\n def forward(self, x: torch.Tensor):\n if self.r > 0 and not self.merged:\n result = nn.Embedding.forward(self, x)\n if self.r > 0:\n after_A = F.embedding(\n x,\n self.lora_A.T,\n self.padding_idx,\n self.max_norm,\n self.norm_type,\n self.scale_grad_by_freq,\n self.sparse,\n )\n result += (after_A @ self.lora_B.T) * self.scaling\n return result\n else:\n return nn.Embedding.forward(self, x)\n\n\nclass LoRAMergedLinear(nn.Linear, LoRALayer):\n \"\"\"\n LoRA where single nn.Linear represents more than one layer (used in some implementations of attention query/key/value projections). Weights of linear layer are set to be frozen per default.\n\n Parameters\n ----------\n in_features\n input dimension, set to the original linear layer input dimension LoRA is replacing\n out_features\n output dimension, set to the original linear layer output dimension LoRA is replacing\n r\n rank r of the low-rank decomposition\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n lora_dropout\n Dropout rate for LoRA\n fan_in_fan_out\n Set this to True if the layer to replace stores weight like (fan_in, fan_out)\n merge_weights\n Merging weights during inference to reduce latency\n\n References\n ----------\n 1. Edward J. Hu*, Yelong Shen*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen,\n \"LoRA: Low-Rank Adaptation of Large Language Models\", 2021\n https://arxiv.org/abs/2106.09685\n 2. Code: https://github.com/microsoft/LoRA\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n r: int = 0,\n lora_alpha: int = 1,\n lora_dropout: float = 0.0,\n enable_lora: List[bool] = [False],\n fan_in_fan_out: bool = False,\n merge_weights: bool = True,\n **kwargs,\n ):\n nn.Linear.__init__(self, in_features, out_features, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights)\n assert out_features % len(enable_lora) == 0, \"The length of enable_lora must divide out_features\"\n self.enable_lora = enable_lora\n self.fan_in_fan_out = fan_in_fan_out\n # Actual trainable parameters\n if r > 0 and any(enable_lora):\n self.lora_A = nn.Parameter(self.weight.new_zeros((r * sum(enable_lora), in_features)))\n self.lora_B = nn.Parameter(\n self.weight.new_zeros((out_features // len(enable_lora) * sum(enable_lora), r))\n ) # weights for Conv1D with groups=sum(enable_lora)\n self.scaling = self.lora_alpha / self.r\n # Freezing the pre-trained weight matrix\n self.weight.requires_grad = False\n # Compute the indices\n self.lora_ind = self.weight.new_zeros((out_features,), dtype=torch.bool).view(len(enable_lora), -1)\n self.lora_ind[enable_lora, :] = True\n self.lora_ind = self.lora_ind.view(-1)\n self.reset_parameters()\n if fan_in_fan_out:\n self.weight.data = self.weight.data.T\n\n def reset_parameters(self):\n nn.Linear.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))\n nn.init.zeros_(self.lora_B)\n\n def zero_pad(self, x):\n result = x.new_zeros((*x.shape[:-1], self.out_features))\n result = result.view(-1, self.out_features)\n result[:, self.lora_ind] = x.reshape(-1, self.out_features // len(self.enable_lora) * sum(self.enable_lora))\n return result.view((*x.shape[:-1], self.out_features))\n\n def train(self, mode: bool = True):\n def T(w):\n return w.T if self.fan_in_fan_out else w\n\n nn.Linear.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n if self.r > 0 and any(self.enable_lora):\n delta_w = F.conv1d(\n self.lora_A.data.unsqueeze(0), self.lora_B.data.unsqueeze(-1), groups=sum(self.enable_lora)\n ).squeeze(0)\n self.weight.data -= self.zero_pad(T(delta_w * self.scaling))\n self.merged = False\n\n def eval(self):\n def T(w):\n return w.T if self.fan_in_fan_out else w\n\n nn.Linear.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n if self.r > 0 and any(self.enable_lora):\n delta_w = F.conv1d(\n self.lora_A.data.unsqueeze(0), self.lora_B.data.unsqueeze(-1), groups=sum(self.enable_lora)\n ).squeeze(0)\n self.weight.data += self.zero_pad(T(delta_w * self.scaling))\n self.merged = True\n\n def forward(self, x: torch.Tensor):\n def T(w):\n return w.T if self.fan_in_fan_out else w\n\n if self.merged:\n return F.linear(x, T(self.weight), bias=self.bias)\n else:\n result = F.linear(x, T(self.weight), bias=self.bias)\n if self.r > 0:\n after_A = F.linear(self.lora_dropout(x), self.lora_A)\n after_B = F.conv1d(\n after_A.transpose(-2, -1), self.lora_B.unsqueeze(-1), groups=sum(self.enable_lora)\n ).transpose(-2, -1)\n result += self.zero_pad(after_B) * self.scaling\n return result\n\n\nclass LoRAConv2d(nn.Conv2d, LoRALayer):\n \"\"\"\n LoRA incorporated in 2d-Convolutional Layer. Weights of convolutional layer are set to be frozen per default.\n\n Parameters\n ----------\n in_channels\n Number of channels in the input image.\n out_channels\n Number of channels produced by the convolution.\n kernel_size\n Size of the convolving kernel.\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n lora_dropout\n Adding dropout to LoRA.\n merge_weights\n Merging weights during inference to reduce latency.\n\n References\n ----------\n 1. Edward J. Hu*, Yelong Shen*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen,\n \"LoRA: Low-Rank Adaptation of Large Language Models\", 2021\n https://arxiv.org/abs/2106.09685\n 2. Code: https://github.com/microsoft/LoRA\n \"\"\"\n\n def __init__(\n self,\n in_channels: int,\n out_channels: int,\n kernel_size: int,\n r: int = 0,\n lora_alpha: int = 1,\n lora_dropout: float = 0.0,\n merge_weights: bool = True,\n **kwargs,\n ):\n nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights)\n assert type(kernel_size) is int\n # Actual trainable parameters\n if r > 0:\n self.lora_A = nn.Parameter(self.weight.new_zeros((r * kernel_size, in_channels * kernel_size)))\n self.lora_B = nn.Parameter(self.weight.new_zeros((out_channels * kernel_size, r * kernel_size)))\n self.scaling = self.lora_alpha / self.r\n # Freezing the pre-trained weight matrix\n self.weight.requires_grad = False\n self.reset_parameters()\n\n def reset_parameters(self):\n nn.Conv2d.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))\n nn.init.zeros_(self.lora_B)\n\n def train(self, mode: bool = True):\n nn.Conv2d.train(self, mode)\n if self.merge_weights and self.merged:\n # Make sure that the weights are not merged\n self.weight.data -= (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling\n self.merged = False\n\n def eval(self):\n nn.Conv2d.eval(self)\n if self.merge_weights and not self.merged:\n # Merge the weights and mark it\n self.weight.data += (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling\n self.merged = True\n\n def forward(self, x: torch.Tensor):\n if self.r > 0 and not self.merged:\n return F.conv2d(\n x,\n self.weight + (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling,\n self.bias,\n self.stride,\n self.padding,\n self.dilation,\n self.groups,\n )\n return nn.Conv2d.forward(self, x)\n\n\nclass ConvLoRALinear(nn.Linear, LoRALayer):\n \"\"\"\n Conv-LoRA incorporated in Linear Layer. Weights of linear layer are set to be frozen per default.\n\n Parameters\n ----------\n in_features\n input dimension, set to the original linear layer input dimension LoRA is replacing.\n out_features\n output dimension, set to the original linear layer output dimension LoRA is replacing.\n r\n rank r of the low-rank decomposition.\n lora_alpha\n Scaling factor. Can be simply set to same value as r as initialization is scaled already.\n lora_dropout\n Dropout probability.\n fan_in_fan_out\n Set this to True if the layer to replace stores weight like (fan_in, fan_out).\n merge_weights\n Merging weights during inference to reduce latency.\n conv_lora_expert_num\n The number of experts in MoE-Conv.\n\n References\n ----------\n 1. Zihan Zhong, Zhiqiang Tang, Tong He, Haoyang Fang, Chun Yuan,\n \"Convolution Meets LoRA: Parameter Efficient Finetuning for Segment Anything Model\", 2024\n https://arxiv.org/abs/2401.17868\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n r: int = 0,\n lora_alpha: int = 1,\n lora_dropout: float = 0.0,\n fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out)\n merge_weights: bool = False,\n conv_lora_expert_num: Optional[int] = None,\n **kwargs,\n ):\n nn.Linear.__init__(self, in_features, out_features, **kwargs)\n LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights)\n\n self.fan_in_fan_out = fan_in_fan_out\n # Actual trainable parameters\n if r > 0:\n self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features)))\n self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r)))\n self.scaling = self.lora_alpha / self.r\n # Freezing the pre-trained weight matrix\n self.weight.requires_grad = False\n\n # MoE-Conv\n topk = 1\n self.lora_moe_gating = MoEGate(M=conv_lora_expert_num, d=self.r, K=topk)\n self.lora_moe_experts = nn.ModuleList([])\n self.upsample_ratios = list(range(1, conv_lora_expert_num + 1))\n for upsample_ratio in self.upsample_ratios:\n expert = nn.Conv2d(in_channels=r, out_channels=r, kernel_size=3, stride=1, padding=1, bias=True)\n expert.bias.data.zero_()\n self.lora_moe_experts.append(nn.Sequential(expert, nn.GELU()))\n self.num_experts = conv_lora_expert_num\n self.multiply_by_gates = False\n\n self.reset_parameters()\n if fan_in_fan_out:\n self.weight.data = self.weight.data.T\n\n def reset_parameters(self):\n nn.Linear.reset_parameters(self)\n if hasattr(self, \"lora_A\"):\n # initialize A the same way as the default for nn.Linear and B to zero\n nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))\n nn.init.zeros_(self.lora_B)\n\n def T(self, w):\n return w.T if self.fan_in_fan_out else w\n\n def forward(self, x: torch.Tensor):\n result = F.linear(x, self.T(self.weight), bias=self.bias)\n if self.r > 0:\n lora_res = self.lora_dropout(x) @ self.lora_A.T\n dim = lora_res.dim()\n if dim == 3:\n B, L, C = lora_res.size()\n H = W = int(math.sqrt(L))\n lora_res = lora_res.reshape(B, H, W, C)\n else:\n H, W = lora_res.size()[1:3]\n\n # Calculate the gating values.\n lora_res = lora_res.permute(0, 3, 1, 2).contiguous()\n gates, moe_loss = self.lora_moe_gating(lora_res)\n\n # Distribute data samples to experts.\n dispatcher = SparseDispatcher(self.num_experts, gates)\n expert_inputs = dispatcher.dispatch(lora_res)\n expert_outputs = []\n for i in range(self.num_experts):\n if len(expert_inputs[i]) == 0:\n continue\n upsample_ratio = self.upsample_ratios[i]\n cur_res = expert_inputs[i]\n if upsample_ratio != 1:\n cur_res = F.interpolate(cur_res, scale_factor=upsample_ratio, mode=\"bicubic\")\n cur_res = self.lora_moe_experts[i](cur_res)\n if upsample_ratio != 1:\n cur_res = F.interpolate(cur_res, size=(int(H), int(W)), mode=\"bicubic\")\n expert_outputs.append(cur_res)\n\n # Combine data samples after processing by each expert.\n temp_lora_res = dispatcher.combine(expert_outputs, multiply_by_gates=self.multiply_by_gates)\n lora_res = lora_res + temp_lora_res\n\n lora_res = lora_res.permute(0, 2, 3, 1).contiguous()\n if dim == 3:\n lora_res = lora_res.reshape(B, L, C)\n result += (lora_res @ self.lora_B.T) * self.scaling\n\n return result, moe_loss\n\n\nclass MoEGate(nn.Module):\n def __init__(self, d, M=4, K=1, noisy_gating=True):\n \"\"\"Constructor\n Args:\n d: input channel dimensionality.\n M: the number of experts.\n K: the number of chosen experts for each forward pass.\n \"\"\"\n super(MoEGate, self).__init__()\n self.M = M\n self.k = K\n self.gap = nn.AdaptiveAvgPool2d((1, 1)) # global average pooling\n\n self.noisy_gating = noisy_gating\n\n self.w_gate = nn.Parameter(torch.zeros(d, M), requires_grad=True)\n self.w_noise = nn.Parameter(torch.zeros(d, M), requires_grad=True)\n\n self.softplus = nn.Softplus()\n self.softmax = nn.Softmax(1)\n self.register_buffer(\"mean\", torch.tensor([0.0]))\n self.register_buffer(\"std\", torch.tensor([1.0]))\n assert self.k <= self.M\n\n def forward(self, feats, loss_coef=1e-2, noise_epsilon=1e-2):\n batch_size = feats.shape[0]\n\n feats_S = self.gap(feats).view(batch_size, -1)\n\n clean_logits = feats_S @ self.w_gate\n if self.noisy_gating and self.training:\n raw_noise_stddev = feats_S @ self.w_noise\n noise_stddev = self.softplus(raw_noise_stddev) + noise_epsilon\n noisy_logits = clean_logits + (torch.randn_like(clean_logits) * noise_stddev)\n logits = noisy_logits\n else:\n logits = clean_logits\n\n top_logits, top_indices = logits.topk(min(self.k + 1, self.M), dim=1)\n top_k_logits = top_logits[:, : self.k]\n top_k_indices = top_indices[:, : self.k]\n top_k_gates = self.softmax(top_k_logits)\n zeros = torch.zeros_like(logits, requires_grad=True).float()\n gates = zeros.scatter(1, top_k_indices, top_k_gates).to(logits.dtype)\n\n if self.noisy_gating and self.k < self.M and self.training:\n load = (self._prob_in_top_k(clean_logits, noisy_logits, noise_stddev, top_logits)).sum(0)\n else:\n load = self._gates_to_load(gates)\n\n importance = gates.sum(0)\n loss = self.cv_squared(importance) + self.cv_squared(load)\n loss *= loss_coef\n\n return gates, loss\n\n def _gates_to_load(self, gates):\n \"\"\"Compute the true load per expert, given the gates.\n The load is the number of examples for which the corresponding gate is >0.\n Args:\n gates: a `Tensor` of shape [batch_size, n]\n Returns:\n a float32 `Tensor` of shape [n]\n \"\"\"\n return (gates > 0).sum(0)\n\n def cv_squared(self, x):\n \"\"\"The squared coefficient of variation of a sample.\n Useful as a loss to encourage a positive distribution to be more uniform.\n Epsilons added for numerical stability.\n Returns 0 for an empty Tensor.\n Args:\n x: a `Tensor`.\n Returns:\n a `Scalar`.\n \"\"\"\n eps = 1e-10\n\n if x.shape[0] == 1:\n return torch.tensor([0], device=x.device, dtype=x.dtype)\n return x.float().var() / (x.float().mean() ** 2 + eps)\n\n def _prob_in_top_k(self, clean_values, noisy_values, noise_stddev, noisy_top_values):\n \"\"\"Helper function to NoisyTopKGating.\n Computes the probability that value is in top k, given different random noise.\n This gives us a way of backpropagating from a loss that balances the number\n of times each expert is in the top k experts per example.\n In the case of no noise, pass in None for noise_stddev, and the result will\n not be differentiable.\n Args:\n clean_values: a `Tensor` of shape [batch, n].\n noisy_values: a `Tensor` of shape [batch, n]. Equal to clean values plus\n normally distributed noise with standard deviation noise_stddev.\n noise_stddev: a `Tensor` of shape [batch, n], or None\n noisy_top_values: a `Tensor` of shape [batch, m].\n \"values\" Output of tf.top_k(noisy_top_values, m). m >= k+1\n Returns:\n a `Tensor` of shape [batch, n].\n \"\"\"\n batch = clean_values.size(0)\n m = noisy_top_values.size(1)\n top_values_flat = noisy_top_values.flatten()\n\n threshold_positions_if_in = torch.arange(batch, device=clean_values.device) * m + self.k\n threshold_if_in = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_in), 1)\n is_in = torch.gt(noisy_values, threshold_if_in)\n threshold_positions_if_out = threshold_positions_if_in - 1\n threshold_if_out = torch.unsqueeze(torch.gather(top_values_flat, 0, threshold_positions_if_out), 1)\n # is each value currently in the top k.\n normal = Normal(self.mean, self.std)\n prob_if_in = normal.cdf((clean_values - threshold_if_in) / noise_stddev)\n prob_if_out = normal.cdf((clean_values - threshold_if_out) / noise_stddev)\n prob = torch.where(is_in, prob_if_in, prob_if_out)\n return prob\n\n\nclass SparseDispatcher(object):\n \"\"\"Helper for implementing a mixture of experts.\n The purpose of this class is to create input minibatches for the\n experts and to combine the results of the experts to form a unified\n output tensor.\n There are two functions:\n dispatch - take an input Tensor and create input Tensors for each expert.\n combine - take output Tensors from each expert and form a combined output\n Tensor. Outputs from different experts for the same batch element are\n summed together, weighted by the provided \"gates\".\n The class is initialized with a \"gates\" Tensor, which specifies which\n batch elements go to which experts, and the weights to use when combining\n the outputs. Batch element b is sent to expert e iff gates[b, e] != 0.\n The inputs and outputs are all two-dimensional [batch, depth].\n Caller is responsible for collapsing additional dimensions prior to\n calling this class and reshaping the output to the original shape.\n See common_layers.reshape_like().\n Example use:\n gates: a float32 `Tensor` with shape `[batch_size, num_experts]`\n inputs: a float32 `Tensor` with shape `[batch_size, input_size]`\n experts: a list of length `num_experts` containing sub-networks.\n dispatcher = SparseDispatcher(num_experts, gates)\n expert_inputs = dispatcher.dispatch(inputs)\n expert_outputs = [experts[i](expert_inputs[i]) for i in range(num_experts)]\n outputs = dispatcher.combine(expert_outputs)\n The preceding code sets the output for a particular example b to:\n output[b] = Sum_i(gates[b, i] * experts[i](inputs[b]))\n This class takes advantage of sparsity in the gate matrix by including in the\n `Tensor`s for expert i only the batch elements for which `gates[b, i] > 0`.\n\n References\n ----------\n 1. Noam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton, Jeff Dean,\n \"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer\", 2017\n https://arxiv.org/abs/1701.06538\n 2. Code: https://github.com/davidmrau/mixture-of-experts/blob/master/moe.py\n \"\"\"\n\n def __init__(self, num_experts, gates):\n \"\"\"Create a SparseDispatcher.\"\"\"\n self._gates = gates\n self._num_experts = num_experts\n # sort experts\n sorted_experts, index_sorted_experts = torch.nonzero(gates).sort(0)\n # drop indices\n _, self._expert_index = sorted_experts.split(1, dim=1)\n # get according batch index for each expert\n self._batch_index = torch.nonzero(gates)[index_sorted_experts[:, 1], 0]\n # calculate num samples that each expert gets\n self._part_sizes = (gates > 0).sum(0).tolist()\n # expand gates to match with self._batch_index\n gates_exp = gates[self._batch_index.flatten()]\n self._nonzero_gates = torch.gather(gates_exp, 1, self._expert_index)\n\n def dispatch(self, inp):\n \"\"\"Create one input Tensor for each expert.\n The `Tensor` for a expert `i` contains the slices of `inp` corresponding\n to the batch elements `b` where `gates[b, i] > 0`.\n Args:\n inp: a `Tensor` of shape \"[batch_size, ]`\n Returns:\n a list of `num_experts` `Tensor`s with shapes\n `[expert_batch_size_i, ]`.\n \"\"\"\n\n # assigns samples to experts whose gate is nonzero\n\n # expand according to batch index so we can just split by _part_sizes\n inp_exp = inp[self._batch_index].squeeze(1) # [bs * num_of chosen experts, dim]\n return torch.split(inp_exp, self._part_sizes, dim=0)\n\n def combine(self, expert_out, multiply_by_gates=True):\n \"\"\"Sum together the expert output, weighted by the gates.\n The slice corresponding to a particular batch element `b` is computed\n as the sum over all experts `i` of the expert output, weighted by the\n corresponding gate values. If `multiply_by_gates` is set to False, the\n gate values are ignored.\n Args:\n expert_out: a list of `num_experts` `Tensor`s, each with shape\n `[expert_batch_size_i, ]`.\n multiply_by_gates: a boolean\n Returns:\n a `Tensor` with shape `[batch_size, ]`.\n \"\"\"\n # apply exp to expert outputs, so we are not longer in log space\n stitched = torch.cat(expert_out, 0).exp()\n\n if multiply_by_gates:\n # stitched = stitched.mul(self._nonzero_gates)\n stitched = stitched.mul(self._nonzero_gates.unsqueeze(-1).unsqueeze(-1))\n # zeros = torch.zeros(self._gates.size(0), expert_out[-1].size(1), requires_grad=True, device=stitched.device)\n zeros = torch.zeros(\n self._gates.size(0),\n expert_out[-1].size()[1],\n expert_out[-1].size()[2],\n expert_out[-1].size()[3],\n requires_grad=True,\n device=stitched.device,\n )\n # combine samples that have been processed by the same k experts\n combined = zeros.index_add(0, self._batch_index, stitched.float())\n # add eps to all zero values in order to avoid nans when going back to log space\n combined[combined == 0] = np.finfo(float).eps\n # back to log space\n return combined.log()\n\n def expert_to_gates(self):\n \"\"\"Gate values corresponding to the examples in the per-expert `Tensor`s.\n Returns:\n a list of `num_experts` one-dimensional `Tensor`s with type `tf.float32`\n and shapes `[expert_batch_size_i]`\n \"\"\"\n # split nonzero gates for each expert\n return torch.split(self._nonzero_gates, self._part_sizes, dim=0)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/augmenter.py",
"content": "import logging\n\nimport torch\nimport torch.nn as nn\nfrom omegaconf import DictConfig\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\n\nfrom .mlp import Unit\n\nlogger = logging.getLogger(__name__)\n\n\nclass VAETransformer(nn.Module):\n def __init__(self, config: DictConfig, in_feautres: int, n_modality: int) -> None:\n super().__init__()\n self.config = config\n self.emb_d = in_feautres\n self.n_modality = n_modality\n logger.debug(f\" VAE Transformer # features {n_modality}, dim {self.emb_d}\")\n\n # encoder\n encoder_layers = TransformerEncoderLayer(self.emb_d, config.n_head, config.tran_hidden, norm_first=True)\n self.transformer_encoder = TransformerEncoder(encoder_layers, config.n_layer)\n\n # encoder linear z\n self.encoder_fc_z_mu = nn.Linear(self.emb_d, self.config.z_dim)\n self.encoder_fc_z_logvar = nn.Linear(self.emb_d, self.config.z_dim)\n\n # decoder linezr z\n self.decoder_fc = nn.Linear(self.config.z_dim, self.emb_d)\n\n # decoder\n decoder_layers = TransformerEncoderLayer(self.emb_d, config.n_head, config.tran_hidden, norm_first=True)\n self.transformer_decoder = TransformerEncoder(decoder_layers, config.n_layer)\n\n self.last_layer = nn.Linear(self.emb_d, self.emb_d)\n\n self.gating = nn.Identity()\n self.init_parameters()\n\n def init_parameters(self):\n self.last_layer.weight.data.zero_()\n self.last_layer.bias.data.zero_()\n\n def reparameterize(self, mu, logvar):\n std = torch.exp(0.5 * logvar)\n eps = torch.randn_like(std)\n return mu + eps * std\n\n def forward(self, X):\n input = X.reshape(-1, self.n_modality, self.emb_d) # [B, # modality, emb dim] torch.Size([8, 3, 1024])\n\n hidden = self.transformer_encoder(input)\n\n z_mu, z_logvar = self.encoder_fc_z_mu(hidden), self.encoder_fc_z_logvar(hidden)\n\n z = self.reparameterize(z_mu, z_logvar)\n\n hidden = self.decoder_fc(z)\n\n noise = self.gating(self.last_layer(self.transformer_decoder(hidden)[:, : self.n_modality, :]))\n recon_x = X.reshape(-1, self.n_modality, self.emb_d) + noise\n\n return recon_x.reshape(len(X), -1), z_mu, z_logvar\n\n\nclass MlpVAE(nn.Module):\n def __init__(self, input_dim, hidden_dim, z_dim=16) -> None:\n super().__init__()\n self.input_dim = input_dim\n self.z_dim = z_dim\n self.hidden_dim = hidden_dim\n\n # Encoder P(Z|X)\n encoder_layers = []\n dims = [input_dim] + hidden_dim\n for i in range(len(dims) - 1):\n encoder_layers.append(\n Unit(\n normalization=\"layer_norm\",\n in_features=dims[i],\n out_features=dims[i + 1],\n activation=\"relu\",\n dropout=0.5,\n )\n )\n self.encoder = nn.Sequential(*encoder_layers)\n\n self.encoder_fc_z_mu = nn.Linear(self.hidden_dim[-1], self.z_dim)\n self.encoder_fc_z_logvar = nn.Linear(self.hidden_dim[-1], self.z_dim)\n\n # Decoder P(X|Z)\n decoder_layers = []\n dims = [input_dim] + hidden_dim + [z_dim]\n\n for i in range(len(dims) - 1, 0, -1):\n decoder_layers.append(\n Unit(\n normalization=\"layer_norm\",\n in_features=dims[i],\n out_features=dims[i - 1],\n activation=\"relu\",\n dropout=0.5,\n )\n )\n self.decoder = nn.Sequential(*decoder_layers)\n\n self.init_parameters()\n\n def init_parameters(self):\n self.decoder[-1].fc.weight.data.zero_()\n self.decoder[-1].fc.bias.data.zero_()\n\n def reparameterize(self, mu, logvar):\n std = torch.exp(0.5 * logvar)\n eps = torch.randn_like(std)\n return mu + eps * std\n\n def forward(self, x):\n hidden = self.encoder(x)\n z_mu, z_logvar = self.encoder_fc_z_mu(hidden), self.encoder_fc_z_logvar(hidden)\n z = self.reparameterize(z_mu, z_logvar)\n\n noise_x = self.decoder(z)\n recon_x = x + noise_x\n return recon_x, z_mu, z_logvar\n\n\nclass Augmenter(nn.Module):\n def __init__(\n self,\n arch_type: str,\n input_dim: int,\n z_dim: int,\n num_layers: int,\n adv_weight: float,\n ) -> None:\n super().__init__()\n logger.debug(\"Initializing Augmenter\")\n self.arch_type = arch_type\n self.input_dim = input_dim\n self.z_dim = z_dim\n self.num_layers = num_layers\n self.adv_weight = adv_weight\n logger.debug(f\"augmenter arch_type: {self.arch_type}\")\n logger.debug(f\"augmenter input_dim: {self.input_dim}\")\n logger.debug(f\"augmenter z_dim: {self.z_dim}\")\n logger.debug(f\"augmenter num_layers: {self.num_layers}\")\n logger.debug(f\"augmenter adv_weight: {self.adv_weight}\")\n if self.arch_type == \"mlp_vae\":\n step = int((self.input_dim - self.z_dim) / (self.num_layers + 1))\n hidden = [*range(self.input_dim - step, self.z_dim + step, -step)]\n self.vae = MlpVAE(input_dim=self.input_dim, hidden_dim=hidden, z_dim=self.z_dim)\n else:\n raise ValueError(f\"Unknown arch_type: {self.arch_type}\")\n\n self.name_to_id = self.get_layer_ids()\n\n def forward(self, x):\n return self.vae(x)\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n All layers have the same id 0 since there is no pre-trained models used here.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/categorical_mlp.py",
"content": "import logging\nfrom typing import Dict, Optional\n\nimport torch\nfrom torch import nn\n\nfrom ..constants import CATEGORICAL, FEATURES, LABEL, LOGITS\nfrom .mlp import MLP\nfrom .utils import init_weights\n\nlogger = logging.getLogger(__name__)\n\n\nclass CategoricalMLP(nn.Module):\n \"\"\"\n MLP for categorical input. The input dimension is automatically computed based on\n the number of categories in each categorical column.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n num_categories: Dict,\n out_features: Optional[int] = None,\n num_layers: Optional[int] = 1,\n activation: Optional[str] = \"gelu\",\n dropout: Optional[float] = 0.5,\n normalization: Optional[str] = \"layer_norm\",\n num_classes: Optional[int] = 0,\n ):\n \"\"\"\n Parameters\n ----------\n prefix\n The model prefix.\n num_categories\n A list of integers. Each one is the number of categories in one categorical column.\n out_features\n Dimension of output features.\n num_layers\n Number of MLP layers.\n activation\n Name of activation function.\n dropout\n Dropout probability.\n normalization\n Name of normalization function.\n num_classes\n Number of classes. 1 for a regression task.\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {prefix} (CategoricalMLP)\")\n self.out_features = out_features\n max_embedding_dim = 100\n embed_exponent = 0.56\n size_factor = 1.0\n self.column_embeddings = nn.ModuleList()\n self.column_mlps = nn.ModuleList()\n assert isinstance(num_categories, dict)\n self.num_categories = num_categories\n\n for num_categories_per_col in num_categories.values():\n embedding_dim_per_col = int(\n size_factor * max(2, min(max_embedding_dim, 1.6 * num_categories_per_col**embed_exponent))\n )\n self.column_embeddings.append(\n nn.Embedding(\n num_embeddings=num_categories_per_col,\n embedding_dim=embedding_dim_per_col,\n )\n )\n\n self.column_mlps.append(\n MLP(\n in_features=embedding_dim_per_col,\n hidden_features=out_features,\n out_features=out_features,\n num_layers=num_layers,\n activation=activation,\n dropout=dropout,\n normalization=normalization,\n )\n )\n\n self.aggregator_mlp = MLP(\n in_features=out_features * len(num_categories),\n hidden_features=out_features * len(num_categories),\n out_features=out_features,\n num_layers=num_layers,\n activation=activation,\n dropout=dropout,\n normalization=normalization,\n )\n\n self.head = nn.Linear(out_features, num_classes) if num_classes > 0 else nn.Identity()\n # init weights\n self.apply(init_weights)\n\n self.prefix = prefix\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def categorical_key(self):\n return f\"{self.prefix}_{CATEGORICAL}\"\n\n @property\n def input_keys(self):\n return [self.categorical_key]\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n assert len(batch[self.categorical_key]) == len(self.column_embeddings)\n features = []\n for categorical_id, embed, mlp in zip(batch[self.categorical_key], self.column_embeddings, self.column_mlps):\n features.append(mlp(embed(categorical_id)))\n cat_features = torch.cat(features, dim=1)\n features = self.aggregator_mlp(cat_features)\n logits = self.head(features)\n return {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n }\n }\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n All layers have the same id 0 since there is no pre-trained models used here.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/clip.py",
"content": "import logging\nfrom typing import Optional\n\nimport torch\nfrom torch import nn\n\nfrom ..constants import (\n COLUMN,\n COLUMN_FEATURES,\n FEATURES,\n IMAGE,\n IMAGE_VALID_NUM,\n LABEL,\n LOGIT_SCALE,\n LOGITS,\n MASKS,\n TEXT_TOKEN_IDS,\n TEXT_VALID_LENGTH,\n)\nfrom .utils import (\n assign_layer_ids,\n get_column_features,\n get_hf_config_and_model,\n get_image_size_mean_std,\n get_pretrained_tokenizer,\n get_text_segment_num,\n get_text_token_max_len,\n init_weights,\n replace_missing_images_with_learnable,\n)\n\nlogger = logging.getLogger(__name__)\n\n\nclass CLIPForImageText(nn.Module):\n \"\"\"\n Support the CLIP model.\n Refer to https://huggingface.co/docs/transformers/model_doc/clip\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n num_classes: Optional[int] = None,\n pretrained: Optional[bool] = True,\n tokenizer_name: Optional[str] = \"clip\",\n has_image: Optional[bool] = True,\n has_text: Optional[bool] = True,\n image_size: Optional[int] = None,\n image_norm: Optional[str] = None,\n image_chan_num: Optional[int] = 3,\n use_learnable_image: Optional[bool] = False,\n max_text_len: Optional[int] = None,\n text_segment_num: Optional[int] = 1,\n is_matching: Optional[bool] = False,\n ):\n \"\"\"\n Load the pretrained CLIP from huggingface transformers.\n\n Parameters\n ----------\n prefix\n The model prefix.\n checkpoint_name\n Name of the checkpoint.\n num_classes\n The number of classes. 1 for a regression task.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n tokenizer_name\n Name of the huggingface tokenizer type.\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {prefix} (CLIPForImageText)\")\n logger.debug(f\"model checkpoint: {checkpoint_name}\")\n self.checkpoint_name = checkpoint_name\n self.num_classes = num_classes\n if is_matching: # init both image and text attributes for matching\n has_image, has_text = True, True\n self.has_image = has_image\n self.has_text = has_text\n\n self.config, self.model = get_hf_config_and_model(checkpoint_name=checkpoint_name, pretrained=pretrained)\n\n if not self.has_image:\n self.model.vision_model = None\n self.model.visual_projection = None\n\n if not self.has_text:\n self.model.text_model = None\n self.model.text_projection = None\n\n self.out_features = self.model.config.projection_dim\n\n self.head = nn.Linear(self.out_features, num_classes) if num_classes else nn.Identity()\n self.head.apply(init_weights)\n\n self.prefix = prefix\n if has_image:\n self.image_size, self.image_mean, self.image_std = get_image_size_mean_std(\n model_name=self.prefix,\n config=self.model.vision_model.config,\n provided_size=image_size,\n provided_norm_type=image_norm,\n support_variable_input_size=False,\n )\n self.use_learnable_image = use_learnable_image\n if self.use_learnable_image:\n self.learnable_image = nn.Parameter(torch.zeros(image_chan_num, self.image_size, self.image_size))\n logger.debug(\"will use a learnable image to replace missing ones\")\n if has_text:\n self.tokenizer_name = tokenizer_name\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=self.checkpoint_name,\n )\n self.max_text_len = get_text_token_max_len(\n provided_max_len=max_text_len,\n config=self.model.text_model.config,\n tokenizer=self.tokenizer,\n checkpoint_name=self.checkpoint_name,\n )\n self.text_segment_num = get_text_segment_num(\n config=self.model.text_model.config,\n provided_segment_num=text_segment_num,\n checkpoint_name=self.checkpoint_name,\n )\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def text_token_ids_key(self):\n return f\"{self.prefix}_{TEXT_TOKEN_IDS}\"\n\n @property\n def text_valid_length_key(self):\n return f\"{self.prefix}_{TEXT_VALID_LENGTH}\"\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def text_column_prefix(self):\n return f\"{self.text_token_ids_key}_{COLUMN}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def text_feature_dim(self):\n return self.model.config.text_config.hidden_size\n\n @property\n def image_feature_dim(self):\n return self.model.config.vision_config.hidden_size\n\n @property\n def input_keys(self):\n ret = []\n if self.has_image:\n ret.extend([self.image_key, self.image_valid_num_key])\n if self.has_text:\n ret.extend([self.text_token_ids_key, self.text_valid_length_key])\n return ret\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n has_image = self.has_image and self.image_key in batch\n has_text = self.has_text and self.text_token_ids_key in batch\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n\n if has_image:\n images = batch[self.image_key]\n image_valid_num = batch[self.image_valid_num_key]\n assert images.dim() == 5\n b, n, c, h, w = images.shape\n steps = torch.arange(0, n).type_as(image_valid_num)\n image_masks = steps.reshape((1, -1)) < image_valid_num.reshape((-1, 1)) # (b, n)\n if self.use_learnable_image:\n images = replace_missing_images_with_learnable(\n images=images,\n image_masks=image_masks,\n learnable_image=self.learnable_image,\n )\n vision_outputs = self.model.vision_model(\n pixel_values=images.reshape((b * n, c, h, w)),\n output_attentions=True,\n output_hidden_states=True,\n )\n image_features = self.model.visual_projection(vision_outputs.pooler_output)\n image_features = image_features.reshape((b, n, -1)) # (b, n, num_features)\n if not self.use_learnable_image:\n image_features = image_features * image_masks[:, :, None].type_as(image_features)\n\n # normalized features\n image_features = image_features / torch.clamp(image_features.norm(dim=-1, keepdim=True), min=1e-6)\n\n # collect image features by image column names\n image_column_features, image_column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.image_column_prefix,\n features=image_features,\n valid_lengths=image_valid_num,\n )\n ret[COLUMN_FEATURES][FEATURES].update(image_column_features)\n ret[COLUMN_FEATURES][MASKS].update(image_column_feature_masks)\n\n image_features = image_features.mean(dim=1) # (b, num_features)\n ret[FEATURES] = image_features\n\n if has_text:\n text_token_ids = batch[self.text_token_ids_key]\n text_valid_length = batch[self.text_valid_length_key]\n steps = torch.arange(0, text_token_ids.shape[1]).type_as(text_valid_length)\n text_masks = (steps.reshape((1, -1)) < text_valid_length.reshape((-1, 1))).type_as(text_token_ids)\n assert torch.equal(text_valid_length, text_masks.sum(dim=-1))\n\n text_outputs = self.model.text_model(\n input_ids=text_token_ids,\n attention_mask=text_masks,\n output_attentions=True,\n output_hidden_states=True,\n )\n text_features = self.model.text_projection(text_outputs.pooler_output) # (b, num_features)\n\n # normalized features\n text_features = text_features / text_features.norm(dim=-1, keepdim=True)\n\n # collect text features by text column names\n text_column_features, text_column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.text_column_prefix,\n features=self.model.text_projection(text_outputs.last_hidden_state),\n valid_lengths=text_valid_length,\n cls_feature=text_features,\n )\n ret[COLUMN_FEATURES][FEATURES].update(text_column_features)\n ret[COLUMN_FEATURES][MASKS].update(text_column_feature_masks)\n ret[FEATURES] = text_features\n\n if self.num_classes:\n if has_image and has_text:\n features = image_features + text_features\n logits = self.head(features)\n ret[FEATURES] = features\n elif has_image:\n logits = self.head(image_features)\n elif has_text:\n logits = self.head(text_features)\n else:\n raise RuntimeError(\"Neither image or text are used. Must have at least one.\")\n ret[LOGITS] = logits\n else:\n ret[LOGIT_SCALE] = self.model.logit_scale.exp()\n if has_image and has_text:\n # cosine similarity as logits\n logits = torch.sum(image_features * text_features, dim=-1)\n ret[LOGITS] = logits\n\n return {self.prefix: ret}\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefixes = [\"model.text_model\", \"model.vision_model\", \"model\"]\n # later model prefixes can't starts with the early ones\n for i, model_pre in enumerate(model_prefixes):\n for model_pre2 in model_prefixes[i + 1 :]:\n if model_pre2.startswith(model_pre):\n raise ValueError(\n f\"{model_pre} is a substring of {model_pre2}. Need to swap them in {model_prefixes}.\"\n )\n\n pre_encoder_patterns = (\"embeddings\", \"pre\")\n post_encoder_patterns = (\"head\", \"final\", \"post\", \"logit\", \"project\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id = {}\n for per_prefix in model_prefixes:\n per_model_name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=per_prefix,\n )\n name_to_id.update(per_model_name_to_id)\n\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/custom_hf_models/modeling_sam_for_conv_lora.py",
"content": "# coding=utf-8\n# Copyright 2023 The Meta AI Authors and The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"PyTorch SAM model.\"\"\"\n\nimport collections\nimport math\nfrom dataclasses import dataclass\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport torch.utils.checkpoint\nfrom torch import Tensor, nn\nfrom transformers.activations import ACT2FN\nfrom transformers.modeling_outputs import BaseModelOutput\nfrom transformers.modeling_utils import PreTrainedModel\nfrom transformers.models.sam.configuration_sam import (\n SamConfig,\n SamMaskDecoderConfig,\n SamPromptEncoderConfig,\n SamVisionConfig,\n)\nfrom transformers.utils import ModelOutput, add_start_docstrings, add_start_docstrings_to_model_forward, logging\n\nlogger = logging.get_logger(__name__)\n\n\nSAM_PRETRAINED_MODEL_ARCHIVE_LIST = [\n \"facebook/sam-vit-huge\",\n \"facebook/sam-vit-large\",\n \"facebook/sam-vit-base\",\n # See all SAM models at https://huggingface.co/models?filter=sam\n]\n\n\n@dataclass\nclass SamVisionEncoderOutput(ModelOutput):\n \"\"\"\n Base class for sam vision model's outputs that also contains image embeddings obtained by applying the projection\n layer to the pooler_output.\n\n Args:\n image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):\n The image embeddings obtained by applying the projection layer to the pooler_output.\n last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):\n Sequence of hidden-states at the output of the last layer of the model.\n hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):\n Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +\n one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.\n\n Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.\n attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):\n Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n \"\"\"\n\n image_embeds: Optional[torch.FloatTensor] = None\n last_hidden_state: torch.FloatTensor = None\n hidden_states: Optional[Tuple[torch.FloatTensor]] = None\n attentions: Optional[Tuple[torch.FloatTensor]] = None\n moe_loss: Optional[torch.FloatTensor] = None\n\n\n@dataclass\nclass SamImageSegmentationOutput(ModelOutput):\n \"\"\"\n Base class for Segment-Anything model's output\n\n Args:\n iou_scores (`torch.FloatTensor` of shape `(batch_size, num_masks)`):\n The iou scores of the predicted masks.\n pred_masks (`torch.FloatTensor` of shape `(batch_size, num_masks, height, width)`):\n The predicted low resolutions masks. Needs to be post-processed by the processor\n vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):\n Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +\n one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.\n\n Hidden-states of the vision model at the output of each layer plus the optional initial embedding outputs.\n vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):\n Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n mask_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):\n Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,\n sequence_length)`.\n\n Attentions weights after the attention softmax, used to compute the weighted average in the self-attention\n heads.\n \"\"\"\n\n iou_scores: torch.FloatTensor = None\n pred_masks: torch.FloatTensor = None\n vision_hidden_states: Optional[Tuple[torch.FloatTensor]] = None\n vision_attentions: Optional[Tuple[torch.FloatTensor]] = None\n mask_decoder_attentions: Optional[Tuple[torch.FloatTensor]] = None\n vision_moe_loss: Optional[torch.FloatTensor] = None\n\n\nclass SamPatchEmbeddings(nn.Module):\n \"\"\"\n This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial\n `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a\n Transformer.\n \"\"\"\n\n def __init__(self, config):\n super().__init__()\n image_size, patch_size = config.image_size, config.patch_size\n num_channels, hidden_size = config.num_channels, config.hidden_size\n image_size = image_size if isinstance(image_size, collections.abc.Iterable) else (image_size, image_size)\n patch_size = patch_size if isinstance(patch_size, collections.abc.Iterable) else (patch_size, patch_size)\n num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])\n self.image_size = image_size\n self.patch_size = patch_size\n self.num_channels = num_channels\n self.num_patches = num_patches\n\n self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size)\n\n def forward(self, pixel_values):\n batch_size, num_channels, height, width = pixel_values.shape\n if num_channels != self.num_channels:\n raise ValueError(\n \"Make sure that the channel dimension of the pixel values match with the one set in the configuration.\"\n )\n if height != self.image_size[0] or width != self.image_size[1]:\n raise ValueError(\n f\"Input image size ({height}*{width}) doesn't match model ({self.image_size[0]}*{self.image_size[1]}).\"\n )\n embeddings = self.projection(pixel_values).permute(0, 2, 3, 1)\n return embeddings\n\n\nclass SamMLPBlock(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.lin1 = nn.Linear(config.hidden_size, config.mlp_dim)\n self.lin2 = nn.Linear(config.mlp_dim, config.hidden_size)\n self.act = ACT2FN[config.hidden_act]\n\n def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:\n hidden_states = self.lin1(hidden_states)\n hidden_states = self.act(hidden_states)\n hidden_states = self.lin2(hidden_states)\n return hidden_states\n\n\n# Copied from transformers.models.convnext.modeling_convnext.ConvNextLayerNorm with ConvNext->Sam\nclass SamLayerNorm(nn.Module):\n r\"\"\"LayerNorm that supports two data formats: channels_last (default) or channels_first.\n The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height,\n width, channels) while channels_first corresponds to inputs with shape (batch_size, channels, height, width).\n \"\"\"\n\n def __init__(self, normalized_shape, eps=1e-6, data_format=\"channels_last\"):\n super().__init__()\n self.weight = nn.Parameter(torch.ones(normalized_shape))\n self.bias = nn.Parameter(torch.zeros(normalized_shape))\n self.eps = eps\n self.data_format = data_format\n if self.data_format not in [\"channels_last\", \"channels_first\"]:\n raise NotImplementedError(f\"Unsupported data format: {self.data_format}\")\n self.normalized_shape = (normalized_shape,)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n if self.data_format == \"channels_last\":\n x = torch.nn.functional.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)\n elif self.data_format == \"channels_first\":\n input_dtype = x.dtype\n x = x.float()\n u = x.mean(1, keepdim=True)\n s = (x - u).pow(2).mean(1, keepdim=True)\n x = (x - u) / torch.sqrt(s + self.eps)\n x = x.to(dtype=input_dtype)\n x = self.weight[:, None, None] * x + self.bias[:, None, None]\n return x\n\n\nclass SamAttention(nn.Module):\n \"\"\"\n SAM's attention layer that allows for downscaling the size of the embedding after projection to queries, keys, and\n values.\n \"\"\"\n\n def __init__(self, config, downsample_rate=None):\n super().__init__()\n self.hidden_size = config.hidden_size\n\n downsample_rate = config.attention_downsample_rate if downsample_rate is None else downsample_rate\n\n self.internal_dim = config.hidden_size // downsample_rate\n self.num_attention_heads = config.num_attention_heads\n if self.internal_dim % config.num_attention_heads != 0:\n raise ValueError(\"num_attention_heads must divide hidden_size.\")\n\n self.q_proj = nn.Linear(self.hidden_size, self.internal_dim)\n self.k_proj = nn.Linear(self.hidden_size, self.internal_dim)\n self.v_proj = nn.Linear(self.hidden_size, self.internal_dim)\n self.out_proj = nn.Linear(self.internal_dim, self.hidden_size)\n\n def _separate_heads(self, hidden_states: Tensor, num_attention_heads: int) -> Tensor:\n batch, point_batch_size, n_tokens, channel = hidden_states.shape\n c_per_head = channel // num_attention_heads\n hidden_states = hidden_states.reshape(batch * point_batch_size, n_tokens, num_attention_heads, c_per_head)\n return hidden_states.transpose(1, 2)\n\n def _recombine_heads(self, hidden_states: Tensor, point_batch_size: int) -> Tensor:\n batch, n_heads, n_tokens, c_per_head = hidden_states.shape\n hidden_states = hidden_states.transpose(1, 2)\n return hidden_states.reshape(batch // point_batch_size, point_batch_size, n_tokens, n_heads * c_per_head)\n\n def forward(self, query: Tensor, key: Tensor, value: Tensor, attention_similarity: Tensor = None) -> Tensor:\n # Input projections\n query = self.q_proj(query)\n key = self.k_proj(key)\n value = self.v_proj(value)\n\n point_batch_size = query.shape[1]\n # Separate into heads\n query = self._separate_heads(query, self.num_attention_heads)\n key = self._separate_heads(key, self.num_attention_heads)\n value = self._separate_heads(value, self.num_attention_heads)\n\n # SamAttention\n _, _, _, c_per_head = query.shape\n attn = query @ key.permute(0, 1, 3, 2) # batch_size * point_batch_size x N_heads x N_tokens x N_tokens\n attn = attn / math.sqrt(c_per_head)\n attn = torch.softmax(attn, dim=-1)\n\n if attention_similarity is not None:\n attn = attn + attention_similarity\n attn = torch.softmax(attn, dim=-1)\n\n # Get output\n out = attn @ value\n out = self._recombine_heads(out, point_batch_size)\n out = self.out_proj(out)\n\n return out\n\n\nclass SamTwoWayAttentionBlock(nn.Module):\n def __init__(self, config, attention_downsample_rate: int = 2, skip_first_layer_pe: bool = False):\n \"\"\"\n A transformer block with four layers:\n (1) self-attention of sparse inputs (2) cross attention of sparse inputs -> dense inputs (3) mlp block on\n sparse inputs (4) cross attention of dense inputs -> sparse inputs\n\n Arguments:\n config (`SamMaskDecoderConfig`):\n The configuration file used to instantiate the block\n attention_downsample_rate (*optionalk*, int, defaults to 2):\n The downsample ratio of the block used to reduce the inner dim of the attention.\n skip_first_layer_pe (*optional*, bool, defaults to `False`):\n Whether or not to skip the addition of the query_point_embedding on the first layer.\n \"\"\"\n super().__init__()\n\n self.hidden_size = config.hidden_size\n self.layer_norm_eps = config.layer_norm_eps\n\n self.self_attn = SamAttention(config, downsample_rate=1)\n self.layer_norm1 = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps)\n\n self.cross_attn_token_to_image = SamAttention(config, downsample_rate=attention_downsample_rate)\n self.layer_norm2 = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps)\n\n self.mlp = SamMLPBlock(config)\n self.layer_norm3 = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps)\n\n self.layer_norm4 = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps)\n self.cross_attn_image_to_token = SamAttention(config, downsample_rate=attention_downsample_rate)\n\n self.skip_first_layer_pe = skip_first_layer_pe\n\n def forward(\n self,\n queries: Tensor,\n keys: Tensor,\n query_point_embedding: Tensor,\n key_point_embedding: Tensor,\n attention_similarity: Tensor,\n output_attentions: bool = False,\n ):\n # Self attention block\n if self.skip_first_layer_pe:\n queries = self.self_attn(query=queries, key=queries, value=queries)\n else:\n query = queries + query_point_embedding\n attn_out = self.self_attn(query=query, key=query, value=queries)\n queries = queries + attn_out\n queries = self.layer_norm1(queries)\n\n # Cross attention block, tokens attending to image embedding\n query = queries + query_point_embedding\n key = keys + key_point_embedding\n\n attn_out = self.cross_attn_token_to_image(\n query=query, key=key, value=keys, attention_similarity=attention_similarity\n )\n queries = queries + attn_out\n\n queries = self.layer_norm2(queries)\n\n # MLP block\n mlp_out = self.mlp(queries)\n queries = queries + mlp_out\n queries = self.layer_norm3(queries)\n\n # Cross attention block, image embedding attending to tokens\n query = queries + query_point_embedding\n key = keys + key_point_embedding\n\n attn_out = self.cross_attn_image_to_token(query=key, key=query, value=queries)\n keys = keys + attn_out\n\n keys = self.layer_norm4(keys)\n\n outputs = (queries, keys)\n\n if output_attentions:\n outputs = outputs + (attn_out,)\n else:\n outputs = outputs + (None,)\n\n return outputs\n\n\nclass SamTwoWayTransformer(nn.Module):\n def __init__(self, config: SamMaskDecoderConfig):\n super().__init__()\n self.config = config\n\n self.num_hidden_layers = config.num_hidden_layers\n self.layers = nn.ModuleList()\n\n for i in range(self.num_hidden_layers):\n self.layers.append(SamTwoWayAttentionBlock(config, skip_first_layer_pe=(i == 0)))\n\n self.final_attn_token_to_image = SamAttention(config)\n self.layer_norm_final_attn = nn.LayerNorm(config.hidden_size)\n\n def forward(\n self,\n point_embeddings: Tensor,\n image_embeddings: Tensor,\n image_positional_embeddings: Tensor,\n attention_similarity: Tensor,\n target_embedding=None,\n output_attentions: Optional[bool] = None,\n output_hidden_states: Optional[bool] = None,\n return_dict: Optional[bool] = None,\n ) -> Union[Tuple, BaseModelOutput]:\n output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n output_hidden_states = (\n output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n )\n return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n all_attentions = ()\n\n if image_embeddings is None:\n raise ValueError(\"You have to specify an image_embedding\")\n\n image_embeddings = image_embeddings.flatten(2).permute(0, 2, 1).unsqueeze(1)\n image_positional_embeddings = image_positional_embeddings.flatten(2).permute(0, 2, 1).unsqueeze(1)\n\n # Prepare queries\n queries = point_embeddings\n keys = image_embeddings\n\n # Apply transformer blocks and final layernorm\n for layer in self.layers:\n if target_embedding is not None:\n queries += target_embedding\n\n queries, keys, attention_outputs = layer(\n queries=queries,\n keys=keys,\n query_point_embedding=point_embeddings,\n key_point_embedding=image_positional_embeddings,\n attention_similarity=attention_similarity,\n output_attentions=output_attentions,\n )\n\n if output_attentions:\n all_attentions = all_attentions + (attention_outputs,)\n\n # Apply the final attenion layer from the points to the image\n query = queries + point_embeddings\n key = keys + image_positional_embeddings\n\n attn_out = self.final_attn_token_to_image(query=query, key=key, value=keys)\n\n queries = queries + attn_out\n queries = self.layer_norm_final_attn(queries)\n return queries, keys, all_attentions\n\n\nclass SamFeedForward(nn.Module):\n def __init__(\n self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int, sigmoid_output: bool = False\n ):\n super().__init__()\n self.num_layers = num_layers\n self.activation = nn.ReLU()\n self.proj_in = nn.Linear(input_dim, hidden_dim)\n self.proj_out = nn.Linear(hidden_dim, output_dim)\n self.layers = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(num_layers - 2)])\n self.sigmoid_output = sigmoid_output\n\n def forward(self, hidden_states):\n hidden_states = self.proj_in(hidden_states)\n hidden_states = self.activation(hidden_states)\n for layer in self.layers:\n hidden_states = self.activation(layer(hidden_states))\n\n hidden_states = self.proj_out(hidden_states)\n if self.sigmoid_output:\n hidden_states = F.sigmoid(hidden_states)\n return hidden_states\n\n\nclass SamMaskDecoder(nn.Module):\n def __init__(self, config: SamMaskDecoderConfig):\n super().__init__()\n\n self.hidden_size = config.hidden_size\n\n self.num_multimask_outputs = config.num_multimask_outputs\n self.num_mask_tokens = config.num_multimask_outputs + 1\n\n self.iou_token = nn.Embedding(1, self.hidden_size)\n self.mask_tokens = nn.Embedding(self.num_mask_tokens, self.hidden_size)\n\n self.transformer = SamTwoWayTransformer(config)\n\n # should we create a new class for this?\n self.upscale_conv1 = nn.ConvTranspose2d(self.hidden_size, self.hidden_size // 4, kernel_size=2, stride=2)\n self.upscale_conv2 = nn.ConvTranspose2d(self.hidden_size // 4, self.hidden_size // 8, kernel_size=2, stride=2)\n self.upscale_layer_norm = SamLayerNorm(self.hidden_size // 4, data_format=\"channels_first\")\n self.activation = nn.GELU()\n\n mlps_list = []\n for _ in range(self.num_mask_tokens):\n mlps_list += [SamFeedForward(self.hidden_size, self.hidden_size, self.hidden_size // 8, 3)]\n self.output_hypernetworks_mlps = nn.ModuleList(mlps_list)\n\n self.iou_prediction_head = SamFeedForward(\n self.hidden_size, config.iou_head_hidden_dim, self.num_mask_tokens, config.iou_head_depth\n )\n\n def forward(\n self,\n image_embeddings: torch.Tensor,\n image_positional_embeddings: torch.Tensor,\n sparse_prompt_embeddings: torch.Tensor,\n dense_prompt_embeddings: torch.Tensor,\n multimask_output: bool,\n output_attentions: Optional[bool] = None,\n attention_similarity: torch.Tensor = None,\n target_embedding: torch.Tensor = None,\n ) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"\n Predict masks given image and prompt embeddings.\n\n Args:\n image_embeddings (`torch.Tensor`):\n the embeddings from the image encoder\n image_positional_embedding (`torch.Tensor`):\n positional encoding with the shape of image_embeddings\n sparse_prompt_embeddings (`torch.Tensor`):\n The embeddings of the points and boxes\n dense_prompt_embeddings (`torch.Tensor`):\n the embeddings of the mask inputs\n multimask_output (bool):\n Whether to return multiple masks or a single mask.\n output_attentions (bool, *optional*):\n Whether or not to return the attentions tensors of all attention layers.\n \"\"\"\n batch_size, num_channels, height, width = image_embeddings.shape\n point_batch_size = sparse_prompt_embeddings.shape[1]\n # Concatenate output tokens\n output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0)\n output_tokens = output_tokens.repeat(batch_size, point_batch_size, 1, 1)\n\n if sparse_prompt_embeddings.sum().item() != 0:\n tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=2)\n else:\n tokens = output_tokens\n point_embeddings = tokens.to(self.iou_token.weight.dtype)\n\n # Expand per-image data in batch direction to be per-point\n image_embeddings = image_embeddings + dense_prompt_embeddings\n image_embeddings = image_embeddings.repeat(point_batch_size, 1, 1, 1)\n image_positional_embeddings = image_positional_embeddings.repeat(point_batch_size, 1, 1, 1)\n\n # Run the transformer, image_positional_embedding are consumed\n point_embedding, image_embeddings, attentions = self.transformer(\n point_embeddings=point_embeddings,\n image_embeddings=image_embeddings,\n image_positional_embeddings=image_positional_embeddings,\n attention_similarity=attention_similarity,\n target_embedding=target_embedding,\n output_attentions=output_attentions,\n )\n iou_token_out = point_embedding[:, :, 0, :]\n mask_tokens_out = point_embedding[:, :, 1 : (1 + self.num_mask_tokens), :]\n\n # Upscale mask embeddings and predict masks using the mask tokens\n image_embeddings = image_embeddings.transpose(2, 3).reshape(\n batch_size * point_batch_size, num_channels, height, width\n )\n\n upscaled_embedding = self.upscale_conv1(image_embeddings)\n upscaled_embedding = self.activation(self.upscale_layer_norm(upscaled_embedding))\n upscaled_embedding = self.activation(self.upscale_conv2(upscaled_embedding))\n\n hyper_in_list = []\n for i in range(self.num_mask_tokens):\n current_mlp = self.output_hypernetworks_mlps[i]\n hyper_in_list += [current_mlp(mask_tokens_out[:, :, i, :])]\n hyper_in = torch.stack(hyper_in_list, dim=2)\n\n _, num_channels, height, width = upscaled_embedding.shape\n upscaled_embedding = upscaled_embedding.reshape(batch_size, point_batch_size, num_channels, height * width)\n masks = (hyper_in @ upscaled_embedding).reshape(batch_size, point_batch_size, -1, height, width)\n\n # Generate mask quality predictions\n iou_pred = self.iou_prediction_head(iou_token_out)\n\n # Select the correct mask or masks for output\n if multimask_output:\n mask_slice = slice(1, None)\n else:\n mask_slice = slice(0, 1)\n masks = masks[:, :, mask_slice, :, :]\n iou_pred = iou_pred[:, :, mask_slice]\n\n outputs = (masks, iou_pred)\n\n if output_attentions:\n outputs = outputs + (attentions,)\n else:\n outputs = outputs + (None,)\n\n return outputs\n\n\nclass SamPositionalEmbedding(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.scale = config.hidden_size // 2\n self.register_buffer(\"positional_embedding\", self.scale * torch.randn((2, config.num_pos_feats)))\n\n def forward(self, input_coords, input_shape=None):\n \"\"\"Positionally encode points that are normalized to [0,1].\"\"\"\n coordinates = input_coords.clone()\n\n if input_shape is not None:\n coordinates[:, :, :, 0] = coordinates[:, :, :, 0] / input_shape[1]\n coordinates[:, :, :, 1] = coordinates[:, :, :, 1] / input_shape[0]\n\n # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape\n coordinates = 2 * coordinates - 1\n coordinates = coordinates.to(self.positional_embedding.dtype)\n coordinates = coordinates @ self.positional_embedding\n coordinates = 2 * np.pi * coordinates\n # outputs d_1 x ... x d_n x channel shape\n return torch.cat([torch.sin(coordinates), torch.cos(coordinates)], dim=-1)\n\n\nclass SamMaskEmbedding(nn.Module):\n def __init__(self, config: SamPromptEncoderConfig):\n super().__init__()\n self.mask_input_channels = config.mask_input_channels // 4\n self.activation = ACT2FN[config.hidden_act]\n self.conv1 = nn.Conv2d(1, self.mask_input_channels, kernel_size=2, stride=2)\n self.conv2 = nn.Conv2d(self.mask_input_channels, config.mask_input_channels, kernel_size=2, stride=2)\n self.conv3 = nn.Conv2d(config.mask_input_channels, config.hidden_size, kernel_size=1)\n self.layer_norm1 = SamLayerNorm(\n self.mask_input_channels, eps=config.layer_norm_eps, data_format=\"channels_first\"\n )\n self.layer_norm2 = SamLayerNorm(\n self.mask_input_channels * 4, eps=config.layer_norm_eps, data_format=\"channels_first\"\n )\n\n def forward(self, masks):\n hidden_states = self.conv1(masks)\n hidden_states = self.layer_norm1(hidden_states)\n hidden_states = self.activation(hidden_states)\n\n hidden_states = self.conv2(hidden_states)\n hidden_states = self.layer_norm2(hidden_states)\n hidden_states = self.activation(hidden_states)\n dense_embeddings = self.conv3(hidden_states)\n return dense_embeddings\n\n\nclass SamPromptEncoder(nn.Module):\n def __init__(self, config: SamPromptEncoderConfig, shared_patch_embedding):\n super().__init__()\n self.shared_embedding = shared_patch_embedding\n self.mask_embed = SamMaskEmbedding(config)\n self.no_mask_embed = nn.Embedding(1, config.hidden_size)\n\n self.image_embedding_size = (config.image_embedding_size, config.image_embedding_size)\n self.input_image_size = config.image_size\n\n self.point_embed = nn.ModuleList(\n [nn.Embedding(1, config.hidden_size) for i in range(config.num_point_embeddings)]\n )\n self.hidden_size = config.hidden_size\n self.not_a_point_embed = nn.Embedding(1, config.hidden_size)\n\n def _embed_points(self, points: torch.Tensor, labels: torch.Tensor, pad: bool) -> torch.Tensor:\n \"\"\"Embeds point prompts.\"\"\"\n points = points + 0.5 # Shift to center of pixel\n if pad:\n target_point_shape = (points.shape[0], points.shape[1], 1, points.shape[-1])\n target_labels_shape = (points.shape[0], points.shape[1], 1)\n padding_point = torch.zeros(target_point_shape, device=points.device)\n padding_label = -torch.ones(target_labels_shape, device=labels.device)\n points = torch.cat([points, padding_point], dim=2)\n labels = torch.cat([labels, padding_label], dim=2)\n input_shape = (self.input_image_size, self.input_image_size)\n point_embedding = self.shared_embedding(points, input_shape)\n\n # torch.where and expanding the labels tensor is required by the ONNX export\n point_embedding = torch.where(labels[..., None] == -1, self.not_a_point_embed.weight, point_embedding)\n\n # This is required for the ONNX export. The dtype, device need to be explicitly\n # specified as otherwise torch.onnx.export interprets as double\n point_embedding = torch.where(\n labels[..., None] != -10,\n point_embedding,\n torch.tensor(0.0, dtype=point_embedding.dtype, device=point_embedding.device),\n )\n\n point_embedding = torch.where(\n (labels == 0)[:, :, :, None],\n point_embedding + self.point_embed[0].weight[None, None, :, :],\n point_embedding,\n )\n\n point_embedding = torch.where(\n (labels == 1)[:, :, :, None],\n point_embedding + self.point_embed[1].weight[None, None, :, :],\n point_embedding,\n )\n\n return point_embedding\n\n def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor:\n \"\"\"Embeds box prompts.\"\"\"\n boxes = boxes + 0.5 # Shift to center of pixel\n batch_size, nb_boxes = boxes.shape[:2]\n coords = boxes.reshape(batch_size, nb_boxes, 2, 2)\n input_shape = (self.input_image_size, self.input_image_size)\n corner_embedding = self.shared_embedding(coords, input_shape)\n corner_embedding[:, :, 0, :] += self.point_embed[2].weight\n corner_embedding[:, :, 1, :] += self.point_embed[3].weight\n return corner_embedding\n\n def forward(\n self,\n input_points: Optional[Tuple[torch.Tensor, torch.Tensor]],\n input_labels: Optional[torch.Tensor],\n input_boxes: Optional[torch.Tensor],\n input_masks: Optional[torch.Tensor],\n ) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"\n Embeds different types of prompts, returning both sparse and dense embeddings.\n\n Args:\n points (`torch.Tensor`, *optional*):\n point coordinates and labels to embed.\n boxes (`torch.Tensor`, *optional*):\n boxes to embed\n masks (`torch.Tensor`, *optional*):\n masks to embed\n \"\"\"\n sparse_embeddings = None\n batch_size = 1\n target_device = self.shared_embedding.positional_embedding.device\n if input_points is not None:\n batch_size, point_batch_size = input_points.shape[:2]\n if input_labels is None:\n raise ValueError(\"If points are provided, labels must also be provided.\")\n point_embeddings = self._embed_points(input_points, input_labels, pad=(input_boxes is None))\n sparse_embeddings = point_embeddings\n if input_boxes is not None:\n batch_size = input_boxes.shape[0]\n box_embeddings = self._embed_boxes(input_boxes)\n if sparse_embeddings is None:\n sparse_embeddings = box_embeddings\n else:\n sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=2)\n if input_masks is not None:\n dense_embeddings = self.mask_embed(input_masks)\n else:\n dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand(\n batch_size, -1, self.image_embedding_size[0], self.image_embedding_size[1]\n )\n\n if sparse_embeddings is None:\n sparse_embeddings = torch.zeros((batch_size, 1, 1, self.hidden_size), device=target_device)\n\n return sparse_embeddings, dense_embeddings\n\n\nclass SamVisionAttention(nn.Module):\n \"\"\"Multi-head Attention block with relative position embeddings.\"\"\"\n\n def __init__(self, config, window_size):\n super().__init__()\n input_size = (\n (config.image_size // config.patch_size, config.image_size // config.patch_size)\n if window_size == 0\n else (window_size, window_size)\n )\n\n self.num_attention_heads = config.num_attention_heads\n head_dim = config.hidden_size // config.num_attention_heads\n self.scale = head_dim**-0.5\n self.dropout = config.attention_dropout\n\n self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.qkv_bias)\n self.proj = nn.Linear(config.hidden_size, config.hidden_size)\n\n self.use_rel_pos = config.use_rel_pos\n if self.use_rel_pos:\n if input_size is None:\n raise ValueError(\"Input size must be provided if using relative positional encoding.\")\n\n # initialize relative positional embeddings\n self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))\n self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))\n\n def get_rel_pos(self, q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:\n \"\"\"\n Get relative positional embeddings according to the relative positions of\n query and key sizes.\n\n Args:\n q_size (int):\n size of the query.\n k_size (int):\n size of key k.\n rel_pos (`torch.Tensor`):\n relative position embeddings (L, channel).\n\n Returns:\n Extracted positional embeddings according to relative positions.\n \"\"\"\n max_rel_dist = int(2 * max(q_size, k_size) - 1)\n # Interpolate rel pos.\n rel_pos_resized = F.interpolate(\n rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1),\n size=max_rel_dist,\n mode=\"linear\",\n )\n rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)\n\n # Scale the coords with short length if shapes for q and k are different.\n q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0)\n k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0)\n relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)\n\n return rel_pos_resized[relative_coords.long()]\n\n def add_decomposed_rel_pos(\n self,\n attn: torch.Tensor,\n query: torch.Tensor,\n rel_pos_h: torch.Tensor,\n rel_pos_w: torch.Tensor,\n q_size: Tuple[int, int],\n k_size: Tuple[int, int],\n ) -> torch.Tensor:\n \"\"\"\n Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.\n https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py\n\n Args:\n attn (`torch.Tensor`):\n attention map.\n query (`torch.Tensor`):\n query q in the attention layer with shape (batch_size, query_height * query_width, channel).\n rel_pos_h (`torch.Tensor`):\n relative position embeddings (Lh, channel) for height axis.\n rel_pos_w (`torch.Tensor`):\n relative position embeddings (Lw, channel) for width axis.\n q_size (tuple):\n spatial sequence size of query q with (query_height, query_width).\n k_size (tuple):\n spatial sequence size of key k with (key_height, key_width).\n\n Returns:\n attn (`torch.Tensor`):\n attention map with added relative positional embeddings.\n \"\"\"\n query_height, query_width = q_size\n key_height, key_width = k_size\n relative_position_height = self.get_rel_pos(query_height, key_height, rel_pos_h)\n relative_position_width = self.get_rel_pos(query_width, key_width, rel_pos_w)\n\n batch_size, _, dim = query.shape\n reshaped_query = query.reshape(batch_size, query_height, query_width, dim)\n rel_h = torch.einsum(\"bhwc,hkc->bhwk\", reshaped_query, relative_position_height)\n rel_w = torch.einsum(\"bhwc,wkc->bhwk\", reshaped_query, relative_position_width)\n attn = attn.reshape(batch_size, query_height, query_width, key_height, key_width)\n attn = attn + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]\n attn = attn.reshape(batch_size, query_height * query_width, key_height * key_width)\n return attn\n\n def forward(self, hidden_states: torch.Tensor, output_attentions=False, output_moe_loss=False) -> torch.Tensor:\n batch_size, height, width, _ = hidden_states.shape\n # qkv with shape (3, batch_size, nHead, height * width, channel)\n ##### modify here for conv-lora\n qkv = self.qkv(hidden_states)\n if output_moe_loss:\n qkv, moe_loss = qkv\n qkv = qkv.reshape(batch_size, height * width, 3, self.num_attention_heads, -1).permute(2, 0, 3, 1, 4)\n # q, k, v with shape (batch_size * nHead, height * width, channel)\n query, key, value = qkv.reshape(3, batch_size * self.num_attention_heads, height * width, -1).unbind(0)\n\n attn_weights = (query * self.scale) @ key.transpose(-2, -1)\n\n if self.use_rel_pos:\n attn_weights = self.add_decomposed_rel_pos(\n attn_weights, query, self.rel_pos_h, self.rel_pos_w, (height, width), (height, width)\n )\n\n attn_weights = torch.nn.functional.softmax(attn_weights, dtype=torch.float32, dim=-1).to(query.dtype)\n\n attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)\n\n attn_output = (attn_probs @ value).reshape(batch_size, self.num_attention_heads, height, width, -1)\n attn_output = attn_output.permute(0, 2, 3, 1, 4).reshape(batch_size, height, width, -1)\n\n attn_output = self.proj(attn_output)\n\n if output_attentions:\n outputs = (attn_output, attn_weights)\n else:\n outputs = (attn_output, None)\n if output_moe_loss:\n outputs += (moe_loss,)\n return outputs\n\n\nclass SamVisionLayer(nn.Module):\n def __init__(self, config, window_size):\n super().__init__()\n self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)\n self.attn = SamVisionAttention(config, window_size)\n self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)\n self.mlp = SamMLPBlock(config)\n self.window_size = window_size\n\n def window_partition(self, hidden_states: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]:\n \"\"\"\n Args:\n Partition into non-overlapping windows with padding if needed.\n hidden_states (tensor): input tokens with [batch_size, height, width, channel]. window_size (int): window\n size.\n\n Returns:\n windows: windows after partition with [batch_size * num_windows, window_size, window_size, channel].\n (pad_height, pad_width): padded height and width before partition\n \"\"\"\n batch_size, height, width, channel = hidden_states.shape\n\n pad_h = (window_size - height % window_size) % window_size\n pad_w = (window_size - width % window_size) % window_size\n hidden_states = F.pad(hidden_states, (0, 0, 0, pad_w, 0, pad_h))\n pad_height, pad_width = height + pad_h, width + pad_w\n\n hidden_states = hidden_states.reshape(\n batch_size, pad_height // window_size, window_size, pad_width // window_size, window_size, channel\n )\n windows = hidden_states.permute(0, 1, 3, 2, 4, 5).contiguous().reshape(-1, window_size, window_size, channel)\n return windows, (pad_height, pad_width)\n\n def window_unpartition(\n self, windows: torch.Tensor, window_size: int, padding_shape: Tuple[int, int], original_shape: Tuple[int, int]\n ) -> torch.Tensor:\n \"\"\"\n Args:\n Window unpartition into original sequences and removing padding.\n hidden_states (tensor):\n input tokens with [batch_size * num_windows, window_size, window_size, channel].\n window_size (int):\n window size.\n padding_shape (Tuple):\n padded height and width (pad_height, pad_width).\n original_shape (Tuple): original height and width (height, width) before padding.\n\n Returns:\n hidden_states: unpartitioned sequences with [batch_size, height, width, channel].\n \"\"\"\n pad_height, pad_width = padding_shape\n height, width = original_shape\n batch_size = windows.shape[0] // (pad_height * pad_width // window_size // window_size)\n hidden_states = windows.reshape(\n batch_size, pad_height // window_size, pad_width // window_size, window_size, window_size, -1\n )\n hidden_states = (\n hidden_states.permute(0, 1, 3, 2, 4, 5).contiguous().reshape(batch_size, pad_height, pad_width, -1)\n )\n\n hidden_states = hidden_states[:, :height, :width, :].contiguous()\n return hidden_states\n\n def forward(\n self,\n hidden_states: torch.Tensor,\n output_attentions: Optional[bool] = False,\n output_moe_loss: Optional[bool] = False,\n ) -> Tuple[torch.FloatTensor]:\n residual = hidden_states\n\n hidden_states = self.layer_norm1(hidden_states)\n # Window partition\n if self.window_size > 0:\n height, width = hidden_states.shape[1], hidden_states.shape[2]\n hidden_states, padding_shape = self.window_partition(hidden_states, self.window_size)\n\n ### modify here\n attn_outputs = self.attn(\n hidden_states=hidden_states, output_attentions=output_attentions, output_moe_loss=output_moe_loss\n )\n if output_moe_loss:\n hidden_states, attn_weights, moe_loss = attn_outputs\n else:\n hidden_states, attn_weights = attn_outputs\n # Reverse window partition\n if self.window_size > 0:\n hidden_states = self.window_unpartition(hidden_states, self.window_size, padding_shape, (height, width))\n\n hidden_states = residual + hidden_states\n layernorm_output = self.layer_norm2(hidden_states)\n hidden_states = hidden_states + self.mlp(layernorm_output)\n\n outputs = (hidden_states,)\n if output_attentions:\n outputs += (attn_weights,)\n else:\n outputs += (None,) # keep attn_weights on the same place\n if output_moe_loss:\n outputs += (moe_loss,)\n\n return outputs\n\n\nclass SamVisionNeck(nn.Module):\n def __init__(self, config: SamVisionConfig):\n super().__init__()\n self.config = config\n\n self.conv1 = nn.Conv2d(config.hidden_size, config.output_channels, kernel_size=1, bias=False)\n self.layer_norm1 = SamLayerNorm(config.output_channels, data_format=\"channels_first\")\n self.conv2 = nn.Conv2d(config.output_channels, config.output_channels, kernel_size=3, padding=1, bias=False)\n self.layer_norm2 = SamLayerNorm(config.output_channels, data_format=\"channels_first\")\n\n def forward(self, hidden_states):\n hidden_states = hidden_states.permute(0, 3, 1, 2)\n hidden_states = self.conv1(hidden_states)\n hidden_states = self.layer_norm1(hidden_states)\n\n hidden_states = self.conv2(hidden_states)\n hidden_states = self.layer_norm2(hidden_states)\n return hidden_states\n\n\nclass SamVisionEncoder(nn.Module):\n def __init__(self, config: SamVisionConfig):\n super().__init__()\n self.config = config\n self.image_size = config.image_size\n\n self.patch_embed = SamPatchEmbeddings(config)\n\n self.pos_embed = None\n if config.use_abs_pos:\n # Initialize absolute positional embedding with pretrain image size.\n self.pos_embed = nn.Parameter(\n torch.zeros(\n 1,\n config.image_size // config.patch_size,\n config.image_size // config.patch_size,\n config.hidden_size,\n )\n )\n\n self.layers = nn.ModuleList()\n for i in range(config.num_hidden_layers):\n layer = SamVisionLayer(\n config,\n window_size=config.window_size if i not in config.global_attn_indexes else 0,\n )\n self.layers.append(layer)\n\n self.neck = SamVisionNeck(config)\n\n self.gradient_checkpointing = False\n\n def get_input_embeddings(self):\n return self.patch_embed\n\n def forward(\n self,\n pixel_values: Optional[torch.FloatTensor] = None,\n output_attentions: Optional[bool] = None,\n output_hidden_states: Optional[bool] = None,\n output_moe_loss: Optional[bool] = None, # MoE loss for Conv-LoRA\n return_dict: Optional[bool] = None,\n ) -> Union[Tuple, SamVisionEncoderOutput]:\n output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n output_hidden_states = (\n output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n )\n return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n if pixel_values is None:\n raise ValueError(\"You have to specify pixel_values\")\n\n hidden_states = self.patch_embed(pixel_values)\n if self.pos_embed is not None:\n hidden_states = hidden_states + self.pos_embed\n\n all_hidden_states = () if output_hidden_states else None\n all_self_attentions = () if output_attentions else None\n all_moe_loss = 0.0 if output_moe_loss else None\n\n for i, layer_module in enumerate(self.layers):\n if output_hidden_states:\n all_hidden_states = all_hidden_states + (hidden_states,)\n\n if self.gradient_checkpointing and self.training:\n\n def create_custom_forward(module):\n def custom_forward(*inputs):\n return module(*inputs, output_attentions)\n\n return custom_forward\n\n layer_outputs = torch.utils.checkpoint.checkpoint(\n create_custom_forward(layer_module),\n hidden_states,\n )\n else:\n layer_outputs = layer_module(\n hidden_states, output_attentions=output_attentions, output_moe_loss=output_moe_loss\n )\n\n hidden_states = layer_outputs[0]\n\n if output_attentions:\n all_self_attentions = all_self_attentions + (layer_outputs[1],)\n\n if output_moe_loss:\n all_moe_loss = all_moe_loss + layer_outputs[2]\n\n if output_hidden_states:\n all_hidden_states = all_hidden_states + (hidden_states,)\n\n hidden_states = self.neck(hidden_states)\n\n if not return_dict:\n outputs = (\n hidden_states,\n all_hidden_states,\n all_self_attentions,\n all_moe_loss,\n )\n # if output_hidden_states:\n # outputs = outputs + (all_hidden_states,)\n # if output_attentions:\n # outputs = outputs + (all_self_attentions,)\n # if output_moe_loss:\n # outputs = outputs + (all_moe_loss,)\n return outputs\n\n return SamVisionEncoderOutput(\n last_hidden_state=hidden_states,\n hidden_states=all_hidden_states,\n attentions=all_self_attentions,\n moe_loss=all_moe_loss, # modify here\n )\n\n\nclass SamPreTrainedModel(PreTrainedModel):\n config_class = SamConfig\n base_model_prefix = \"sam\"\n main_input_name = \"pixel_values\"\n\n def _init_weights(self, module):\n std = self.config.initializer_range\n if isinstance(module, (nn.Linear, nn.Conv2d, nn.ConvTranspose2d)):\n module.weight.data.normal_(mean=0.0, std=std)\n if module.bias is not None:\n module.bias.data.zero_()\n elif isinstance(module, nn.Embedding):\n module.weight.data.normal_(mean=0.0, std=std)\n if module.padding_idx is not None:\n module.weight.data[module.padding_idx].zero_()\n\n\nSAM_START_DOCSTRING = r\"\"\"\n This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the\n library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads\n etc.)\n\n This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.\n Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage\n and behavior.\n\n Parameters:\n config ([`SamConfig`]): Model configuration class with all the parameters of the model.\n Initializing with a config file does not load the weights associated with the model, only the\n configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n\"\"\"\n\n\nSAM_INPUTS_DOCSTRING = r\"\"\"\n Args:\n pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n Pixel values. Pixel values can be obtained using [`SamProcessor`]. See [`SamProcessor.__call__`] for\n details.\n input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`):\n Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much\n better results. The points can be obtained by passing a list of list of list to the processor that will\n create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the\n second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict\n per input point), the third dimension is the number of points per segmentation mask (it is possible to pass\n multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal)\n coordinates of the point. If a different number of points is passed either for each image, or for each\n mask, the processor will create \"PAD\" points that will correspond to the (0, 0) coordinate, and the\n computation of the embedding will be skipped for these points using the labels.\n input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`):\n Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the\n official implementation, there are 3 types of labels\n\n - `1`: the point is a point that contains the object of interest\n - `0`: the point is a point that does not contain the object of interest\n - `-1`: the point corresponds to the background\n\n We added the label:\n\n - `-10`: the point is a padding point, thus should be ignored by the prompt encoder\n\n The padding labels should be automatically done by the processor.\n input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`):\n Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to\n much better generated masks. The boxes can be obtained by passing a list of list of list to the processor,\n that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch\n size, the number of boxes per image and the coordinates of the top left and bottom right point of the box.\n In the order (`x1`, `y1`, `x2`, `y2`):\n\n - `x1`: the x coordinate of the top left point of the input box\n - `y1`: the y coordinate of the top left point of the input box\n - `x2`: the x coordinate of the bottom right point of the input box\n - `y2`: the y coordinate of the bottom right point of the input box\n\n input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`):\n SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to\n generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be\n manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`).\n\n image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`):\n Image embeddings, this is used by the mask decder to generate masks and iou scores. For more memory\n efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings`\n method, and then feed them to the `forward` method instead of feeding the `pixel_values`.\n multimask_output (`bool`, *optional*):\n In the original implementation and paper, the model always outputs 3 masks per image (or per point / per\n bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the\n \"best\" mask, by specifying `multimask_output=False`.\n attention_similarity (`torch.FloatTensor`, *optional*):\n Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the\n model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048).\n target_embedding (`torch.FloatTensor`, *optional*):\n Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case\n the model is used for personalization as introduced in [PerSAM](https://arxiv.org/abs/2305.03048).\n output_attentions (`bool`, *optional*):\n Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n tensors for more detail.\n output_hidden_states (`bool`, *optional*):\n Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n more detail.\n return_dict (`bool`, *optional*):\n Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n\"\"\"\n\n\n@add_start_docstrings(\n \"Segment Anything Model (SAM) for generating segmentation masks, given an input image and \",\n \" optional 2D location and bounding boxes.\",\n SAM_START_DOCSTRING,\n)\nclass SamModel(SamPreTrainedModel):\n _tied_weights_keys = [\"prompt_encoder.shared_embedding.positional_embedding\"]\n\n def __init__(self, config):\n super().__init__(config)\n self.shared_image_embedding = SamPositionalEmbedding(config.vision_config)\n\n self.vision_encoder = SamVisionEncoder(config.vision_config)\n self.prompt_encoder = SamPromptEncoder(config.prompt_encoder_config, self.shared_image_embedding)\n self.mask_decoder = SamMaskDecoder(config.mask_decoder_config)\n\n self.post_init()\n\n def get_input_embeddings(self):\n return self.vision_encoder.get_input_embeddings()\n\n def get_image_wide_positional_embeddings(self):\n size = self.config.prompt_encoder_config.image_embedding_size\n target_device = self.shared_image_embedding.positional_embedding.device\n target_dtype = self.shared_image_embedding.positional_embedding.dtype\n grid = torch.ones((size, size), device=target_device, dtype=target_dtype)\n y_embed = grid.cumsum(dim=0) - 0.5\n x_embed = grid.cumsum(dim=1) - 0.5\n y_embed = y_embed / size\n x_embed = x_embed / size\n\n positional_embedding = self.shared_image_embedding(torch.stack([x_embed, y_embed], dim=-1))\n return positional_embedding.permute(2, 0, 1).unsqueeze(0) # channel x height x width\n\n @torch.no_grad()\n def get_image_embeddings(\n self,\n pixel_values,\n output_attentions: Optional[bool] = None,\n output_hidden_states: Optional[bool] = None,\n return_dict: Optional[bool] = None,\n ):\n r\"\"\"\n Returns the image embeddings by passing the pixel values through the vision encoder.\n\n Args:\n pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n Input pixel values\n output_attentions (`bool`, *optional*):\n Whether or not to return the attentions tensors of all attention layers.\n output_hidden_states (`bool`, *optional*):\n Whether or not to return the hidden states of all layers.\n return_dict (`bool`, *optional*):\n Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n\n \"\"\"\n vision_output = self.vision_encoder(\n pixel_values,\n output_attentions=output_attentions,\n output_hidden_states=output_hidden_states,\n return_dict=return_dict,\n )\n image_embeddings = vision_output[0]\n return image_embeddings\n\n @torch.no_grad()\n def get_prompt_embeddings(\n self,\n input_points: Optional[torch.FloatTensor] = None,\n input_labels: Optional[torch.LongTensor] = None,\n input_boxes: Optional[torch.FloatTensor] = None,\n input_masks: Optional[torch.LongTensor] = None,\n ):\n r\"\"\"\n Returns the prompt embeddings by passing the input points, labels, boxes and masks through the prompt encoder.\n\n Args:\n input_points (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_points_per_image, 2)`):\n Optional input points for the prompt encoder. The padding of the point is automatically done by the\n processor. `point_batch_size` refers to the number of masks that we want the model to predict per\n point. The model will output `point_batch_size` times 3 masks in total.\n input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points_per_image)`):\n Optional input labels for the prompt encoder. The padding of the labels is automatically done by the\n processor, or can be fed by the user.\n input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes_per_image, 4)`):\n Optional input boxes for the prompt encoder. The padding of the boxes is automatically done by the\n processor. users can also pass manually the input boxes.\n input_masks (`torch.LongTensor` of shape `(batch_size, image_size, image_size)`):\n Optional input masks for the prompt encoder.\n \"\"\"\n prompt_output = self.prompt_encoder(\n input_points=input_points,\n input_labels=input_labels,\n input_boxes=input_boxes,\n input_masks=input_masks,\n )\n return prompt_output\n\n @add_start_docstrings_to_model_forward(SAM_INPUTS_DOCSTRING)\n def forward(\n self,\n pixel_values: Optional[torch.FloatTensor] = None,\n input_points: Optional[torch.FloatTensor] = None,\n input_labels: Optional[torch.LongTensor] = None,\n input_boxes: Optional[torch.FloatTensor] = None,\n input_masks: Optional[torch.LongTensor] = None,\n image_embeddings: Optional[torch.FloatTensor] = None,\n multimask_output: bool = True,\n attention_similarity: Optional[torch.FloatTensor] = None,\n target_embedding: Optional[torch.FloatTensor] = None,\n output_attentions: Optional[bool] = None,\n output_hidden_states: Optional[bool] = None,\n output_moe_loss: Optional[bool] = None, # MoE loss for Conv-LoRA\n return_dict=None,\n **kwargs,\n ) -> List[Dict[str, torch.Tensor]]:\n r\"\"\"\n Example:\n\n ```python\n >>> from PIL import Image\n >>> import requests\n >>> from transformers import AutoModel, AutoProcessor\n\n >>> model = AutoModel.from_pretrained(\"facebook/sam-vit-base\")\n >>> processor = AutoProcessor.from_pretrained(\"facebook/sam-vit-base\")\n\n >>> img_url = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png\"\n >>> raw_image = Image.open(requests.get(img_url, stream=True).raw).convert(\"RGB\")\n >>> input_points = [[[400, 650]]] # 2D location of a window on the car\n >>> inputs = processor(images=raw_image, input_points=input_points, return_tensors=\"pt\")\n\n >>> # Get segmentation mask\n >>> outputs = model(**inputs)\n\n >>> # Postprocess masks\n >>> masks = processor.post_process_masks(\n ... outputs.pred_masks, inputs[\"original_sizes\"], inputs[\"reshaped_input_sizes\"]\n ... )\n ```\n \"\"\"\n output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n output_hidden_states = (\n output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n )\n return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n if pixel_values is None and image_embeddings is None:\n raise ValueError(\"Either pixel_values or image_embeddings must be provided.\")\n\n if pixel_values is not None and image_embeddings is not None:\n raise ValueError(\"Only one of pixel_values and image_embeddings can be provided.\")\n\n if input_points is not None and len(input_points.shape) != 4:\n raise ValueError(\n \"The input_points must be a 4D tensor. Of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.\",\n \" got {}.\".format(input_points.shape),\n )\n if input_boxes is not None and len(input_boxes.shape) != 3:\n raise ValueError(\n \"The input_points must be a 3D tensor. Of shape `batch_size`, `nb_boxes`, `4`.\",\n \" got {}.\".format(input_boxes.shape),\n )\n if input_points is not None and input_boxes is not None:\n point_batch_size = input_points.shape[1]\n box_batch_size = input_boxes.shape[1]\n if point_batch_size != box_batch_size:\n raise ValueError(\n \"You should provide as many bounding boxes as input points per box. Got {} and {}.\".format(\n point_batch_size, box_batch_size\n )\n )\n\n image_positional_embeddings = self.get_image_wide_positional_embeddings()\n # repeat with batch size\n batch_size = pixel_values.shape[0] if pixel_values is not None else image_embeddings.shape[0]\n image_positional_embeddings = image_positional_embeddings.repeat(batch_size, 1, 1, 1)\n\n vision_attentions = None\n vision_hidden_states = None\n vision_moe_loss = None\n\n if pixel_values is not None:\n vision_outputs = self.vision_encoder(\n pixel_values,\n output_attentions=output_attentions,\n output_hidden_states=output_hidden_states,\n output_moe_loss=output_moe_loss,\n return_dict=return_dict,\n )\n image_embeddings = vision_outputs[0]\n\n if output_hidden_states:\n vision_hidden_states = vision_outputs[1]\n if output_attentions:\n # vision_attentions = vision_outputs[-1]\n vision_attentions = vision_outputs[2]\n if output_moe_loss:\n vision_moe_loss = vision_outputs[-1]\n\n if input_points is not None and input_labels is None:\n input_labels = torch.ones_like(input_points[:, :, :, 0], dtype=torch.int, device=input_points.device)\n\n if input_points is not None and image_embeddings.shape[0] != input_points.shape[0]:\n raise ValueError(\n \"The batch size of the image embeddings and the input points must be the same. \",\n \"Got {} and {} respectively.\".format(image_embeddings.shape[0], input_points.shape[0]),\n \" if you want to pass multiple points for the same image, make sure that you passed \",\n \" input_points of shape (batch_size, point_batch_size, num_points_per_image, 3) and \",\n \" input_labels of shape (batch_size, point_batch_size, num_points_per_image)\",\n )\n\n sparse_embeddings, dense_embeddings = self.prompt_encoder(\n input_points=input_points,\n input_labels=input_labels,\n input_boxes=input_boxes,\n input_masks=input_masks,\n )\n\n low_res_masks, iou_predictions, mask_decoder_attentions = self.mask_decoder(\n image_embeddings=image_embeddings,\n image_positional_embeddings=image_positional_embeddings,\n sparse_prompt_embeddings=sparse_embeddings,\n dense_prompt_embeddings=dense_embeddings,\n multimask_output=multimask_output,\n attention_similarity=attention_similarity,\n target_embedding=target_embedding,\n output_attentions=output_attentions,\n )\n\n if not return_dict:\n # output = (iou_predictions, low_res_masks)\n # if output_hidden_states:\n # output = output + (vision_hidden_states,)\n\n # if output_attentions:\n # output = output + (vision_attentions, mask_decoder_attentions)\n output = (\n iou_predictions,\n low_res_masks,\n vision_hidden_states,\n vision_attentions,\n mask_decoder_attentions,\n vision_moe_loss,\n )\n return output\n\n return SamImageSegmentationOutput(\n iou_scores=iou_predictions,\n pred_masks=low_res_masks,\n vision_hidden_states=vision_hidden_states,\n vision_attentions=vision_attentions,\n mask_decoder_attentions=mask_decoder_attentions,\n vision_moe_loss=vision_moe_loss,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/custom_transformer.py",
"content": "import enum\nimport math\nimport warnings\nfrom typing import Callable, Dict, List, Optional, Tuple, Union, cast\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import Tensor, nn\n\nfrom .utils import init_weights\n\nModuleType = Union[str, Callable[..., nn.Module]]\n_INTERNAL_ERROR_MESSAGE = \"Internal error. Please, open an issue.\"\n\n\ndef _is_glu_activation(activation: ModuleType):\n return isinstance(activation, str) and activation.endswith(\"glu\") or activation in [ReGLU, GEGLU]\n\n\ndef _make_nn_module(module_type: ModuleType, *args) -> nn.Module:\n if isinstance(module_type, str):\n if module_type == \"reglu\":\n return ReGLU()\n elif module_type == \"geglu\":\n return GEGLU()\n elif module_type == \"gelu\":\n return nn.GELU()\n elif module_type == \"relu\":\n return nn.ReLU()\n elif module_type == \"leaky_relu\":\n return nn.LeakyReLU()\n elif module_type == \"layer_norm\":\n return nn.LayerNorm(*args)\n else:\n try:\n cls = getattr(nn, module_type)\n except AttributeError as err:\n raise ValueError(f\"Failed to construct the module {module_type} with the arguments {args}\") from err\n return cls(*args)\n else:\n return module_type(*args)\n\n\ndef _all_or_none(values):\n return all(x is None for x in values) or all(x is not None for x in values)\n\n\ndef reglu(x: Tensor) -> Tensor:\n \"\"\"The ReGLU activation function from [1].\n\n References:\n ----------\n [1] Noam Shazeer, \"GLU Variants Improve Transformer\", 2020\n \"\"\"\n assert x.shape[-1] % 2 == 0\n a, b = x.chunk(2, dim=-1)\n return a * F.relu(b)\n\n\ndef geglu(x: Tensor) -> Tensor:\n \"\"\"The GEGLU activation function from [1].\n\n References:\n ----------\n [1] Noam Shazeer, \"GLU Variants Improve Transformer\", 2020\n \"\"\"\n assert x.shape[-1] % 2 == 0\n a, b = x.chunk(2, dim=-1)\n return a * F.gelu(b)\n\n\nclass ReGLU(nn.Module):\n \"\"\"\n The ReGLU activation function from [1].\n\n References:\n ----------\n [1] Noam Shazeer, \"GLU Variants Improve Transformer\", 2020\n \"\"\"\n\n def forward(self, x: Tensor) -> Tensor:\n return reglu(x)\n\n\nclass GEGLU(nn.Module):\n \"\"\"\n The GEGLU activation function from [1].\n\n References:\n ----------\n [1] Noam Shazeer, \"GLU Variants Improve Transformer\", 2020\n \"\"\"\n\n def forward(self, x: Tensor) -> Tensor:\n return geglu(x)\n\n\nclass CLSToken(nn.Module):\n \"\"\"[CLS]-token for BERT-like inference.\n\n To learn about the [CLS]-based inference, see [1].\n\n When used as a module, the [CLS]-token is appended **to the end** of each item in\n the batch.\n\n References:\n ----------\n [1] Jacob Devlin, Ming-Wei Chang, Kenton Lee, Kristina Toutanova \"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\" 2018\n \"\"\"\n\n def __init__(self, token_dim: int, initialization: str) -> None:\n \"\"\"\n Args:\n token_dim: the size of token\n initialization: initialization policy for parameters. Must be one of\n :code:`['uniform', 'normal']`. Let :code:`s = d ** -0.5`. Then, the\n corresponding distributions are :code:`Uniform(-s, s)` and :code:`Normal(0, s)`. In\n the paper [gorishniy2021revisiting], the 'uniform' initialization was\n used.\n\n References:\n * [gorishniy2021revisiting] Yury Gorishniy, Ivan Rubachev, Valentin Khrulkov, Artem Babenko \"Revisiting Deep Learning Models for Tabular Data\", 2021\n \"\"\"\n super().__init__()\n initialization_ = _TokenInitialization.from_str(initialization)\n self.weight = nn.Parameter(Tensor(token_dim))\n initialization_.apply(self.weight, token_dim)\n\n def expand(self, *leading_dimensions: int) -> Tensor:\n \"\"\"Expand (repeat) the underlying [CLS]-token to a tensor with the given leading dimensions.\n\n A possible use case is building a batch of [CLS]-tokens. See `_CLSToken` for\n examples of usage.\n\n Note:\n Under the hood, the `torch.Tensor.expand` method is applied to the\n underlying :code:`weight` parameter, so gradients will be propagated as\n expected.\n\n Args:\n leading_dimensions: the additional new dimensions\n\n Returns:\n tensor of the shape :code:`(*leading_dimensions, len(self.weight))`\n \"\"\"\n if not leading_dimensions:\n return self.weight\n new_dims = (1,) * (len(leading_dimensions) - 1)\n return self.weight.view(*new_dims, -1).expand(*leading_dimensions, -1)\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Append self **to the end** of each item in the batch (see `_CLSToken`).\"\"\"\n return torch.cat([x, self.expand(len(x), 1)], dim=1)\n\n\nclass _TokenInitialization(enum.Enum):\n UNIFORM = \"uniform\"\n NORMAL = \"normal\"\n\n @classmethod\n def from_str(cls, initialization: str) -> \"_TokenInitialization\":\n try:\n return cls(initialization)\n except ValueError:\n valid_values = [x.value for x in _TokenInitialization]\n raise ValueError(f\"initialization must be one of {valid_values}\")\n\n def apply(self, x: Tensor, d: int) -> None:\n d_sqrt_inv = 1 / math.sqrt(d)\n if self == _TokenInitialization.UNIFORM:\n # used in the paper \"Revisiting Deep Learning Models for Tabular Data\";\n # is equivalent to `nn.init.kaiming_uniform_(x, a=math.sqrt(5))` (which is\n # used by torch to initialize nn.Linear.weight, for example)\n nn.init.uniform_(x, a=-d_sqrt_inv, b=d_sqrt_inv)\n elif self == _TokenInitialization.NORMAL:\n nn.init.normal_(x, std=d_sqrt_inv)\n\n\nclass MultiheadAttention(nn.Module):\n \"\"\"Multihead Attention (self-/cross-) with optional 'linear' attention.\n\n To learn more about Multihead Attention, see [1]. See the implementation\n of `Transformer` and the examples below to learn how to use the compression technique\n from [2] to speed up the module when the number of tokens is large.\n\n References:\n ----------\n [1] Jacob Devlin, Ming-Wei Chang, Kenton Lee, Kristina Toutanova \"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding\" 2018\n [2] Sinong Wang, Belinda Z. Li, Madian Khabsa, Han Fang, Hao Ma \"Linformer: Self-Attention with Linear Complexity\", 2020\n \"\"\"\n\n def __init__(\n self,\n *,\n token_dim: int,\n num_heads: int,\n dropout: float,\n bias: bool,\n initialization: str,\n ) -> None:\n \"\"\"\n Parameters\n ----------\n token_dim:\n the token size. Must be a multiple of :code:`num_heads`.\n num_heads:\n the number of heads. If greater than 1, then the module will have\n an addition output layer (so called \"mixing\" layer).\n dropout:\n dropout rate for the attention map. The dropout is applied to\n *probabilities* and do not affect logits.\n bias:\n if `True`, then input (and output, if presented) layers also have bias.\n `True` is a reasonable default choice.\n initialization:\n initialization for input projection layers. Must be one of\n :code:`['kaiming', 'xavier']`. `kaiming` is a reasonable default choice.\n\n Raises\n ----------\n AssertionError: if requirements for the inputs are not met.\n \"\"\"\n super().__init__()\n if num_heads > 1:\n assert token_dim % num_heads == 0, \"token_dim must be a multiple of num_heads\"\n assert initialization in [\"kaiming\", \"xavier\"]\n\n self.W_q = nn.Linear(token_dim, token_dim, bias)\n self.W_k = nn.Linear(token_dim, token_dim, bias)\n self.W_v = nn.Linear(token_dim, token_dim, bias)\n self.W_out = nn.Linear(token_dim, token_dim, bias) if num_heads > 1 else None\n self.num_heads = num_heads\n self.dropout = nn.Dropout(dropout) if dropout else None\n\n for m in [self.W_q, self.W_k, self.W_v]:\n # the \"xavier\" branch tries to follow torch.nn.MultiheadAttention;\n # the second condition checks if W_v plays the role of W_out; the latter one\n # is initialized with Kaiming in torch\n if initialization == \"xavier\" and (m is not self.W_v or self.W_out is not None):\n # gain is needed since W_qkv is represented with 3 separate layers (it\n # implies different fan_out)\n nn.init.xavier_uniform_(m.weight, gain=1 / math.sqrt(2))\n if m.bias is not None:\n nn.init.zeros_(m.bias)\n if self.W_out is not None:\n nn.init.zeros_(self.W_out.bias)\n\n def _reshape(self, x: Tensor) -> Tensor:\n batch_size, num_tokens, d = x.shape\n head_dim = d // self.num_heads\n return (\n x.reshape(batch_size, num_tokens, self.num_heads, head_dim)\n .transpose(1, 2)\n .reshape(batch_size * self.num_heads, num_tokens, head_dim)\n )\n\n def forward(\n self,\n x_q: Tensor,\n x_kv: Tensor,\n key_compression: Optional[nn.Linear],\n value_compression: Optional[nn.Linear],\n ) -> Tuple[Tensor, Dict[str, Tensor]]:\n \"\"\"Perform the forward pass.\n\n Parameters\n ----------\n x_q:\n query tokens\n x_kv:\n key-value tokens\n key_compression:\n Linformer-style compression for keys\n value_compression:\n Linformer-style compression for values\n\n Returns:\n ----------\n (tokens, attention_stats)\n \"\"\"\n assert _all_or_none([key_compression, value_compression]), (\n \"If key_compression is (not) None, then value_compression must (not) be None\"\n )\n q, k, v = self.W_q(x_q), self.W_k(x_kv), self.W_v(x_kv)\n for tensor in [q, k, v]:\n assert tensor.shape[-1] % self.num_heads == 0, _INTERNAL_ERROR_MESSAGE\n if key_compression is not None:\n k = key_compression(k.transpose(1, 2)).transpose(1, 2)\n v = value_compression(v.transpose(1, 2)).transpose(1, 2) # type: ignore\n\n batch_size = len(q)\n head_dim_key = k.shape[-1] // self.num_heads\n head_dim_value = v.shape[-1] // self.num_heads\n n_q_tokens = q.shape[1]\n\n q = self._reshape(q)\n k = self._reshape(k)\n attention_logits = q @ k.transpose(1, 2) / math.sqrt(head_dim_key)\n attention_probs = F.softmax(attention_logits, dim=-1)\n if self.dropout is not None:\n attention_probs = self.dropout(attention_probs)\n x = attention_probs @ self._reshape(v)\n x = (\n x.reshape(batch_size, self.num_heads, n_q_tokens, head_dim_value)\n .transpose(1, 2)\n .reshape(batch_size, n_q_tokens, self.num_heads * head_dim_value)\n )\n if self.W_out is not None:\n x = self.W_out(x)\n return x, {\n \"attention_logits\": attention_logits,\n \"attention_probs\": attention_probs,\n }\n\n\nclass AdditiveAttention(nn.Module):\n \"\"\"Additive Attention with linear complexity to input sequence length.\n\n Additive attention was proposed and used in FastFormer.\n See Ref. [1] for details.\n This implementation is motivated by: https://github.com/jrzaurin/pytorch-widedeep.git\n\n References:\n ----------\n [1] Wu, Chuhan, et al. \"Fastformer: Additive attention can be all you need.\" arXiv preprint arXiv:2108.09084 (2021).\n \"\"\"\n\n def __init__(\n self,\n *,\n token_dim: int,\n num_heads: int,\n dropout: float,\n bias: bool,\n share_qv_weights: bool,\n initialization: str,\n ) -> None:\n \"\"\"\n Parameters\n ----------\n token_dim:\n the token size. Must be a multiple of :code:`num_heads`.\n num_heads:\n the number of heads. If greater than 1, then the module will have\n an addition output layer (so called \"mixing\" layer).\n dropout:\n dropout rate for the attention map. The dropout is applied to\n *probabilities* and do not affect logits.\n bias:\n if `True`, then input (and output, if presented) layers also have bias.\n `True` is a reasonable default choice.\n share_qv_weights:\n if 'True', then value and query transformation parameters are shared.\n initialization:\n initialization for input projection layers. Must be one of\n :code:`['kaiming', 'xavier']`. `kaiming` is a reasonable default choice.\n \"\"\"\n super().__init__()\n\n assert token_dim % num_heads == 0, \"token_dim must be a multiple of num_heads\"\n assert initialization in [\"kaiming\", \"xavier\"]\n\n self.head_dim = token_dim // num_heads\n self.num_heads = num_heads\n self.share_qv_weights = share_qv_weights\n self.dropout = nn.Dropout(dropout)\n trainable = []\n if share_qv_weights:\n self.qv_proj = nn.Linear(token_dim, token_dim, bias=bias)\n trainable.extend([self.qv_proj])\n else:\n self.q_proj = nn.Linear(token_dim, token_dim, bias=bias)\n self.v_proj = nn.Linear(token_dim, token_dim, bias=bias)\n trainable.extend([self.q_proj, self.v_proj])\n\n self.k_proj = nn.Linear(token_dim, token_dim, bias=bias)\n self.W_q = nn.Linear(token_dim, num_heads)\n self.W_k = nn.Linear(token_dim, num_heads)\n self.r_out = nn.Linear(token_dim, token_dim)\n trainable.extend([self.k_proj, self.W_q, self.W_k, self.r_out])\n\n if initialization == \"xavier\":\n self.apply(init_weights)\n else:\n for m in trainable:\n if m.bias is not None:\n nn.init.zeros_(m.bias)\n\n def forward(\n self,\n x_q: Tensor,\n x_kv: Tensor,\n *args, # Not used. just to make the input consistent with MultiheadAttention.\n ) -> Tuple[Tensor, Dict[str, Tensor]]:\n batch_size, n_q_tokens, token_dim = x_q.shape\n batch_size, n_k_tokens, token_dim = x_kv.shape\n\n q = self.qv_proj(x_q) if self.share_qv_weights else self.q_proj(x_q)\n v = self.qv_proj(x_kv) if self.share_qv_weights else self.v_proj(x_kv)\n k = self.k_proj(x_kv)\n\n alphas = (self.W_q(q) / math.sqrt(self.head_dim)).softmax(dim=1)\n q_r = q.reshape(batch_size, n_q_tokens, self.num_heads, self.head_dim)\n global_query = torch.einsum(\" b s h, b s h d -> b h d\", alphas, q_r)\n global_query = global_query.reshape(batch_size, self.num_heads * self.head_dim).unsqueeze(1)\n\n p = k * global_query\n\n betas = (self.W_k(p) / math.sqrt(self.head_dim)).softmax(dim=1)\n p_r = p.reshape(batch_size, n_k_tokens, self.num_heads, self.head_dim)\n global_key = torch.einsum(\" b s h, b s h d -> b h d\", betas, p_r)\n global_key = global_key.reshape(batch_size, self.num_heads * self.head_dim).unsqueeze(1)\n\n u = v * global_key\n output = q + self.dropout(self.r_out(u))\n\n return output, {\n \"query_weight\": alphas,\n \"key_weight\": betas,\n }\n\n\nclass Custom_Transformer(nn.Module):\n \"\"\"Transformer with extra features.\n\n This module is the backbone of `FTTransformer`.\"\"\"\n\n WARNINGS = {\"first_prenormalization\": True, \"prenormalization\": True}\n\n class FFN(nn.Module):\n \"\"\"The Feed-Forward Network module used in every `Transformer` block.\"\"\"\n\n def __init__(\n self,\n *,\n token_dim: int,\n d_hidden: int,\n bias_first: bool,\n bias_second: bool,\n dropout: float,\n activation: ModuleType,\n ):\n super().__init__()\n self.linear_first = nn.Linear(\n token_dim,\n d_hidden * (2 if _is_glu_activation(activation) else 1),\n bias_first,\n )\n self.activation = _make_nn_module(activation)\n self.dropout = nn.Dropout(dropout)\n self.linear_second = nn.Linear(d_hidden, token_dim, bias_second)\n\n def forward(self, x: Tensor) -> Tensor:\n x = self.linear_first(x)\n x = self.activation(x)\n x = self.dropout(x)\n x = self.linear_second(x)\n return x\n\n class Head(nn.Module):\n \"\"\"The final module of the `Transformer` that performs BERT-like inference.\"\"\"\n\n def __init__(\n self,\n *,\n d_in: int,\n bias: bool,\n activation: ModuleType,\n normalization: ModuleType,\n d_out: int,\n ):\n super().__init__()\n self.normalization = _make_nn_module(normalization, d_in)\n self.activation = _make_nn_module(activation)\n self.linear = nn.Linear(d_in, d_out, bias)\n\n def forward(self, x: Tensor) -> Tensor:\n x = x[:, -1]\n x = self.normalization(x)\n x = self.activation(x)\n x = self.linear(x)\n return x\n\n def __init__(\n self,\n *,\n token_dim: int,\n num_blocks: int,\n attention_num_heads: int,\n attention_dropout: float,\n attention_initialization: str,\n attention_normalization: str,\n ffn_hidden_size: int,\n ffn_dropout: float,\n ffn_activation: str,\n ffn_normalization: str,\n residual_dropout: float,\n prenormalization: bool,\n first_prenormalization: bool,\n last_layer_query_idx: Union[None, List[int], slice],\n num_tokens: Optional[int],\n kv_compression_ratio: Optional[float],\n kv_compression_sharing: Optional[str],\n head_activation: ModuleType,\n head_normalization: ModuleType,\n d_out: int,\n projection: Optional[bool] = False,\n additive_attention: Optional[bool] = False,\n share_qv_weights: Optional[bool] = False,\n ) -> None:\n \"\"\"\n Parameters\n ----------\n token_dim\n The size of one token for `_CategoricalFeatureTokenizer`.\n num_blocks\n Number of the `FT_Transformer` blocks, which should be non-negative.\n attention_num_heads\n Number of attention heads in each `FT_Transformer` block, which should be positive.\n attention_dropout\n Dropout ratio for the Multi Headed Attention module.\n attention_initialization\n Weights initialization scheme for Multi Headed Attention module.\n attention_normalization\n Normalization policy for attention layers. \"layer_norm\" is a good default.\n ffn_hidden_size\n Number of the hidden nodes of the linear layers in the Feed-Forward Network module.\n ffn_dropout\n Dropout ratio of the hidden nodes of the linear layers in the Feed-Forward Network module.\n ffn_activation\n Activation function type for the Feed-Forward Network module.\n ffn_normalization\n Normalization scheme of the Feed-Forward Network module.\n residual_dropout\n Dropout ratio for the linear layers in FT_Transformer block.\n prenormalization, first_prenormalization\n Prenormalization to stabilize the training.\n num_tokens\n Number of tokens of the input sequence.\n kv_compression_ratio\n The compression ration to reduce the input sequence length.\n kv_compression_sharing\n If `true` the projections will share weights.\n head_activation\n Activation function type of the MLP layer.\n head_normalization\n Normalization scheme of the MLP layer.\n d_out\n Output dimension.\n projection\n Whether to use a project head.\n additive_attention\n If 'true' the transformer will use additive attention with linear complexity to sequence length.\n share_qv_weights\n if 'true', then value and query transformation parameters are shared in additive attention.\n \"\"\"\n super().__init__()\n if isinstance(last_layer_query_idx, int):\n raise ValueError(\n \"last_layer_query_idx must be None, list[int] or slice. \"\n f\"Do you mean last_layer_query_idx=[{last_layer_query_idx}] ?\"\n )\n if not prenormalization:\n assert not first_prenormalization, (\n \"If `prenormalization` is False, then `first_prenormalization` must be False\"\n )\n assert _all_or_none([num_tokens, kv_compression_ratio, kv_compression_sharing]), (\n \"If any of the following arguments is (not) None, then all of them must (not) be None: \"\n \"num_tokens, kv_compression_ratio, kv_compression_sharing\"\n )\n assert additive_attention or not share_qv_weights, (\n \"If `share_qv_weights` is True, then `additive_attention` must be True\"\n )\n assert kv_compression_sharing in [None, \"headwise\", \"key-value\", \"layerwise\"]\n if not prenormalization:\n if self.WARNINGS[\"prenormalization\"]:\n warnings.warn(\n \"prenormalization is set to False. Are you sure about this? \"\n \"The training can become less stable. \"\n \"You can turn off this warning by tweaking the \"\n \"rtdl.Transformer.WARNINGS dictionary.\",\n UserWarning,\n )\n assert not first_prenormalization, (\n \"If prenormalization is False, then first_prenormalization is ignored and must be set to False\"\n )\n if prenormalization and first_prenormalization and self.WARNINGS[\"first_prenormalization\"]:\n warnings.warn(\n \"first_prenormalization is set to True. Are you sure about this? \"\n \"For example, the vanilla FTTransformer with \"\n \"first_prenormalization=True performs SIGNIFICANTLY worse. \"\n \"You can turn off this warning by tweaking the \"\n \"rtdl.Transformer.WARNINGS dictionary.\",\n UserWarning,\n )\n\n def make_kv_compression():\n assert num_tokens and kv_compression_ratio, _INTERNAL_ERROR_MESSAGE # for mypy\n # https://github.com/pytorch/fairseq/blob/1bba712622b8ae4efb3eb793a8a40da386fe11d0/examples/linformer/linformer_src/modules/multihead_linear_attention.py#L83\n return nn.Linear(num_tokens, int(num_tokens * kv_compression_ratio), bias=False)\n\n self.shared_kv_compression = (\n make_kv_compression() if kv_compression_ratio and kv_compression_sharing == \"layerwise\" else None\n )\n\n self.prenormalization = prenormalization\n self.last_layer_query_idx = last_layer_query_idx\n\n self.blocks = nn.ModuleList([])\n for layer_idx in range(num_blocks):\n layer = nn.ModuleDict(\n {\n \"attention\": AdditiveAttention(\n token_dim=token_dim,\n num_heads=attention_num_heads,\n dropout=attention_dropout,\n bias=True,\n share_qv_weights=share_qv_weights,\n initialization=attention_initialization,\n )\n if additive_attention\n else MultiheadAttention(\n token_dim=token_dim,\n num_heads=attention_num_heads,\n dropout=attention_dropout,\n bias=True,\n initialization=attention_initialization,\n ),\n \"ffn\": Custom_Transformer.FFN(\n token_dim=token_dim,\n d_hidden=ffn_hidden_size,\n bias_first=True,\n bias_second=True,\n dropout=ffn_dropout,\n activation=ffn_activation,\n ),\n \"attention_residual_dropout\": nn.Dropout(residual_dropout),\n \"ffn_residual_dropout\": nn.Dropout(residual_dropout),\n \"output\": nn.Identity(), # for hooks-based introspection\n }\n )\n if layer_idx or not prenormalization or first_prenormalization:\n layer[\"attention_normalization\"] = _make_nn_module(attention_normalization, token_dim)\n layer[\"ffn_normalization\"] = _make_nn_module(ffn_normalization, token_dim)\n if kv_compression_ratio and self.shared_kv_compression is None:\n layer[\"key_compression\"] = make_kv_compression()\n if kv_compression_sharing == \"headwise\":\n layer[\"value_compression\"] = make_kv_compression()\n else:\n assert kv_compression_sharing == \"key-value\", _INTERNAL_ERROR_MESSAGE\n self.blocks.append(layer)\n\n self.head = (\n Custom_Transformer.Head(\n d_in=token_dim,\n d_out=d_out,\n bias=True,\n activation=head_activation, # type: ignore\n normalization=head_normalization if prenormalization else \"Identity\",\n )\n if projection\n else nn.Identity()\n )\n\n def _get_kv_compressions(self, layer):\n return (\n (self.shared_kv_compression, self.shared_kv_compression)\n if self.shared_kv_compression is not None\n else (layer[\"key_compression\"], layer[\"value_compression\"])\n if \"key_compression\" in layer and \"value_compression\" in layer\n else (layer[\"key_compression\"], layer[\"key_compression\"])\n if \"key_compression\" in layer\n else (None, None)\n )\n\n def _start_residual(self, layer, stage, x):\n assert stage in [\"attention\", \"ffn\"], _INTERNAL_ERROR_MESSAGE\n x_residual = x\n if self.prenormalization:\n norm_key = f\"{stage}_normalization\"\n if norm_key in layer:\n x_residual = layer[norm_key](x_residual)\n return x_residual\n\n def _end_residual(self, layer, stage, x, x_residual):\n assert stage in [\"attention\", \"ffn\"], _INTERNAL_ERROR_MESSAGE\n x_residual = layer[f\"{stage}_residual_dropout\"](x_residual)\n x = x + x_residual\n if not self.prenormalization:\n x = layer[f\"{stage}_normalization\"](x)\n return x\n\n def forward(self, x: Tensor) -> Tensor:\n assert x.ndim == 3, \"The input must have 3 dimensions: (n_objects, num_tokens, token_dim)\"\n for layer_idx, layer in enumerate(self.blocks):\n layer = cast(nn.ModuleDict, layer)\n\n query_idx = self.last_layer_query_idx if layer_idx + 1 == len(self.blocks) else None\n x_residual = self._start_residual(layer, \"attention\", x)\n x_residual, _ = layer[\"attention\"](\n x_residual if query_idx is None else x_residual[:, query_idx],\n x_residual,\n *self._get_kv_compressions(layer),\n )\n if query_idx is not None:\n x = x[:, query_idx]\n x = self._end_residual(layer, \"attention\", x, x_residual)\n\n x_residual = self._start_residual(layer, \"ffn\", x)\n x_residual = layer[\"ffn\"](x_residual)\n x = self._end_residual(layer, \"ffn\", x, x_residual)\n x = layer[\"output\"](x)\n\n x = self.head(x)\n\n return x\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/document_transformer.py",
"content": "import inspect\nimport logging\nfrom typing import Callable, Dict, List, Optional, Tuple, Union, cast\n\nfrom transformers import logging as hf_logging\n\nfrom ..constants import (\n ATTENTION_MASK,\n BBOX,\n COLUMN_FEATURES,\n FEATURES,\n IMAGE,\n INPUT_IDS,\n LOGITS,\n MASKS,\n PIXEL_VALUES,\n TOKEN_TYPE_IDS,\n)\nfrom .hf_text import HFAutoModelForTextPrediction\nfrom .utils import get_column_features, get_image_size_mean_std\n\nhf_logging.set_verbosity_error()\n\nlogger = logging.getLogger(__name__)\n\n\nclass DocumentTransformer(HFAutoModelForTextPrediction):\n \"\"\"\n Document Classification with Huggingface backbones. Inherit from HFAutoModelForTextPrediction.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str = \"microsoft/layoutlmv3-base\",\n num_classes: Optional[int] = 0,\n pooling_mode: Optional[str] = \"cls\",\n gradient_checkpointing: Optional[bool] = False,\n low_cpu_mem_usage: Optional[bool] = False,\n pretrained: Optional[bool] = True,\n tokenizer_name: Optional[str] = \"hf_auto\",\n image_size: Optional[int] = None,\n image_norm: Optional[str] = None,\n ):\n \"\"\"\n Load a pretrained huggingface layout-aware document transformer backbone.\n\n Parameters\n ----------\n prefix\n The model prefix.\n checkpoint_name\n Name of the checkpoint. We support loading checkpoint from\n Huggingface Models list: https://huggingface.co/models\n For example, you can use layout-aware models:\n - microsoft/layoutlmv3-base\n - microsoft/layoutlm-base-uncased\n - microsoft/xdoc-base\n - microsoft/layoutxlm-base\n - microsoft/layoutlmv2-base-uncased\n you may also use text focused transformers:\n - 'microsoft/deberta-v3-base'\n - 'bert-base-uncased'\n num_classes\n The number of classes. 1 for a regression task.\n pooling_mode\n The pooling mode for the Transformer. Can be \"cls\", or \"mean\"\n gradient_checkpointing\n Whether to enable gradient checkpointing\n low_cpu_mem_usage\n Whether to turn on the optimization of reducing the peak CPU memory usage when loading the pretrained model.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n tokenizer_name\n Name of the huggingface tokenizer type.\n image_norm\n How to normalize an image. We now support:\n - inception\n Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm\n - imagenet\n Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm\n - clip\n Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and\n std (0.26862954, 0.26130258, 0.27577711), used for CLIP.\n image_size\n The provided width / height of a square image.\n \"\"\"\n logger.debug(f\"initializing {prefix} (DocumentTransformer)\")\n logger.debug(f\"model checkpoint: {checkpoint_name}\")\n super().__init__(\n prefix=prefix,\n checkpoint_name=checkpoint_name,\n num_classes=num_classes,\n pooling_mode=pooling_mode,\n gradient_checkpointing=gradient_checkpointing,\n low_cpu_mem_usage=low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=tokenizer_name,\n )\n self.image_size, self.image_mean, self.image_std = get_image_size_mean_std(\n model_name=self.prefix,\n config=self.config,\n provided_size=image_size,\n provided_norm_type=image_norm,\n )\n self.is_text_only_flag = self.is_text_only()\n\n if self.is_text_only_flag:\n logger.debug(f\"Checkpoint: {checkpoint_name} uses the text data only for classification.\")\n\n def is_text_only(self):\n \"\"\"\n Check the tokenizer to see if it is a text only tokenizer.\n\n Parameters\n ----------\n tokenizer\n The tokenizer to be used.\n\n Returns\n -------\n True if the tokenizer only accept text, otherwise, False.\n \"\"\"\n model_args = list(inspect.signature(self.tokenizer.__call__).parameters.keys())\n # Tokenizers of document foundation models usually have a \"boxes\" argument.\n if \"boxes\" not in model_args:\n return True\n else:\n return False\n\n @property\n def text_attention_mask_key(self):\n return f\"{self.prefix}_{ATTENTION_MASK}\"\n\n @property\n def text_bbox_key(self):\n return f\"{self.prefix}_{BBOX}\"\n\n @property\n def document_pixel_value_key(self):\n return f\"{self.prefix}_{PIXEL_VALUES}\"\n\n def update_input_data(\n self,\n input_data: dict,\n batch: dict,\n keys: dict,\n ):\n \"\"\"\n Update the model input data based on the model argument.\n For example, microsoft/layoutlm-base-uncased has a \"bbox\" argument;\n microsoft/layoutlmv3-base has arguments: \"bbox\" and \"image\".\n Text only bert does not have these two arguments.\n\n Parameters\n ----------\n input_data\n A dictionary containing the model input data.\n batch:\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n keys:\n A dictionary containing the model arguments and corresponding batch keys.\n \"\"\"\n model_args = list(inspect.signature(self.model.forward).parameters.keys())\n for key, value in keys.items():\n if key in model_args:\n input_data.update({key: batch[value]})\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n input_data = {}\n self.update_input_data(\n input_data,\n batch,\n keys={\n INPUT_IDS: self.text_token_ids_key,\n TOKEN_TYPE_IDS: self.text_segment_ids_key,\n ATTENTION_MASK: self.text_attention_mask_key,\n BBOX: self.text_bbox_key,\n PIXEL_VALUES: self.document_pixel_value_key,\n IMAGE: self.document_pixel_value_key,\n },\n )\n\n text_masks = batch[self.text_attention_mask_key]\n\n outputs = self.model(**input_data)\n\n if self.pooling_mode == \"cls\":\n pooled_features = outputs.last_hidden_state[:, 0, :]\n elif self.pooling_mode == \"mean\":\n # In some models, last_hidden_state is the concatenation of document image features and text features.\n pooled_features = outputs.last_hidden_state.mean(1)\n else:\n raise NotImplementedError(f\"Pooling mode={self.pooling_mode} is not supported.\")\n\n logits = self.head(pooled_features)\n\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n column_features, column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.text_column_prefix,\n features=outputs.last_hidden_state,\n valid_lengths=sum(text_masks),\n cls_feature=pooled_features,\n )\n ret[COLUMN_FEATURES][FEATURES].update(column_features)\n ret[COLUMN_FEATURES][MASKS].update(column_feature_masks)\n\n ret.update(\n {\n LOGITS: logits,\n FEATURES: pooled_features,\n }\n )\n\n return {self.prefix: ret}\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/ft_transformer.py",
"content": "import logging\nimport os\nimport tempfile\nfrom typing import Dict, List, Optional\n\nimport torch\nfrom torch import Tensor, nn\n\nfrom ..constants import CATEGORICAL, FEATURES, LABEL, LOGITS, NUMERICAL\nfrom .custom_transformer import CLSToken, Custom_Transformer, _TokenInitialization\nfrom .utils import init_weights\n\nlogger = logging.getLogger(__name__)\n\n\nclass CategoricalFeatureTokenizer(nn.Module):\n \"\"\"\n Feature tokenizer for categorical features in tabular data.\n It transforms the input categorical features to tokens (embeddings).\n\n The categorical features usually refers to discrete features.\n \"\"\"\n\n def __init__(\n self,\n num_categories: List[int],\n token_dim: int,\n bias: Optional[bool] = True,\n initialization: Optional[str] = \"normal\",\n ) -> None:\n \"\"\"\n Parameters\n ----------\n num_categories:\n A list of integers. Each one is the number of categories in one categorical column.\n token_dim:\n The size of one token.\n bias:\n If `True`, for each feature, an additional trainable vector will be added to the\n embedding regardless of feature value. Notablly, the bias are not shared between features.\n initialization:\n Initialization policy for parameters. Must be one of `['uniform', 'normal']`.\n\n References\n ----------\n 1. Yury Gorishniy, Ivan Rubachev, Valentin Khrulkov, Artem Babenko,\n \"Revisiting Deep Learning Models for Tabular Data\", NeurIPS 2021\n https://arxiv.org/pdf/2106.11959.pdf\n 2. Code: https://github.com/Yura52/tabular-dl-revisiting-models\n \"\"\"\n super().__init__()\n\n self.num_categories = num_categories\n category_offsets = torch.tensor([0] + num_categories[:-1]).cumsum(0)\n\n self.register_buffer(\"category_offsets\", category_offsets, persistent=False)\n self.embeddings = nn.Embedding(sum(num_categories), token_dim)\n self.bias = nn.Parameter(Tensor(len(num_categories), token_dim)) if bias else None\n initialization_ = _TokenInitialization.from_str(initialization)\n\n for parameter in [self.embeddings.weight, self.bias]:\n if parameter is not None:\n initialization_.apply(parameter, token_dim)\n\n @property\n def num_tokens(self) -> int:\n \"\"\"The number of tokens.\"\"\"\n return len(self.num_categories)\n\n @property\n def token_dim(self) -> int:\n \"\"\"The size of one token.\"\"\"\n return self.embeddings.embedding_dim\n\n def forward(self, x: Tensor) -> Tensor:\n x = self.embeddings(x + self.category_offsets[None])\n\n if self.bias is not None:\n x = x + self.bias[None]\n\n return x\n\n\nclass Periodic(nn.Module):\n def __init__(\n self,\n in_features: int,\n d_embedding: int,\n trainable: Optional[bool] = True,\n initialization: Optional[str] = \"normal\",\n sigma: Optional[float] = 1.0,\n ):\n \"\"\"\n Parameters\n ----------\n in_features\n Input feature size.\n d_embedding\n Output feature size, should be an even number.\n trainable\n Determine whether the coefficients needed to be updated.\n initialization\n Initialization scheme.\n sigma\n Standard deviation used for initialization='normal'\n\n Reference:\n ----------\n 1. Code: https://github.com/Yura52/tabular-dl-num-embeddings\n 2. Paper: On Embeddings for Numerical Features in Tabular Deep Learning, https://arxiv.org/abs/2203.05556\n \"\"\"\n super().__init__()\n\n assert d_embedding % 2 == 0, \"d_embedding mod 2 should be 0, current d_embedding is {}\".format(d_embedding)\n\n if initialization == \"log-linear\":\n coefficients = sigma ** (torch.arange(d_embedding // 2) / (d_embedding // 2))\n coefficients = coefficients[None].repeat(in_features, 1)\n elif initialization == \"normal\":\n coefficients = torch.normal(0.0, sigma, (in_features, d_embedding // 2))\n\n if trainable:\n self.coefficients = nn.Parameter(coefficients)\n else:\n self.register_buffer(\"coefficients\", coefficients)\n\n def cos_sin(self, x: Tensor):\n return torch.cat([torch.cos(x), torch.sin(x)], -1)\n\n def forward(self, x: Tensor):\n assert x.ndim == 2, \"Periodic should only be applied to first layer i.e. ndim==2\"\n return self.cos_sin(2 * torch.pi * self.coefficients[None] * x[..., None])\n\n\nclass NLinear(nn.Module):\n def __init__(self, n: int, d_in: int, d_out: int, bias: bool = True):\n super().__init__()\n self.weight = nn.Parameter(Tensor(n, d_in, d_out))\n self.bias = nn.Parameter(Tensor(n, d_out)) if bias else None\n with torch.no_grad():\n for i in range(n):\n layer = nn.Linear(d_in, d_out)\n self.weight[i] = layer.weight.T\n if self.bias is not None:\n self.bias[i] = layer.bias\n\n def forward(self, x):\n assert x.ndim == 3, \"Error input dimension, should be 3, but given {}\".format(x.ndim)\n x = x[..., None] * self.weight[None]\n x = x.sum(-2)\n if self.bias is not None:\n x = x + self.bias[None]\n return x\n\n\nclass NLinearMemoryEfficient(nn.Module):\n def __init__(self, n: int, d_in: int, d_out: int):\n super().__init__()\n self.layers = nn.ModuleList([nn.Linear(d_in, d_out) for _ in range(n)])\n\n def forward(self, x):\n return torch.stack([l(x[:, i]) for i, l in enumerate(self.layers)], 1)\n\n\nclass NLayerNorm(nn.Module):\n def __init__(self, n_features: int, d: int):\n super().__init__()\n self.weight = nn.Parameter(torch.ones(n_features, d))\n self.bias = nn.Parameter(torch.zeros(n_features, d))\n\n def forward(self, x: Tensor):\n assert x.ndim == 3\n x = (x - x.mean(-1, keepdim=True)) / x.std(-1, keepdim=True)\n x = self.weight * x + self.bias\n return x\n\n\nclass NumericalFeatureTokenizer(nn.Module):\n \"\"\"\n Numerical tokenizer for numerical features in tabular data.\n It transforms the input numerical features to tokens (embeddings).\n\n The numerical features usually refers to continuous features.\n\n It consists of two steps:\n 1. each feature is multiplied by a trainable vector i.e., weights,\n 2. another trainable vector is added i.e., bias.\n\n Note that each feature has its separate pair of trainable vectors,\n i.e. the vectors are not shared between features.\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n token_dim: int,\n bias: Optional[bool] = True,\n initialization: Optional[str] = \"normal\",\n ):\n \"\"\"\n Parameters\n ----------\n in_features:\n Dimension of input features i.e. the number of continuous (scalar) features\n token_dim:\n The size of one token.\n bias:\n If `True`, for each feature, an additional trainable vector will be added to the\n embedding regardless of feature value. Notablly, the bias are not shared between features.\n initialization:\n Initialization policy for parameters. Must be one of `['uniform', 'normal']`.\n\n References\n ----------\n Yury Gorishniy, Ivan Rubachev, Valentin Khrulkov, Artem Babenko,\n \"Revisiting Deep Learning Models for Tabular Data\", 2021\n https://arxiv.org/pdf/2106.11959.pdf\n \"\"\"\n super().__init__()\n\n initialization_ = _TokenInitialization.from_str(initialization)\n self.weight = nn.Parameter(Tensor(in_features, token_dim))\n self.bias = nn.Parameter(Tensor(in_features, token_dim)) if bias else None\n for parameter in [self.weight, self.bias]:\n if parameter is not None:\n initialization_.apply(parameter, token_dim)\n\n @property\n def num_tokens(self) -> int:\n \"\"\"The number of tokens.\"\"\"\n return len(self.weight)\n\n @property\n def token_dim(self) -> int:\n \"\"\"The size of one token.\"\"\"\n return self.weight.shape[1]\n\n def forward(\n self,\n x: Tensor,\n ) -> Tensor:\n x = self.weight[None] * x[..., None]\n if self.bias is not None:\n x = x + self.bias[None]\n\n return x\n\n\nclass AutoDis(nn.Module):\n \"\"\"\n Paper (the version is important): https://arxiv.org/abs/2012.08986v2\n Code: https://github.com/mindspore-ai/models/tree/bdf2d8bcf11fe28e4ad3060cf2ddc818eacd8597/research/recommend/autodis\n We borrow the implementations from: https://github.com/Yura52/tabular-dl-num-embeddings/blob/main/bin/train4.py\n The paper is significantly different from the code (it looks like the code\n implements the first version of the paper). We implement the second version\n here. Not all technical details are given for the second version, so what we do\n here can be different from what authors actually did.\n Anyway, AutoDis (v2) is essentially the following sequence of layers (applied from\n left to right): [Linear(no bias), LeakyReLU, Linear(no bias), Softmax, Linear]\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n d_embedding: int,\n n_meta_embeddings: int,\n temperature: Optional[float] = 3.0,\n ):\n super().__init__()\n self.first_layer = NumericalFeatureTokenizer(\n in_features=in_features,\n token_dim=n_meta_embeddings,\n bias=False,\n initialization=\"uniform\",\n )\n self.leaky_relu = nn.LeakyReLU()\n self.second_layer = NLinear(in_features, n_meta_embeddings, n_meta_embeddings, False)\n self.softmax = nn.Softmax(-1)\n self.temperature = temperature\n # \"meta embeddings\" from the paper are just a linear layer\n self.third_layer = NLinear(in_features, n_meta_embeddings, d_embedding, False)\n # 0.01 is taken from the source code\n nn.init.uniform_(self.third_layer.weight, 0.01)\n\n def forward(self, x: Tensor):\n x = self.first_layer(x)\n x = self.leaky_relu(x)\n x = self.second_layer(x)\n x = self.softmax(x / self.temperature)\n x = self.third_layer(x)\n return x\n\n\nclass NumEmbeddings(nn.Module):\n def __init__(\n self,\n in_features: int,\n embedding_arch: List[str],\n d_embedding: Optional[int] = None,\n memory_efficient: Optional[bool] = False,\n ):\n \"\"\"\n Parameters\n ----------\n in_features\n Input feature size.\n embedding_arch\n A list containing the names of embedding layers.\n Currently support:\n {'linear', 'shared_linear', 'autodis', 'positional', 'relu', 'leaky_relu', 'layernorm'}\n To use the embedding schemes summarized in Table 3 of 'On Embeddings for Numerical Features in Tabular Deep Learning' (https://arxiv.org/abs/2203.05556)\n By setting the embedding_arch as follows:\n 1. `L`: ['linear']\n 2. `LR`: ['linear', 'relu']\n 3. `LRLR`: ['linear', 'relu', 'linear', 'relu']\n 4. `P`: ['positional']\n 5. `PL`: ['positional', 'linear']\n 6. `PLR`: ['positional', 'linear', 'relu']\n 7. `PLRLR`: ['positional', 'linear', 'relu', 'linear', 'relu']\n 8. `AutoDis`: ['autodis']\n 9. `Leaky Gates` in [ref.3]: ['linear', 'leaky_relu']\n Notably, in `L` (i.e. embedding_arch=['linear']) for numerical transformer,\n it identical as the original feature_tokenzier in FT_Transformer (c.f. Figure 2.a in https://arxiv.org/pdf/2106.11959.pdf).\n d_embedding:\n Dimension of the embeddings.\n The output shape should be [batch_size, number_of_numerical_featurs, d_embedding]\n memory_efficient:\n Use efficient linear layer scheme if True. Default is False.\n\n Reference:\n ----------\n 1. Code: https://github.com/Yura52/tabular-dl-num-embeddings\n 2. Paper: On Embeddings for Numerical Features in Tabular Deep Learning, https://arxiv.org/abs/2203.05556\n 3. Paper: Simple Modifications to Improve Tabular Neural Networks: https://arxiv.org/pdf/2108.03214\n \"\"\"\n\n super().__init__()\n assert embedding_arch\n assert set(embedding_arch) <= {\n \"linear\",\n \"shared_linear\",\n \"autodis\",\n \"positional\",\n \"relu\",\n \"leaky_relu\",\n \"layernorm\",\n }\n\n if any(x in embedding_arch for x in [\"linear\", \"shared_linear\", \"autodis\"]):\n assert d_embedding is not None\n\n assert embedding_arch.count(\"positional\") <= 1\n\n if \"autodis\" in embedding_arch:\n embedding_arch = [\"autodis\"]\n\n NLinear_ = NLinearMemoryEfficient if memory_efficient else NLinear\n layers: list[nn.Module] = []\n\n if embedding_arch[0] == \"linear\":\n layers.append(\n NumericalFeatureTokenizer(\n in_features=in_features, token_dim=d_embedding, bias=True, initialization=\"normal\"\n )\n )\n elif embedding_arch[0] == \"positional\":\n layers.append(\n Periodic(\n in_features=in_features,\n d_embedding=d_embedding,\n trainable=True,\n initialization=\"normal\",\n sigma=1.0,\n )\n )\n elif embedding_arch[0] == \"autodis\":\n layers.append(\n AutoDis(\n in_features=in_features,\n d_embedding=d_embedding,\n n_meta_embeddings=d_embedding,\n temperature=3.0,\n )\n )\n else:\n layers.append(\n nn.Identity(),\n )\n\n for x in embedding_arch[1:]:\n layers.append(\n nn.ReLU()\n if x == \"relu\"\n else nn.LeakyReLU()\n if x == \"leaky_relu\"\n else NLinear_(in_features, d_embedding, d_embedding)\n if x == \"linear\"\n else nn.Linear(d_embedding, d_embedding)\n if x == \"shared_linear\"\n else NLayerNorm(in_features, d_embedding)\n if x == \"layernorm\"\n else nn.Identity()\n )\n\n assert not isinstance(layers[-1], nn.Identity)\n\n self.d_embedding = d_embedding\n self.in_features = in_features\n self.layers = nn.Sequential(*layers)\n\n @property\n def num_tokens(self) -> int:\n \"\"\"The number of tokens.\"\"\"\n y = self.forward(torch.ones(1, self.in_features))\n return y.shape[1]\n\n @property\n def token_dim(self) -> int:\n \"\"\"The size of one token.\"\"\"\n y = self.forward(torch.ones(1, self.in_features))\n return y.shape[-1]\n\n def forward(self, x):\n return self.layers(x)\n\n\nclass FT_Transformer(nn.Module):\n \"\"\"\n FT-Transformer for categorical tabular features.\n The input dimension is automatically computed based on\n the number of categories in each categorical column.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n num_numerical_columns: int,\n num_categories: Dict,\n numerical_fill_values: Dict,\n embedding_arch: List[str],\n token_dim: int,\n hidden_size: Optional[int] = 192,\n hidden_features: Optional[int] = 192,\n num_classes: Optional[int] = 0,\n token_bias: Optional[bool] = True,\n token_initialization: Optional[str] = \"normal\",\n num_blocks: Optional[int] = 0,\n attention_num_heads: Optional[int] = 8,\n attention_initialization: Optional[str] = \"kaiming\",\n attention_normalization: Optional[str] = \"layer_norm\",\n attention_dropout: Optional[str] = 0.2,\n residual_dropout: Optional[str] = 0.0,\n ffn_activation: Optional[str] = \"reglu\",\n ffn_normalization: Optional[str] = \"layer_norm\",\n ffn_hidden_size: Optional[str] = 6,\n ffn_dropout: Optional[str] = 0.0,\n prenormalization: Optional[bool] = True,\n first_prenormalization: Optional[bool] = False,\n kv_compression_ratio: Optional[float] = None,\n kv_compression_sharing: Optional[str] = None,\n head_activation: Optional[str] = \"relu\",\n head_normalization: Optional[str] = \"layer_norm\",\n additive_attention: Optional[bool] = False,\n share_qv_weights: Optional[bool] = False,\n pooling_mode: Optional[str] = \"cls\",\n checkpoint_name: str = None,\n pretrained: bool = False,\n ) -> None:\n \"\"\"\n Parameters\n ----------\n prefix\n The model prefix.\n num_categories\n A list of integers. Each one is the number of categories in one categorical column.\n token_dim\n The size of one token after categorical/numerical tokenizers.\n hidden_size\n The embedding dimension of the transformer backbone.\n out_features\n Dimension of output features.\n num_classes\n Number of classes. 1 for a regression task.\n token_bias\n If `True`, for each feature, an additional trainable vector will be added in `_CategoricalFeatureTokenizer`\n to the embedding regardless of feature value. Notably, the bias are not shared between features.\n token_initialization\n Initialization policy for parameters in `_CategoricalFeatureTokenizer` and `_CLSToke`.\n Must be one of `['uniform', 'normal']`.\n num_blocks\n Number of the `FT_Transformer` blocks, which should be non-negative.\n attention_num_heads\n Number of attention heads in each `FT_Transformer` block, which should be positive.\n attention_initialization\n Weights initialization scheme for Multi Headed Attention module.\n attention_dropout\n Dropout ratio for the Multi Headed Attention module.\n residual_dropout\n Dropout ratio for the linear layers in FT_Transformer block.\n ffn_activation\n Activation function type for the Feed-Forward Network module.\n ffn_normalization\n Normalization scheme of the Feed-Forward Network module.\n ffn_hidden_size\n Number of the hidden nodes of the linear layers in the Feed-Forward Network module.\n ffn_dropout\n Dropout ratio of the hidden nodes of the linear layers in the Feed-Forward Network module.\n prenormalization, first_prenormalization\n Prenormalization to stabilize the training.\n kv_compression_ratio\n The compression ration to reduce the input sequence length.\n kv_compression_sharing\n If `true` the projections will share weights.\n head_activation\n Activation function type of the MLP layer.\n head_normalization\n Normalization scheme of the MLP layer.\n additive_attention\n If 'true' the transformer will use additive attention with linear complexity to sequence length.\n share_qv_weights\n if 'true', then value and query transformation parameters are shared in additive attention.\n pooling_mode\n The pooling mode for the Transformer. Can be \"cls\", or \"mean\"\n\n References\n ----------\n 1. Yury Gorishniy, Ivan Rubachev, Valentin Khrulkov, Artem Babenko,\n \"Revisiting Deep Learning Models for Tabular Data\", 2021\n https://arxiv.org/pdf/2106.11959.pdf\n 2. Code: https://github.com/Yura52/tabular-dl-revisiting-models\n \"\"\"\n\n super().__init__()\n logger.debug(f\"initializing {prefix} (FT_Transformer)\")\n assert num_categories or num_numerical_columns > 0, \"there must be categorical columns or numerical columns\"\n assert token_dim > 0, \"token_dim must be positive\"\n assert num_blocks >= 0, \"num_blocks must be non-negative\"\n assert attention_num_heads > 0, \"attention_num_heads must be positive\"\n assert token_initialization in [\"uniform\", \"normal\"], \"initialization must be uniform or normal\"\n\n self.prefix = prefix\n self.out_features = hidden_size\n self.pooling_mode = pooling_mode\n\n self.categorical_feature_tokenizer = None\n self.numerical_feature_tokenizer = None\n\n if num_categories:\n self.num_categories = num_categories\n self.categorical_feature_tokenizer = CategoricalFeatureTokenizer(\n num_categories=list(num_categories.values()),\n token_dim=token_dim,\n bias=token_bias,\n initialization=token_initialization,\n )\n self.categorical_adapter = nn.Linear(token_dim, hidden_size)\n\n if num_numerical_columns > 0:\n self.numerical_fill_values = numerical_fill_values\n self.numerical_feature_tokenizer = NumEmbeddings(\n in_features=num_numerical_columns,\n d_embedding=token_dim,\n embedding_arch=embedding_arch,\n )\n self.numerical_adapter = nn.Linear(token_dim, hidden_size)\n\n self.transformer = Custom_Transformer(\n token_dim=hidden_size,\n num_blocks=num_blocks,\n attention_num_heads=attention_num_heads,\n attention_dropout=attention_dropout,\n attention_initialization=attention_initialization,\n attention_normalization=attention_normalization,\n ffn_hidden_size=ffn_hidden_size,\n ffn_dropout=ffn_dropout,\n ffn_activation=ffn_activation,\n ffn_normalization=ffn_normalization,\n residual_dropout=residual_dropout,\n prenormalization=prenormalization,\n first_prenormalization=first_prenormalization,\n last_layer_query_idx=None,\n num_tokens=None,\n kv_compression_ratio=kv_compression_ratio,\n kv_compression_sharing=kv_compression_sharing,\n head_activation=head_activation,\n head_normalization=head_normalization,\n d_out=hidden_features,\n projection=False,\n additive_attention=additive_attention,\n share_qv_weights=share_qv_weights,\n )\n\n self.head = Custom_Transformer.Head(\n d_in=hidden_size,\n d_out=num_classes,\n bias=True,\n activation=head_activation,\n normalization=head_normalization,\n )\n\n self.cls_token = CLSToken(\n token_dim=hidden_size,\n initialization=\"uniform\",\n )\n\n # init tokenizer and head weights\n if self.numerical_feature_tokenizer:\n self.numerical_adapter.apply(init_weights)\n if self.categorical_feature_tokenizer:\n self.categorical_adapter.apply(init_weights)\n self.head.apply(init_weights)\n # init transformer backbone from provided checkpoint\n from ..utils.download import download\n\n if pretrained and checkpoint_name:\n if os.path.exists(checkpoint_name):\n ckpt = torch.load(checkpoint_name) # nosec B614\n else:\n with tempfile.TemporaryDirectory() as tmpdirname:\n checkpoint_path = os.path.join(tmpdirname, \"./ft_transformer_pretrained.ckpt\")\n download(checkpoint_name, checkpoint_path)\n ckpt = torch.load(checkpoint_path) # nosec B614\n self.transformer.load_state_dict(ckpt[\"state_dict\"])\n\n self.name_to_id = self.get_layer_ids()\n\n @property\n def categorical_key(self):\n return f\"{self.prefix}_{CATEGORICAL}\"\n\n @property\n def numerical_key(self):\n return f\"{self.prefix}_{NUMERICAL}\"\n\n @property\n def input_keys(self):\n input_keys = []\n if self.categorical_feature_tokenizer:\n input_keys.append(self.categorical_key)\n if self.numerical_feature_tokenizer:\n input_keys.append(self.numerical_key)\n return input_keys\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(self, batch: dict):\n \"\"\"\n\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n multimodal_features = []\n if self.categorical_feature_tokenizer:\n categorical_inputs = []\n for categorical_input in batch[self.categorical_key]:\n categorical_inputs.append(categorical_input)\n categorical_inputs = torch.stack(categorical_inputs, dim=1)\n\n categorical_features = self.categorical_feature_tokenizer(categorical_inputs)\n categorical_features = self.categorical_adapter(categorical_features)\n multimodal_features.append(categorical_features)\n if self.numerical_feature_tokenizer:\n numerical_features = self.numerical_feature_tokenizer(batch[self.numerical_key])\n numerical_features = self.numerical_adapter(numerical_features)\n multimodal_features.append(numerical_features)\n\n multimodal_features = torch.cat(multimodal_features, dim=1)\n multimodal_features = self.cls_token(multimodal_features)\n features = self.transformer(multimodal_features)\n logits = self.head(features)\n\n if self.pooling_mode == \"cls\":\n features = features[:, -1, :]\n elif self.pooling_mode == \"mean\":\n features = features.mean(dim=1)\n else:\n raise NotImplementedError(f\"Pooling mode={self.pooling_mode} is not supported.\")\n\n output = {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n }\n }\n\n return output\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n All layers have the same id 0 since there is no pre-trained models used here.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/fusion/__init__.py",
"content": "from .base import AbstractMultimodalFusionModel\nfrom .fusion_mlp import MultimodalFusionMLP\nfrom .fusion_ner import MultimodalFusionNER\nfrom .fusion_transformer import MultimodalFusionTransformer\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/fusion/base.py",
"content": "import logging\nfrom abc import ABC, abstractclassmethod, abstractmethod\nfrom typing import Optional\n\nfrom torch import nn\n\nlogger = logging.getLogger(__name__)\n\n\nclass AbstractMultimodalFusionModel(ABC, nn.Module):\n \"\"\"\n An abstract class to fuse different models' features (single-modal and multimodal).\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n models: list,\n aux_loss_weight: Optional[float] = None,\n ):\n super().__init__()\n\n self.prefix = prefix\n self.aux_loss_weight = aux_loss_weight\n self.model = nn.ModuleList(models)\n\n @property\n @abstractmethod\n def label_key(self):\n pass\n\n @abstractmethod\n def forward(self, *args, **kwargs):\n pass\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end.\n\n It assumes that each individual model has the \"name_to_id\" attribute storing\n the already computed model's layer ids. This function only collects those layer ids.\n It also add prefixes for each model's parameter names since the fusion model wraps\n those individual models, making the name scope changed. Configuring the optimizer\n requires a full name of each parameter.\n\n The layers defined in this class, e.g., head, adapter,\n and, fusion_mlp, have id 0.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n names = [n for n, _ in self.named_parameters()]\n\n outer_layer_names = [n for n in names if not n.startswith(model_prefix)]\n name_to_id = {}\n logger.debug(f\"outer layers are treated as head: {outer_layer_names}\")\n for n in outer_layer_names:\n name_to_id[n] = 0\n\n for i, per_model in enumerate(self.model):\n per_model_prefix = f\"{model_prefix}.{i}\"\n if not hasattr(per_model, \"name_to_id\"):\n raise ValueError(f\"name_to_id attribute is missing in model: {per_model.__class__.__name__}\")\n for n, layer_id in per_model.name_to_id.items():\n full_n = f\"{per_model_prefix}.{n}\"\n name_to_id[full_n] = layer_id\n\n # double check each parameter has been assigned an id\n for n in names:\n assert n in name_to_id\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/fusion/fusion_mlp.py",
"content": "import logging\nfrom typing import List, Optional\n\nimport torch\nfrom torch import nn\n\nfrom ...constants import (\n AUG_LOGITS,\n FEATURES,\n LABEL,\n LOGITS,\n MULTIMODAL_FEATURES,\n MULTIMODAL_FEATURES_POST_AUG,\n MULTIMODAL_FEATURES_PRE_AUG,\n ORI_LOGITS,\n VAE_MEAN,\n VAE_VAR,\n WEIGHT,\n)\nfrom ..mlp import MLP\nfrom ..utils import init_weights, run_model\nfrom .base import AbstractMultimodalFusionModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultimodalFusionMLP(AbstractMultimodalFusionModel):\n \"\"\"\n Use MLP to fuse different models' features (single-modal and multimodal).\n Specifically, it adapts the features of each model to specified dimensions,\n concatenates the adapted features, and fuses the features through MLP.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n models: list,\n hidden_features: List[int],\n num_classes: int,\n adapt_in_features: Optional[str] = None,\n activation: Optional[str] = \"gelu\",\n dropout: Optional[float] = 0.5,\n normalization: Optional[str] = \"layer_norm\",\n aux_loss_weight: Optional[float] = None,\n ):\n \"\"\"\n Parameters\n ----------\n prefix\n The fusion model's prefix\n models\n The individual models whose output features will be fused.\n hidden_features\n A list of integers representing the hidden feature dimensions. For example,\n [512, 128, 64] indicates three hidden MLP layers with their corresponding output\n feature dimensions.\n num_classes\n The number of classes.\n adapt_in_features\n Choice of how to adapt the features of each model. We now support\n - min\n Adapt all features to the minimum dimension. For example, if three models have\n feature dimensions [512, 768, 64], it will linearly map all the features to\n dimension 64.\n - max\n Adapt all features to the maximum dimension. For example, if three models have\n feature dimensions are [512, 768, 64], it will linearly map all the features to\n dimension 768.\n activation\n Name of activation function.\n dropout\n Dropout probability.\n normalization\n Name of normalization function.\n aux_loss_weight\n The weight of individual models. For example, if we fuse the features of ViT, CLIP, and BERT,\n The loss will be computed as \"loss = fusion_loss + aux_loss_weight(vit_loss + clip_loss + bert_loss)\".\n Basically, it supports adding an auxiliary loss for each individual model.\n \"\"\"\n super().__init__(\n prefix=prefix,\n models=models,\n aux_loss_weight=aux_loss_weight,\n )\n logger.debug(f\"initializing {prefix} (MultimodalFusionMLP)\")\n if aux_loss_weight is not None:\n assert aux_loss_weight >= 0\n logger.debug(f\"auxiliary loss weight: {aux_loss_weight}\")\n self.num_classes = num_classes\n self.augmenter = None\n\n raw_in_features = [per_model.out_features for per_model in models]\n if adapt_in_features is not None:\n if adapt_in_features == \"min\":\n base_in_feat = min(raw_in_features)\n elif adapt_in_features == \"max\":\n base_in_feat = max(raw_in_features)\n else:\n raise ValueError(f\"unknown adapt_in_features: {adapt_in_features}\")\n\n self.adapter = nn.ModuleList([nn.Linear(in_feat, base_in_feat) for in_feat in raw_in_features])\n\n in_features = base_in_feat * len(raw_in_features)\n else:\n self.adapter = nn.ModuleList([nn.Identity() for _ in range(len(raw_in_features))])\n in_features = sum(raw_in_features)\n\n assert len(self.adapter) == len(self.model)\n self.augmenter_in_features = in_features\n\n fusion_mlp = []\n for per_hidden_features in hidden_features:\n fusion_mlp.append(\n MLP(\n in_features=in_features,\n hidden_features=per_hidden_features,\n out_features=per_hidden_features,\n num_layers=1,\n activation=activation,\n dropout=dropout,\n normalization=normalization,\n )\n )\n in_features = per_hidden_features\n self.fusion_mlp = nn.Sequential(*fusion_mlp)\n # in_features has become the latest hidden size\n self.head = nn.Linear(in_features, num_classes)\n # init weights\n self.adapter.apply(init_weights)\n self.fusion_mlp.apply(init_weights)\n self.head.apply(init_weights)\n\n self.out_features = in_features\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def input_keys(self):\n input_keys = []\n for m in self.model:\n assert hasattr(m, \"input_keys\"), f\"invalid model {type(m)}, which doesn't have a 'input_keys' attribute\"\n input_keys += m.input_keys\n return input_keys\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(\n self,\n *args,\n ):\n \"\"\"\n\n Parameters\n ----------\n *args\n A list of torch.Tensor(s) containing the input mini-batch data. The fusion model doesn't need to\n directly access the mini-batch data since it aims to fuse the individual models'\n output features.\n\n Returns\n -------\n If \"aux_loss_weight\" is None, it returns dictionary containing the fusion model's logits and\n features. Otherwise, it returns a list of dictionaries collecting all the models' output,\n including the fusion model's.\n \"\"\"\n multimodal_features = []\n multimodal_logits = []\n multimodal_features_pre_aug = None\n multimodal_features_post_aug = None\n vae_mean = None\n vae_var = None\n offset = 0\n for per_model, per_adapter in zip(self.model, self.adapter):\n per_model_args = args[offset : offset + len(per_model.input_keys)]\n batch = dict(zip(per_model.input_keys, per_model_args))\n per_output = run_model(per_model, batch)\n multimodal_features.append(\n per_adapter(per_output[per_model.prefix][FEATURES].to(per_adapter.weight.dtype))\n )\n multimodal_logits.append(per_output[per_model.prefix][LOGITS])\n offset += len(per_model.input_keys)\n\n # make sure the returned multimodal features contain unimodal encoder features\n multimodal_features_ret = multimodal_features\n multimodal_features = torch.cat(multimodal_features, dim=1)\n batch_size = multimodal_features.shape[0]\n if self.training and self.augmenter is not None:\n multimodal_features_pre_aug = multimodal_features.detach().clone() # [bs, dim]\n multimodal_features_post_aug, vae_mean, vae_var = self.augmenter(multimodal_features_pre_aug)\n multimodal_features_post_aug_clone = multimodal_features_post_aug.clone()\n multimodal_features_post_aug_clone.register_hook(lambda grad: -grad * self.augmenter.adv_weight)\n multimodal_features = torch.cat([multimodal_features, multimodal_features_post_aug_clone], dim=0)\n\n features = self.fusion_mlp(multimodal_features)\n logits = self.head(features)\n ori_logits = logits[:batch_size].detach() # detach the original logits when computing the consistency loss\n aug_logits = logits[batch_size:]\n\n return (\n features,\n logits,\n multimodal_logits,\n multimodal_features_ret,\n multimodal_features_pre_aug,\n multimodal_features_post_aug,\n ori_logits,\n aug_logits,\n vae_mean,\n vae_var,\n )\n\n def get_output_dict(\n self,\n features: torch.Tensor,\n logits: torch.Tensor,\n multimodal_logits: List[torch.Tensor],\n multimodal_features: List[torch.Tensor],\n multimodal_features_pre_aug: torch.Tensor,\n multimodal_features_post_aug: torch.Tensor,\n ori_logits: torch.Tensor,\n aug_logits: torch.Tensor,\n vae_mean: torch.Tensor,\n vae_var: torch.Tensor,\n ):\n fusion_output = {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n MULTIMODAL_FEATURES: multimodal_features,\n MULTIMODAL_FEATURES_PRE_AUG: multimodal_features_pre_aug,\n MULTIMODAL_FEATURES_POST_AUG: multimodal_features_post_aug,\n ORI_LOGITS: ori_logits,\n AUG_LOGITS: aug_logits,\n VAE_MEAN: vae_mean,\n VAE_VAR: vae_var,\n }\n }\n # filter out None\n fusion_output = {self.prefix: {k: v for k, v in fusion_output[self.prefix].items() if v is not None}}\n if self.aux_loss_weight is not None:\n output = {}\n for per_model, per_logits in zip(self.model, multimodal_logits):\n per_output = {per_model.prefix: {}}\n per_output[per_model.prefix][WEIGHT] = torch.tensor(self.aux_loss_weight).to(per_logits.dtype)\n per_output[per_model.prefix][LOGITS] = per_logits\n output.update(per_output)\n fusion_output[self.prefix].update({WEIGHT: torch.tensor(1.0).to(logits)})\n output.update(fusion_output)\n return output\n else:\n return fusion_output\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/fusion/fusion_ner.py",
"content": "import logging\nfrom typing import List, Optional\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\n\nfrom ...constants import FEATURES, LABEL, LOGITS, NER_ANNOTATION, NER_TEXT, TOKEN_WORD_MAPPING, WORD_OFFSETS\nfrom ..mlp import MLP\nfrom ..utils import run_model\nfrom .base import AbstractMultimodalFusionModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultimodalFusionNER(AbstractMultimodalFusionModel):\n \"\"\"\n Use MLP to fuse different models' features (single-modal and multimodal) for NER.\n Specifically, it adapts the features of each model to specified dimensions,\n concatenates the adapted features, and fuses the features through MLP.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n models: list,\n hidden_features: List[int],\n num_classes: int,\n adapt_in_features: str = \"max\",\n activation: Optional[str] = \"gelu\",\n dropout_prob: Optional[float] = 0.5,\n normalization: Optional[str] = \"layer_norm\",\n loss_weight: Optional[float] = None,\n ):\n \"\"\"\n Parameters\n ----------\n prefix\n The fusion model's prefix\n models\n The individual models whose output features will be fused.\n hidden_features\n A list of integers representing the hidden feature dimensions. For example,\n [512, 128, 64] indicates three hidden MLP layers with their corresponding output\n feature dimensions.\n num_classes\n The number of classes.\n adapt_in_features\n Choice of how to adapt the features of each model. We now support\n - min\n Adapt all features to the minimum dimension. For example, if three models have\n feature dimensions [512, 768, 64], it will linearly map all the features to\n dimension 64.\n - max\n Adapt all features to the maximum dimension. For example, if three models have\n feature dimensions are [512, 768, 64], it will linearly map all the features to\n dimension 768.\n activation\n Name of activation function.\n dropout_prob\n Dropout probability.\n normalization\n Name of normalization function.\n loss_weight\n The weight of individual models.\n \"\"\"\n super().__init__(\n prefix=prefix,\n models=models,\n loss_weight=loss_weight,\n )\n logger.debug(\"initializing MultimodalFusionNER\")\n\n if loss_weight is not None:\n assert loss_weight > 0\n self.num_classes = num_classes\n\n self.ner_model = None\n self.tokenizer = None\n other_models = []\n for per_model in models:\n if per_model.prefix != NER_TEXT:\n other_models.append(per_model)\n else:\n self.ner_model = per_model\n self.tokenizer = per_model.tokenizer\n self.other_models = nn.ModuleList(other_models)\n raw_in_features = [per_model.out_features for per_model in models if per_model.prefix != NER_TEXT]\n if adapt_in_features is not None:\n if adapt_in_features == \"min\":\n base_in_feat = min(raw_in_features)\n elif adapt_in_features == \"max\":\n base_in_feat = max(raw_in_features)\n else:\n raise ValueError(f\"unknown adapt_in_features: {adapt_in_features}\")\n self.adapter = nn.ModuleList([nn.Linear(in_feat, base_in_feat) for in_feat in raw_in_features])\n in_features = base_in_feat * len(raw_in_features)\n else:\n self.adapter = nn.ModuleList([nn.Identity() for _ in range(len(raw_in_features))])\n in_features = sum(raw_in_features)\n assert len(self.adapter) == len(self.other_models)\n fusion_mlp = []\n for per_hidden_features in hidden_features:\n fusion_mlp.append(\n MLP(\n in_features=in_features,\n hidden_features=per_hidden_features,\n out_features=per_hidden_features,\n num_layers=1,\n activation=activation,\n dropout_prob=dropout_prob,\n normalization=normalization,\n )\n )\n in_features = per_hidden_features\n self.fusion_mlp = nn.Sequential(*fusion_mlp)\n self.head = nn.Linear(in_features + self.ner_model.out_features, num_classes)\n self.out_features = in_features\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def label_key(self):\n return f\"{NER_TEXT}_{LABEL}\"\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data. The fusion model doesn't need to\n directly access the mini-batch data since it aims to fuse the individual models'\n output features.\n Returns\n -------\n It returns dictionary containing the fusion model's logits and features.\n \"\"\"\n multimodal_features = []\n ner_output = run_model(self.ner_model, batch)\n for per_model, per_adapter in zip(self.other_models, self.adapter):\n per_output = run_model(per_model, batch)\n multimodal_features.append(per_adapter(per_output[per_model.prefix][FEATURES]))\n\n features = self.fusion_mlp(torch.cat(multimodal_features, dim=1))\n features = features.unsqueeze(dim=1).repeat(1, ner_output[self.ner_model.prefix][FEATURES].size()[1], 1)\n features = torch.cat((ner_output[self.ner_model.prefix][FEATURES], features), dim=-1)\n\n logits = self.head(features)\n logits_label = torch.argmax(F.log_softmax(logits, dim=-1), dim=-1)\n fusion_output = {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n NER_ANNOTATION: logits_label,\n TOKEN_WORD_MAPPING: ner_output[self.ner_model.prefix][TOKEN_WORD_MAPPING],\n WORD_OFFSETS: ner_output[self.ner_model.prefix][WORD_OFFSETS],\n }\n }\n\n return fusion_output\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/fusion/fusion_transformer.py",
"content": "import logging\nfrom typing import Optional\n\nimport torch\nfrom torch import nn\n\nfrom ...constants import FEATURES, LABEL, LOGITS, WEIGHT\nfrom ..custom_transformer import CLSToken, Custom_Transformer\nfrom ..utils import init_weights, run_model\nfrom .base import AbstractMultimodalFusionModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultimodalFusionTransformer(AbstractMultimodalFusionModel):\n \"\"\"\n Use Transformer to fuse different models' features (single-modal and multimodal).\n Specifically, it adapts the features of each model to specified dimensions,\n concatenates the adapted features, and fuses the features through Transformer.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n models: list,\n hidden_features: int,\n num_classes: int,\n num_blocks: Optional[int] = 0,\n attention_num_heads: Optional[int] = 8,\n attention_initialization: Optional[str] = \"kaiming\",\n attention_normalization: Optional[str] = \"layer_norm\",\n attention_dropout: Optional[str] = 0.2,\n residual_dropout: Optional[str] = 0.0,\n ffn_activation: Optional[str] = \"reglu\",\n ffn_normalization: Optional[str] = \"layer_norm\",\n ffn_hidden_size: Optional[str] = 192,\n ffn_dropout: Optional[str] = 0.0,\n prenormalization: Optional[bool] = True,\n first_prenormalization: Optional[bool] = False,\n kv_compression_ratio: Optional[float] = None,\n kv_compression_sharing: Optional[str] = None,\n head_activation: Optional[str] = \"relu\",\n head_normalization: Optional[str] = \"layer_norm\",\n adapt_in_features: Optional[str] = None,\n aux_loss_weight: Optional[float] = None,\n additive_attention: Optional[bool] = False,\n share_qv_weights: Optional[bool] = False,\n ):\n \"\"\"\n Parameters\n ----------\n prefix\n The fusion model's prefix\n models\n The individual models whose output features will be fused.\n hidden_features\n A list of integers representing the hidden feature dimensions. For example,\n [512, 128, 64] indicates three hidden MLP layers with their corresponding output\n feature dimensions.\n num_classes\n The number of classes.\n num_blocks\n Number of the `FT_Transformer` blocks, which should be non-negative.\n attention_num_heads\n Number of attention heads in each `FT_Transformer` block, which should be positive.\n attention_dropout\n Dropout ratio for the Multi Headed Attention module.\n attention_initialization\n Weights initialization scheme for Multi Headed Attention module.\n attention_normalization\n Normalization policy for attention layers. \"layer_norm\" is a good default.\n residual_dropout\n Dropout ratio for the linear layers in FT_Transformer block.\n ffn_hidden_size\n Number of the hidden nodes of the linear layers in the Feed-Forward Network module.\n ffn_dropout\n Dropout ratio of the hidden nodes of the linear layers in the Feed-Forward Network module.\n ffn_activation\n Activation function type for the Feed-Forward Network module.\n ffn_normalization\n Normalization scheme of the Feed-Forward Network module.\n prenormalization, first_prenormalization\n Prenormalization to stabilize the training.\n kv_compression_ratio\n The compression ration to reduce the input sequence length.\n kv_compression_sharing\n If `true` the projections will share weights.\n head_activation\n Activation function type of the MLP layer.\n head_normalization\n Normalization scheme of the MLP layer.\n adapt_in_features\n Choice of how to adapt the features of each model. We now support\n - min\n Adapt all features to the minimum dimension. For example, if three models have\n feature dimensions [512, 768, 64], it will linearly map all the features to\n dimension 64.\n - max\n Adapt all features to the maximum dimension. For example, if three models have\n feature dimensions are [512, 768, 64], it will linearly map all the features to\n dimension 768.\n aux_loss_weight\n The weight of individual models. For example, if we fuse the features of ViT, CLIP, and BERT,\n The loss will be computed as \"loss = fusion_loss + aux_loss_weight(vit_loss + clip_loss + bert_loss)\".\n Basically, it supports adding an auxiliary loss for each individual model.\n additive_attention\n If 'true' the transformer will use additive attention with linear complexity to sequence length.\n share_qv_weights\n if 'true', then value and query transformation parameters are shared in additive attention.\n \"\"\"\n super().__init__(\n prefix=prefix,\n models=models,\n aux_loss_weight=aux_loss_weight,\n )\n logger.debug(f\"initializing {prefix} (MultimodalFusionTransformer)\")\n if aux_loss_weight is not None:\n assert aux_loss_weight >= 0\n\n raw_in_features = [per_model.out_features for per_model in models]\n\n if adapt_in_features == \"min\":\n base_in_feat = min(raw_in_features)\n elif adapt_in_features == \"max\":\n base_in_feat = max(raw_in_features)\n else:\n raise ValueError(f\"unknown adapt_in_features: {adapt_in_features}\")\n\n self.adapter = nn.ModuleList([nn.Linear(in_feat, base_in_feat) for in_feat in raw_in_features])\n\n in_features = base_in_feat\n\n assert len(self.adapter) == len(self.model)\n\n self.fusion_transformer = Custom_Transformer(\n token_dim=in_features,\n num_blocks=num_blocks,\n attention_num_heads=attention_num_heads,\n attention_dropout=attention_dropout,\n attention_initialization=attention_initialization,\n attention_normalization=attention_normalization,\n ffn_hidden_size=ffn_hidden_size,\n ffn_dropout=ffn_dropout,\n ffn_activation=ffn_activation,\n ffn_normalization=ffn_normalization,\n residual_dropout=residual_dropout,\n prenormalization=prenormalization,\n first_prenormalization=first_prenormalization,\n last_layer_query_idx=None,\n num_tokens=None,\n kv_compression_ratio=kv_compression_ratio,\n kv_compression_sharing=kv_compression_sharing,\n head_activation=head_activation,\n head_normalization=head_normalization,\n d_out=hidden_features,\n projection=False,\n additive_attention=additive_attention,\n share_qv_weights=share_qv_weights,\n )\n\n self.head = Custom_Transformer.Head(\n d_in=in_features,\n d_out=num_classes,\n bias=True,\n activation=head_activation,\n normalization=head_normalization,\n )\n\n self.cls_token = CLSToken(\n token_dim=in_features,\n initialization=\"uniform\",\n )\n\n self.out_features = in_features\n\n # init weights\n self.adapter.apply(init_weights)\n self.head.apply(init_weights)\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(\n self,\n batch: dict,\n ):\n multimodal_features = []\n output = {}\n for per_model, per_adapter in zip(self.model, self.adapter):\n per_output = run_model(per_model, batch)\n multimodal_feature = per_adapter(per_output[per_model.prefix][FEATURES])\n if multimodal_feature.ndim == 2:\n multimodal_feature = torch.unsqueeze(multimodal_feature, dim=1)\n multimodal_features.append(multimodal_feature)\n\n if self.aux_loss_weight is not None:\n per_output[per_model.prefix].update(\n {WEIGHT: torch.tensor(self.aux_loss_weight).to(multimodal_features[0])}\n )\n output.update(per_output)\n\n multimodal_features = torch.cat(multimodal_features, dim=1)\n multimodal_features = self.cls_token(multimodal_features)\n features = self.fusion_transformer(multimodal_features)\n\n logits = self.head(features)\n fusion_output = {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n }\n }\n if self.aux_loss_weight is not None:\n fusion_output[self.prefix].update({WEIGHT: torch.tensor(1.0).to(logits)})\n output.update(fusion_output)\n return output\n else:\n return fusion_output\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/hf_text.py",
"content": "import logging\nfrom typing import Dict, List, Optional, Tuple\n\nimport torch\nfrom torch import nn\nfrom transformers import logging as hf_logging\nfrom transformers.models.t5 import T5PreTrainedModel\n\nfrom ..constants import (\n COLUMN,\n COLUMN_FEATURES,\n FEATURES,\n LABEL,\n LOGITS,\n MASKS,\n TEXT_SEGMENT_IDS,\n TEXT_TOKEN_IDS,\n TEXT_VALID_LENGTH,\n)\nfrom .utils import (\n DummyLayer,\n assign_layer_ids,\n get_column_features,\n get_hf_config_and_model,\n get_pretrained_tokenizer,\n get_text_segment_num,\n get_text_token_max_len,\n init_weights,\n)\n\nhf_logging.set_verbosity_error()\n\nlogger = logging.getLogger(__name__)\n\n\nclass HFAutoModelForTextPrediction(nn.Module):\n \"\"\"\n Support huggingface text backbones.\n Refer to https://github.com/huggingface/transformers\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str = \"microsoft/deberta-v3-base\",\n num_classes: Optional[int] = 0,\n pooling_mode: Optional[str] = \"cls\",\n gradient_checkpointing: Optional[bool] = False,\n low_cpu_mem_usage: Optional[bool] = False,\n pretrained: Optional[bool] = True,\n tokenizer_name: Optional[str] = \"hf_auto\",\n max_text_len: Optional[int] = None,\n text_segment_num: Optional[int] = 1,\n use_fast: Optional[bool] = True,\n ):\n \"\"\"\n Load a pretrained huggingface text transformer backbone.\n\n Parameters\n ----------\n prefix\n The model prefix.\n checkpoint_name\n Name of the checkpoint or the local directory of a custom checkpoint.\n We support loading checkpoint from\n Huggingface Models list: https://huggingface.co/models\n For example, you may use\n English backbones:\n - 'microsoft/deberta-v3-base'\n - 'bert-base-uncased'\n - 'google/electra-base-discriminator'\n - 'distilroberta-base'\n Multilingual backbones:\n - 'microsoft/mdeberta-v3-base'\n - 'xlm-roberta-base'\n num_classes\n The number of classes. 1 for a regression task.\n pooling_mode\n The pooling mode for the Transformer. Can be \"cls\", or \"mean\"\n gradient_checkpointing\n Whether to enable gradient checkpointing\n low_cpu_mem_usage\n Whether to turn on the optimization of reducing the peak CPU memory usage when loading the pretrained model.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n tokenizer_name\n Name of the huggingface tokenizer type.\n max_text_len\n The maximum length of text tokens.\n text_segment_num\n The number of text segments.\n use_fast\n Use a fast Rust-based tokenizer if it is supported for a given model.\n If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead.\n See: https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoTokenizer.from_pretrained.use_fast\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {prefix} (HFAutoModelForTextPrediction)\")\n logger.debug(f\"model checkpoint: {checkpoint_name}\")\n self.checkpoint_name = checkpoint_name\n self.num_classes = num_classes\n\n self.config, self.model = get_hf_config_and_model(\n checkpoint_name=checkpoint_name, pretrained=pretrained, low_cpu_mem_usage=low_cpu_mem_usage\n )\n self.tokenizer_name = tokenizer_name\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=self.checkpoint_name,\n use_fast=use_fast,\n )\n self.max_text_len = get_text_token_max_len(\n provided_max_len=max_text_len,\n config=self.config,\n tokenizer=self.tokenizer,\n checkpoint_name=self.checkpoint_name,\n )\n self.text_segment_num = get_text_segment_num(\n config=self.config,\n provided_segment_num=text_segment_num,\n checkpoint_name=self.checkpoint_name,\n )\n\n if isinstance(self.model, T5PreTrainedModel):\n self.is_t5 = True\n # Remove the decoder in T5. We will only use the T5 encoder for extracting the embeddings\n del self.model.decoder\n else:\n self.is_t5 = False\n\n self.gradient_checkpointing = gradient_checkpointing\n if gradient_checkpointing:\n self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={\"use_reentrant\": False})\n if self.is_t5:\n self.dummy_layer = DummyLayer()\n\n self.out_features = self.model.config.hidden_size\n\n self.head = nn.Linear(self.out_features, num_classes) if num_classes else nn.Identity()\n self.head.apply(init_weights)\n\n self.prefix = prefix\n self.pooling_mode = pooling_mode\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n if hasattr(self.model.config, \"type_vocab_size\") and self.model.config.type_vocab_size <= 1:\n # Disable segment ids for models like RoBERTa\n self.disable_seg_ids = True\n else:\n self.disable_seg_ids = False\n\n @property\n def text_token_ids_key(self):\n return f\"{self.prefix}_{TEXT_TOKEN_IDS}\"\n\n @property\n def text_segment_ids_key(self):\n return f\"{self.prefix}_{TEXT_SEGMENT_IDS}\"\n\n @property\n def text_valid_length_key(self):\n return f\"{self.prefix}_{TEXT_VALID_LENGTH}\"\n\n @property\n def input_keys(self):\n return [self.text_token_ids_key, self.text_segment_ids_key, self.text_valid_length_key]\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def text_column_prefix(self):\n return f\"{self.text_token_ids_key}_{COLUMN}\"\n\n @property\n def text_feature_dim(self):\n return self.model.config.hidden_size\n\n def forward(\n self,\n text_token_ids: torch.Tensor,\n text_segment_ids: torch.Tensor,\n text_valid_length: torch.Tensor,\n text_column_names: Optional[List[str]] = None,\n text_column_indices: Optional[List[torch.Tensor]] = None,\n ):\n \"\"\"\n Parameters\n ----------\n text_token_ids : torch.Tensor\n Indices of input sequence tokens in the vocabulary.\n text_segment_ids : torch.Tensor\n Indices of input sequence segments.\n text_valid_length : torch.Tensor\n Valid length of the input text sequence.\n text_column_names : list of str, optional\n Names of the text columns.\n text_column_indices : list of torch.Tensor, optional\n Start and stop indices of the text columns.\n\n Returns\n -------\n A tuple that contains (pooled_features, logits, column_features, column_feature_masks)\n \"\"\"\n if self.disable_seg_ids:\n text_segment_ids = None\n\n steps = torch.arange(0, text_token_ids.shape[1]).type_as(text_valid_length)\n text_masks = (steps.reshape((1, -1)) < text_valid_length.reshape((-1, 1))).type_as(text_token_ids)\n\n if self.is_t5:\n # For the T5 model, we will only use the encoder to encode the sentence. This is adopted in\n # \"Sentence-T5 (ST5): Scalable Sentence Encoders from Pre-trained Text-to-Text Models\"\n # (https://aclanthology.org/2022.findings-acl.146.pdf).\n inputs_embeds = self.model.encoder.embed_tokens(text_token_ids)\n if self.gradient_checkpointing:\n # FIXME(?) This is a hack! We added a DummyLayer to ensure that the\n # gradient checkpointing will assign output layer as require_grad=True\n # Reference: https://discuss.pytorch.org/t/checkpoint-with-no-grad-requiring-inputs-problem/19117/9\n inputs_embeds = self.dummy_layer(inputs_embeds)\n outputs = self.model.encoder(\n inputs_embeds=inputs_embeds,\n attention_mask=text_masks,\n )\n else:\n if \"token_type_ids\" in self.tokenizer.model_input_names:\n outputs = self.model(\n input_ids=text_token_ids,\n token_type_ids=text_segment_ids,\n attention_mask=text_masks,\n )\n else:\n outputs = self.model(\n input_ids=text_token_ids,\n attention_mask=text_masks,\n )\n if self.pooling_mode == \"cls\":\n pooled_features = outputs.last_hidden_state[:, 0, :]\n elif self.pooling_mode == \"mean\":\n pooled_features = (outputs.last_hidden_state * text_masks.unsqueeze(-1)).sum(1)\n sum_mask = text_masks.unsqueeze(-1).sum(1)\n sum_mask = torch.clamp(sum_mask, min=1e-9)\n pooled_features = pooled_features / sum_mask\n else:\n raise NotImplementedError(f\"Pooling mode={self.pooling_mode} is not supported.\")\n\n logits = self.head(pooled_features)\n last_hidden_state = outputs.last_hidden_state\n\n batch = {\n self.text_token_ids_key: text_token_ids,\n self.text_segment_ids_key: text_segment_ids,\n self.text_valid_length_key: text_valid_length,\n }\n if text_column_names:\n assert len(text_column_names) == len(text_column_indices), \"invalid text column inputs\"\n for idx, name in enumerate(text_column_names):\n batch[name] = text_column_indices[idx]\n column_features, column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.text_column_prefix,\n features=last_hidden_state,\n valid_lengths=text_valid_length,\n cls_feature=pooled_features,\n )\n\n if column_features == {} or column_feature_masks == {}:\n return pooled_features, logits\n else:\n return pooled_features, logits, column_features, column_feature_masks\n\n def get_output_dict(\n self,\n pooled_features: torch.Tensor,\n logits: torch.Tensor,\n column_features: Optional[Dict[str, torch.Tensor]] = None,\n column_feature_masks: Optional[Dict[str, torch.Tensor]] = None,\n ):\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n if column_features != None:\n ret[COLUMN_FEATURES][FEATURES].update(column_features)\n ret[COLUMN_FEATURES][MASKS].update(column_feature_masks)\n ret[LOGITS] = logits\n ret[FEATURES] = pooled_features\n return {self.prefix: ret}\n\n def get_layer_ids(self):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n In the AutoModel scenario, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\n \"embeddings\",\n \"LayerNorm\",\n \"wte\",\n \"wpe\",\n \"shared.weight\",\n \"encoder.conv.conv\",\n \"relative_attention_bias\",\n \"dummy_layer\",\n )\n post_encoder_patterns = (\"head\", \"pooler\", \"ln_f\", \"final_layer_norm\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n\n def save(self, save_path: str = \"./\"):\n self.model.save_pretrained(save_path)\n self.tokenizer.save_pretrained(save_path)\n logger.info(f\"Model weights and tokenizer for {self.prefix} are saved to {save_path}.\")\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/meta_transformer.py",
"content": "import logging\nfrom typing import Dict, Optional\n\nimport torch\nfrom timm import create_model\nfrom timm.models.vision_transformer import Block\nfrom torch import nn\n\nfrom ..constants import (\n CATEGORICAL,\n FEATURES,\n IMAGE,\n IMAGE_VALID_NUM,\n LABEL,\n LOGITS,\n NUMERICAL,\n TEXT_SEGMENT_IDS,\n TEXT_TOKEN_IDS,\n TEXT_VALID_LENGTH,\n)\nfrom .custom_transformer import CLSToken\nfrom .ft_transformer import CategoricalFeatureTokenizer, NumEmbeddings\nfrom .utils import (\n assign_layer_ids,\n get_hf_config_and_model,\n get_image_size_mean_std,\n get_pretrained_tokenizer,\n get_text_segment_num,\n get_text_token_max_len,\n init_weights,\n replace_missing_images_with_learnable,\n)\n\nlogger = logging.getLogger(__name__)\n\n\nclass MetaTransformer(nn.Module):\n def __init__(\n self,\n prefix: str,\n num_classes: int,\n checkpoint_path: str,\n model_version: str,\n has_image: bool,\n has_text: bool,\n num_numerical_columns: int,\n num_categories: Dict,\n numerical_fill_values: Dict,\n image_size: Optional[int] = None,\n image_norm: Optional[str] = None,\n image_chan_num: Optional[int] = 3,\n use_learnable_image: Optional[bool] = False,\n max_text_len: Optional[int] = None,\n text_segment_num: Optional[int] = 1,\n ):\n super().__init__()\n logger.debug(f\"initializing {prefix} (MetaTransformer)\")\n self.prefix = prefix\n self.checkpoint_name = checkpoint_path\n\n if model_version == \"base\":\n dim = 768\n num_heads = 12\n layer_num = 12\n elif model_version == \"large\":\n dim = 1024\n num_heads = 16\n layer_num = 24\n else:\n raise ValueError(f\"Unsupported model version: {model_version}. Options are 'base' and 'large'.\")\n\n self.model = nn.Sequential(\n *[\n Block(\n dim=dim,\n num_heads=num_heads,\n mlp_ratio=4.0,\n qkv_bias=True,\n norm_layer=nn.LayerNorm,\n act_layer=nn.GELU,\n )\n for i in range(layer_num)\n ]\n )\n\n checkpoint = torch.load(checkpoint_path) # nosec B614\n self.checkpoint_path = checkpoint_path\n self.model.load_state_dict(checkpoint, strict=True)\n\n self.head = nn.Linear(dim, num_classes) if num_classes else nn.Identity()\n\n self.cls_token = CLSToken(token_dim=dim, initialization=\"uniform\")\n self.config = None\n\n self.tokenizer = None\n self.text_adaptor = None\n self.image_tokenizer = None\n self.image_adaptor = None\n self.categorical_feature_tokenizer = None\n self.categorical_adapter = None\n self.numerical_feature_tokenizer = None\n self.numerical_adapter = None\n\n # if has_text or has_image:\n # clip_ckpt = \"openai/clip-vit-base-patch32\"\n # _, clip_model = get_hf_config_and_model(checkpoint_name=clip_ckpt, pretrained=True)\n\n if has_text:\n checkpoint_name = \"microsoft/deberta-v3-base\"\n _, text_model = get_hf_config_and_model(checkpoint_name=checkpoint_name, pretrained=True)\n self.text_config = text_model.config\n # refer to https://github.com/invictus717/MetaTransformer/blob/master/Data2Seq/Data2Seq.py#L28\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=\"hf_auto\",\n checkpoint_name=checkpoint_name,\n )\n self.text_embed = text_model.embeddings\n self.text_adaptor = nn.Linear(self.text_config.hidden_size, dim)\n self.tokenizer_name = \"hf_auto\"\n self.max_text_len = get_text_token_max_len(\n provided_max_len=max_text_len,\n config=self.text_config,\n tokenizer=self.tokenizer,\n checkpoint_name=checkpoint_name,\n )\n self.text_segment_num = get_text_segment_num(\n config=self.text_config,\n provided_segment_num=text_segment_num,\n checkpoint_name=checkpoint_name,\n )\n if has_image:\n image_model = create_model(\"timm/vit_base_patch16_224.mae\", pretrained=True)\n self.image_config = image_model.default_cfg\n self.patch_embed = image_model.patch_embed\n self.image_adaptor = nn.Linear(image_model.embed_dim, dim)\n self.image_size, self.image_mean, self.image_std = get_image_size_mean_std(\n model_name=self.prefix,\n config=self.image_config,\n provided_size=image_size,\n provided_norm_type=image_norm,\n support_variable_input_size=False,\n )\n self.use_learnable_image = use_learnable_image\n if self.use_learnable_image:\n self.learnable_image = nn.Parameter(torch.zeros(image_chan_num, self.image_size, self.image_size))\n logger.debug(\"will use a learnable image to replace missing ones\")\n\n if num_categories:\n self.num_categories = num_categories\n self.categorical_feature_tokenizer = CategoricalFeatureTokenizer(\n num_categories=list(num_categories.values()),\n token_dim=dim,\n bias=True,\n initialization=\"normal\",\n )\n self.categorical_adapter = nn.Linear(dim, dim)\n\n if num_numerical_columns > 0:\n self.num_numerical_columns = num_numerical_columns\n self.numerical_fill_values = numerical_fill_values\n self.numerical_feature_tokenizer = NumEmbeddings(\n in_features=num_numerical_columns,\n d_embedding=dim,\n embedding_arch=[\"linear\"],\n )\n self.numerical_adapter = nn.Linear(dim, dim)\n\n self.out_features = dim\n\n # init weights\n self.head.apply(init_weights)\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def text_token_ids_key(self):\n return f\"{self.prefix}_{TEXT_TOKEN_IDS}\"\n\n @property\n def text_valid_length_key(self):\n return f\"{self.prefix}_{TEXT_VALID_LENGTH}\"\n\n @property\n def text_segment_ids_key(self):\n return f\"{self.prefix}_{TEXT_SEGMENT_IDS}\"\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def categorical_key(self):\n return f\"{self.prefix}_{CATEGORICAL}\"\n\n @property\n def numerical_key(self):\n return f\"{self.prefix}_{NUMERICAL}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(\n self,\n batch: dict,\n ):\n multimodal_features = []\n if self.image_tokenizer:\n images = batch[self.image_key]\n image_valid_num = batch[self.image_valid_num_key]\n b, n, c, h, w = images.shape\n steps = torch.arange(0, n).type_as(image_valid_num)\n image_masks = steps.reshape((1, -1)) < image_valid_num.reshape((-1, 1)) # (b, n)\n if self.use_learnable_image:\n images = replace_missing_images_with_learnable(\n images=images,\n image_masks=image_masks,\n learnable_image=self.learnable_image,\n )\n image_embeddings = self.patch_embed(images.reshape((b * n, c, h, w))) # (b*n, l, d)\n assert image_embeddings.ndim == 3\n image_embeddings = self.image_adaptor(image_embeddings)\n multimodal_features.append(image_embeddings)\n if self.text_adaptor: # text tokenizer is used in text processor\n text_token_ids = batch[self.text_token_ids_key]\n text_valid_length = batch[self.text_valid_length_key]\n steps = torch.arange(0, text_token_ids.shape[1]).type_as(text_valid_length)\n text_masks = (steps.reshape((1, -1)) < text_valid_length.reshape((-1, 1))).type_as(text_token_ids)\n # text_embeddings = self.text_embeddings(batch[self.text_token_ids_key]) # (b, l, d)\n input_ids = text_token_ids\n inputs_embeds = None\n attention_mask = text_masks\n position_ids = None\n if \"token_type_ids\" in self.tokenizer.model_input_names:\n token_type_ids = batch[self.text_segment_ids_key]\n else:\n token_type_ids = None\n\n if input_ids is not None and inputs_embeds is not None:\n raise ValueError(\"You cannot specify both input_ids and inputs_embeds at the same time\")\n elif input_ids is not None:\n input_shape = input_ids.size()\n elif inputs_embeds is not None:\n input_shape = inputs_embeds.size()[:-1]\n else:\n raise ValueError(\"You have to specify either input_ids or inputs_embeds\")\n\n device = input_ids.device if input_ids is not None else inputs_embeds.device\n\n if attention_mask is None:\n attention_mask = torch.ones(input_shape, device=device)\n if token_type_ids is None:\n token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)\n\n text_embeddings = self.text_embed(\n input_ids=input_ids,\n token_type_ids=token_type_ids,\n position_ids=position_ids,\n mask=attention_mask,\n inputs_embeds=inputs_embeds,\n )\n text_embeddings = self.text_adaptor(text_embeddings)\n assert text_embeddings.ndim == 3\n multimodal_features.append(text_embeddings)\n if self.categorical_feature_tokenizer:\n categorical_inputs = []\n for categorical_input in batch[self.categorical_key]:\n categorical_inputs.append(categorical_input)\n categorical_inputs = torch.stack(categorical_inputs, dim=1)\n\n categorical_features = self.categorical_feature_tokenizer(categorical_inputs)\n categorical_features = self.categorical_adapter(categorical_features) # (b, l, d)\n assert categorical_features.ndim == 3\n multimodal_features.append(categorical_features)\n if self.numerical_feature_tokenizer:\n numerical_features = self.numerical_feature_tokenizer(batch[self.numerical_key])\n numerical_features = self.numerical_adapter(numerical_features)\n assert numerical_features.ndim == 3\n multimodal_features.append(numerical_features)\n\n multimodal_features = torch.cat(multimodal_features, dim=1)\n multimodal_features = self.cls_token(multimodal_features)\n features = self.model(multimodal_features)\n pooled_features = features[:, -1, :] # CLSToken append the cls token to the sequence tail\n logits = self.head(pooled_features)\n ret = {\n LOGITS: logits,\n FEATURES: pooled_features,\n }\n return {self.prefix: ret}\n\n def get_layer_ids(self):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n In the AutoModel scenario, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\n \"embeddings\",\n \"LayerNorm\",\n \"wte\",\n \"wpe\",\n \"shared.weight\",\n \"encoder.conv.conv\",\n \"relative_attention_bias\",\n \"dummy_layer\",\n )\n post_encoder_patterns = (\"head\", \"pooler\", \"ln_f\", \"final_layer_norm\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/mlp.py",
"content": "from typing import Optional\n\nimport numpy as np\nimport torch\nfrom torch import nn\n\nALL_ACT_LAYERS = {\n \"leaky_relu\": nn.LeakyReLU,\n \"gelu\": nn.GELU,\n \"relu\": nn.ReLU,\n}\n\n\nclass GhostBatchNorm(nn.Module):\n \"\"\"\n Ghost Batch Normalization.\n It allows the use of large batch sizes,\n but with batch normalization parameters calculated on smaller sub-batches.\n\n [1] Train longer, generalize better: closing the generalization gap in large batch training of neural networks : https://arxiv.org/abs/1705.08741\n [2] Simple Modifications to Improve Tabular Neural Networks: https://arxiv.org/pdf/2108.03214\n \"\"\"\n\n def __init__(\n self,\n input_dim: int,\n virtual_batch_size: Optional[int] = 64,\n momentum: Optional[float] = 0.01,\n ):\n super(GhostBatchNorm, self).__init__()\n\n self.input_dim = input_dim\n self.virtual_batch_size = virtual_batch_size\n self.bn = nn.BatchNorm1d(self.input_dim, momentum=momentum)\n\n def forward(self, x):\n chunks = x.chunk(int(np.ceil(x.shape[0] / self.virtual_batch_size)), 0)\n res = [self.bn(x_) for x_ in chunks]\n\n return torch.cat(res, dim=0)\n\n\nclass Unit(nn.Module):\n \"\"\"\n One MLP layer. It orders the operations as: norm -> fc -> act_fn -> dropout\n \"\"\"\n\n def __init__(\n self,\n normalization: str,\n in_features: int,\n out_features: int,\n activation: str,\n dropout: float,\n ):\n \"\"\"\n Parameters\n ----------\n normalization\n Name of activation function.\n in_features\n Dimension of input features.\n out_features\n Dimension of output features.\n activation\n Name of activation function.\n dropout\n Dropout probability.\n \"\"\"\n super().__init__()\n if normalization == \"layer_norm\":\n self.norm = nn.LayerNorm(in_features)\n elif normalization == \"batch_norm\":\n self.norm = nn.BatchNorm1d(in_features)\n elif normalization == \"ghost_batch_norm\":\n self.norm = GhostBatchNorm(in_features)\n else:\n raise ValueError(f\"unknown normalization: {normalization}\")\n self.fc = nn.Linear(in_features, out_features)\n self.act_fn = ALL_ACT_LAYERS[activation]()\n self.dropout = nn.Dropout(dropout)\n\n def forward(self, x):\n # pre normalization\n x = self.norm(x)\n x = self.fc(x)\n x = self.act_fn(x)\n x = self.dropout(x)\n return x\n\n\nclass MLP(nn.Module):\n \"\"\"\n Multi-layer perceptron (MLP). If the hidden or output feature dimension is\n not provided, we assign it the input feature dimension.\n \"\"\"\n\n def __init__(\n self,\n in_features: int,\n hidden_features: Optional[int] = None,\n out_features: Optional[int] = None,\n num_layers: Optional[int] = 1,\n activation: Optional[str] = \"gelu\",\n dropout: Optional[float] = 0.5,\n normalization: Optional[str] = \"layer_norm\",\n ):\n \"\"\"\n Parameters\n ----------\n in_features\n Dimension of input features.\n hidden_features\n Dimension of hidden features.\n out_features\n Dimension of output features.\n num_layers\n Number of layers.\n activation\n Name of activation function.\n dropout\n Dropout probability.\n normalization\n Name of normalization function.\n \"\"\"\n super().__init__()\n out_features = out_features or in_features\n hidden_features = hidden_features or in_features\n\n layers = []\n for _ in range(num_layers):\n per_unit = Unit(\n normalization=normalization,\n in_features=in_features,\n out_features=hidden_features,\n activation=activation,\n dropout=dropout,\n )\n in_features = hidden_features\n layers.append(per_unit)\n if out_features != hidden_features:\n self.fc_out = nn.Linear(hidden_features, out_features)\n else:\n self.fc_out = None\n self.layers = nn.Sequential(*layers)\n\n def forward(self, x):\n x = self.layers(x)\n if self.fc_out is not None:\n return self.fc_out(x)\n else:\n return x\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/mmdet_image.py",
"content": "import logging\nimport os\nimport time\nimport warnings\nfrom typing import Optional\n\nimport torch\nfrom torch import nn\n\nfrom ..constants import BBOX, BBOX_FORMATS, COLUMN, IMAGE, IMAGE_VALID_NUM, LABEL, XYXY\nfrom .utils import freeze_model_layers, lookup_mmdet_config, update_mmdet_config\n\ntry:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n import mmcv\n import mmdet\n import mmengine\n from mmdet.registry import MODELS\n from mmengine.runner import load_checkpoint\nexcept ImportError as e:\n mmcv = None\n mmdet = None\n mmengine = None\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass MMDetAutoModelForObjectDetection(nn.Module):\n \"\"\"\n Support MMDET object detection models.\n Refer to https://github.com/open-mmlab/mmdetection\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n config_file: Optional[str] = None,\n classes: Optional[list] = None,\n pretrained: Optional[bool] = True,\n output_bbox_format: Optional[str] = XYXY,\n frozen_layers: Optional[list] = None,\n ):\n \"\"\"\n Load a pretrained object detector from MMdetection.\n\n Parameters\n ----------\n prefix\n The prefix of the MMdetAutoModelForObjectDetection model.\n checkpoint_name\n Name of the mmdet checkpoint.\n classes\n All classes in this dataset.\n pretrained\n Whether using the pretrained mmdet models. If pretrained=True, download the pretrained model.\n \"\"\"\n from ..utils import check_if_packages_installed\n\n check_if_packages_installed(package_names=[\"mmcv\", \"mmengine\", \"mmdet\"])\n\n super().__init__()\n self.prefix = prefix\n self.pretrained = pretrained\n self.checkpoint = None\n self.checkpoint_name = checkpoint_name\n self.config_file = config_file\n self.classes = classes\n # Based on our offline benchmarking results, instead of freezing layers,\n # Setting backbone to a smaller learning rate achieves better results,\n self.frozen_layers = frozen_layers\n self._assign_backbone_layers()\n\n self.device = None\n\n if output_bbox_format.lower() in BBOX_FORMATS:\n self.output_bbox_format = output_bbox_format.lower()\n else:\n raise ValueError(\n f\"Not supported bounding box output format for object detection: {output_bbox_format}. All supported bounding box output formats are: {BBOX_FORMATS}.\"\n )\n\n # TODO: Config only init (without checkpoint)\n\n self._get_checkpoint_and_config_file(checkpoint_name=checkpoint_name, config_file=config_file)\n self._load_config()\n\n self._update_classes(classes)\n self._load_checkpoint(self.checkpoint_file)\n\n freeze_model_layers(self.model, self.frozen_layers)\n\n def _reset_classes(self, classes: list):\n temp_ckpt_file = f\"temp_ckpt_{int(time.time() * 1000)}.pth\"\n self._save_weights(temp_ckpt_file)\n self._update_classes(classes)\n self._load_checkpoint()\n os.remove(temp_ckpt_file)\n\n def _update_classes(self, classes: Optional[list] = None):\n if classes:\n self.num_classes = len(classes)\n self.classes = classes\n update_mmdet_config(key=\"num_classes\", value=self.num_classes, config=self.config)\n else:\n self.num_classes = lookup_mmdet_config(key=\"num_classes\", config=self.config)\n if not self.num_classes:\n self.num_classes = 1\n warnings.warn(\n f\"num_classes is not provided and is set to default value {self.num_classes} and this may cause error. Please provide sample_data_path in predictor's initialization.\"\n )\n self.classes = None\n self.id2label = dict(zip(range(self.num_classes), range(self.num_classes)))\n\n def _load_checkpoint(self, checkpoint_file):\n # build model and load pretrained weights\n from mmdet.utils import register_all_modules\n\n register_all_modules() # https://github.com/open-mmlab/mmdetection/issues/9719\n\n self.model = MODELS.build(self.config.model)\n # yolox use self.config.model.data_preprocessor, yolov3 use self.config.data_preprocessor\n self.data_preprocessor = MODELS.build(\n self.config.data_preprocessor\n if \"data_preprocessor\" in self.config\n else self.config.model.data_preprocessor\n )\n\n if self.pretrained and checkpoint_file is not None: # TODO: enable training from scratch\n self.checkpoint = load_checkpoint(self.model, checkpoint_file, map_location=\"cpu\")\n\n # save the config and classes in the model for convenience\n self.model.cfg = self.config\n if self.classes:\n self.model.CLASSES = self.classes\n else:\n if self.checkpoint and \"CLASSES\" in self.checkpoint.get(\"meta\", {}):\n warnings.simplefilter(\"once\")\n warnings.warn(\n f\"Using classes provided in checkpoints: {self.checkpoint['meta']['CLASSES']}. Provide data while init MultiModalPredictor if this is not expected.\"\n )\n self.model.CLASSES = self.checkpoint[\"meta\"][\"CLASSES\"]\n else:\n warnings.warn(\n f\"CLASSES is not provided and this may cause error. Please provide sample_data_path in predictor's initialization.\"\n )\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id <= 0]\n\n def set_data_preprocessor_device(self):\n if not self.device:\n self.device = next(self.model.parameters()).device\n if self.device != self.data_preprocessor.device:\n self.data_preprocessor.to(self.device)\n\n def save(self, save_path: str = \"./\", tokenizers: Optional[dict] = None):\n weights_save_path = os.path.join(save_path, \"model.pth\")\n configs_save_path = os.path.join(save_path, \"config.py\")\n\n self._save_weights(save_path=weights_save_path)\n self._save_configs(save_path=configs_save_path)\n\n return save_path\n\n def _save_weights(self, save_path=None):\n if not save_path:\n save_path = f\"./{self.checkpoint_name}_autogluon.pth\"\n\n torch.save({\"state_dict\": self.model.state_dict(), \"meta\": {\"CLASSES\": self.model.CLASSES}}, save_path) # nosec B614\n\n def _save_configs(self, save_path=None):\n if not save_path:\n save_path = f\"./{self.checkpoint_name}_autogluon.py\"\n\n self.config.dump(save_path)\n\n def _get_checkpoint_and_config_file(self, checkpoint_name: str = None, config_file: str = None):\n from mim.commands import download as mimdownload\n\n from ..utils import download, get_pretrain_configs_dir\n\n logger.debug(f\"initializing {checkpoint_name}\")\n\n if not checkpoint_name:\n checkpoint_name = self.checkpoint_name\n if not config_file:\n config_file = self.config_file\n\n mmdet_configs_dir = get_pretrain_configs_dir(subfolder=\"detection\")\n\n AG_CUSTOM_MODELS = {\n \"faster_rcnn_r50_fpn_1x_voc0712\": {\n \"url\": \"https://automl-mm-bench.s3.amazonaws.com/voc_script/faster_rcnn_r50_fpn_1x_voc0712_20220320_192712-54bef0f3.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"voc\", \"faster_rcnn_r50_fpn_1x_voc0712.py\"),\n },\n \"yolox_nano\": {\n \"url\": \"https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_nano.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_nano_8xb8-300e_coco.py\"),\n \"source\": \"MegVii\",\n },\n \"yolox_tiny\": {\n \"url\": \"https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_tiny_8x8_300e_coco/yolox_tiny_8x8_300e_coco_20211124_171234-b4047906.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_tiny_8xb8-300e_coco.py\"),\n },\n \"yolox_s\": {\n \"url\": \"https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_s_8x8_300e_coco/yolox_s_8x8_300e_coco_20211121_095711-4592a793.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_s_8xb8-300e_coco.py\"),\n },\n \"yolox_m\": {\n \"url\": \"https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_m.pth\", # Megvii weight, need more verifications\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_m_8xb8-300e_coco.py\"),\n \"source\": \"MegVii\",\n },\n \"yolox_l\": {\n \"url\": \"https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_l_8x8_300e_coco/yolox_l_8x8_300e_coco_20211126_140236-d3bd2b23.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_l_8xb8-300e_coco.py\"),\n },\n \"yolox_l_objects365\": { # TODO: update with better pretrained weights\n \"url\": \"https://automl-mm-bench.s3.amazonaws.com/object_detection/checkpoints/yolox/yolox_l_objects365_temp.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_l_8xb8-300e_coco.py\"),\n },\n \"yolox_x\": {\n \"url\": \"https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_x_8x8_300e_coco/yolox_x_8x8_300e_coco_20211126_140254-1ef88d67.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"yolox\", \"yolox_x_8xb8-300e_coco.py\"),\n },\n \"dino_swinl_tta\": {\n \"url\": \"https://github.com/RistoranteRist/mmlab-weights/releases/download/dino-swinl/dino-5scale_swin-l_8xb2-36e_coco-5486e051.pth\",\n \"config_file\": os.path.join(mmdet_configs_dir, \"dino\", \"dino_swinl_tta.py\"),\n },\n }\n\n if os.path.isfile(checkpoint_name):\n checkpoint_file = checkpoint_name\n elif os.path.isdir(checkpoint_name):\n checkpoint_file = os.path.join(checkpoint_name, \"model.pth\")\n config_file = os.path.join(checkpoint_name, \"config.py\")\n else:\n if checkpoint_name in AG_CUSTOM_MODELS:\n # TODO: add sha1_hash\n checkpoint_file = download(\n url=AG_CUSTOM_MODELS[checkpoint_name][\"url\"],\n )\n if (\n \"source\" in AG_CUSTOM_MODELS[checkpoint_name]\n and AG_CUSTOM_MODELS[checkpoint_name][\"source\"] == \"MegVii\"\n ):\n checkpoint_file = self.convert_megvii_yolox(checkpoint_file)\n else:\n # download config and checkpoint files using openmim\n checkpoint_file = mimdownload(package=\"mmdet\", configs=[checkpoint_name], dest_root=\".\")[0]\n\n if config_file:\n if not os.path.isfile(config_file):\n raise ValueError(f\"Invalid checkpoint_name ({checkpoint_name}) or config_file ({config_file}): \")\n else:\n if checkpoint_name in AG_CUSTOM_MODELS:\n config_file = AG_CUSTOM_MODELS[checkpoint_name][\"config_file\"]\n else:\n try:\n # download config and checkpoint files using openmim\n mimdownload(package=\"mmdet\", configs=[checkpoint_name], dest_root=\".\")\n config_file = checkpoint_name + \".py\"\n except Exception as e:\n raise ValueError(f\"Invalid checkpoint_name ({checkpoint_name}) or config_file ({config_file}): \")\n\n self.checkpoint_name = checkpoint_name\n self.checkpoint_file = checkpoint_file\n self.config_file = config_file\n\n def _load_config(self):\n # read config files\n if isinstance(self.config_file, str):\n self.config = mmengine.Config.fromfile(self.config_file)\n else:\n if not isinstance(self.config_file, dict):\n raise ValueError(\n f\"The variable config_file has type {type(self.config_file)}.\"\n f\"Detection Model's config_file should either be a str of file path, or a dict as config.\"\n )\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def image_feature_dim(self):\n return self.model.num_features\n\n def forward(\n self,\n batch,\n mode,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n mode\n \"loss\" or \"predict\". TODO: support \"tensor\"\n https://github.com/open-mmlab/mmdetection/blob/main/mmdet/models/detectors/base.py#L58C1\n\n Returns\n -------\n A dictionary with bounding boxes.\n \"\"\"\n\n self.set_data_preprocessor_device()\n data = self.data_preprocessor(batch)\n rets = self.model(\n inputs=data[\"inputs\"],\n data_samples=data[\"data_samples\"],\n mode=mode,\n )\n\n if mode == \"loss\":\n return rets\n elif mode == \"predict\":\n # for detailed data structure, see https://github.com/open-mmlab/mmdetection/blob/main/mmdet/structures/det_data_sample.py\n return [{BBOX: ret.pred_instances, LABEL: ret.gt_instances} for ret in rets]\n else:\n raise ValueError(f\"{mode} mode is not supported.\")\n\n def _parse_losses(self, losses):\n return self.model._parse_losses(losses)\n\n def _assign_backbone_layers(self):\n \"\"\"\n Backbone layers are only assigned when we use low lr for backbone and high lr for others\n TODO: Add hyperparameter controlling this if later find any models requiring this setting other than dino.\n \"\"\"\n if \"dino\" in self.checkpoint_name.lower():\n self.backbone_layers = [\"backbone\"]\n else:\n self.backbone_layers = None\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n\n # two stage lr: backbone v.s. else, or head v.s. else\n if self.backbone_layers:\n return self.get_layer_ids_by_backbone()\n else:\n return self.get_layer_ids_by_head()\n\n def get_layer_ids_by_backbone(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Currently only head to 0 others to 1.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n backbones = self.backbone_layers\n\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n for backbone in backbones:\n if backbone in n:\n name_to_id[n] = 1\n\n return name_to_id\n\n def get_layer_ids_by_head(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Currently only head to 0 others to 1.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n # for some models, use head lr in \"head\" of bbox_head\n # now support: yolov3, faster_rcnn, deformable_detr, yolox, vfnet, centernet, cascade_rcnn, detr, htc, atss, ssd\n registered_head_layers_patterns = [\n \"bbox_head.fc_cls\",\n \"bbox_head.fc_reg\",\n \"bbox_head.convs_pred\",\n \"bbox_head.cls_branches\",\n \"bbox_head.multi_level_conv_cls\",\n \"bbox_head.multi_level_conv_reg\",\n \"bbox_head.multi_level_conv_obj\",\n \"bbox_head.vfnet_cls\",\n \"bbox_head.heatmap_head\",\n \"bbox_head.atss_cls\",\n \"bbox_head.cls_convs\",\n ]\n # for other models, use head lr in whole bbox_head\n default_head_layers_patterns = [\"bbox_head\"]\n\n head_registered = False\n is_yolox = False\n for n, _ in self.named_parameters():\n name_to_id[n] = 1\n for pattern in registered_head_layers_patterns:\n if pattern in n:\n name_to_id[n] = 0\n head_registered = True\n if \"bbox_head.multi_level_conv_cls\" in n:\n is_yolox = True\n\n if not head_registered:\n for n, _ in self.named_parameters():\n name_to_id[n] = 1\n for pattern in default_head_layers_patterns:\n if pattern in n:\n name_to_id[n] = 0\n\n if is_yolox and \"use_layer_id\" in self.config:\n name_to_id = self.get_yolox_layer_ids()\n\n return name_to_id\n\n def get_yolox_layer_ids(self):\n # logic not straight forward, need to print out the model to understand\n name_to_value = {}\n for name, _ in self.named_parameters():\n n = name\n n = n.replace(\"backbone\", \"0\")\n n = n.replace(\"neck\", \"1\")\n n = n.replace(\"bbox_head\", \"2\")\n\n # backbone\n n = n.replace(\"stem\", \"0\")\n\n # neck\n n = n.replace(\"reduce_layers\", \"0\")\n n = n.replace(\"top_down_blocks\", \"1\")\n n = n.replace(\"downsamples\", \"2\")\n n = n.replace(\"bottom_up_blocks\", \"3\")\n n = n.replace(\"out_convs\", \"4\")\n\n n = n.replace(\"main_conv\", \"0\")\n n = n.replace(\"short_conv\", \"1\")\n n = n.replace(\"final_conv\", \"2\")\n n = n.replace(\"blocks\", \"3\")\n\n # bbox_head\n n = n.replace(\"multi_level_cls_convs\", \"0\")\n n = n.replace(\"multi_level_reg_convs\", \"0\")\n n = n.replace(\"multi_level_conv_cls\", \"1\")\n n = n.replace(\"multi_level_conv_reg\", \"1\")\n n = n.replace(\"multi_level_conv_obj\", \"1\")\n\n value = int(\"\".join(c for c in n if c.isdigit()).ljust(8, \"0\"))\n name_to_value[name] = value\n\n values = list(set(name_to_value.values()))\n values.sort(reverse=True)\n value_to_id = dict(zip(values, range(len(values))))\n\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = value_to_id[name_to_value[n]]\n return name_to_id\n\n def convert_megvii_yolox(self, source_path):\n \"\"\"\n Convert YOLOX in megvii naming to mmdetection naming.\n Using code script from: https://github.com/haiyang-tju/dl_tools/blob/master/megvii_nano_2_mmdet.py\n \"\"\"\n sd = source_path\n\n model_dict = torch.load(sd, map_location=torch.device(\"cpu\")) # nosec B614\n if \"state_dict\" in model_dict:\n model_dict = model_dict[\"state_dict\"]\n if \"model\" in model_dict:\n model_dict = model_dict[\"model\"]\n\n new_dict = dict()\n for k, v in model_dict.items():\n new_k = k\n\n if \"backbone.backbone.\" in k:\n new_k = k.replace(\"backbone.backbone.\", \"backbone.\")\n if \"backbone.dark2.\" in new_k:\n new_k = new_k.replace(\"backbone.dark2.\", \"backbone.stage1.\")\n if \"backbone.dark3.\" in new_k:\n new_k = new_k.replace(\"backbone.dark3.\", \"backbone.stage2.\")\n if \"backbone.dark4.\" in new_k:\n new_k = new_k.replace(\"backbone.dark4.\", \"backbone.stage3.\")\n if \"backbone.dark5.\" in new_k:\n new_k = new_k.replace(\"backbone.dark5.\", \"backbone.stage4.\")\n if \"dconv.\" in new_k:\n new_k = new_k.replace(\"dconv.\", \"depthwise_conv.\")\n if \"pconv.\" in new_k:\n new_k = new_k.replace(\"pconv.\", \"pointwise_conv.\")\n if \"backbone.stage1.1.conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.stage1.1.conv1.\", \"backbone.stage1.1.main_conv.\")\n if \"backbone.stage1.1.conv2.\" in new_k:\n new_k = new_k.replace(\"backbone.stage1.1.conv2.\", \"backbone.stage1.1.short_conv.\")\n if \"backbone.stage1.1.conv3.\" in new_k:\n new_k = new_k.replace(\"backbone.stage1.1.conv3.\", \"backbone.stage1.1.final_conv.\")\n if \".m.\" in new_k:\n new_k = new_k.replace(\".m.\", \".blocks.\")\n if \"backbone.stage2.1.conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.stage2.1.conv1.\", \"backbone.stage2.1.main_conv.\")\n if \"backbone.stage2.1.conv2.\" in new_k:\n new_k = new_k.replace(\"backbone.stage2.1.conv2.\", \"backbone.stage2.1.short_conv.\")\n if \"backbone.stage2.1.conv3.\" in new_k:\n new_k = new_k.replace(\"backbone.stage2.1.conv3.\", \"backbone.stage2.1.final_conv.\")\n if \"backbone.stage3.1.conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.stage3.1.conv1.\", \"backbone.stage3.1.main_conv.\")\n if \"backbone.stage3.1.conv2.\" in new_k:\n new_k = new_k.replace(\"backbone.stage3.1.conv2.\", \"backbone.stage3.1.short_conv.\")\n if \"backbone.stage3.1.conv3.\" in new_k:\n new_k = new_k.replace(\"backbone.stage3.1.conv3.\", \"backbone.stage3.1.final_conv.\")\n if \"backbone.stage4.2.conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.stage4.2.conv1.\", \"backbone.stage4.2.main_conv.\")\n if \"backbone.stage4.2.conv2.\" in new_k:\n new_k = new_k.replace(\"backbone.stage4.2.conv2.\", \"backbone.stage4.2.short_conv.\")\n if \"backbone.stage4.2.conv3.\" in new_k:\n new_k = new_k.replace(\"backbone.stage4.2.conv3.\", \"backbone.stage4.2.final_conv.\")\n if \"backbone.lateral_conv0.\" in new_k:\n new_k = new_k.replace(\"backbone.lateral_conv0.\", \"neck.reduce_layers.0.\")\n if \"backbone.reduce_conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.reduce_conv1.\", \"neck.reduce_layers.1.\")\n if \"backbone.C3_p4.\" in new_k:\n new_k = new_k.replace(\"backbone.C3_p4.\", \"neck.top_down_blocks.0.\")\n if \"neck.top_down_blocks.0.conv1.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.0.conv1.\", \"neck.top_down_blocks.0.main_conv.\")\n if \"neck.top_down_blocks.0.conv2.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.0.conv2.\", \"neck.top_down_blocks.0.short_conv.\")\n if \"neck.top_down_blocks.0.conv3.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.0.conv3.\", \"neck.top_down_blocks.0.final_conv.\")\n if \"backbone.C3_p3.\" in new_k:\n new_k = new_k.replace(\"backbone.C3_p3.\", \"neck.top_down_blocks.1.\")\n if \"neck.top_down_blocks.1.conv1.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.1.conv1.\", \"neck.top_down_blocks.1.main_conv.\")\n if \"neck.top_down_blocks.1.conv2.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.1.conv2.\", \"neck.top_down_blocks.1.short_conv.\")\n if \"neck.top_down_blocks.1.conv3.\" in new_k:\n new_k = new_k.replace(\"neck.top_down_blocks.1.conv3.\", \"neck.top_down_blocks.1.final_conv.\")\n\n if \"backbone.bu_conv2.\" in new_k:\n new_k = new_k.replace(\"backbone.bu_conv2.\", \"neck.downsamples.0.\")\n if \"backbone.bu_conv1.\" in new_k:\n new_k = new_k.replace(\"backbone.bu_conv1.\", \"neck.downsamples.1.\")\n\n if \"backbone.C3_n3.\" in new_k:\n new_k = new_k.replace(\"backbone.C3_n3.\", \"neck.bottom_up_blocks.0.\")\n if \"neck.bottom_up_blocks.0.conv1.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.0.conv1.\", \"neck.bottom_up_blocks.0.main_conv.\")\n if \"neck.bottom_up_blocks.0.conv2.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.0.conv2.\", \"neck.bottom_up_blocks.0.short_conv.\")\n if \"neck.bottom_up_blocks.0.conv3.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.0.conv3.\", \"neck.bottom_up_blocks.0.final_conv.\")\n if \"backbone.C3_n4.\" in new_k:\n new_k = new_k.replace(\"backbone.C3_n4.\", \"neck.bottom_up_blocks.1.\")\n if \"neck.bottom_up_blocks.1.conv1.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.1.conv1.\", \"neck.bottom_up_blocks.1.main_conv.\")\n if \"neck.bottom_up_blocks.1.conv2.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.1.conv2.\", \"neck.bottom_up_blocks.1.short_conv.\")\n if \"neck.bottom_up_blocks.1.conv3.\" in new_k:\n new_k = new_k.replace(\"neck.bottom_up_blocks.1.conv3.\", \"neck.bottom_up_blocks.1.final_conv.\")\n\n if \"head.stems.\" in new_k:\n new_k = new_k.replace(\"head.stems.\", \"neck.out_convs.\")\n if \"head.cls_convs.\" in new_k:\n new_k = new_k.replace(\"head.cls_convs.\", \"bbox_head.multi_level_cls_convs.\")\n if \"head.reg_convs.\" in new_k:\n new_k = new_k.replace(\"head.reg_convs.\", \"bbox_head.multi_level_reg_convs.\")\n if \"head.cls_preds.\" in new_k:\n new_k = new_k.replace(\"head.cls_preds.\", \"bbox_head.multi_level_conv_cls.\")\n if \"head.reg_preds.\" in new_k:\n new_k = new_k.replace(\"head.reg_preds.\", \"bbox_head.multi_level_conv_reg.\")\n if \"head.obj_preds.\" in new_k:\n new_k = new_k.replace(\"head.obj_preds.\", \"bbox_head.multi_level_conv_obj.\")\n\n if \"bbox_head.multi_level_conv_cls.\" in new_k:\n if self.classes:\n new_dict[new_k] = v[: len(self.classes), ...] # there take the num_classes\n else:\n new_dict[new_k] = v\n else:\n new_dict[new_k] = v\n\n data = {\"state_dict\": new_dict}\n\n target_directory = os.path.splitext(sd)[0] + f\"_cvt.pth\"\n torch.save(data, target_directory) # nosec B614\n\n return target_directory\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/mmocr_text_detection.py",
"content": "import logging\nfrom typing import Optional\n\ntry:\n from mmcv.parallel import scatter\nexcept ImportError:\n mmcv = None\ntry:\n from mmocr.utils.model import revert_sync_batchnorm\nexcept ImportError:\n mmocr = None\nfrom torch import nn\n\nfrom ..constants import BBOX, COLUMN, COLUMN_FEATURES, FEATURES, IMAGE, IMAGE_VALID_NUM, LABEL, LOGITS, MASKS\nfrom .utils import assign_layer_ids, get_column_features, get_mmocr_config_and_model, get_model_head\n\nlogger = logging.getLogger(__name__)\n\n\nclass MMOCRAutoModelForTextDetection(nn.Module):\n \"\"\"\n Support MMOCR object detection models.\n Refer to https://github.com/open-mmlab/mmocr\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n num_classes: Optional[int] = None,\n pretrained: Optional[bool] = True,\n ):\n \"\"\"\n Load a pretrained ocr text detection detector from MMOCR.\n\n Parameters\n ----------\n prefix\n The prefix of the MMdetAutoModelForTextDetection model.\n checkpoint_name\n Name of the mmdet checkpoint.\n num_classes\n The number of classes.\n pretrained\n Whether using the pretrained mmdet models. If pretrained=True, download the pretrained model.\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {checkpoint_name}\")\n self.checkpoint_name = checkpoint_name\n self.pretrained = pretrained\n\n self.config, self.model = get_mmocr_config_and_model(checkpoint_name)\n self.model = revert_sync_batchnorm(self.model)\n self.model.cfg = self.config\n self.prefix = prefix\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def image_feature_dim(self):\n return self.model.num_features\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with bounding boxes.\n \"\"\"\n\n data = batch[self.image_key]\n # single image\n data[\"img_metas\"] = [img_metas.data[0] for img_metas in data[\"img_metas\"]]\n data[\"img\"] = [img.data[0] for img in data[\"img\"]]\n\n device = next(self.model.parameters()).device # model device\n if next(self.model.parameters()).is_cuda:\n # scatter to specified GPU\n data = scatter(data, [device])[0]\n\n results = self.model(return_loss=False, rescale=True, **data)\n\n ret = {BBOX: results[0][\"boundary_result\"]}\n return {self.prefix: ret}\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Setting all layers as the same id 0 for now.\n TODO: Need to investigate mmocr's model definitions\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/mmocr_text_recognition.py",
"content": "import logging\nfrom typing import List, Optional\n\ntry:\n from mmcv.parallel import scatter\nexcept ImportError:\n mmcv = None\ntry:\n from mmocr.utils.model import revert_sync_batchnorm\nexcept ImportError:\n mmocr = None\nfrom torch import nn\n\nfrom ..constants import (\n COLUMN,\n COLUMN_FEATURES,\n FEATURES,\n IMAGE,\n IMAGE_VALID_NUM,\n LABEL,\n LOGITS,\n MASKS,\n SCORE,\n TEXT,\n)\nfrom .utils import assign_layer_ids, get_column_features, get_mmocr_config_and_model, get_model_head\n\nlogger = logging.getLogger(__name__)\n\n\nclass MMOCRAutoModelForTextRecognition(nn.Module):\n \"\"\"\n Support MMOCR text recognition models.\n Refer to https://github.com/open-mmlab/mmocr\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n num_classes: Optional[int] = None,\n pretrained: Optional[bool] = True,\n ):\n \"\"\"\n Load a pretrained ocr text recognition detector from MMOCR.\n\n Parameters\n ----------\n prefix\n The prefix of the MMdetAutoModelForTextRecognition model.\n checkpoint_name\n Name of the mmdet checkpoint.\n num_classes\n The number of classes.\n pretrained\n Whether using the pretrained mmocr models. If pretrained=True, download the pretrained model.\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {checkpoint_name}\")\n self.checkpoint_name = checkpoint_name\n self.pretrained = pretrained\n\n self.config, self.model = get_mmocr_config_and_model(checkpoint_name)\n self.model = revert_sync_batchnorm(self.model)\n self.model.cfg = self.config\n self.prefix = prefix\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def image_feature_dim(self):\n return self.model.num_features\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with bounding boxes.\n \"\"\"\n\n data = batch[self.image_key]\n # single image\n if isinstance(data[\"img_metas\"], List):\n data[\"img_metas\"] = [img_metas.data[0] for img_metas in data[\"img_metas\"]]\n else:\n data[\"img_metas\"] = data[\"img_metas\"].data\n\n if isinstance(data[\"img\"], List):\n data[\"img\"] = [img.data[0] for img in data[\"img\"]]\n else:\n data[\"img\"] = data[\"img\"].data\n\n device = next(self.model.parameters()).device # model device\n if next(self.model.parameters()).is_cuda:\n # scatter to specified GPU\n data = scatter(data, [device])[0]\n\n results = self.model(return_loss=False, rescale=True, **data)\n\n ret = {TEXT: results[0][\"text\"], SCORE: results[0][\"score\"]}\n return {self.prefix: ret}\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Setting all layers as the same id 0 for now.\n TODO: Need to investigate mmocr's model definitions\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/ner_text.py",
"content": "import logging\nfrom typing import Dict, List, Optional, Tuple\n\nimport torch\nimport torch.nn.functional as F\nfrom transformers import logging as hf_logging\n\nfrom ..constants import (\n COLUMN_FEATURES,\n FEATURES,\n LOGITS,\n MASKS,\n NER_ANNOTATION,\n TOKEN_WORD_MAPPING,\n WORD_OFFSETS,\n)\nfrom .hf_text import HFAutoModelForTextPrediction\nfrom .utils import assign_layer_ids, get_column_features, get_pretrained_tokenizer\n\nhf_logging.set_verbosity_error()\n\nlogger = logging.getLogger(__name__)\n\n\nclass HFAutoModelForNER(HFAutoModelForTextPrediction):\n \"\"\"\n Named entity recognition with huggingface backbones. Inherit from HFAutoModelForTextPrediction.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str = \"microsoft/deberta-v3-base\",\n num_classes: Optional[int] = 0,\n pooling_mode: Optional[str] = \"cls\",\n gradient_checkpointing: Optional[bool] = False,\n low_cpu_mem_usage: Optional[bool] = False,\n pretrained: Optional[bool] = True,\n tokenizer_name: Optional[str] = \"hf_auto\",\n ):\n \"\"\"\n Load a pretrained huggingface text transformer backbone.\n Parameters\n ----------\n prefix\n The model prefix.\n checkpoint_name\n Name of the checkpoint. We support loading checkpoint from\n Huggingface Models list: https://huggingface.co/models\n For example, you may use\n English backbones:\n - 'bert-base-cased'\n num_classes\n The number of classes. 1 for a regression task.\n pooling_mode\n The pooling mode to be used, it is not used in the NER task.\n gradient_checkpointing\n Whether to enable gradient checkpointing\n low_cpu_mem_usage\n Whether to turn on the optimization of reducing the peak CPU memory usage when loading the pretrained model.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n tokenizer_name\n Name of the huggingface tokenizer type.\n \"\"\"\n logger.debug(f\"initializing {checkpoint_name}\")\n super().__init__(\n prefix=prefix,\n checkpoint_name=checkpoint_name,\n num_classes=num_classes,\n pooling_mode=pooling_mode,\n gradient_checkpointing=gradient_checkpointing,\n low_cpu_mem_usage=low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=tokenizer_name,\n )\n\n if self.config.model_type in {\"gpt2\", \"roberta\"}:\n # Refer to this PR: https://github.com/huggingface/transformers/pull/12116\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=self.checkpoint_name,\n add_prefix_space=True,\n )\n\n # some checkpoint such as deberta does not specify model_max_length\n # here, we reset it using model config.\n if hasattr(self.model.config, \"max_position_embeddings\"):\n self.tokenizer.model_max_length = self.model.config.max_position_embeddings\n if hasattr(self.model.config, \"n_positions\"):\n self.tokenizer.model_max_length = self.model.config.n_positions\n\n @property\n def input_keys(self):\n return [\n self.text_token_ids_key,\n self.text_segment_ids_key,\n self.text_valid_length_key,\n self.text_token_word_mapping_key,\n self.text_word_offsets_key,\n ]\n\n @property\n def text_token_word_mapping_key(self):\n return f\"{self.prefix}_{TOKEN_WORD_MAPPING}\"\n\n @property\n def text_word_offsets_key(self):\n return f\"{self.prefix}_{WORD_OFFSETS}\"\n\n def forward(\n self,\n text_token_ids: torch.Tensor,\n text_segment_ids: torch.Tensor,\n text_valid_length: torch.Tensor,\n token_word_mapping: torch.Tensor,\n word_offsets: torch.Tensor,\n text_column_names: Optional[List[str]] = None,\n text_column_indices: Optional[List[torch.Tensor]] = None,\n ):\n \"\"\"\n Parameters\n ----------\n text_token_ids : torch.Tensor\n Indices of input sequence tokens in the vocabulary.\n text_segment_ids : torch.Tensor\n Indices of input sequence segments.\n text_valid_length : torch.Tensor\n Valid length of the input text sequence.\n token_word_mapping : torch.Tensor\n Mapping the named entities to task specific labels.\n word_offsets : torch.Tensor\n Locations of the named entities.\n text_column_names : list of str, optional\n Names of the text columns.\n text_column_indices : list of torch.Tensor, optional\n Start and stop indices of the text columns.\n\n Returns\n -------\n A tuple that contains (sequence_output, logits, logits_label, token_word_mapping, word_offsets, column_features, column_feature_masks)\n \"\"\"\n if self.disable_seg_ids:\n text_segment_ids = None\n steps = torch.arange(0, text_token_ids.shape[1]).type_as(text_valid_length)\n text_masks = (steps.reshape((1, -1)) < text_valid_length.reshape((-1, 1))).type_as(text_token_ids)\n\n if self.is_t5:\n # For the T5 model, we will only use the encoder to encode the sentence. This is adopted in\n # \"Sentence-T5 (ST5): Scalable Sentence Encoders from Pre-trained Text-to-Text Models\"\n # (https://aclanthology.org/2022.findings-acl.146.pdf)!\n inputs_embeds = self.model.encoder.embed_tokens(text_token_ids)\n outputs = self.model.encoder(\n inputs_embeds=inputs_embeds,\n attention_mask=text_masks,\n )\n else:\n outputs = self.model(\n input_ids=text_token_ids,\n token_type_ids=text_segment_ids,\n attention_mask=text_masks,\n )\n\n sequence_output = outputs.last_hidden_state\n batch_size, max_len, feat_dim = sequence_output.shape\n valid_output = torch.zeros(batch_size, max_len, feat_dim, dtype=torch.float32)\n\n pooled_features = outputs.last_hidden_state[:, 0, :]\n\n logits = self.head(sequence_output)\n\n logits_label = torch.argmax(F.log_softmax(logits, dim=-1), dim=-1)\n\n batch = {\n self.text_token_ids_key: text_token_ids,\n self.text_segment_ids_key: text_segment_ids,\n self.text_valid_length_key: text_valid_length,\n }\n if text_column_names:\n assert len(text_column_names) == len(text_column_indices), \"invalid text column inputs\"\n batch.update(**dict(zip(text_column_names, text_column_indices)))\n column_features, column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.text_column_prefix,\n features=outputs.last_hidden_state,\n valid_lengths=text_valid_length,\n cls_feature=pooled_features,\n )\n\n if column_features == {} or column_feature_masks == {}:\n return sequence_output, logits, logits_label, token_word_mapping, word_offsets\n else:\n return (\n sequence_output,\n logits,\n logits_label,\n token_word_mapping,\n word_offsets,\n column_features,\n column_feature_masks,\n )\n\n def get_output_dict(\n self,\n sequence_output: torch.Tensor,\n logits: torch.Tensor,\n logits_label: torch.Tensor,\n token_word_mapping: torch.Tensor,\n word_offsets: torch.Tensor,\n column_features: Optional[Dict[str, torch.Tensor]] = None,\n column_feature_masks: Optional[Dict[str, torch.Tensor]] = None,\n ):\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n if column_features != None:\n ret[COLUMN_FEATURES][FEATURES].update(column_features)\n ret[COLUMN_FEATURES][MASKS].update(column_feature_masks)\n\n ret.update(\n {\n LOGITS: logits,\n FEATURES: sequence_output, # input of ner fusion model\n NER_ANNOTATION: logits_label,\n TOKEN_WORD_MAPPING: token_word_mapping,\n WORD_OFFSETS: word_offsets,\n }\n )\n\n return {self.prefix: ret}\n\n def get_layer_ids(self):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n In the AutoModel scenario, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\n \"embeddings\",\n \"LayerNorm\",\n \"wte\",\n \"wpe\",\n \"shared.weight\",\n \"encoder.conv.conv\",\n \"relative_attention_bias\",\n \"dummy_layer\",\n \"mask_emb\",\n \"word_embedding.weight\",\n )\n post_encoder_patterns = (\"head\", \"pooler\", \"ln_f\", \"final_layer_norm\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/numerical_mlp.py",
"content": "import logging\nfrom typing import Dict, List, Optional\n\nfrom torch import nn\n\nfrom ..constants import FEATURES, LABEL, LOGITS, NUMERICAL\nfrom .ft_transformer import NumEmbeddings\nfrom .mlp import MLP\nfrom .utils import init_weights\n\nlogger = logging.getLogger(__name__)\n\n\nclass NumericalMLP(nn.Module):\n \"\"\"\n MLP for numerical input.\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n in_features: int,\n hidden_features: Optional[int] = None,\n out_features: Optional[int] = None,\n num_layers: Optional[int] = 1,\n activation: Optional[str] = \"leaky_relu\",\n dropout: Optional[float] = 0.5,\n normalization: Optional[str] = \"layer_norm\",\n num_classes: Optional[int] = 0,\n token_dim: Optional[int] = 8,\n embedding_arch: Optional[List[str]] = None,\n numerical_fill_values: Optional[Dict] = None,\n ):\n \"\"\"\n Parameters\n ----------\n prefix\n The model prefix.\n in_features\n Dimension of input features.\n hidden_features\n Dimension of hidden features.\n out_features\n Dimension of output features.\n num_layers\n Number of MLP layers.\n activation\n Name of activation function.\n dropout\n Dropout probability.\n normalization\n Name of normalization function.\n num_classes\n Number of classes. 1 for a regression task.\n token_dim\n The size of one token for `NumericalEmbedding`.\n embedding_arch\n A list containing the names of embedding layers.\n Currently support:\n {'linear', 'shared_linear', 'autodis', 'positional', 'relu', 'layernorm'}\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {prefix} (NumericalMLP)\")\n self.out_features = out_features\n self.numerical_fill_values = numerical_fill_values\n\n self.numerical_feature_tokenizer = (\n NumEmbeddings(\n in_features=in_features,\n d_embedding=token_dim,\n embedding_arch=embedding_arch,\n )\n if embedding_arch is not None\n else nn.Identity()\n )\n\n in_features = in_features * token_dim if embedding_arch is not None else in_features\n\n self.mlp = MLP(\n in_features=in_features,\n hidden_features=hidden_features,\n out_features=out_features,\n num_layers=num_layers,\n activation=activation,\n dropout=dropout,\n normalization=normalization,\n )\n self.head = nn.Linear(out_features, num_classes) if num_classes > 0 else nn.Identity()\n # init weights\n self.apply(init_weights)\n\n self.prefix = prefix\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def numerical_key(self):\n return f\"{self.prefix}_{NUMERICAL}\"\n\n @property\n def input_keys(self):\n return [self.numerical_key]\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n def forward(\n self,\n batch: dict,\n ):\n \"\"\"\n\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n features = self.numerical_feature_tokenizer(batch[self.numerical_key])\n features = features.flatten(1, 2) if features.ndim == 3 else features\n features = self.mlp(features)\n logits = self.head(features)\n\n return {\n self.prefix: {\n LOGITS: logits,\n FEATURES: features,\n }\n }\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n All layers have the same id 0 since there is no pre-trained models used here.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n, _ in self.named_parameters():\n name_to_id[n] = 0\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/sam.py",
"content": "import logging\nfrom typing import Dict, List, Optional, Tuple\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom transformers import SamConfig\n\nfrom ..constants import CLASS_LABEL, CLASS_LOGITS, COLUMN, IMAGE, IMAGE_VALID_NUM, LABEL, LOGITS, MASK_LABEL, MOE_LOSS\nfrom .adaptation_layers import ConvLoRALinear\nfrom .custom_hf_models.modeling_sam_for_conv_lora import SamImageSegmentationOutput, SamModel\nfrom .utils import assign_layer_ids, freeze_model_layers, image_mean_std\n\nlogger = logging.getLogger(__name__)\n\n\ndef multi_class_mask_decoder_forward(\n self,\n image_embeddings: torch.Tensor,\n image_positional_embeddings: torch.Tensor,\n sparse_prompt_embeddings: torch.Tensor,\n dense_prompt_embeddings: torch.Tensor,\n multimask_output: bool,\n output_attentions: Optional[bool] = None,\n attention_similarity: torch.Tensor = None,\n target_embedding: torch.Tensor = None,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"\n Modify the forward method of SamMaskDecoder for multi-class semantic segmentation\n based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/sam/modeling_sam.py#L468.\n\n Args:\n image_embeddings (`torch.Tensor`):\n the embeddings from the image encoder\n image_positional_embedding (`torch.Tensor`):\n positional encoding with the shape of image_embeddings\n sparse_prompt_embeddings (`torch.Tensor`):\n The embeddings of the points and boxes\n dense_prompt_embeddings (`torch.Tensor`):\n the embeddings of the mask inputs\n multimask_output (bool):\n Whether to return multiple masks or a single mask.\n output_attentions (bool, *optional*):\n Whether or not to return the attentions tensors of all attention layers.\n \"\"\"\n batch_size, num_channels, height, width = image_embeddings.shape\n point_batch_size = sparse_prompt_embeddings.shape[1]\n # Concatenate output tokens\n output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0)\n output_tokens = output_tokens.repeat(batch_size, point_batch_size, 1, 1)\n\n if sparse_prompt_embeddings.sum().item() != 0:\n tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=2)\n else:\n tokens = output_tokens\n point_embeddings = tokens.to(self.iou_token.weight.dtype)\n\n # Expand per-image data in batch direction to be per-point\n image_embeddings = image_embeddings + dense_prompt_embeddings\n image_embeddings = image_embeddings.repeat(point_batch_size, 1, 1, 1)\n image_positional_embeddings = image_positional_embeddings.repeat(point_batch_size, 1, 1, 1)\n\n # Run the transformer, image_positional_embedding are consumed\n point_embedding, image_embeddings, attentions = self.transformer(\n point_embeddings=point_embeddings,\n image_embeddings=image_embeddings,\n image_positional_embeddings=image_positional_embeddings,\n attention_similarity=attention_similarity,\n target_embedding=target_embedding,\n output_attentions=output_attentions,\n )\n iou_token_out = point_embedding[:, :, 0, :]\n mask_tokens_out = point_embedding[:, :, 1 : (1 + self.num_mask_tokens), :]\n\n # Upscale mask embeddings and predict masks using the mask tokens\n image_embeddings = image_embeddings.transpose(2, 3).reshape(\n batch_size * point_batch_size, num_channels, height, width\n )\n\n upscaled_embedding = self.upscale_conv1(image_embeddings)\n upscaled_embedding = self.activation(self.upscale_layer_norm(upscaled_embedding))\n upscaled_embedding = self.activation(self.upscale_conv2(upscaled_embedding))\n\n ################ Modify the original code. We aim at returning a single mask for each object.\n ################ Original logic is to return multiple masks.\n ################ So we use an MLP network to process all the mask proposals instead of multiple networks.\n # hyper_in_list = []\n # for i in range(self.num_mask_tokens):\n # current_mlp = self.output_hypernetworks_mlps[i]\n # hyper_in_list += [current_mlp(mask_tokens_out[:, :, i, :])]\n # hyper_in = torch.stack(hyper_in_list, dim=2)\n hyper_in = self.output_hypernetworks_mlps[0](mask_tokens_out)\n ################\n\n _, num_channels, height, width = upscaled_embedding.shape\n upscaled_embedding = upscaled_embedding.reshape(batch_size, point_batch_size, num_channels, height * width)\n masks = (hyper_in @ upscaled_embedding).reshape(\n batch_size, point_batch_size, -1, height, width\n ) # bs, 1, num_tokens, h, w\n\n ################ New added class prediction logic.\n class_predictions = self.output_classifier_mlps(mask_tokens_out).reshape(\n batch_size, point_batch_size, -1, self.num_classes + 1\n )\n ################\n\n # Generate mask quality predictions\n iou_pred = self.iou_prediction_head(iou_token_out)\n\n outputs = (masks, iou_pred)\n\n if output_attentions:\n outputs = outputs + (attentions,)\n else:\n outputs = outputs + (None,)\n\n return outputs + (class_predictions,)\n\n\ndef multi_class_sam_model_forward(\n self,\n pixel_values: Optional[torch.FloatTensor] = None,\n input_points: Optional[torch.FloatTensor] = None,\n input_labels: Optional[torch.LongTensor] = None,\n input_boxes: Optional[torch.FloatTensor] = None,\n input_masks: Optional[torch.LongTensor] = None,\n image_embeddings: Optional[torch.FloatTensor] = None,\n multimask_output: bool = True,\n attention_similarity: Optional[torch.FloatTensor] = None,\n target_embedding: Optional[torch.FloatTensor] = None,\n output_attentions: Optional[bool] = None,\n output_hidden_states: Optional[bool] = None,\n output_moe_loss: Optional[bool] = None, # MoE loss for Conv-LoRA\n return_dict=None,\n **kwargs,\n) -> List[Dict[str, torch.Tensor]]:\n r\"\"\"\n Modify the forward method of SamModel for multi-class semantic segmentation\n based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/sam/modeling_sam.py#L1279.\n\n \"\"\"\n output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n output_hidden_states = (\n output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n )\n return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n if pixel_values is None and image_embeddings is None:\n raise ValueError(\"Either pixel_values or image_embeddings must be provided.\")\n\n if pixel_values is not None and image_embeddings is not None:\n raise ValueError(\"Only one of pixel_values and image_embeddings can be provided.\")\n\n if input_points is not None and len(input_points.shape) != 4:\n raise ValueError(\n \"The input_points must be a 4D tensor. Of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.\",\n \" got {}.\".format(input_points.shape),\n )\n if input_boxes is not None and len(input_boxes.shape) != 3:\n raise ValueError(\n \"The input_points must be a 3D tensor. Of shape `batch_size`, `nb_boxes`, `4`.\",\n \" got {}.\".format(input_boxes.shape),\n )\n if input_points is not None and input_boxes is not None:\n point_batch_size = input_points.shape[1]\n box_batch_size = input_boxes.shape[1]\n if point_batch_size != box_batch_size:\n raise ValueError(\n \"You should provide as many bounding boxes as input points per box. Got {} and {}.\".format(\n point_batch_size, box_batch_size\n )\n )\n\n image_positional_embeddings = self.get_image_wide_positional_embeddings()\n # repeat with batch size\n batch_size = pixel_values.shape[0] if pixel_values is not None else image_embeddings.shape[0]\n image_positional_embeddings = image_positional_embeddings.repeat(batch_size, 1, 1, 1)\n\n vision_attentions = None\n vision_hidden_states = None\n vision_moe_loss = None\n\n if pixel_values is not None:\n vision_outputs = self.vision_encoder(\n pixel_values,\n output_attentions=output_attentions,\n output_hidden_states=output_hidden_states,\n output_moe_loss=output_moe_loss,\n return_dict=return_dict,\n )\n image_embeddings = vision_outputs[0]\n\n if output_hidden_states:\n vision_hidden_states = vision_outputs[1]\n if output_attentions:\n # vision_attentions = vision_outputs[-1]\n vision_attentions = vision_outputs[2]\n if output_moe_loss:\n vision_moe_loss = vision_outputs[-1]\n\n if input_points is not None and input_labels is None:\n input_labels = torch.ones_like(input_points[:, :, :, 0], dtype=torch.int, device=input_points.device)\n\n if input_points is not None and image_embeddings.shape[0] != input_points.shape[0]:\n raise ValueError(\n \"The batch size of the image embeddings and the input points must be the same. \",\n \"Got {} and {} respectively.\".format(image_embeddings.shape[0], input_points.shape[0]),\n \" if you want to pass multiple points for the same image, make sure that you passed \",\n \" input_points of shape (batch_size, point_batch_size, num_points_per_image, 3) and \",\n \" input_labels of shape (batch_size, point_batch_size, num_points_per_image)\",\n )\n\n sparse_embeddings, dense_embeddings = self.prompt_encoder(\n input_points=input_points,\n input_labels=input_labels,\n input_boxes=input_boxes,\n input_masks=input_masks,\n )\n\n low_res_masks, iou_predictions, mask_decoder_attentions, class_predictions = self.mask_decoder(\n image_embeddings=image_embeddings,\n image_positional_embeddings=image_positional_embeddings,\n sparse_prompt_embeddings=sparse_embeddings,\n dense_prompt_embeddings=dense_embeddings,\n multimask_output=multimask_output,\n attention_similarity=attention_similarity,\n target_embedding=target_embedding,\n output_attentions=output_attentions,\n )\n\n if not return_dict:\n output = (\n iou_predictions,\n low_res_masks,\n vision_hidden_states,\n vision_attentions,\n mask_decoder_attentions,\n vision_moe_loss,\n )\n return output\n\n return (\n SamImageSegmentationOutput(\n iou_scores=iou_predictions,\n pred_masks=low_res_masks,\n vision_hidden_states=vision_hidden_states,\n vision_attentions=vision_attentions,\n mask_decoder_attentions=mask_decoder_attentions,\n vision_moe_loss=vision_moe_loss,\n ),\n ################ New added. Return class predictions as well.\n class_predictions,\n ################\n )\n\n\nclass SAMForSemanticSegmentation(nn.Module):\n \"\"\"\n Support SAM for semantic segmentation.\n Refer to https://huggingface.co/docs/transformers/main/model_doc/sam\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n num_classes: int = 1,\n pretrained: Optional[bool] = True,\n frozen_layers: Optional[list] = None,\n num_mask_tokens: int = 1,\n image_norm: Optional[str] = None,\n ):\n \"\"\"\n Load a pretrained Segment Anything Model (SAM).\n\n Parameters\n ----------\n prefix\n The prefix of the SAMForSemanticSegmentation model.\n checkpoint_name\n Name of the SAM checkpoint.\n num_classes\n The number of classes\n pretrained\n Whether using the pretrained SAM models. If pretrained=True, download the pretrained model.\n frozen_layers\n A list of substrings of frozen layers' names.\n num_mask_tokens\n The number of mask proposals.\n image_norm\n How to normalize an image. We now support:\n - inception\n Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm\n - imagenet\n Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm\n - clip\n Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and\n std (0.26862954, 0.26130258, 0.27577711), used for CLIP.\n \"\"\"\n\n super().__init__()\n self.prefix = prefix\n self.pretrained = pretrained\n self.checkpoint_name = checkpoint_name\n self.num_classes = num_classes\n self.frozen_layers = frozen_layers\n\n self.device = None\n self.name_to_id = {}\n\n self._load_checkpoint(checkpoint_name)\n\n freeze_model_layers(self.model, self.frozen_layers)\n\n self.image_size = self.model.vision_encoder.image_size\n self.config = self.model.config\n self.image_mean, self.image_std = image_mean_std(image_norm)\n\n self.model.mask_decoder.num_mask_tokens = num_mask_tokens\n mask_token_data = self.model.mask_decoder.mask_tokens.weight.data[0]\n self.model.mask_decoder.mask_tokens = nn.Embedding(num_mask_tokens, self.model.mask_decoder.hidden_size)\n for i in range(num_mask_tokens):\n self.model.mask_decoder.mask_tokens.weight.data[i] = mask_token_data\n hyper_mlps = self.model.mask_decoder.output_hypernetworks_mlps[0]\n self.model.mask_decoder.output_hypernetworks_mlps = nn.ModuleList([hyper_mlps])\n if num_classes > 1:\n self.model.mask_decoder.num_classes = num_classes\n self.model.mask_decoder.output_classifier_mlps = nn.Linear(\n self.model.mask_decoder.hidden_size, num_classes + 1\n )\n\n mask_decoder_forward = multi_class_mask_decoder_forward.__get__(\n self.model.mask_decoder, self.model.mask_decoder.__class__\n )\n setattr(self.model.mask_decoder, \"forward\", mask_decoder_forward)\n\n sam_model_forward = multi_class_sam_model_forward.__get__(self.model, self.model.__class__)\n setattr(self.model, \"forward\", sam_model_forward)\n\n # for Conv-LoRA\n self.output_moe_loss = False\n\n def _load_checkpoint(self, checkpoint_name):\n if self.pretrained:\n self.model = SamModel.from_pretrained(checkpoint_name)\n else:\n configuration = SamConfig(name_or_path=checkpoint_name)\n self.model = SamModel(configuration)\n\n def save(self, save_path: str = \"./\"):\n self.model.save_pretrained(save_path)\n logger.info(f\"Model weights for {self.prefix} is saved to {save_path}.\")\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def image_feature_dim(self):\n return self.model.num_features\n\n @property\n def mask_label_key(self):\n return f\"{self.prefix}_{MASK_LABEL}\"\n\n @property\n def class_label_key(self):\n return f\"{self.prefix}_{CLASS_LABEL}\"\n\n def train(self, mode: bool = True):\n super().train(mode)\n for module in self.modules():\n if isinstance(module, ConvLoRALinear):\n self.output_moe_loss = True\n return self\n\n return self\n\n def forward(\n self,\n batch,\n ):\n \"\"\"\n Parameters\n ----------\n batch\n A dictionary containing the input mini-batch data.\n We need to use the keys with the model prefix to index required data.\n\n Returns\n -------\n A dictionary with mask predictions.\n \"\"\"\n # binary\n if self.num_classes == 1:\n rets = self.model(batch[self.image_key], multimask_output=False, output_moe_loss=self.output_moe_loss)\n pred_masks = rets.pred_masks[:, 0, :, :, :]\n pred_masks = F.interpolate(\n pred_masks, (self.image_size, self.image_size), mode=\"bilinear\", align_corners=False\n )\n if self.training:\n rets_dict = {self.prefix: {LOGITS: pred_masks}}\n else:\n rets_dict = {self.prefix: {LOGITS: pred_masks, LABEL: batch[self.label_key]}}\n # multi-class\n else:\n rets = self.model(batch[self.image_key], multimask_output=False, output_moe_loss=self.output_moe_loss)\n rets, class_predictions = rets\n pred_masks = rets.pred_masks[:, 0, :, :, :]\n pred_classes = class_predictions[:, 0, :, :]\n pred_masks = F.interpolate(\n pred_masks, (self.image_size, self.image_size), mode=\"bilinear\", align_corners=False\n )\n if self.training:\n rets_dict = {self.prefix: {LOGITS: pred_masks, CLASS_LOGITS: pred_classes}}\n else:\n rets_dict = {\n self.prefix: {\n LOGITS: pred_masks,\n CLASS_LOGITS: pred_classes,\n LABEL: batch[self.label_key],\n }\n }\n if self.output_moe_loss:\n rets_dict[self.prefix].update({MOE_LOSS: rets.vision_moe_loss})\n return rets_dict\n\n def get_layer_ids(self):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n In the AutoModel scenario, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\n \"vision_encoder\",\n \"prompt_encoder\",\n )\n post_encoder_patterns = (\"mask_decoder\",)\n\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/t_few.py",
"content": "import logging\nfrom functools import lru_cache\nfrom typing import Dict, List, Optional, Tuple\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom transformers import AutoConfig, AutoModelForSeq2SeqLM\nfrom transformers import logging as hf_logging\n\nfrom ..constants import (\n CHOICES_IDS,\n COLUMN,\n COLUMN_FEATURES,\n FEATURES,\n LABEL,\n LM_TARGET,\n LOGITS,\n MASKS,\n TEMPLATE_LOGITS,\n TEXT_SEGMENT_IDS,\n TEXT_TOKEN_IDS,\n TEXT_VALID_LENGTH,\n)\nfrom .utils import (\n DummyLayer,\n assign_layer_ids,\n get_column_features,\n get_pretrained_tokenizer,\n get_text_segment_num,\n get_text_token_max_len,\n)\n\nhf_logging.set_verbosity_error()\n\nlogger = logging.getLogger(__name__)\n\n\n@lru_cache(None)\ndef warn_once(logger, msg: str):\n logger.warning(msg)\n\n\nclass TFewModel(nn.Module):\n \"\"\"\n Implementation of T-Few (https://arxiv.org/pdf/2205.05638.pdf).\n Refer to https://github.com/r-three/t-few\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str = \"bigscience/T0_3B\",\n num_classes: Optional[int] = 0,\n length_norm: float = 1.0, # Normalizes length to adjust for length bias in target template\n unlikely_loss: float = 1.0, # Adds loss term that lowers probability of incorrect outputs\n mc_loss: float = 1.0, # Adds multiple choice cross entropy loss\n gradient_checkpointing: Optional[bool] = False,\n low_cpu_mem_usage: Optional[bool] = False,\n pretrained: Optional[bool] = True,\n tokenizer_name: Optional[str] = \"hf_auto\",\n max_text_len: Optional[int] = None,\n text_segment_num: Optional[int] = 1,\n ):\n \"\"\"\n Load a pretrained T5-based text transformer backbone.\n\n Parameters\n ----------\n prefix\n The model prefix.\n checkpoint_name\n Name of the checkpoint. We support loading T5ForConditionalGeneration checkpoints from\n Huggingface Models list: https://huggingface.co/models.\n We recommend using T0 backbones. For example, you may use\n - 'bigscience/T0_3B'\n - 'bigscience/T0p'\n - 'bigscience/T0pp'\n num_classes\n The number of classes. 1 for a regression task.\n length_norm\n Normalizes length to adjust for length bias in target template\n unlikely_loss\n Adds loss term that lowers probability of incorrect outputs\n mc_loss\n Adds multiple choice cross entropy loss\n gradient_checkpointing\n Whether to enable gradient checkpointing\n low_cpu_mem_usage\n Whether to turn on the optimization of reducing the peak CPU memory usage when loading the pretrained model.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n tokenizer_name\n Name of the huggingface tokenizer type.\n \"\"\"\n super().__init__()\n logger.debug(f\"initializing {checkpoint_name}\")\n\n self.checkpoint_name = checkpoint_name\n self.num_classes = num_classes\n\n self.config = AutoConfig.from_pretrained(checkpoint_name) # nosec B615\n\n if pretrained:\n self.model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_name, low_cpu_mem_usage=low_cpu_mem_usage) # nosec B615\n else:\n self.model = AutoModelForSeq2SeqLM.from_config(self.config)\n\n self.tokenizer_name = tokenizer_name\n self.tokenizer = get_pretrained_tokenizer(\n tokenizer_name=self.tokenizer_name,\n checkpoint_name=self.checkpoint_name,\n )\n self.max_text_len = get_text_token_max_len(\n provided_max_len=max_text_len,\n config=self.config,\n tokenizer=self.tokenizer,\n checkpoint_name=self.checkpoint_name,\n )\n self.text_segment_num = get_text_segment_num(\n config=self.config,\n provided_segment_num=text_segment_num,\n checkpoint_name=self.checkpoint_name,\n )\n self.eos_token = self.tokenizer.eos_token\n self.out_features = (\n self.model.config.hidden_size\n ) # required attribute for some features, e.g. data augmentation\n\n self.gradient_checkpointing = gradient_checkpointing\n if gradient_checkpointing:\n self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={\"use_reentrant\": False})\n self.dummy_layer = DummyLayer()\n\n self.prefix = prefix\n\n self.mc_loss = mc_loss\n self.unlikely_loss = unlikely_loss\n self.length_norm = length_norm\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def text_token_ids_key(self):\n return f\"{self.prefix}_{TEXT_TOKEN_IDS}\"\n\n @property\n def text_segment_ids_key(self):\n return f\"{self.prefix}_{TEXT_SEGMENT_IDS}\"\n\n @property\n def text_valid_length_key(self):\n return f\"{self.prefix}_{TEXT_VALID_LENGTH}\"\n\n @property\n def input_keys(self):\n return [self.text_token_ids_key, self.text_valid_length_key, self.choices_key]\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def choices_key(self):\n return f\"{self.prefix}_{CHOICES_IDS}\"\n\n @property\n def text_column_prefix(self):\n return f\"{self.text_token_ids_key}_{COLUMN}\"\n\n @property\n def text_feature_dim(self):\n return self.model.config.hidden_size\n\n def forward(\n self,\n text_token_ids: torch.Tensor,\n text_valid_length: torch.Tensor,\n choices_ids: Optional[torch.Tensor] = None,\n text_column_names: Optional[List[str]] = None,\n text_column_indices: Optional[List[torch.Tensor]] = None,\n ):\n \"\"\"\n Parameters\n ----------\n text_token_ids : torch.Tensor\n Indices of input sequence tokens in the vocabulary.\n text_valid_length : torch.Tensor\n Valid length of the input text sequence.\n choices_ids : torch.Tensor, optional\n The choices ids for multiple-choices tasks.\n text_column_names : list of str, optional\n Names of the text columns.\n text_column_indices : list of torch.Tensor, optional\n Start and stop indices of the text columns.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n # TODO: Bad style, check for choices in multimodal.data. Split TemplateEngine into TextTemplateEngine and LabelTemplateEngine.\n\n if not choices_ids.numel():\n warn_once(\n logger,\n msg=\"No target choices found in batch. Ensure that 'data.templates_turn_on=True' and that a valid preset or custom templates are provided.\",\n )\n warn_once(logger, msg=\"Fallback to numerical representation of classes...\")\n choices_ids = (\n self.tokenizer([str(i) for i in range(self.num_classes)], return_tensors=\"pt\", padding=True)[\n \"input_ids\"\n ]\n .repeat(text_token_ids.size(0), 1, 1)\n .to(text_token_ids)\n )\n\n assert choices_ids.size(1) == self.num_classes, (\n f\"Number of target choices is different from number of classes, but they must be the same. Please check template.\"\n )\n\n bs = text_token_ids.size(0)\n # TODO(?) Currently does not support mixed-task batching, but can be added by adjusting the label_templates dict.\n\n bs, num_choices = choices_ids.size()[:2]\n flat_choices_ids = choices_ids.flatten(0, 1)\n\n text_masks = (text_token_ids != self.tokenizer.pad_token_id).float()\n\n inputs_embeds = self.model.encoder.embed_tokens(text_token_ids)\n\n if self.gradient_checkpointing:\n inputs_embeds = self.dummy_layer(inputs_embeds)\n\n # Forward input through the encoder\n encoder_hidden_states_or = self.model.encoder(inputs_embeds=inputs_embeds, attention_mask=text_masks)[0]\n encoder_hidden_states = encoder_hidden_states_or.unsqueeze(dim=1).repeat(1, num_choices, 1, 1).flatten(0, 1)\n\n attention_mask = text_masks.unsqueeze(dim=1).repeat(1, num_choices, 1).flatten(0, 1)\n decoder_input_ids = torch.cat([torch.zeros_like(flat_choices_ids[:, :1]), flat_choices_ids[:, :-1]], dim=1)\n decoder_attention_mask = (decoder_input_ids == decoder_input_ids).float()\n # Forward encoder output and target template as input for decoder\n model_output = self.model(\n attention_mask=attention_mask,\n encoder_outputs=[encoder_hidden_states],\n decoder_input_ids=decoder_input_ids,\n decoder_attention_mask=decoder_attention_mask,\n )\n\n model_output = model_output.logits\n\n target_template_logits = model_output # Decoder Logits over the vocabulary for target template sequence\n\n lm_target = flat_choices_ids - 100 * (flat_choices_ids == self.tokenizer.pad_token_id).long()\n # Calculate entropy of target templates' logits to target template, i.e. how close the target template is to what\n # the model would predict, going from sentence start token (target_template_logits) to sentence end token (\n # lm_target)\n choices_scores = (\n F.cross_entropy(target_template_logits.flatten(0, 1), lm_target.flatten(0, 1), reduction=\"none\")\n .view(bs, num_choices, -1)\n .sum(dim=-1)\n )\n # Add length normalization to adjust for target templates of different length\n if self.length_norm > 0:\n choices_scores = choices_scores / torch.pow(\n (choices_ids != self.tokenizer.pad_token_id).sum(dim=-1), self.length_norm\n )\n # Use the entropy score as the class \"logit\" scoring of T-Few.\n choices_scores = -choices_scores\n\n # FIXME(?) Not sure having column features with the decoder vocabulary logits in T-Few makes sense\n batch = {\n self.text_token_ids_key: text_token_ids,\n self.text_valid_length_key: text_valid_length,\n self.choices_key: choices_ids,\n }\n if text_column_names:\n assert len(text_column_names) == len(text_column_indices), \"invalid text column inputs\"\n for idx, name in enumerate(text_column_names):\n batch[name] = text_column_indices[idx]\n column_features, column_feature_masks = get_column_features(\n batch=batch,\n column_name_prefix=self.text_column_prefix,\n features=model_output,\n valid_lengths=text_valid_length,\n )\n\n # needed to ensure compatibility to encoder-only pipelines\n features = encoder_hidden_states_or[:, 0, :]\n logits = choices_scores\n\n target_template_logits = target_template_logits.view(bs, num_choices, *target_template_logits.size()[1:])\n lm_target = lm_target.view(bs, num_choices, *lm_target.size()[1:])\n\n if column_features == {} or column_feature_masks == {}:\n return features, logits, target_template_logits, lm_target\n else:\n return features, logits, target_template_logits, lm_target, column_features, column_feature_masks\n\n def get_output_dict(\n self,\n features: torch.Tensor,\n logits: torch.Tensor,\n target_template_logits: torch.Tensor,\n lm_target: torch.Tensor,\n column_features: Optional[Dict[str, torch.Tensor]] = None,\n column_feature_masks: Optional[Dict[str, torch.Tensor]] = None,\n ):\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n if column_features != None:\n ret[COLUMN_FEATURES][FEATURES].update(column_features)\n ret[COLUMN_FEATURES][MASKS].update(column_feature_masks)\n\n ret.update(\n {\n LOGITS: logits, # needed for default crossentropy loss\n TEMPLATE_LOGITS: target_template_logits, # needed for unlikelihood loss\n LM_TARGET: lm_target, # needed for lm loss\n FEATURES: features,\n }\n )\n\n return {self.prefix: ret}\n\n def get_layer_ids(self):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n In the AutoModel scenario, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\n \"embeddings\",\n \"LayerNorm\",\n \"wte\",\n \"wpe\",\n \"shared.weight\",\n \"encoder.conv.conv\",\n \"dummy_layer\",\n )\n post_encoder_patterns = (\"head\", \"pooler\", \"ln_f\", \"final_layer_norm\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 1\n\n for name, id in name_to_id.items(): # no layer should be assigned zero id as zero id is finetuned\n if id == 0:\n name_to_id[name] = 1\n\n return name_to_id\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/timm_image.py",
"content": "import json\nimport logging\nimport os\nfrom typing import Dict, List, Optional\n\nimport torch\nfrom timm import create_model\nfrom timm.layers.linear import Linear\nfrom torch import nn\n\nfrom ..constants import COLUMN, COLUMN_FEATURES, FEATURES, IMAGE, IMAGE_VALID_NUM, LABEL, LOGITS, MASKS\nfrom .utils import (\n assign_layer_ids,\n get_column_features,\n get_image_size_mean_std,\n get_model_head,\n replace_missing_images_with_learnable,\n)\n\nlogger = logging.getLogger(__name__)\n\n\n# Stores the class names of the timm backbones that support variable input size. You can add more backbones to the list.\nSUPPORT_VARIABLE_INPUT_SIZE_TIMM_CLASSES = {\"convnext\", \"efficientnet\", \"mobilenetv3\", \"regnet\", \"resnet\"}\n\n\nclass TimmAutoModelForImagePrediction(nn.Module):\n \"\"\"\n Support TIMM image backbones.\n Refer to https://github.com/rwightman/pytorch-image-models\n \"\"\"\n\n def __init__(\n self,\n prefix: str,\n checkpoint_name: str,\n num_classes: Optional[int] = 0,\n mix_choice: Optional[str] = \"all_logits\",\n pretrained: Optional[bool] = True,\n image_size: Optional[int] = None,\n image_norm: Optional[str] = None,\n image_chan_num: Optional[int] = 3,\n use_learnable_image: Optional[bool] = False,\n ):\n \"\"\"\n Load a pretrained image backbone from TIMM.\n\n Parameters\n ----------\n prefix\n The prefix of the TimmAutoModelForImagePrediction model.\n checkpoint_name\n Name of the timm checkpoint, or local parent directory of the saved finetuned timm weights and config.\n num_classes\n The number of classes. 1 for a regression task.\n mix_choice\n Choice used for mixing multiple images. We now support.\n - all_images\n The images are directly averaged and passed to the model.\n - all_logits\n The logits output from individual images are averaged to generate the final output.\n pretrained\n Whether using the pretrained timm models. If pretrained=True, download the pretrained model.\n image_norm\n How to normalize an image. We now support:\n - inception\n Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm\n - imagenet\n Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm\n - clip\n Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and\n std (0.26862954, 0.26130258, 0.27577711), used for CLIP.\n image_size\n The provided width / height of a square image.\n \"\"\"\n super().__init__()\n # In TIMM, if num_classes==0, then create_model would automatically set self.model.head = nn.Identity()\n logger.debug(f\"initializing {prefix} (TimmAutoModelForImagePrediction)\")\n logger.debug(f\"model checkpoint: {checkpoint_name}\")\n if os.path.exists(checkpoint_name):\n checkpoint_path = f\"{checkpoint_name}/pytorch_model.bin\"\n try:\n with open(f\"{checkpoint_name}/config.json\") as f:\n self.config = json.load(f)\n pretrained_cfg = self.config.get(\"pretrained_cfg\", {})\n for k, v in pretrained_cfg.items():\n if k not in self.config:\n self.config[k] = v\n self.checkpoint_name = self.config.get(\"architecture\", None)\n self.model = create_model(self.checkpoint_name, checkpoint_path=checkpoint_path, num_classes=0)\n # create a head with new num_classes\n self.head = (\n Linear(in_features=self.config[\"num_features\"], out_features=num_classes)\n if num_classes > 0\n else nn.Identity()\n )\n self.num_classes = num_classes if num_classes is not None else 0\n except:\n raise ValueError(f\"Timm model path {checkpoint_name} does not exist or model is invalid.\")\n else:\n self.checkpoint_name = checkpoint_name\n self.model = create_model(checkpoint_name, pretrained=pretrained, num_classes=num_classes)\n self.head = get_model_head(model=self.model)\n self.config = self.model.default_cfg\n self.num_classes = self.model.num_classes\n\n self.pretrained = pretrained\n self.out_features = self.model.num_features\n self.global_pool = self.model.global_pool if hasattr(self.model, \"global_pool\") else None\n self.model.reset_classifier(0) # remove the internal head\n\n self.mix_choice = mix_choice\n logger.debug(f\"mix_choice: {mix_choice}\")\n\n self.prefix = prefix\n self.image_size, self.image_mean, self.image_std = get_image_size_mean_std(\n model_name=self.prefix,\n config=self.config,\n provided_size=image_size,\n provided_norm_type=image_norm,\n support_variable_input_size=self.support_variable_input_size(),\n )\n self.image_chan_num = image_chan_num\n self.use_learnable_image = use_learnable_image\n if self.use_learnable_image:\n self.learnable_image = nn.Parameter(torch.zeros(image_chan_num, self.image_size, self.image_size))\n logger.debug(\"will use a learnable image to replace missing ones\")\n\n self.name_to_id = self.get_layer_ids()\n self.head_layer_names = [n for n, layer_id in self.name_to_id.items() if layer_id == 0]\n\n @property\n def image_key(self):\n return f\"{self.prefix}_{IMAGE}\"\n\n @property\n def image_valid_num_key(self):\n return f\"{self.prefix}_{IMAGE_VALID_NUM}\"\n\n @property\n def label_key(self):\n return f\"{self.prefix}_{LABEL}\"\n\n @property\n def input_keys(self):\n return [self.image_key, self.image_valid_num_key]\n\n @property\n def image_column_prefix(self):\n return f\"{self.image_key}_{COLUMN}\"\n\n @property\n def image_feature_dim(self):\n return self.model.num_features\n\n def support_variable_input_size(self):\n \"\"\"Whether the TIMM image support images sizes that are different from the default used in the backbones\"\"\"\n if \"test_input_size\" in self.config and self.config[\"test_input_size\"] != self.config[\"input_size\"]:\n return True\n cls_name = type(self.model).__name__.lower()\n for k in SUPPORT_VARIABLE_INPUT_SIZE_TIMM_CLASSES:\n if cls_name in k:\n return True\n return False\n\n def forward(\n self,\n images: torch.FloatTensor,\n image_valid_num: torch.Tensor,\n image_column_names: Optional[List[str]] = None,\n image_column_indices: Optional[List[torch.Tensor]] = None,\n ):\n \"\"\"\n Parameters\n ----------\n images : torch.FloatTensor\n A tensor in [N, C, H, W] layout to represent the images.\n image_valid_num : torch.Tensor\n A tensor that describes valid number of input images.\n image_column_names : list of str, optional\n A list of strings that indicates names of the image columns.\n image_column_indices : list of torch.Tensor, optional\n A list of tensors that indicates start and stop indices of the image columns.\n\n Returns\n -------\n A dictionary with logits and features.\n \"\"\"\n column_features = column_feature_masks = dict()\n if self.mix_choice == \"all_images\": # mix inputs\n mixed_images = (\n images.sum(dim=1) / torch.clamp(image_valid_num, min=1e-6)[:, None, None, None]\n ) # mixed shape: (b, 3, h, w)\n features = self.model(mixed_images)\n if self.num_classes > 0:\n logits = self.head(features)\n else:\n logits = features\n\n elif self.mix_choice == \"all_logits\": # mix outputs\n b, n, c, h, w = images.shape\n steps = torch.arange(0, n).type_as(image_valid_num)\n image_masks = steps.reshape((1, -1)) < image_valid_num.reshape((-1, 1)) # (b, n)\n\n if self.use_learnable_image:\n images = replace_missing_images_with_learnable(\n images=images,\n image_masks=image_masks,\n learnable_image=self.learnable_image,\n )\n features = self.model(images.reshape((b * n, c, h, w))) # (b*n, num_features)\n if self.num_classes > 0:\n logits = self.head(features)\n logits = logits.reshape((b, n, -1)) # (b, n, num_classes)\n # reshape features after head prediction\n features = features.reshape((b, n, -1)) # (b, n, num_features)\n\n if not self.use_learnable_image:\n features = features * image_masks[:, :, None].type_as(features) # (b, n, num_features)\n\n # need to collect column features before summing them\n if image_column_names:\n assert len(image_column_names) == len(image_column_indices), \"invalid image column inputs\"\n # collect features by image column names\n column_features, column_feature_masks = get_column_features(\n batch=dict(zip(image_column_names, image_column_indices)),\n column_name_prefix=self.image_column_prefix,\n features=features,\n valid_lengths=image_valid_num,\n )\n\n if self.use_learnable_image:\n features = features.mean(dim=1)\n else:\n features = features.sum(dim=1) / torch.clamp(image_valid_num, min=1e-6)[:, None] # (b, num_features)\n if self.num_classes > 0:\n if self.use_learnable_image:\n logits = logits.mean(dim=1)\n else:\n logits = logits * image_masks[:, :, None].type_as(logits) # (b, n, num_classes)\n logits = logits.sum(dim=1) / torch.clamp(image_valid_num, min=1e-6)[:, None] # (b, num_classes)\n else:\n logits = features\n\n else:\n raise ValueError(f\"unknown mix_choice: {self.mix_choice}\")\n\n return features, logits, column_features, column_feature_masks\n\n def get_output_dict(\n self,\n features: torch.Tensor,\n logits: torch.Tensor,\n column_features: Optional[Dict[str, torch.Tensor]] = None,\n column_feature_masks: Optional[Dict[str, torch.Tensor]] = None,\n ):\n ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}\n\n if column_features is not None and len(column_features) > 0:\n assert column_feature_masks is not None and len(column_features) == len(column_feature_masks)\n ret[COLUMN_FEATURES][FEATURES].update(column_features)\n ret[COLUMN_FEATURES][MASKS].update(column_feature_masks)\n\n ret[FEATURES] = features\n if self.num_classes > 0:\n ret[LOGITS] = logits\n\n return {self.prefix: ret}\n\n def get_layer_ids(\n self,\n ):\n \"\"\"\n Assign an id to each layer. Layer ids will be used in layer-wise lr decay.\n Basically, id gradually increases when going from the output end to\n the input end. The layers defined in this class, e.g., head, have id 0.\n\n Due to different backbone architectures in TIMM, this function may not always return the correct result.\n Thus, you can use \"print(json.dumps(name_to_id, indent=2))\" to manually check whether\n the layer ids are reasonable.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n model_prefix = \"model\"\n pre_encoder_patterns = (\"embed\", \"cls_token\", \"stem\", \"bn1\", \"conv1\")\n post_encoder_patterns = (\"head\", \"norm\", \"bn2\")\n names = [n for n, _ in self.named_parameters()]\n\n name_to_id, names = assign_layer_ids(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_pre=model_prefix,\n )\n\n if len(names) > 0:\n logger.debug(f\"outer layers are treated as head: {names}\")\n for n in names:\n assert n not in name_to_id\n name_to_id[n] = 0\n\n return name_to_id\n\n def dump_config(\n self,\n config_path: str,\n ):\n \"\"\"\n Save TIMM image model configs to a local file.\n\n Parameters\n ----------\n config_path:\n A file to where the config is written to.\n \"\"\"\n from ..utils import filter_timm_pretrained_cfg\n\n config = {}\n pretrained_cfg = filter_timm_pretrained_cfg(self.config, remove_source=True, remove_null=True)\n # set some values at root config level\n config[\"architecture\"] = pretrained_cfg.pop(\"architecture\")\n config[\"num_classes\"] = self.num_classes\n config[\"num_features\"] = self.out_features\n\n global_pool_type = getattr(self, \"global_pool\", None)\n if isinstance(global_pool_type, str) and global_pool_type:\n config[\"global_pool\"] = global_pool_type\n\n config[\"pretrained_cfg\"] = pretrained_cfg\n\n with open(config_path, \"w\") as f:\n json.dump(config, f, indent=2)\n logger.info(f\"Timm config saved to {config_path}.\")\n\n def save(self, save_path: str = \"./\", tokenizers: Optional[dict] = None):\n weights_path = f\"{save_path}/pytorch_model.bin\"\n torch.save(self.model.state_dict(), weights_path) # nosec B614\n logger.info(f\"Model {self.prefix} weights saved to {weights_path}.\")\n config_path = f\"{save_path}/config.json\"\n self.dump_config(config_path)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/models/utils.py",
"content": "import functools\nimport logging\nimport re\nimport warnings\nfrom typing import Dict, List, Optional, Tuple\n\nimport timm\nimport torch\nimport torch._dynamo\nimport torch.nn.functional as F\nfrom omegaconf import DictConfig, OmegaConf\nfrom timm.data.constants import (\n IMAGENET_DEFAULT_MEAN,\n IMAGENET_DEFAULT_STD,\n IMAGENET_INCEPTION_MEAN,\n IMAGENET_INCEPTION_STD,\n)\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\nfrom transformers import AutoConfig, AutoModel, AutoTokenizer, BertTokenizer, CLIPTokenizer, ElectraTokenizer\nfrom transformers.models.mask2former.modeling_mask2former import Mask2FormerLoss\n\nfrom ..constants import (\n ALL_MODALITIES,\n CATEGORICAL,\n CATEGORICAL_MLP,\n CLASS_LOGITS,\n CLIP,\n CLIP_IMAGE_MEAN,\n CLIP_IMAGE_STD,\n DOCUMENT,\n DOCUMENT_TRANSFORMER,\n FT_TRANSFORMER,\n FUSION_MLP,\n FUSION_NER,\n FUSION_TRANSFORMER,\n HF_TEXT,\n IMAGE,\n LOGITS,\n META_TRANSFORMER,\n MMDET_IMAGE,\n MMOCR_TEXT_DET,\n MMOCR_TEXT_RECOG,\n NER_TEXT,\n NUMERICAL,\n NUMERICAL_MLP,\n OCR,\n PEFT_ADDITIVE_STRATEGIES,\n REGRESSION,\n SAM,\n SEMANTIC_MASK,\n SEMANTIC_SEGMENTATION,\n SEMANTIC_SEGMENTATION_IMG,\n T_FEW,\n TEXT,\n TEXT_NER,\n TIMM_IMAGE,\n)\nfrom .adaptation_layers import ConvLoRALinear, IA3Linear, IA3LoRALinear, LoRALinear\n\nlogger = logging.getLogger(__name__)\n\n\nALL_TOKENIZERS = {\n \"bert\": BertTokenizer,\n \"clip\": CLIPTokenizer,\n \"electra\": ElectraTokenizer,\n \"hf_auto\": AutoTokenizer,\n}\n\n\nclass DummyLayer(nn.Module):\n \"\"\"\n DummyLayer to ensure that the gradient checkpointing will assign output layer as require_grad=True.\n Reference: https://discuss.pytorch.org/t/checkpoint-with-no-grad-requiring-inputs-problem/19117/9\n \"\"\"\n\n def __init__(self):\n super().__init__()\n self.dummy_bias = torch.ones(1, dtype=torch.float32, requires_grad=True)\n\n def forward(self, x):\n return x + self.dummy_bias.to(x) - self.dummy_bias.to(x)\n\n\ndef init_weights(module: nn.Module):\n \"\"\"\n Initialize one module. It uses xavier_norm to initialize nn.Embedding\n and xavier_uniform to initialize nn.Linear's weight.\n\n Parameters\n ----------\n module\n A Pytorch nn.Module.\n \"\"\"\n if isinstance(module, nn.Embedding):\n nn.init.xavier_normal_(module.weight)\n elif isinstance(module, nn.Linear):\n nn.init.xavier_uniform_(module.weight)\n if module.bias is not None:\n nn.init.zeros_(module.bias)\n elif isinstance(module, nn.LayerNorm):\n module.bias.data.zero_()\n module.weight.data.fill_(1.0)\n\n\ndef assign_encoder_layer_ids(\n encoder_names: List[List[str]],\n):\n \"\"\"\n Assign ids to encoder layers. The encoder may contain several blocks e.g., block1 and block2.\n This function iterates through all the layers of each block from the input end towards the output end.\n It increases 1 on the layer id when the detected digit in a layer name changes.\n\n Parameters\n ----------\n encoder_names\n Encoder layer names.\n\n Returns\n -------\n name_to_id\n The encoder layer-to-id mapping.\n encoder_layer_num\n The encoder layer number.\n \"\"\"\n name_to_id = {}\n cur_id = 0\n for i, group_names in enumerate(encoder_names):\n last_inferred_id = -1\n for n in group_names:\n detect_id = False\n n_splits = n.split(\".\")\n\n for split in n_splits:\n # the first digit encountered is used to infer layer id\n if split.isdigit():\n inferred_id = int(split)\n # increase at most 1 one time\n if inferred_id != last_inferred_id:\n cur_id += 1 # layer.0 -> layer_id 1\n last_inferred_id = inferred_id\n\n name_to_id[n] = cur_id\n detect_id = True\n break\n\n if detect_id is False:\n raise ValueError(f\"parameter name: {n} not has no id inside\")\n\n if len(name_to_id) > 0:\n encoder_layer_num = max(name_to_id.values())\n else:\n encoder_layer_num = 0\n return name_to_id, encoder_layer_num\n\n\ndef assign_non_encoder_layer_ids(\n non_encoder_names: List[str],\n layer_id: int,\n):\n \"\"\"\n Assign the provided id to non-encoder layers.\n\n Parameters\n ----------\n non_encoder_names\n Names layers not belonging to an encoder.\n layer_id\n provided id.\n\n Returns\n -------\n A dictionary mapping the layer names (keys) to their ids (values).\n \"\"\"\n name_to_id = {}\n for n in non_encoder_names:\n name_to_id[n] = layer_id\n return name_to_id\n\n\ndef split_encoder_non_encoder(names: List[str], post_encoder_patterns: Tuple[str, ...]):\n \"\"\"\n Group layer names into two types: encoder and non-encoder.\n A layer belongs to encoder if its name contains at least one digit.\n It uses this rule since a model's encoder in Pytorch's implementation\n is generally wrapped by nn.Sequential() or nn.ModuleList(),\n which produce digits in layer names.\n\n Parameters\n ----------\n names\n Model layer names.\n Returns\n -------\n encoder_names\n A list of encoder layer names.\n non_encoder_names\n A list of non-encoder layer names.\n \"\"\"\n encoder_names = []\n non_encoder_names = []\n for n in names:\n is_encoder = False\n if any(p in n for p in post_encoder_patterns):\n non_encoder_names.append(n)\n continue\n for i in n.split(\".\"):\n if i.isdigit():\n encoder_names.append(n)\n is_encoder = True\n break\n if not is_encoder:\n non_encoder_names.append(n)\n\n return encoder_names, non_encoder_names\n\n\ndef group_param_names(\n names: List[str],\n pre_encoder_patterns: Tuple[str, ...],\n post_encoder_patterns: Tuple[str, ...],\n model_prefix: Optional[str] = None,\n):\n \"\"\"\n Group layer names into three types: pre-encoder, encoder, and post-encoder.\n If \"model_prefix\" is provided, the selected layer names must start with it.\n In this case, the left names will be returned for the next-time processing.\n This function first extracts the first-level children modules' names and\n classify them into encoder and non-encoder layers. Note that an encoder may\n consist of several manually named children modules, e.g., block1 and block2.\n The non-encoder layers are further subdivided into pre-encoder and post-encoder.\n\n Parameters\n ----------\n names\n Model layer names\n pre_encoder_patterns\n Patterns to identify a layer as a pre-encoder layer. If a layer name contains one pattern,\n the layer will be grouped into pre-encoder layers.\n post_encoder_patterns\n Patterns to identify a layer as a post-encoder layer. If a layer name contains one pattern,\n the layer will be grouped into post-encoder layers.\n model_prefix\n A prefix to filter layer names. Only layer names starting with it will be selected.\n Returns\n -------\n left_names\n The layer names left for the next-time processing.\n encoder_names_grouped\n Encoder layer names.\n pre_encoder_names\n Names of layers before the encoder.\n post_encoder_names\n Names of layers after the encoder.\n \"\"\"\n # two set of patterns can't have intersections\n assert all(pre_p not in post_encoder_patterns for pre_p in pre_encoder_patterns)\n\n left_names = []\n # in case model_prefix is provided, e.g., the clip model with image and text branches\n selected_names = []\n for n in names:\n if model_prefix is not None and not n.startswith(model_prefix):\n left_names.append(n)\n else:\n selected_names.append(n)\n\n # split blocks at the first level\n children_prefix = []\n for n in selected_names:\n if model_prefix is not None:\n child_name = n[len(model_prefix) + 1 :].split(\".\")[0]\n child_prefix = f\"{model_prefix}.{child_name}\"\n else:\n child_prefix = n.split(\".\")[0]\n if child_prefix not in children_prefix:\n children_prefix.append(child_prefix)\n\n encoder_names_grouped = []\n non_encoder_names = []\n for child_prefix in children_prefix:\n per_names_group = [n for n in selected_names if n.startswith(child_prefix)]\n per_encoder_names, per_non_encoder_names = split_encoder_non_encoder(per_names_group, post_encoder_patterns)\n encoder_names_grouped.append(per_encoder_names)\n non_encoder_names.extend(per_non_encoder_names)\n\n pre_encoder_names = []\n post_encoder_names = []\n for n in non_encoder_names:\n if any(p in n for p in pre_encoder_patterns):\n pre_encoder_names.append(n)\n elif any(p in n for p in post_encoder_patterns):\n post_encoder_names.append(n)\n else:\n raise ValueError(f\"parameter name: {n} belong to neither pre or post encoder names\")\n\n # only process left names in next iteration\n return left_names, encoder_names_grouped, pre_encoder_names, post_encoder_names\n\n\ndef reverse_layer_ids(\n encoder_name_to_id: dict,\n pre_enocder_name_to_id: dict,\n post_enocder_name_to_id: dict,\n):\n \"\"\"\n The layer ids need to increase when going from the output end to the input end.\n We need to reverse the ids which were originally assigned in a decreasing order.\n\n Parameters\n ----------\n encoder_name_to_id\n The layer-to-id mapping of encoder layers.\n pre_enocder_name_to_id\n The layer-to-id mapping of pre-encoder layers.\n post_enocder_name_to_id\n The layer-to-id mapping of post-encoder layers.\n\n Returns\n -------\n The layer-to-id mapping of all layers with layer ids reversed.\n \"\"\"\n name_to_id = {**pre_enocder_name_to_id, **encoder_name_to_id, **post_enocder_name_to_id}\n if len(name_to_id) > 0:\n layer_num = max(name_to_id.values())\n # if no post encoder layers, the minimum layer id should be 1\n if len(post_enocder_name_to_id) == 0:\n layer_num += 1\n for n, layer_id in name_to_id.items():\n name_to_id[n] = layer_num - layer_id\n\n return name_to_id\n\n\ndef assign_layer_ids(\n names: List[str],\n pre_encoder_patterns: Tuple[str, ...],\n post_encoder_patterns: Tuple[str, ...],\n model_pre: Optional[str] = None,\n):\n \"\"\"\n Assign ids to all layers. It splits a model into three parts: pre-encoder, encoder, and post-encoder.\n Encoder is generally a stack of multiple similar layers, such as transformer layers. Since encoder is\n generally wrapped by nn.Sequential() or nn.ModuleList(), its inside layer names contain digits.\n It sets 0 as the ids of all post-encoder layers and a maximum id (layer_num) for the all the pre-encoder\n layers. The encoder layers have decreasing ids from the input to the output ends.\n\n Parameters\n ----------\n names\n model layer names.\n pre_encoder_patterns\n Patterns to identify a layer as a pre-encoder layer. If a layer name contains one pattern,\n the layer will be grouped into pre-encoder layers.\n post_encoder_patterns\n Patterns to identify a layer as a post-encoder layer. If a layer name contains one pattern,\n the layer will be grouped into post-encoder layers.\n model_pre\n The layer names' prefix. Only the layer names with this prefix will be assigned ids. The left\n layer names will be returned.\n\n Returns\n -------\n name_to_id\n A dictionary mapping the layer names (keys) to their ids (values).\n left_names\n The layer names not starting with the \"model_pre\".\n \"\"\"\n try:\n left_names, encoder_names, pre_encoder_names, post_encoder_names = group_param_names(\n names=names,\n pre_encoder_patterns=pre_encoder_patterns,\n post_encoder_patterns=post_encoder_patterns,\n model_prefix=model_pre,\n )\n # add a constraint\n if len(encoder_names) == 0 and len(pre_encoder_names) != 0:\n raise ValueError(f\"encoder_names is empty, but pre_encoder_names has values: {pre_encoder_names}\")\n\n encoder_name_to_id, encoder_layer_num = assign_encoder_layer_ids(\n encoder_names=encoder_names,\n )\n\n pre_encoder_name_to_id = assign_non_encoder_layer_ids(non_encoder_names=pre_encoder_names, layer_id=0)\n\n post_encoder_name_to_id = assign_non_encoder_layer_ids(\n non_encoder_names=post_encoder_names, layer_id=encoder_layer_num + 1\n )\n\n name_to_id = reverse_layer_ids(\n encoder_name_to_id=encoder_name_to_id,\n pre_enocder_name_to_id=pre_encoder_name_to_id,\n post_enocder_name_to_id=post_encoder_name_to_id,\n )\n except Exception as e:\n logger.debug(\n f\"When calling assign_layer_ids(), it catches exception: {e}. All the layers will use the same layer_id.\"\n )\n name_to_id = dict()\n left_names = names\n\n return name_to_id, left_names\n\n\ndef get_column_features(\n batch: Dict[str, torch.Tensor],\n column_name_prefix: str,\n features: torch.Tensor,\n valid_lengths: torch.Tensor,\n cls_feature: Optional[torch.Tensor] = None,\n):\n \"\"\"\n Index the features of one column defined by `column_name_prefix`.\n This function can be used to index both image and text features.\n The features have shape (b, n, d), where n can be the image number or\n text token number. One column corresponds to a subset of\n the n images or text tokens. One column name can only appear once in the return.\n\n Parameters\n ----------\n batch\n The batch input containing the feature column information, i.e., indexes.\n column_name_prefix\n The column name prefix of one modality (image or text).\n features\n The features of columns whose names starts with column_name_prefix.\n valid_lengths\n The valid image number or text token number of each sample in a batch.\n cls_feature\n The cls feature containing information from all feature columns.\n\n Returns\n -------\n The column features with masks. If the column has no valid features, its\n mask is 0.\n \"\"\"\n column_features = {}\n feature_masks = {}\n\n cut_idx = len(column_name_prefix) + 1\n if cls_feature is not None:\n all_column_names = []\n # create a zero mask to do logical_or with each column's mask\n joint_mask = torch.zeros(features.shape[0]).to(features) # (b,)\n for key in batch:\n if key.startswith(column_name_prefix):\n per_col_features = []\n per_col_masks = torch.zeros(features.shape[0]).to(features) # (b,)\n assert batch[key].ndim == 2 and batch[key].shape[1] == 2\n for i, per_sample_col_idx in enumerate(batch[key]):\n start_idx = per_sample_col_idx[0]\n end_idx = per_sample_col_idx[1]\n if start_idx < end_idx:\n assert end_idx <= valid_lengths[i]\n per_col_features.append(features[i, start_idx:end_idx].mean(dim=0))\n per_col_masks[i] = 1\n else: # the column has no valid image/text.\n per_col_features.append(torch.zeros_like(features[0, 0]))\n per_col_masks[i] = 0\n column_name = key[cut_idx:]\n column_features[column_name] = torch.stack(per_col_features, dim=0) # (b, num_features)\n feature_masks[column_name] = per_col_masks # (b,)\n if cls_feature is not None:\n all_column_names.append(column_name)\n joint_mask = torch.logical_or(joint_mask, per_col_masks)\n\n # all the columns of one model's input share the model's cls feature\n if (\n cls_feature is not None and len(all_column_names) > 0\n ): # some models', e.g, timm_image, output doesn't have the cls feature.\n # remove the individual column features since these column features not independent\n for column_name in all_column_names:\n column_features.pop(column_name)\n feature_masks.pop(column_name)\n\n joint_column_name = \"_\".join(all_column_names)\n column_features[joint_column_name] = cls_feature\n feature_masks[joint_column_name] = joint_mask.to(features)\n\n # print(f\"column_features: {column_features}\")\n return column_features, feature_masks\n\n\ndef create_adaptation(peft: str, layer: nn.Module, lora_r: int, lora_alpha: int, **kwargs):\n \"\"\"\n Creates a model adaptation module (IA3, LoRA, IA3_LoRA) given a linear layer.\n\n Parameters\n ----------\n peft\n Name of the adaptation module.\n layer\n The layer the adaptation module should be applied to.\n lora_r\n The rank r of the low-rank decomposition.\n lora_alpha\n The scaling factor. Can be set to same value as r in\n most cases, as initialization is scaled already.\n filter\n Apply loRA only to linear layers filtered by name (e.g. \"query.\").\n If None, loRA is applied to all linear Layers in module.\n module_filter\n Apply loRA only to modules filtered by name (e.g. \".*EncDecAttention|.*DenseReluDense\")\n If None, loRA is considered for all modules\n\n Returns\n -------\n Model with injected LoRA modules.\n \"\"\"\n if \"ia3_lora\" in peft:\n return IA3LoRALinear(\n layer.in_features, layer.out_features, r=lora_r, lora_alpha=lora_alpha, merge_weights=False\n )\n elif \"conv_lora\" in peft:\n return ConvLoRALinear(\n layer.in_features,\n layer.out_features,\n r=lora_r,\n lora_alpha=lora_alpha,\n merge_weights=False,\n conv_lora_expert_num=kwargs[\"conv_lora_expert_num\"],\n )\n elif \"ia3\" in peft:\n return IA3Linear(layer.in_features, layer.out_features, merge_weights=False)\n elif \"lora\" in peft:\n return LoRALinear(layer.in_features, layer.out_features, r=lora_r, lora_alpha=lora_alpha, merge_weights=False)\n elif peft is not None:\n raise NotImplementedError(\n f\"The efficient finetuning strategy '{peft}'\"\n f\" is not supported. We only support\"\n f\" {', '.join(PEFT_ADDITIVE_STRATEGIES)}.\"\n )\n\n\ndef inject_adaptation_to_linear_layer(\n model: nn.Module,\n peft: str,\n lora_r: int = None,\n lora_alpha: int = None,\n filter: Optional[List[str]] = None,\n module_filter: Optional[List[str]] = None,\n extra_trainable_params: Optional[List[str]] = None,\n **kwargs,\n) -> nn.Module:\n \"\"\"\n Injects trainable adatio Low-Rank decomposition matrices (LoRA) into linear\n layers of a PyTorch model. Used for efficient fine-tuning of large\n pre-trained models.\n\n Parameters\n ----------\n model\n A PyTorch model.\n peft\n Efficient finetuning method that should be applied.\n lora_r\n The rank r of the low-rank decomposition.\n lora_alpha\n The scaling factor. Can be set to same value as r in\n most cases, as initialization is scaled already.\n filter\n Apply loRA only to linear layers filtered by name (e.g. \"query.\").\n If None, loRA is applied to all linear Layers in module.\n module_filter\n Apply loRA only to modules filtered by name (e.g. \".*EncDecAttention|.*DenseReluDense\")\n If None, loRA is considered for all modules\n extra_trainable_params\n Not to apply loRA to modules filtered by name, and these modules are not frozen during training (e.g. \"mask_decoder\").\n If None, all the modules except for those applied loRA are frozen.\n Returns\n -------\n Model with injected LoRA modules.\n \"\"\"\n for m_name, module in dict(model.named_modules()).items():\n if extra_trainable_params and any(re.match(filter_layer, m_name) for filter_layer in extra_trainable_params):\n continue\n if hasattr(model, \"frozen_layers\") and any(\n re.search(filter_layer, m_name) for filter_layer in model.frozen_layers\n ):\n continue\n if not module_filter or any(re.match(filter_module, m_name) for filter_module in module_filter):\n for c_name, layer in dict(module.named_children()).items():\n if not filter or any(re.match(filter_layer, c_name) for filter_layer in filter):\n assert isinstance(layer, nn.Linear), (\n f\"LoRA can only be applied to torch.nn.Linear, but {layer} is {type(layer)}.\"\n )\n adaptation_layer = create_adaptation(peft, layer, lora_r, lora_alpha, **kwargs)\n adaptation_layer.weight = layer.weight\n adaptation_layer.bias = layer.bias\n setattr(module, c_name, adaptation_layer)\n\n return model # return model to enable method chaining\n\n\ndef get_model_head(model: nn.Module):\n \"\"\"\n Return the model's head. Different models may have different head names.\n\n Parameters\n ----------\n model\n A Pytorch model.\n\n Returns\n -------\n The model's head.\n \"\"\"\n if hasattr(model, \"head\"):\n head = model.head # move the head outside\n elif hasattr(model, \"last_linear\"):\n head = model.last_linear\n elif hasattr(model, \"fc\"):\n head = model.fc\n elif hasattr(model, \"classifier\"):\n head = model.classifier\n else:\n raise ValueError(f\"Model {type(model)} doesn't have head. Need to check its implementation.\")\n\n return head.fc if hasattr(head, \"fc\") else head\n\n\ndef get_hf_config_and_model(\n checkpoint_name: str, pretrained: Optional[bool] = True, low_cpu_mem_usage: Optional[bool] = False\n):\n \"\"\"\n Get a Huggingface config and model based on a checkpoint name.\n\n Parameters\n ----------\n checkpoint_name\n A model checkpoint name or a local path that saves a custom checkpoint.\n pretrained\n Whether using the pretrained weights. If pretrained=True, download the pretrained model.\n low_cpu_mem_usage\n Whether to turn on the optimization of reducing the peak CPU memory usage when loading the pretrained model.\n\n Returns\n -------\n A Huggingface config and model.\n \"\"\"\n config = AutoConfig.from_pretrained(checkpoint_name) # nosec B615\n\n if pretrained:\n model = AutoModel.from_pretrained(checkpoint_name, low_cpu_mem_usage=low_cpu_mem_usage) # nosec B615\n else:\n model = AutoModel.from_config(config)\n # Explicitly set the model to train mode after loading as by default it is in eval mode\n # See issue #4965\n model.train()\n return config, model\n\n\ndef apply_sigmoid(output: Dict):\n \"\"\"\n Apply the sigmoid to logits.\n\n Parameters\n ----------\n output\n The model output dict.\n\n Returns\n -------\n The output with logits transformed by sigmoid.\n \"\"\"\n for k, v in output.items():\n output[k][LOGITS] = torch.sigmoid(v[LOGITS].float())\n return output\n\n\ndef apply_multi_class_semantic_seg_postprocess(output: Dict):\n \"\"\"\n Apply the semantic postprocessing to logits.\n\n Parameters\n ----------\n output\n The model output dict.\n\n Returns\n -------\n The output with post-proceesed semantic masks.\n \"\"\"\n\n def semantic_inference(mask_cls, mask_pred):\n \"\"\"\n Post-processing mask prediction for multi-class semantic segmentation inference based on https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/maskformer_model.py\n\n Args:\n mask_cls (`torch.Tensor`):\n Class logits. A tensor of shape `(num_queries, num_classes + 1)` (include the \"no object\" category).\n\n mask_pred (`torch.Tensor`):\n Mask logits. A tensor of shape `(num_queries, height, width)`.\n\n Returns:\n semseg (`torch.Tensor`): The processed mask prediction. A tensor of shape `(num_classes, height, width)`.\n\n References:\n [1] https://arxiv.org/abs/2107.06278\n [2] https://arxiv.org/abs/2112.01527\n \"\"\"\n mask_cls = F.softmax(mask_cls, dim=-1)[..., :-1]\n mask_pred = mask_pred.sigmoid()\n semseg = torch.einsum(\"qc,qhw->chw\", mask_cls, mask_pred)\n return semseg\n\n for k, v in output.items():\n pred_classes = output[k][CLASS_LOGITS]\n pred_masks = output[k][LOGITS]\n semantic_masks = []\n for mask_cls_result, mask_pred_result in zip(\n pred_classes, pred_masks\n ): # bs, num_q, num_class and bs, num_q, h, w\n per_sample_semantic_masks = semantic_inference(mask_cls_result, mask_pred_result) # num_class, h, w\n semantic_masks.append(per_sample_semantic_masks)\n semantic_masks = torch.stack(semantic_masks, dim=0)\n output[k][SEMANTIC_MASK] = semantic_masks\n return output\n\n\ndef get_model_postprocess_fn(problem_type: str, loss_func: _Loss):\n \"\"\"\n Get the postprocessing function for the model outputs.\n\n Parameters\n ----------\n problem_type\n The problem type, e.g., classification or regression.\n loss_func\n The loss function used in training.\n\n Returns\n -------\n The postprocessing function.\n \"\"\"\n postprocess_func = None\n if problem_type == REGRESSION:\n if isinstance(loss_func, nn.BCEWithLogitsLoss):\n postprocess_func = apply_sigmoid\n elif problem_type == SEMANTIC_SEGMENTATION:\n if isinstance(loss_func, Mask2FormerLoss):\n postprocess_func = apply_multi_class_semantic_seg_postprocess\n else:\n postprocess_func = apply_sigmoid\n\n return postprocess_func\n\n\ndef get_mmocr_config_and_model(checkpoint_name: str):\n \"\"\"\n Get an MMOCR config and model based on a checkpoint name.\n\n Parameters\n ----------\n checkpoint_name\n A model checkpoint name.\n\n Returns\n -------\n An MMOCR config and model.\n \"\"\"\n from ..utils import check_if_packages_installed\n\n check_if_packages_installed(package_names=[\"mmcv\"])\n try:\n import mmocr\n from mmocr.models import build_detector\n except ImportError:\n mmocr = None\n from mim.commands.download import download\n\n checkpoints = download(package=\"mmocr\", configs=[checkpoint_name], dest_root=\".\")\n\n # read config files\n check_if_packages_installed(problem_type=OCR)\n config_file = checkpoint_name + \".py\"\n if isinstance(config_file, str):\n config = mmcv.Config.fromfile(config_file)\n\n # build model and load pretrained weights\n checkpoint = checkpoints[0]\n model = build_detector(config.model, test_cfg=config.get(\"test_cfg\"))\n if checkpoint is not None:\n checkpoint = load_checkpoint(model, checkpoint, map_location=\"cpu\")\n return config, model\n\n\ndef lookup_mmdet_config(key, config):\n if key in config:\n return config[key]\n for subconfig in config.values():\n if isinstance(subconfig, dict):\n result = lookup_mmdet_config(key, subconfig)\n if result is not None:\n return result\n elif isinstance(subconfig, list):\n for subsubconfig in subconfig:\n if isinstance(subsubconfig, dict):\n result = lookup_mmdet_config(key, subsubconfig)\n if result is not None:\n return result\n return None\n\n\ndef update_mmdet_config(key, value, config):\n for k, subconfig in config.items():\n if key == k:\n config[k] = value\n elif isinstance(subconfig, dict):\n update_mmdet_config(key, value, subconfig)\n elif isinstance(subconfig, list):\n for subsubconfig in subconfig:\n if isinstance(subsubconfig, dict):\n update_mmdet_config(key, value, subsubconfig)\n\n\ndef run_model(model: nn.Module, batch: dict, trt_model: Optional[nn.Module] = None):\n from ..utils.onnx import OnnxModule\n from .document_transformer import DocumentTransformer\n from .fusion.fusion_mlp import MultimodalFusionMLP\n from .hf_text import HFAutoModelForTextPrediction\n from .t_few import TFewModel\n from .timm_image import TimmAutoModelForImagePrediction\n\n supported_models = (\n TimmAutoModelForImagePrediction,\n HFAutoModelForTextPrediction,\n MultimodalFusionMLP,\n TFewModel,\n OnnxModule,\n )\n pure_model = model\n if isinstance(model, torch._dynamo.eval_frame.OptimizedModule):\n pure_model = model._orig_mod\n if isinstance(model, nn.DataParallel):\n pure_model = model.module\n if isinstance(pure_model, OnnxModule):\n for k in batch:\n # HACK input data types in ONNX\n if batch[k].dtype == torch.int32:\n batch[k] = batch[k].to(torch.int64)\n # DocumentTransformer inherited from HFAutoModelForTextPrediction\n if (not isinstance(pure_model, DocumentTransformer)) and isinstance(pure_model, supported_models):\n input_vec = [batch[k] for k in pure_model.input_keys]\n column_names, column_values = [], []\n for k in batch.keys():\n has_image_column_prefix = isinstance(pure_model, TimmAutoModelForImagePrediction) and k.startswith(\n pure_model.image_column_prefix\n )\n has_text_column_prefix = isinstance(pure_model, HFAutoModelForTextPrediction) and k.startswith(\n pure_model.text_column_prefix\n )\n if has_image_column_prefix or has_text_column_prefix:\n column_names.append(k)\n column_values.append(batch[k])\n if column_names != [] and column_values != []:\n input_vec.append(column_names)\n input_vec.append(column_values)\n if isinstance(pure_model, OnnxModule):\n # OnnxModule doesn't support multi-gpu yet\n output_vec = pure_model(*tuple(input_vec))\n else:\n output_vec = model(*tuple(input_vec))\n\n output = pure_model.get_output_dict(*output_vec)\n else:\n output = model(batch)\n return output\n\n\ndef freeze_model_layers(model, frozen_layers):\n \"\"\"\n Freeze model layers with pattern in frozen_layers.\n\n Parameters\n ----------\n model\n The pytorch model.\n frozen_layers\n A list of substrings of frozen layers' names.\n\n e.g. if frozen_layers = [\"backbone\", \"neck\"],\n all layers including \"backbone\" or \"neck\" in the name will be frozen.\n \"\"\"\n\n if not frozen_layers:\n return\n\n is_frozen_layer = lambda n: any(bb in n for bb in frozen_layers)\n\n for n, p in model.named_parameters():\n if is_frozen_layer(n):\n p.requires_grad = False\n\n\ndef get_pretrained_tokenizer(\n tokenizer_name: str,\n checkpoint_name: str,\n use_fast: Optional[bool] = True,\n add_prefix_space: Optional[bool] = None,\n):\n \"\"\"\n Load the tokenizer for a pre-trained huggingface checkpoint.\n\n Parameters\n ----------\n tokenizer_name\n The tokenizer type, e.g., \"bert\", \"clip\", \"electra\", and \"hf_auto\".\n checkpoint_name\n Name of a pre-trained checkpoint.\n use_fast\n Use a fast Rust-based tokenizer if it is supported for a given model.\n If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead.\n See: https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoTokenizer.from_pretrained.use_fast\n\n Returns\n -------\n A tokenizer instance.\n \"\"\"\n try:\n tokenizer_class = ALL_TOKENIZERS[tokenizer_name]\n if add_prefix_space is None:\n return tokenizer_class.from_pretrained(checkpoint_name, use_fast=use_fast)\n else:\n return tokenizer_class.from_pretrained(\n checkpoint_name, use_fast=use_fast, add_prefix_space=add_prefix_space\n )\n except TypeError as e:\n try:\n tokenizer = BertTokenizer.from_pretrained(checkpoint_name) # nosec B615\n warnings.warn(\n f\"Current checkpoint {checkpoint_name} does not support AutoTokenizer. \"\n \"Switch to BertTokenizer instead.\",\n UserWarning,\n )\n return tokenizer\n except:\n raise e\n\n\ndef extract_value_from_config(\n config: Dict,\n keys: Tuple[str, ...],\n):\n \"\"\"\n Traverse a config dictionary to get some hyper-parameter's value.\n\n Parameters\n ----------\n config\n A config dictionary.\n keys\n The possible names of a hyper-parameter.\n\n Returns\n -------\n The hyper-parameter value.\n \"\"\"\n result = []\n for k, v in config.items():\n if k in keys:\n result.append(v)\n elif isinstance(v, dict):\n result += extract_value_from_config(v, keys)\n else:\n pass\n\n return result\n\n\ndef extract_image_hparams_from_config(model_name: str, config):\n \"\"\"\n Extract some default hyper-parameters, e.g., image size, mean, and std,\n from a pre-trained (timm or huggingface) checkpoint.\n\n Parameters\n ----------\n model_name\n Name of model.\n config\n Config of a pre-trained checkpoint.\n\n Returns\n -------\n image_size\n Image width/height.\n mean\n Image normalization mean.\n std\n Image normalizaiton std.\n \"\"\"\n if model_name.lower().startswith((TIMM_IMAGE, META_TRANSFORMER)):\n image_size = config[\"input_size\"][-1]\n image_mean = config[\"mean\"]\n image_std = config[\"std\"]\n elif model_name.lower().startswith((CLIP, DOCUMENT_TRANSFORMER)):\n extracted = extract_value_from_config(\n config=config.to_diff_dict(),\n keys=(\"image_size\",),\n )\n if len(extracted) == 0:\n image_size = None\n elif len(extracted) >= 1:\n image_size = extracted[0]\n if isinstance(image_size, tuple):\n image_size = image_size[-1]\n else:\n raise ValueError(f\" more than one image_size values are detected: {extracted}\")\n image_mean = None\n image_std = None\n else:\n raise ValueError(f\"Unknown image processor prefix: {model_name}\")\n return image_size, image_mean, image_std\n\n\ndef image_mean_std(norm_type: str):\n \"\"\"\n Get image normalization mean and std by its name.\n\n Parameters\n ----------\n norm_type\n Name of image normalization.\n\n Returns\n -------\n Normalization mean and std.\n \"\"\"\n if norm_type == \"inception\":\n return IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD\n elif norm_type == \"imagenet\":\n return IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD\n elif norm_type == \"clip\":\n return CLIP_IMAGE_MEAN, CLIP_IMAGE_STD\n else:\n raise ValueError(f\"unknown image normalization: {norm_type}\")\n\n\ndef get_image_size_mean_std(\n model_name: str,\n config,\n provided_size: int,\n provided_norm_type: str,\n support_variable_input_size: Optional[bool] = False,\n):\n image_size, image_mean, image_std = extract_image_hparams_from_config(\n model_name=model_name,\n config=config,\n )\n if support_variable_input_size and provided_size is not None:\n # We have detected that the model supports using an image size that is\n # different from the pretrained model, e.g., ConvNets with global pooling\n if provided_size < image_size:\n logger.warning(\n f\"The provided image size={provided_size} is smaller than the default size \"\n f\"of the pretrained backbone, which is {image_size}. \"\n f\"Detailed configuration of the backbone is in {config}. \"\n f\"You may like to double check your configuration.\"\n )\n image_size = provided_size\n elif provided_size is not None and provided_size != image_size:\n logger.warning(\n f\"The model does not support using an image size that is different from the default size. \"\n f\"Provided image size={provided_size}. Default size={image_size}. \"\n f\"Detailed model configuration={config}. We have ignored the provided image size.\"\n )\n\n if image_size is None:\n if provided_size is not None:\n image_size = provided_size\n logger.debug(f\"using provided image size: {image_size}.\")\n else:\n raise ValueError(\"image size is missing.\")\n else:\n logger.debug(f\"using detected image size: {image_size}\")\n\n if image_mean is None or image_std is None:\n if provided_norm_type is not None:\n image_mean, image_std = image_mean_std(provided_norm_type)\n logger.debug(f\"using provided normalization: {provided_norm_type}.\")\n else:\n raise ValueError(\"image normalization mean and std are missing.\")\n else:\n logger.debug(f\"using detected image normalization: {image_mean} and {image_std}.\")\n\n return image_size, image_mean, image_std\n\n\ndef get_text_segment_num(config, provided_segment_num: int, checkpoint_name: str):\n extracted = extract_value_from_config(config=config.to_diff_dict(), keys=(\"type_vocab_size\",))\n if len(extracted) == 0:\n default_segment_num = 1\n elif len(extracted) == 1:\n default_segment_num = extracted[0]\n else:\n raise ValueError(f\" more than one type_vocab_size values are detected: {extracted}\")\n\n if default_segment_num <= 0:\n default_segment_num = 1\n\n if provided_segment_num < default_segment_num:\n warnings.warn(\n f\"provided text_segment_num: {provided_segment_num} \"\n f\"is smaller than {checkpoint_name}'s default: {default_segment_num}\"\n )\n text_segment_num = min(provided_segment_num, default_segment_num)\n assert text_segment_num >= 1\n logger.debug(f\"text segment num: {text_segment_num}\")\n\n return text_segment_num\n\n\ndef get_text_token_max_len(provided_max_len, config, tokenizer, checkpoint_name):\n \"\"\"\n Compute the allowable max length of token sequences.\n\n Parameters\n ----------\n provided_max_len\n The provided max length.\n config\n Model config.\n tokenizer\n Text tokenizer.\n checkpoint_name\n Name of checkpoint.\n\n Returns\n -------\n Token sequence max length.\n \"\"\"\n if hasattr(config, \"relative_attention\") and config.relative_attention:\n default_max_len = tokenizer.model_max_length\n elif hasattr(config, \"position_embedding_type\") and \"relative\" in config.position_embedding_type:\n default_max_len = tokenizer.model_max_length\n elif hasattr(config, \"max_position_embeddings\"):\n default_max_len = config.max_position_embeddings\n else:\n default_max_len = tokenizer.model_max_length\n\n if provided_max_len is None or provided_max_len <= 0:\n max_len = default_max_len\n else:\n if provided_max_len < default_max_len:\n if default_max_len < 10**6: # Larger than this value usually means infinite.\n warnings.warn(\n f\"provided max length: {provided_max_len} \"\n f\"is smaller than {checkpoint_name}'s default: {default_max_len}\"\n )\n max_len = min(provided_max_len, default_max_len)\n\n logger.debug(f\"text max length: {max_len}\")\n\n return max_len\n\n\ndef replace_missing_images_with_learnable(\n images: torch.Tensor,\n image_masks,\n learnable_image: nn.Parameter,\n):\n b, n, c, h, w = images.shape\n assert learnable_image.shape == (c, h, w)\n for i in range(b):\n for j in range(n):\n if not image_masks[i][j]: # False means a missing image\n images[i][j] = learnable_image\n\n return images\n\n\ndef select_model(\n config: DictConfig,\n df_preprocessor,\n strict: Optional[bool] = True,\n):\n \"\"\"\n Filter model config through the detected modalities in the training data.\n If MultiModalFeaturePreprocessor can't detect some modality,\n this function will remove the models that use this modality. This function is to\n maximize the user flexibility in defining the config.\n For example, if one uses the default, including hf_text and timm_image, as the model config template\n but the training data don't have images, this function will filter out timm_image.\n\n Parameters\n ----------\n config\n A DictConfig object. The model config should be accessible by \"config.model\"\n df_preprocessor\n A MultiModalFeaturePreprocessor object, which has called .fit() on the training data.\n Column names of the same modality are grouped into one list. If a modality's list is empty,\n it means the training data don't have this modality.\n strict\n If False, allow retaining one model when partial modalities are available for that model.\n\n Returns\n -------\n Config with some unused models removed.\n \"\"\"\n data_status = {}\n for per_modality in ALL_MODALITIES:\n data_status[per_modality] = False\n if len(df_preprocessor.image_feature_names) > 0:\n data_status[IMAGE] = True\n if len(df_preprocessor.text_feature_names) > 0:\n data_status[TEXT] = True\n if len(df_preprocessor.categorical_feature_names) > 0:\n data_status[CATEGORICAL] = True\n if len(df_preprocessor.numerical_feature_names) > 0:\n data_status[NUMERICAL] = True\n if len(df_preprocessor.ner_feature_names) > 0:\n data_status[TEXT_NER] = True\n if len(df_preprocessor.document_feature_names) > 0:\n data_status[DOCUMENT] = True\n if len(df_preprocessor.semantic_segmentation_feature_names) > 0:\n data_status[SEMANTIC_SEGMENTATION_IMG] = True\n\n names = config.model.names\n if isinstance(names, str):\n names = [names]\n selected_model_names = []\n fusion_model_name = []\n for model_name in names:\n model_config = getattr(config.model, model_name)\n strict = getattr(model_config, \"requires_all_dtypes\", strict)\n if not model_config.data_types:\n fusion_model_name.append(model_name)\n continue\n model_data_status = [data_status[d_type] for d_type in model_config.data_types]\n if all(model_data_status):\n selected_model_names.append(model_name)\n else:\n if any(model_data_status) and not strict:\n selected_model_names.append(model_name)\n # update data types to be consistent with detected\n model_config.data_types = [d_type for d_type in model_config.data_types if data_status[d_type]]\n else:\n delattr(config.model, model_name)\n\n if len(selected_model_names) == 0:\n raise ValueError(\"No model is available for this dataset.\")\n # only allow no more than 1 fusion model\n if len(fusion_model_name) > 1:\n raise ValueError(f\"More than one fusion models `{fusion_model_name}` are detected, but only one is allowed.\")\n\n if len(selected_model_names) > 1:\n assert len(fusion_model_name) == 1\n selected_model_names.extend(fusion_model_name)\n elif len(fusion_model_name) == 1 and hasattr(config.model, fusion_model_name[0]):\n delattr(config.model, fusion_model_name[0])\n\n config.model.names = selected_model_names\n logger.debug(f\"selected models: {selected_model_names}\")\n for model_name in selected_model_names:\n logger.debug(f\"model dtypes: {getattr(config.model, model_name).data_types}\")\n\n # clean up unused model configs\n model_keys = list(config.model.keys())\n for model_name in model_keys:\n if model_name not in selected_model_names + [\"names\"]:\n delattr(config.model, model_name)\n\n return config\n\n\ndef create_model(\n model_name: str,\n model_config: DictConfig,\n num_classes: Optional[int] = 0,\n classes: Optional[list] = None,\n num_numerical_columns: Optional[int] = None,\n num_categories: Optional[Dict] = None,\n numerical_fill_values: Optional[Dict] = None,\n pretrained: Optional[bool] = True,\n is_matching: Optional[bool] = False,\n):\n \"\"\"\n Create a single model.\n\n Parameters\n ----------\n model_name\n Name of the model.\n model_config\n Config of the model.\n num_classes\n The class number for a classification task. It should be 1 for a regression task.\n classes\n All classes in this dataset.\n num_numerical_columns\n The number of numerical columns in the training dataframe.\n num_categories\n The category number for each categorical column in the training dataframe.\n numerical_fill_values\n If numerical values are null, fill them with these.\n pretrained\n Whether using the pretrained timm models. If pretrained=True, download the pretrained model.\n is_matching\n Whether the model is used for semantic matching.\n\n Returns\n -------\n A model.\n \"\"\"\n if model_name.lower().startswith(CLIP):\n from .clip import CLIPForImageText\n\n model = CLIPForImageText(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n pretrained=pretrained,\n tokenizer_name=model_config.tokenizer_name,\n has_image=IMAGE in model_config.data_types,\n has_text=TEXT in model_config.data_types,\n image_size=model_config.image_size,\n image_norm=model_config.image_norm,\n image_chan_num=model_config.image_chan_num,\n use_learnable_image=model_config.use_learnable_image,\n max_text_len=model_config.max_text_len,\n text_segment_num=model_config.text_segment_num,\n is_matching=is_matching,\n )\n elif model_name.lower().startswith(TIMM_IMAGE):\n from .timm_image import TimmAutoModelForImagePrediction\n\n model = TimmAutoModelForImagePrediction(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n mix_choice=model_config.mix_choice,\n pretrained=pretrained,\n image_size=model_config.image_size,\n image_norm=model_config.image_norm,\n image_chan_num=model_config.image_chan_num,\n use_learnable_image=model_config.use_learnable_image,\n )\n elif model_name.lower().startswith(HF_TEXT):\n from .hf_text import HFAutoModelForTextPrediction\n\n model = HFAutoModelForTextPrediction(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n pooling_mode=model_config.pooling_mode,\n gradient_checkpointing=model_config.gradient_checkpointing,\n low_cpu_mem_usage=model_config.low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=model_config.tokenizer_name,\n max_text_len=model_config.max_text_len,\n text_segment_num=model_config.text_segment_num,\n use_fast=model_config.use_fast,\n )\n elif model_name.lower().startswith(T_FEW):\n from .t_few import TFewModel\n\n model = TFewModel(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n length_norm=model_config.length_norm, # Normalizes length to adjust for length bias in target template\n unlikely_loss=model_config.unlikely_loss, # Adds loss term that lowers probability of incorrect outputs\n mc_loss=model_config.mc_loss, # Adds multiple choice cross entropy loss\n num_classes=num_classes,\n gradient_checkpointing=model_config.gradient_checkpointing,\n low_cpu_mem_usage=model_config.low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=model_config.tokenizer_name,\n max_text_len=model_config.max_text_len,\n text_segment_num=model_config.text_segment_num,\n )\n elif model_name.lower().startswith(NUMERICAL_MLP):\n from .numerical_mlp import NumericalMLP\n\n model = NumericalMLP(\n prefix=model_name,\n in_features=num_numerical_columns,\n hidden_features=model_config.hidden_size,\n out_features=model_config.hidden_size,\n num_layers=model_config.num_layers,\n activation=model_config.activation,\n dropout=model_config.dropout,\n normalization=model_config.normalization,\n token_dim=model_config.token_dim,\n embedding_arch=model_config.embedding_arch,\n num_classes=num_classes,\n numerical_fill_values=numerical_fill_values,\n )\n elif model_name.lower().startswith(CATEGORICAL_MLP):\n from .categorical_mlp import CategoricalMLP\n\n model = CategoricalMLP(\n prefix=model_name,\n num_categories=num_categories,\n out_features=model_config.hidden_size,\n num_layers=model_config.num_layers,\n activation=model_config.activation,\n dropout=model_config.dropout,\n normalization=model_config.normalization,\n num_classes=num_classes,\n )\n elif model_name.lower().startswith(DOCUMENT_TRANSFORMER):\n from .document_transformer import DocumentTransformer\n\n model = DocumentTransformer(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n pooling_mode=model_config.pooling_mode,\n gradient_checkpointing=model_config.gradient_checkpointing,\n low_cpu_mem_usage=model_config.low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=model_config.tokenizer_name,\n image_size=model_config.image_size,\n image_norm=model_config.image_norm,\n )\n elif model_name.lower().startswith(MMDET_IMAGE):\n from .mmdet_image import MMDetAutoModelForObjectDetection\n\n model = MMDetAutoModelForObjectDetection(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n config_file=model_config.config_file,\n classes=classes,\n pretrained=pretrained,\n output_bbox_format=model_config.output_bbox_format,\n frozen_layers=model_config.frozen_layers,\n )\n elif model_name.lower().startswith(MMOCR_TEXT_DET):\n from .mmocr_text_detection import MMOCRAutoModelForTextDetection\n\n model = MMOCRAutoModelForTextDetection(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n )\n elif model_name.lower().startswith(MMOCR_TEXT_RECOG):\n from .mmocr_text_recognition import MMOCRAutoModelForTextRecognition\n\n model = MMOCRAutoModelForTextRecognition(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n )\n elif model_name.lower().startswith(NER_TEXT):\n from .ner_text import HFAutoModelForNER\n\n model = HFAutoModelForNER(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n gradient_checkpointing=model_config.gradient_checkpointing,\n low_cpu_mem_usage=model_config.low_cpu_mem_usage,\n pretrained=pretrained,\n tokenizer_name=model_config.tokenizer_name,\n )\n elif model_name.lower().startswith(FUSION_MLP):\n from .fusion import MultimodalFusionMLP\n\n model = functools.partial(\n MultimodalFusionMLP,\n prefix=model_name,\n hidden_features=model_config.hidden_sizes,\n num_classes=num_classes,\n adapt_in_features=model_config.adapt_in_features,\n activation=model_config.activation,\n dropout=model_config.dropout,\n normalization=model_config.normalization,\n aux_loss_weight=model_config.aux_loss_weight,\n )\n elif model_name.lower().startswith(FUSION_NER):\n from .fusion import MultimodalFusionNER\n\n model = functools.partial(\n MultimodalFusionNER,\n prefix=model_name,\n hidden_features=model_config.hidden_sizes,\n num_classes=num_classes,\n adapt_in_features=model_config.adapt_in_features,\n activation=model_config.activation,\n dropout_prob=model_config.drop_rate,\n normalization=model_config.normalization,\n loss_weight=model_config.weight if hasattr(model_config, \"weight\") else None,\n )\n elif model_name.lower().startswith(FUSION_TRANSFORMER):\n from .fusion import MultimodalFusionTransformer\n\n model = functools.partial(\n MultimodalFusionTransformer,\n prefix=model_name,\n hidden_features=model_config.hidden_size,\n num_classes=num_classes,\n num_blocks=model_config.num_blocks,\n attention_num_heads=model_config.attention_num_heads,\n ffn_hidden_size=model_config.ffn_hidden_size,\n attention_dropout=model_config.attention_dropout,\n residual_dropout=model_config.residual_dropout,\n ffn_dropout=model_config.ffn_dropout,\n attention_normalization=model_config.normalization,\n ffn_normalization=model_config.normalization,\n head_normalization=model_config.normalization,\n ffn_activation=model_config.ffn_activation,\n head_activation=model_config.head_activation,\n adapt_in_features=model_config.adapt_in_features,\n aux_loss_weight=model_config.aux_loss_weight,\n additive_attention=model_config.additive_attention,\n share_qv_weights=model_config.share_qv_weights,\n )\n elif model_name.lower().startswith(FT_TRANSFORMER):\n from .ft_transformer import FT_Transformer\n\n model = FT_Transformer(\n prefix=model_name,\n num_numerical_columns=num_numerical_columns,\n num_categories=num_categories,\n numerical_fill_values=numerical_fill_values,\n embedding_arch=model_config.embedding_arch,\n token_dim=model_config.token_dim,\n hidden_size=model_config.hidden_size,\n hidden_features=model_config.hidden_size,\n num_classes=num_classes,\n num_blocks=model_config.num_blocks,\n attention_num_heads=model_config.attention_num_heads,\n attention_dropout=model_config.attention_dropout,\n attention_normalization=model_config.normalization,\n ffn_hidden_size=model_config.ffn_hidden_size,\n ffn_dropout=model_config.ffn_dropout,\n ffn_normalization=model_config.normalization,\n ffn_activation=model_config.ffn_activation,\n residual_dropout=model_config.residual_dropout,\n head_normalization=model_config.normalization,\n head_activation=model_config.head_activation,\n additive_attention=model_config.additive_attention,\n share_qv_weights=model_config.share_qv_weights,\n pooling_mode=model_config.pooling_mode,\n checkpoint_name=model_config.checkpoint_name,\n pretrained=pretrained,\n )\n elif model_name.lower().startswith(SAM):\n from .sam import SAMForSemanticSegmentation\n\n model = SAMForSemanticSegmentation(\n prefix=model_name,\n checkpoint_name=model_config.checkpoint_name,\n num_classes=num_classes,\n pretrained=pretrained,\n frozen_layers=model_config.frozen_layers,\n num_mask_tokens=model_config.num_mask_tokens,\n image_norm=model_config.image_norm,\n )\n elif model_name.lower().startswith(META_TRANSFORMER):\n from .meta_transformer import MetaTransformer\n\n model = MetaTransformer(\n prefix=model_name,\n checkpoint_path=model_config.checkpoint_path,\n num_classes=num_classes,\n model_version=model_config.model_version,\n has_image=IMAGE in model_config.data_types,\n has_text=TEXT in model_config.data_types,\n num_numerical_columns=num_numerical_columns,\n num_categories=num_categories,\n numerical_fill_values=numerical_fill_values,\n image_size=model_config.image_size,\n image_norm=model_config.image_norm,\n image_chan_num=model_config.image_chan_num,\n use_learnable_image=model_config.use_learnable_image,\n max_text_len=model_config.max_text_len,\n text_segment_num=model_config.text_segment_num,\n )\n else:\n raise ValueError(f\"unknown model name: {model_name}\")\n\n return model\n\n\ndef create_fusion_model(\n config: DictConfig,\n num_classes: Optional[int] = None,\n classes: Optional[list] = None,\n num_numerical_columns: Optional[int] = None,\n num_categories: Optional[Dict] = None,\n numerical_fill_values: Optional[Dict] = None,\n pretrained: Optional[bool] = True,\n):\n \"\"\"\n Create models. It supports the auto models of huggingface text and timm image.\n Multimodal models, e.g., CLIP, should be added case-by-case since their configs and usages\n may be different. It uses MLP for the numerical features, categorical features, and late-fusion.\n\n Parameters\n ----------\n config\n A DictConfig object. The model config should be accessible by \"config.model\".\n num_classes\n The class number for a classification task. It should be 1 for a regression task.\n classes\n All classes in this dataset.\n num_numerical_columns\n The number of numerical columns in the training dataframe.\n num_categories\n The category number for each categorical column in the training dataframe.\n numerical_fill_values\n If numerical values are null, fill them with these.\n pretrained\n Whether using the pretrained timm models. If pretrained=True, download the pretrained model.\n\n Returns\n -------\n A Pytorch model.\n \"\"\"\n names = config.model.names\n if isinstance(names, str):\n names = [names]\n # make sure no duplicate model names\n assert len(names) == len(set(names))\n logger.debug(f\"output_shape: {num_classes}\")\n names = sorted(names)\n config.model.names = names\n single_models = []\n fusion_model = None\n\n for model_name in names:\n model_config = getattr(config.model, model_name)\n model = create_model(\n model_name=model_name,\n model_config=model_config,\n num_classes=num_classes,\n classes=classes,\n num_numerical_columns=num_numerical_columns,\n num_categories=num_categories,\n numerical_fill_values=numerical_fill_values,\n pretrained=pretrained,\n )\n\n if isinstance(model, functools.partial): # fusion model\n if fusion_model is None:\n fusion_model = model\n else:\n raise ValueError(\n f\"More than one fusion models are detected in {names}. Only one fusion model is allowed.\"\n )\n else: # single model\n if config.optim.peft is not None:\n model = apply_peft_adaptation(model, config)\n single_models.append(model)\n\n if len(single_models) > 1:\n # must have one fusion model if there are multiple independent models\n model = fusion_model(models=single_models)\n elif len(single_models) == 1:\n model = single_models[0]\n else:\n raise ValueError(f\"No available models for {names}\")\n\n # build augmenter for multimodal data augmentation\n if config.optim.lemda.turn_on:\n from .fusion import MultimodalFusionMLP\n\n assert isinstance(model, MultimodalFusionMLP)\n from .augmenter import Augmenter\n\n augmenter = Augmenter(\n arch_type=config.optim.lemda.arch_type,\n input_dim=model.augmenter_in_features,\n z_dim=config.optim.lemda.z_dim,\n num_layers=config.optim.lemda.num_layers,\n adv_weight=config.optim.lemda.adv_weight,\n )\n model.augmenter = augmenter\n\n return model\n\n\ndef apply_peft_adaptation(model: nn.Module, config: DictConfig) -> nn.Module:\n \"\"\"\n Apply an adaptation to the model for efficient fine-tuning.\n\n Parameters\n ----------\n model\n A PyTorch model.\n config:\n A DictConfig object. The optimization config should be accessible by \"config.optimization\".\n \"\"\"\n if config.optim.peft in PEFT_ADDITIVE_STRATEGIES:\n model = inject_adaptation_to_linear_layer(\n model=model,\n peft=config.optim.peft,\n lora_r=config.optim.lora.r,\n lora_alpha=config.optim.lora.alpha,\n module_filter=config.optim.lora.module_filter,\n filter=config.optim.lora.filter,\n extra_trainable_params=config.optim.extra_trainable_params,\n conv_lora_expert_num=config.optim.lora.conv_lora_expert_num,\n )\n model.name_to_id = model.get_layer_ids() # Need to update name to id dictionary.\n\n return model\n\n\ndef modify_duplicate_model_names(\n learner,\n postfix: str,\n blacklist: List[str],\n):\n \"\"\"\n Modify a learner's model names if they exist in a blacklist.\n\n Parameters\n ----------\n learner\n A BaseLearner object.\n postfix\n The postfix used to change the duplicate names.\n blacklist\n A list of names. The provided learner can't use model names in the list.\n\n Returns\n -------\n The learner guaranteed has no duplicate model names with the blacklist names.\n \"\"\"\n model_names = []\n for n in learner._config.model.names:\n if n in blacklist:\n new_name = f\"{n}_{postfix}\"\n assert new_name not in blacklist\n assert new_name not in learner._config.model.names\n # modify model prefix\n if n == learner._model.prefix:\n learner._model.prefix = new_name\n else:\n assert isinstance(learner._model.model, nn.ModuleList)\n for per_model in learner._model.model:\n if n == per_model.prefix:\n per_model.prefix = new_name\n break\n # modify data processor prefix\n for per_modality_processors in learner._data_processors.values():\n for per_processor in per_modality_processors:\n if n == per_processor.prefix:\n per_processor.prefix = new_name\n # modify model config keys\n setattr(learner._config.model, new_name, getattr(learner._config.model, n))\n delattr(learner._config.model, n)\n\n model_names.append(new_name)\n else:\n model_names.append(n)\n\n learner._config.model.names = model_names\n\n return learner\n\n\ndef list_timm_models(pretrained=True):\n return timm.list_models(pretrained=pretrained)\n\n\ndef is_lazy_weight_tensor(p: torch.Tensor) -> bool:\n from torch.nn.parameter import UninitializedParameter\n\n if isinstance(p, UninitializedParameter):\n warnings.warn(\n \"A layer with UninitializedParameter was found. \"\n \"Thus, the total number of parameters detected may be inaccurate.\"\n )\n return True\n return False\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/__init__.py",
"content": "from .lit_distiller import DistillerLitModule\nfrom .lit_matcher import MatcherLitModule\nfrom .lit_mmdet import MMDetLitModule\nfrom .lit_module import LitModule\nfrom .lit_ner import NerLitModule\nfrom .lit_semantic_seg import SemanticSegmentationLitModule\nfrom .losses import get_aug_loss_func, get_loss_func, get_matcher_loss_func, get_matcher_miner_func\nfrom .metrics import (\n CustomHitRate,\n compute_ranking_score,\n compute_score,\n get_minmax_mode,\n get_stopping_threshold,\n get_torchmetric,\n infer_metrics,\n)\nfrom .utils import get_norm_layer_param_names, get_peft_param_names\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/deepspeed.py",
"content": "# Slightly adapted file of lightning.pytorch.strategies.deepspeed to not init deepspeed using lightning's deepspeed inference workaround as this has higher memory requirements and can result in OOM during inference. Instead fallback to initialization using `deepspeed.initialize`.\n# TODO: Support deepspeed_inference, custom kernels, and quantization for fast inference.\n\n# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport logging\nfrom typing import Any, Dict, Generator, List, Mapping, Optional, Tuple, Union, cast\n\nimport lightning.pytorch as pl\nimport torch\nfrom lightning.pytorch.accelerators.cuda import CUDAAccelerator\nfrom lightning.pytorch.overrides.base import _LightningModuleWrapperBase, _LightningPrecisionModuleWrapperBase\nfrom lightning.pytorch.plugins.environments.cluster_environment import ClusterEnvironment\nfrom lightning.pytorch.plugins.precision import PrecisionPlugin\nfrom lightning.pytorch.strategies import DeepSpeedStrategy\nfrom lightning.pytorch.utilities import GradClipAlgorithmType\nfrom lightning.pytorch.utilities.exceptions import MisconfigurationException\nfrom lightning.pytorch.utilities.model_helpers import is_overridden\nfrom lightning.pytorch.utilities.rank_zero import rank_zero_warn\nfrom lightning.pytorch.utilities.types import _PATH\n\n\nclass CustomDeepSpeedStrategy(DeepSpeedStrategy):\n \"\"\"\n Provides capabilities to run training using the DeepSpeed library, with training optimizations for large\n billion parameter models. `For more information: https://pytorch-\n lightning.readthedocs.io/en/stable/advanced/model_parallel.html#deepspeed`.\n\n .. warning:: ``DeepSpeedStrategy`` is in beta and subject to change.\n\n Defaults have been set to enable ZeRO-Offload and some have been taken from the link below.\n These defaults have been set generally, but may require tuning for optimum performance based on your model size.\n `For more information: https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training`.\n \"\"\"\n\n def __init__(\n self,\n accelerator: Optional[\"pl.accelerators.accelerator.Accelerator\"] = None,\n zero_optimization: bool = True,\n stage: int = 2,\n remote_device: str = \"cpu\",\n offload_optimizer: bool = False,\n offload_parameters: bool = False,\n offload_params_device: str = \"cpu\",\n nvme_path: str = \"/local_nvme\",\n params_buffer_count: int = 5,\n params_buffer_size: int = 100_000_000,\n max_in_cpu: int = 1_000_000_000,\n offload_optimizer_device: str = \"cpu\",\n optimizer_buffer_count: int = 4,\n block_size: int = 1048576,\n queue_depth: int = 8,\n single_submit: bool = False,\n overlap_events: bool = True,\n thread_count: int = 1,\n pin_memory: bool = False,\n sub_group_size: int = 1_000_000_000_000,\n contiguous_gradients: bool = True,\n overlap_comm: bool = True,\n allgather_partitions: bool = True,\n reduce_scatter: bool = True,\n allgather_bucket_size: int = 200_000_000,\n reduce_bucket_size: int = 200_000_000,\n zero_allow_untested_optimizer: bool = True,\n logging_batch_size_per_gpu: Union[str, int] = \"auto\",\n config: Optional[Union[_PATH, Dict[str, Any]]] = None,\n logging_level: int = logging.WARN,\n parallel_devices: Optional[List[torch.device]] = None,\n cluster_environment: Optional[ClusterEnvironment] = None,\n loss_scale: float = 0,\n initial_scale_power: int = 16,\n loss_scale_window: int = 1000,\n hysteresis: int = 2,\n min_loss_scale: int = 1,\n partition_activations: bool = False,\n cpu_checkpointing: bool = False,\n contiguous_memory_optimization: bool = False,\n synchronize_checkpoint_boundary: bool = False,\n load_full_weights: bool = False,\n precision_plugin: Optional[PrecisionPlugin] = None,\n process_group_backend: Optional[str] = None,\n ) -> None:\n \"\"\"\n Parameters\n ----------\n zero_optimization\n Enable ZeRO optimization. This is only compatible with precision=\"16-mixed\".\n\n stage\n Different stages of the ZeRO Optimizer. 0 is disabled,\n 1 is optimizer state partitioning, 2 is optimizer+gradient state partitioning,\n 3 is optimizer+gradient_parameter partitioning using the infinity engine.\n\n remote_device\n Device to instantiate the model on initially (``cpu`` or ``nvme``).\n\n offload_optimizer\n Enable offloading optimizer memory and computation to CPU or NVMe\n based on ``offload_optimizer_device``.\n\n offload_parameters\n When using ZeRO Stage 3, Enable offloading parameter memory and computation\n to CPU or NVMe based on ``offload_params_device``.\n\n offload_params_device\n When offloading parameters choose the device to offload to, ``cpu`` or ``nvme``.\n\n offload_optimizer_device\n When offloading optimizer state choose the device to offload to,\n ``cpu`` or ``nvme``.\n\n params_buffer_count\n Number of buffers in buffer pool for\n parameter offloading when ``offload_params_device`` is ``nvme``.\n\n params_buffer_size\n Size of buffers in buffer pool for parameter offloading\n when ``offload_params_device`` is ``nvme``.\n\n max_in_cpu\n Number of parameter elements to maintain in CPU memory when offloading to NVMe is enabled.\n\n nvme_path\n Filesystem path for NVMe device for optimizer/parameter state offloading.\n\n optimizer_buffer_count\n Number of buffers in buffer pool for optimizer state offloading\n when ``offload_optimizer_device`` is set to to ``nvme``.\n This should be at least the number of states maintained per parameter by the optimizer.\n For example, Adam optimizer has 4 states (parameter, gradient, momentum, and variance).\n\n block_size\n When using NVMe Offloading, the I/O block size in bytes.\n\n queue_depth\n When using NVMe Offloading, the I/O queue depth.\n\n single_submit\n When using NVMe Offloading,\n submit requests to storage device as multiple individual requests,\n as opposed to one block of requests.\n\n overlap_events\n When using NVMe Offloading,\n submit requests to storage device in an overlapped fashion\n without waiting for completion of earlier requests.\n\n thread_count\n When using NVMe Offloading,\n Intra-request parallelism for each read/write submitted by a user thread.\n\n pin_memory\n When using ZeRO stage 3, pin optimizer state memory on CPU.\n This could boost throughput at the cost of extra memory overhead.\n\n sub_group_size\n When using ZeRO stage 3, defines the number of parameters\n within a sub group to offload at a time.\n Smaller numbers require more communication, but improve memory efficiency.\n\n contiguous_gradients\n Copies gradients to a continuous buffer as they are produced.\n Avoids memory fragmentation during backwards. Useful when training large models.\n\n overlap_comm\n Overlap the reduction (synchronization) of gradients with the backwards computation.\n This is a speed optimization when training across multiple GPUs/machines.\n\n allgather_partitions: All gather updated parameters at the end of training step,\n instead of using a series of broadcast collectives.\n\n reduce_scatter: Use reduce/scatter instead of allreduce to average gradients.\n\n allgather_bucket_size\n Number of elements to allgather at once.\n Used to limit the memory required for larger model sizes, with a tradeoff with speed.\n\n reduce_bucket_size\n Number of elements to reduce at once.\n Used to limit the memory required for larger model sizes, with a tradeoff with speed.\n\n zero_allow_untested_optimizer\n Allow untested optimizers to be used with ZeRO. Currently only Adam is a\n DeepSpeed supported optimizer when using ZeRO.\n\n logging_batch_size_per_gpu\n Config used in DeepSpeed to calculate verbose timing for logging\n on a per sample per second basis (only displayed if logging=logging.INFO).\n If set to \"auto\", the plugin tries to infer this from\n the train DataLoader's BatchSampler, else defaults to 1.\n To obtain accurate logs when using datasets that do not support batch samplers,\n set this to the actual per gpu batch size (trainer.batch_size).\n\n config\n Pass in a deepspeed formatted config dict,\n or path to a deepspeed config: https://www.deepspeed.ai/docs/config-json.\n All defaults will be ignored if a config is passed in.\n\n logging_level\n Set logging level for deepspeed.\n\n loss_scale\n Loss scaling value for FP16 training.\n 0.0 results in dynamic loss scaling, otherwise static.\n\n initial_scale_power\n Power of the initial dynamic loss scale value. Loss scale is computed\n by ``2^initial_scale_power``.\n\n loss_scale_window\n Window in which to raise/lower the dynamic FP16 loss scaling value.\n\n hysteresis\n FP16 Delay shift in Dynamic Loss scaling.\n\n min_loss_scale\n The minimum FP16 dynamic loss scaling value.\n\n partition_activations\n Enables partition activation when used with ZeRO stage 3 and model parallelism.\n Still requires you to wrap your forward functions in deepspeed.checkpointing.checkpoint.\n See `deepspeed tutorial\n `_.\n\n cpu_checkpointing\n Offloads partitioned activations to CPU if ``partition_activations`` is enabled.\n\n contiguous_memory_optimization\n Copies partitioned activations so that they are contiguous in memory.\n Not supported by all models.\n\n synchronize_checkpoint_boundary\n Insert :func:`torch.cuda.synchronize` at each checkpoint boundary.\n\n load_full_weights\n True when loading a single checkpoint file containing the model state dict\n when using ZeRO Stage 3. This differs from the DeepSpeed checkpoint which contains shards\n per worker.\n \"\"\"\n super().__init__(\n accelerator=accelerator,\n parallel_devices=parallel_devices,\n cluster_environment=cluster_environment,\n precision_plugin=precision_plugin,\n process_group_backend=process_group_backend,\n zero_optimization=zero_optimization,\n stage=stage,\n remote_device=remote_device,\n offload_optimizer=offload_optimizer,\n offload_parameters=offload_parameters,\n params_buffer_count=params_buffer_count,\n params_buffer_size=params_buffer_size,\n max_in_cpu=max_in_cpu,\n offload_optimizer_device=offload_optimizer_device,\n optimizer_buffer_count=optimizer_buffer_count,\n block_size=block_size,\n queue_depth=queue_depth,\n single_submit=single_submit,\n overlap_events=overlap_events,\n thread_count=thread_count,\n pin_memory=pin_memory,\n sub_group_size=sub_group_size,\n contiguous_gradients=contiguous_gradients,\n overlap_comm=overlap_comm,\n allgather_partitions=allgather_partitions,\n reduce_scatter=reduce_scatter,\n allgather_bucket_size=allgather_bucket_size,\n reduce_bucket_size=reduce_bucket_size,\n zero_allow_untested_optimizer=zero_allow_untested_optimizer,\n logging_batch_size_per_gpu=logging_batch_size_per_gpu,\n config=config,\n logging_level=logging_level,\n loss_scale=loss_scale,\n initial_scale_power=initial_scale_power,\n loss_scale_window=loss_scale_window,\n hysteresis=hysteresis,\n min_loss_scale=min_loss_scale,\n partition_activations=partition_activations,\n cpu_checkpointing=cpu_checkpointing,\n contiguous_memory_optimization=contiguous_memory_optimization,\n synchronize_checkpoint_boundary=synchronize_checkpoint_boundary,\n load_full_weights=load_full_weights,\n )\n\n def init_deepspeed(self) -> None:\n assert self.lightning_module is not None\n # deepspeed handles gradient clipping internally\n if is_overridden(\"configure_gradient_clipping\", self.lightning_module, pl.LightningModule):\n rank_zero_warn(\n \"Since DeepSpeed handles gradient clipping internally, the default\"\n \" `LightningModule.configure_gradient_clipping` implementation will not actually clip gradients.\"\n \" The hook will still be called. Consider setting\"\n \" `Trainer(gradient_clip_val=..., gradient_clip_algorithm='norm')`\"\n \" which will use the internal mechanism.\"\n )\n\n if self.lightning_module.trainer.gradient_clip_algorithm == GradClipAlgorithmType.VALUE:\n raise MisconfigurationException(\"DeepSpeed does not support clipping gradients by value.\")\n\n if not isinstance(self.accelerator, CUDAAccelerator):\n raise MisconfigurationException(\n f\"DeepSpeed strategy is only supported on GPU but `{self.accelerator.__class__.__name__}` is used.\"\n )\n\n accumulation_scheduler = self.lightning_module.trainer.accumulation_scheduler\n\n if accumulation_scheduler.epochs != [0]:\n raise MisconfigurationException(\n \"DeepSpeed currently does not support different `accumulate_grad_batches` at different epochs.\"\n )\n\n assert isinstance(self.model, (pl.LightningModule, _LightningPrecisionModuleWrapperBase))\n model = _LightningModuleWrapperBase(pl_module=self.model)\n\n self._initialize_deepspeed_train(model)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_distiller.py",
"content": "import logging\nfrom typing import Callable, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport torch\nimport torch.nn.functional as F\nimport torchmetrics\nfrom lightning.pytorch.utilities import grad_norm\nfrom omegaconf import DictConfig\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\nfrom torchmetrics.aggregation import BaseAggregator\n\nfrom ..constants import FEATURES, LOGITS, WEIGHT\nfrom ..models.utils import run_model\nfrom .lr import apply_layerwise_lr_decay, apply_single_lr, apply_two_stages_lr, get_lr_scheduler\nfrom .utils import get_optimizer\n\nlogger = logging.getLogger(__name__)\n\n\nclass DistillerLitModule(pl.LightningModule):\n \"\"\"\n Knowledge distillation loops for training and evaluation. This module is independent of\n the model definition. This class inherits from the Pytorch Lightning's LightningModule:\n https://lightning.ai/docs/pytorch/stable/common/lightning_module.html\n \"\"\"\n\n def __init__(\n self,\n student_model: nn.Module,\n teacher_model: nn.Module,\n matches: List[DictConfig],\n critics: nn.ModuleList,\n baseline_funcs: nn.ModuleList,\n hard_label_weight: float,\n soft_label_weight: float,\n softmax_regression_weight: float,\n temperature: float,\n output_feature_loss_weight: float,\n optim_type: Optional[str] = None,\n lr_choice: Optional[str] = None,\n lr_schedule: Optional[str] = None,\n lr: Optional[float] = None,\n lr_decay: Optional[float] = None,\n end_lr: Optional[Union[float, int]] = None,\n lr_mult: Optional[Union[float, int]] = None,\n weight_decay: Optional[float] = None,\n warmup_steps: Optional[int] = None,\n hard_label_loss_func: Optional[_Loss] = None,\n soft_label_loss_func: Optional[_Loss] = None,\n softmax_regression_loss_func: Optional[_Loss] = None,\n output_feature_adaptor: Optional[nn.Module] = None,\n output_feature_loss_func: Optional[_Loss] = None,\n rkd_loss_func: Optional[nn.Module] = None,\n validation_metric: Optional[torchmetrics.Metric] = None,\n validation_metric_name: Optional[str] = None,\n custom_metric_func: Callable = None,\n test_metric: Optional[torchmetrics.Metric] = None,\n track_grad_norm: Optional[Union[int, str]] = -1,\n **kwargs,\n ):\n \"\"\"\n Parameters\n ----------\n student_model\n The student model in knowledge distillation.\n teacher_model\n The teacher model in knowledge distillation.\n matches\n Teacher/student layer matches to compute the intermediate loss.\n critics\n The critics used in computing mutual information loss.\n baseline_funcs\n The baseline functions used in computing mutual information loss.\n hard_label_weight\n Weight for hard label loss.\n soft_label_weight\n Weight for soft label loss.\n softmax_regression_weight_label_weight\n Weight for softmax regression loss. Ref: https://www.adrianbulat.com/downloads/ICLR2021/knowledge_distillation_via_softmax_regression_representation_learning.pdf\n temperature\n A scalar to scale teacher and student logits in soft label loss.\n output_feature_loss_weight\n Weight for output_feature layer's loss.\n optim_type\n Optimizer type. We now support:\n - adamw\n - adam\n - sgd\n lr_choice\n How to set each layer's learning rate. If not specified, the default is a single\n learnng rate for all layers. Otherwise, we now support two choices:\n - two_stages\n The layers in the pretrained models have a small learning rate (lr * lr_mult),\n while the newly added head layers use the provided learning rate.\n - layerwise_decay\n The layers have decreasing learning rate from the output end to the input end.\n The intuition is that later layers are more task-related, hence larger learning rates.\n lr_schedule\n Learning rate schedule. We now support:\n - cosine_decay\n Linear warmup followed by cosine decay\n - polynomial_decay\n Linear warmup followed by polynomial decay\n lr\n Learning rate.\n lr_decay\n The learning rate decay factor (0, 1). It is used only when lr_choice is \"layerwise_decay\".\n end_lr\n The final learning rate after decay.\n lr_mult\n The learning rate multiplier (0, 1). It is used only when lr_choice is \"two_stages\".\n weight_decay\n The weight decay to regularize layer weights' l2 norm.\n warmup_steps\n How many steps to warmup learning rate. If a float (0, 1), it would represent the\n percentage of steps over all the training steps. The actual number is calculated as\n \"int(warmup_steps * max_steps)\". If an integer, it would be the exact step number.\n hard_label_loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss(), for hard labels.\n soft_label_loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss(), for soft labels.\n softmax_regression_loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss(), for softmax regression.\n Refer to: https://www.adrianbulat.com/downloads/ICLR2021/knowledge_distillation_via_softmax_regression_representation_learning.pdf\n output_feature_adaptor\n A Pytorch Module, e.g. nn.Linear, for adapting student output feature to the shape of teacher's.\n output_feature_loss_func\n A Pytorch loss module, e.g., nn.MSELoss(), for output_feature distance between teacher and student.\n rkd_loss_func\n A Pytorch loss module, i.e., RKDLoss in .rkd_loss, for rkd loss of output_feature between teacher and student.\n validation_metric\n A torchmetrics module used in the validation stage, e.g., torchmetrics.Accuracy().\n validation_metric_name\n Name of validation metric in case that validation_metric is a aggregation metric,\n e.g., torchmetrics.MeanMetric, whose name can't reflect the real metric name.\n custom_metric_func\n A customized metric function in case that torchmetrics doesn't have the metric.\n It is generally used together with torchmetrics' aggregators, e.g., torchmetrics.MeanMetric.\n Refer to https://github.com/PyTorchLightning/metrics/blob/master/torchmetrics/aggregation.py\n test_metric\n A torchmetrics module used in the test stage, e.g., torchmetrics.Accuracy().\n track_grad_norm\n Track the p-norm of gradients during training. May be set to âinfâ infinity-norm.\n If using Automatic Mixed Precision (AMP), the gradients will be unscaled before logging them.\n \"\"\"\n super().__init__()\n self.optim_type = optim_type\n self.save_hyperparameters(\n ignore=[\n \"student_model\",\n \"teacher_model\",\n \"validation_metric\",\n \"hard_label_loss_func\",\n \"soft_label_loss_func\",\n \"custom_metric_func\",\n \"test_metric\",\n \"matches\",\n \"critics\",\n \"baseline_funcs\",\n \"output_feature_adaptor\",\n \"output_feature_loss_func\",\n \"rkd_loss_func\",\n \"loss_func\",\n \"mixup_fn\",\n ]\n )\n if matches:\n assert len(matches) == len(critics)\n assert len(matches) == len(baseline_funcs)\n self.student_model = student_model\n self.teacher_model = teacher_model\n self.matches = matches\n self.critics = critics\n self.baseline_funcs = baseline_funcs\n self.validation_metric = validation_metric\n self.validation_metric_name = f\"val_{validation_metric_name}\"\n self.temperature = temperature\n self.hard_label_weight = hard_label_weight\n self.soft_label_weight = soft_label_weight\n self.softmax_regression_weight = softmax_regression_weight\n self.output_feature_loss_weight = output_feature_loss_weight\n self.hard_label_loss_func = hard_label_loss_func\n self.soft_label_loss_func = soft_label_loss_func\n self.softmax_regression_loss_func = softmax_regression_loss_func\n self.output_feature_loss_func = output_feature_loss_func\n if isinstance(validation_metric, BaseAggregator) and custom_metric_func is None:\n raise ValueError(\n f\"validation_metric {validation_metric} is an aggregation metric,\"\n \"which must be used with a customized metric function.\"\n )\n self.custom_metric_func = custom_metric_func\n\n self.output_feature_adaptor = output_feature_adaptor\n self.rkd_loss_func = rkd_loss_func\n self.track_grad_norm = track_grad_norm\n\n def _compute_hard_label_loss(\n self,\n output: dict,\n label: torch.Tensor,\n ):\n loss = 0\n for per_output in output.values():\n weight = per_output[WEIGHT] if WEIGHT in per_output else 1\n loss += (\n self.hard_label_loss_func(\n input=per_output[LOGITS].squeeze(dim=1),\n target=label,\n )\n * weight\n )\n\n return loss\n\n def _compute_soft_label_loss(\n self,\n student_output: dict,\n teacher_output: dict,\n ):\n student_logits = student_output[self.student_model.prefix][LOGITS].squeeze(dim=1)\n soft_labels = teacher_output[self.teacher_model.prefix][LOGITS].squeeze(dim=1)\n student_logits = student_logits / self.temperature\n soft_labels = soft_labels / self.temperature\n\n if isinstance(self.soft_label_loss_func, nn.CrossEntropyLoss):\n soft_labels = F.softmax(soft_labels, dim=-1)\n\n loss = self.soft_label_loss_func(\n input=student_logits,\n target=soft_labels,\n )\n return loss\n\n def _compute_output_feature_loss(\n self,\n student_output: dict,\n teacher_output: dict,\n ):\n student_result = student_output[self.student_model.prefix][FEATURES].squeeze(dim=1)\n teacher_result = teacher_output[self.teacher_model.prefix][FEATURES].squeeze(dim=1)\n\n student_result = self.output_feature_adaptor(student_result)\n\n loss = self.output_feature_loss_func(\n input=student_result,\n target=teacher_result,\n )\n return loss\n\n def _compute_rkd_loss(\n self,\n student_output: dict,\n teacher_output: dict,\n ):\n student_result = student_output[self.student_model.prefix][FEATURES].squeeze(dim=1)\n teacher_result = teacher_output[self.teacher_model.prefix][FEATURES].squeeze(dim=1)\n\n student_result = self.output_feature_adaptor(student_result)\n\n loss = self.rkd_loss_func(\n feature_student=student_result,\n feature_teacher=teacher_result,\n )\n\n return loss\n\n def _compute_softmax_regression_loss(\n self,\n student_output: dict,\n teacher_output: dict,\n ):\n student_feature = student_output[self.student_model.prefix][FEATURES].squeeze(dim=1)\n student_feature = self.output_feature_adaptor(student_feature)\n\n student_logits = self.teacher_model.head(student_feature)\n soft_labels = teacher_output[self.teacher_model.prefix][LOGITS].squeeze(dim=1)\n\n student_logits = student_logits\n soft_labels = soft_labels\n\n if isinstance(self.softmax_regression_loss_func, nn.CrossEntropyLoss):\n soft_labels = F.softmax(soft_labels, dim=-1)\n\n loss = self.softmax_regression_loss_func(\n input=student_logits,\n target=soft_labels,\n )\n return loss\n\n def _compute_loss(\n self,\n student_output: dict,\n teacher_output: dict,\n label: torch.Tensor,\n ):\n loss = 0\n hard_label_loss = self._compute_hard_label_loss(\n output=student_output,\n label=label,\n )\n loss += hard_label_loss * self.hard_label_weight\n\n if self.soft_label_weight > 0:\n soft_label_loss = self._compute_soft_label_loss(\n student_output=student_output,\n teacher_output=teacher_output,\n )\n loss += soft_label_loss * self.soft_label_weight\n\n if self.softmax_regression_weight > 0:\n softmax_regression_loss = self._compute_softmax_regression_loss(\n student_output=student_output,\n teacher_output=teacher_output,\n )\n loss += softmax_regression_loss * self.softmax_regression_weight\n\n if self.output_feature_loss_weight > 0:\n output_feature_loss = self._compute_output_feature_loss(\n student_output=student_output,\n teacher_output=teacher_output,\n )\n loss += output_feature_loss * self.output_feature_loss_weight\n\n if self.rkd_loss_func.distance_loss_weight > 0 or self.rkd_loss_func.angle_loss_weight > 0:\n rkd_loss = self._compute_rkd_loss(\n student_output=student_output,\n teacher_output=teacher_output,\n )\n loss += rkd_loss\n\n return loss\n\n def _compute_metric_score(\n self,\n metric: torchmetrics.Metric,\n custom_metric_func: Callable,\n logits: torch.Tensor,\n label: torch.Tensor,\n ):\n if isinstance(\n metric,\n (\n torchmetrics.classification.BinaryAUROC,\n torchmetrics.classification.BinaryAveragePrecision,\n torchmetrics.classification.BinaryF1Score,\n ),\n ):\n prob = F.softmax(logits.float(), dim=1)\n metric.update(preds=prob[:, 1], target=label) # only for binary classification\n elif isinstance(metric, BaseAggregator):\n metric.update(custom_metric_func(logits, label))\n else:\n metric.update(logits.squeeze(dim=1), label)\n\n def _shared_step(\n self,\n batch: dict,\n ):\n student_output = run_model(self.student_model, batch)\n self.teacher_model.eval()\n with torch.no_grad():\n teacher_output = run_model(self.teacher_model, batch)\n label = batch[self.student_model.label_key]\n loss = self._compute_loss(\n student_output=student_output,\n teacher_output=teacher_output,\n label=label,\n )\n return student_output, loss\n\n def training_step(self, batch, batch_idx):\n \"\"\"\n Per training step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#training-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the training loss.\n batch_idx\n Index of mini-batch.\n\n Returns\n -------\n Average loss of the mini-batch data.\n \"\"\"\n _, loss = self._shared_step(batch)\n self.log(\"train_loss\", loss)\n return loss\n\n def validation_step(self, batch, batch_idx):\n \"\"\"\n Per validation step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#validation-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the validation loss and metric.\n The validation metric is used for top k model selection and early stopping.\n batch_idx\n Index of mini-batch.\n \"\"\"\n student_output, loss = self._shared_step(batch)\n # By default, on_step=False and on_epoch=True\n self.log(\"val_loss\", loss)\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n logits=student_output[self.student_model.prefix][LOGITS],\n label=batch[self.student_model.label_key],\n )\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n\n def configure_optimizers(self):\n \"\"\"\n Configure optimizer. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#configure-optimizers\n Returns\n -------\n [optimizer]\n Optimizer.\n [sched]\n Learning rate scheduler.\n \"\"\"\n kwargs = dict(\n model=self.student_model,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n if self.hparams.lr_choice == \"two_stages\":\n logger.debug(\"applying 2-stage learning rate...\")\n grouped_parameters = apply_two_stages_lr(\n lr_mult=self.hparams.lr_mult,\n return_params=True,\n **kwargs,\n )\n elif self.hparams.lr_choice == \"layerwise_decay\":\n logger.debug(\"applying layerwise learning rate decay...\")\n grouped_parameters = apply_layerwise_lr_decay(\n lr_decay=self.hparams.lr_decay,\n **kwargs,\n )\n else:\n logger.debug(\"applying single learning rate...\")\n grouped_parameters = apply_single_lr(\n **kwargs,\n )\n\n if self.critics: # to handle None\n for per_model_critics in self.critics:\n for per_critic in per_model_critics:\n critics_parameters = apply_single_lr(\n model=per_critic,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n grouped_parameters.extend(critics_parameters)\n\n if self.baseline_funcs: # to handle None\n for per_model_baseline_funcs in self.baseline_funcs:\n for per_baseline_func in per_model_baseline_funcs:\n baseline_func_params = apply_single_lr(\n model=per_baseline_func,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n grouped_parameters.extend(baseline_func_params)\n\n adaptor_params = apply_single_lr(\n model=self.output_feature_adaptor,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n grouped_parameters.extend(adaptor_params)\n\n optimizer = get_optimizer(\n optim_type=self.hparams.optim_type,\n optimizer_grouped_parameters=grouped_parameters,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n\n logger.debug(f\"trainer.max_steps: {self.trainer.max_steps}\")\n if self.trainer.max_steps is None or -1:\n max_steps = (\n len(self.trainer.datamodule.train_dataloader())\n * self.trainer.max_epochs\n // self.trainer.accumulate_grad_batches\n )\n logger.debug(\n f\"len(trainer.datamodule.train_dataloader()): {len(self.trainer.datamodule.train_dataloader())}\"\n )\n logger.debug(f\"trainer.max_epochs: {self.trainer.max_epochs}\")\n logger.debug(f\"trainer.accumulate_grad_batches: {self.trainer.accumulate_grad_batches}\")\n else:\n max_steps = self.trainer.max_steps\n\n logger.debug(f\"max steps: {max_steps}\")\n\n warmup_steps = self.hparams.warmup_steps\n if isinstance(warmup_steps, float):\n warmup_steps = int(max_steps * warmup_steps)\n\n logger.debug(f\"warmup steps: {warmup_steps}\")\n logger.debug(f\"lr_schedule: {self.hparams.lr_schedule}\")\n scheduler = get_lr_scheduler(\n optimizer=optimizer,\n num_max_steps=max_steps,\n num_warmup_steps=warmup_steps,\n lr_schedule=self.hparams.lr_schedule,\n end_lr=self.hparams.end_lr,\n )\n\n sched = {\"scheduler\": scheduler, \"interval\": \"step\"}\n logger.debug(\"done configuring optimizer and scheduler\")\n return [optimizer], [sched]\n\n def on_before_optimizer_step(self, optimizer):\n # If using mixed precision, the gradients are already unscaled here\n if self.track_grad_norm != -1:\n self.log_dict(grad_norm(self, norm_type=self.track_grad_norm))\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_matcher.py",
"content": "import logging\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport torch\nimport torchmetrics\nfrom lightning.pytorch.utilities import grad_norm\nfrom omegaconf import DictConfig\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\nfrom torchmetrics.aggregation import BaseAggregator\n\nfrom ..constants import FEATURES, LOGIT_SCALE, PROBABILITY, QUERY, RESPONSE\nfrom ..models.utils import run_model\nfrom ..utils.matcher import compute_matching_probability\nfrom .losses import MultiNegativesSoftmaxLoss, generate_metric_learning_labels\nfrom .lr import apply_layerwise_lr_decay, apply_single_lr, apply_two_stages_lr, get_lr_scheduler\nfrom .metrics import CustomHitRate\nfrom .utils import get_optimizer\n\nlogger = logging.getLogger(__name__)\n\n\nclass MatcherLitModule(pl.LightningModule):\n \"\"\"\n Control the loops for training, evaluation, and prediction. This module is independent of\n the model definition. This class inherits from the Pytorch Lightning's LightningModule:\n https://lightning.ai/docs/pytorch/stable/common/lightning_module.html\n \"\"\"\n\n def __init__(\n self,\n query_model: nn.Module,\n response_model: nn.Module,\n signature: Optional[str] = None,\n match_label: Optional[int] = None,\n matches: Optional[List[DictConfig]] = None,\n optim_type: Optional[str] = None,\n lr_choice: Optional[str] = None,\n lr_schedule: Optional[str] = None,\n lr: Optional[float] = None,\n lr_decay: Optional[float] = None,\n end_lr: Optional[Union[float, int]] = None,\n lr_mult: Optional[Union[float, int]] = None,\n weight_decay: Optional[float] = None,\n warmup_steps: Optional[int] = None,\n loss_func: Optional[_Loss] = None,\n miner_func: Optional[_Loss] = None,\n validation_metric: Optional[torchmetrics.Metric] = None,\n validation_metric_name: Optional[str] = None,\n custom_metric_func: Callable = None,\n test_metric: Optional[torchmetrics.Metric] = None,\n track_grad_norm: Optional[Union[int, str]] = -1,\n ):\n \"\"\"\n Parameters\n ----------\n query_model\n The query model.\n response_model\n The response model.\n signature\n query or response.\n match_label\n The label of match class.\n matches\n A list of DictConfigs, each of which defines one pair of feature column match and the configs\n to compute the matching loss.\n optim_type\n Optimizer type. We now support:\n - adamw\n - adam\n - sgd\n lr_choice\n How to set each layer's learning rate. If not specified, the default is a single\n learnng rate for all layers. Otherwise, we now support two choices:\n - two_stages\n The layers in the pretrained models have a small learning rate (lr * lr_mult),\n while the newly added head layers use the provided learning rate.\n - layerwise_decay\n The layers have decreasing learning rate from the output end to the input end.\n The intuition is that later layers are more task-related, hence larger learning rates.\n lr_schedule\n Learning rate schedule. We now support:\n - cosine_decay\n Linear warmup followed by cosine decay\n - polynomial_decay\n Linear warmup followed by polynomial decay\n lr\n Learning rate.\n lr_decay\n The learning rate decay factor (0, 1). It is used only when lr_choice is \"layerwise_decay\".\n end_lr\n The final learning rate after decay.\n lr_mult\n The learning rate multiplier (0, 1). It is used only when lr_choice is \"two_stages\".\n weight_decay\n The weight decay to regularize layer weights' l2 norm.\n warmup_steps\n How many steps to warmup learning rate. If a float (0, 1), it would represent the\n percentage of steps over all the training steps. The actual number is calculated as\n \"int(warmup_steps * max_steps)\". If an integer, it would be the exact step number.\n loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss().\n validation_metric\n A torchmetrics module used in the validation stage, e.g., torchmetrics.Accuracy().\n validation_metric_name\n Name of validation metric in case that validation_metric is a aggregation metric,\n e.g., torchmetrics.MeanMetric, whose name can't reflect the real metric name.\n custom_metric_func\n A customized metric function in case that torchmetrics doesn't have the metric.\n It is generally used together with torchmetrics' aggregators, e.g., torchmetrics.MeanMetric.\n Refer to https://github.com/PyTorchLightning/metrics/blob/master/torchmetrics/aggregation.py\n test_metric\n A torchmetrics module used in the test stage, e.g., torchmetrics.Accuracy().\n track_grad_norm\n Track the p-norm of gradients during training. May be set to âinfâ infinity-norm.\n If using Automatic Mixed Precision (AMP), the gradients will be unscaled before logging them.\n \"\"\"\n super().__init__()\n self.save_hyperparameters(\n ignore=[\n \"query_model\",\n \"response_model\",\n \"validation_metric\",\n \"test_metric\",\n \"loss_func\",\n \"miner_func\",\n \"matches\",\n ]\n )\n self.query_model = query_model\n self.response_model = response_model\n if signature:\n assert signature in [QUERY, RESPONSE]\n self.signature = signature\n self.validation_metric = validation_metric\n self.validation_metric_name = f\"val_{validation_metric_name}\"\n\n if isinstance(validation_metric, BaseAggregator) and custom_metric_func is None:\n raise ValueError(\n f\"validation_metric {validation_metric} is an aggregation metric,\"\n f\"which must be used with a customized metric function.\"\n )\n self.custom_metric_func = custom_metric_func\n\n self.matches = matches\n self.match_label = match_label\n self.reverse_prob = match_label == 0\n\n logger.debug(f\"match label: {match_label}\")\n logger.debug(f\"reverse probability: {self.reverse_prob}\")\n\n self.loss_func = loss_func\n self.miner_func = miner_func\n self.track_grad_norm = track_grad_norm\n\n def _compute_loss(\n self,\n query_embeddings: torch.Tensor,\n response_embeddings: torch.Tensor,\n label: torch.Tensor,\n logit_scale: Optional[torch.tensor] = None,\n ):\n assert query_embeddings.shape == response_embeddings.shape\n\n if isinstance(self.loss_func, MultiNegativesSoftmaxLoss):\n loss = self.loss_func(\n features_a=query_embeddings,\n features_b=response_embeddings,\n logit_scale=logit_scale,\n rank=self.global_rank,\n world_size=self.trainer.world_size,\n )\n else:\n embeddings = torch.cat([query_embeddings, response_embeddings], dim=0) # (b*2, d)\n\n metric_learning_labels = generate_metric_learning_labels(\n num_samples=len(query_embeddings),\n match_label=self.match_label,\n labels=label,\n )\n indices_tuple = self.miner_func(\n embeddings=embeddings,\n labels=metric_learning_labels,\n )\n loss = self.loss_func(\n embeddings=embeddings,\n labels=metric_learning_labels,\n indices_tuple=indices_tuple,\n )\n\n return loss\n\n @staticmethod\n def _compute_metric_score(\n metric: torchmetrics.Metric,\n custom_metric_func: Callable,\n label: torch.Tensor,\n query_embeddings: torch.Tensor,\n response_embeddings: torch.Tensor,\n logit_scale: Optional[torch.Tensor] = None,\n reverse_prob: Optional[bool] = False,\n ):\n if isinstance(metric, BaseAggregator):\n metric.update(custom_metric_func(query_embeddings, response_embeddings, label))\n elif isinstance(metric, CustomHitRate):\n metric.update(\n batch_query_embeds=query_embeddings,\n batch_response_embeds=response_embeddings,\n logit_scale=logit_scale if logit_scale else None,\n )\n else:\n metric.update(\n compute_matching_probability(\n embeddings1=query_embeddings,\n embeddings2=response_embeddings,\n reverse_prob=reverse_prob,\n ),\n label,\n )\n\n def _get_label(self, batch: Dict):\n label = None\n if self.response_model.label_key in batch:\n label = batch[self.response_model.label_key]\n return label\n\n def _shared_step(\n self,\n batch: Dict,\n ):\n query_outputs = run_model(self.query_model, batch)[self.query_model.prefix]\n query_embeddings = query_outputs[FEATURES]\n\n response_outputs = run_model(self.response_model, batch)[self.response_model.prefix]\n response_embeddings = response_outputs[FEATURES]\n\n logit_scale = (response_outputs[LOGIT_SCALE] if LOGIT_SCALE in response_outputs else None,)\n\n if isinstance(logit_scale, tuple):\n logit_scale = logit_scale[0]\n\n loss = self._compute_loss(\n query_embeddings=query_embeddings,\n response_embeddings=response_embeddings,\n label=self._get_label(batch),\n logit_scale=logit_scale,\n )\n return query_embeddings, response_embeddings, logit_scale, loss\n\n def training_step(self, batch, batch_idx):\n \"\"\"\n Per training step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#training-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the training loss.\n batch_idx\n Index of mini-batch.\n\n Returns\n -------\n Average loss of the mini-batch data.\n \"\"\"\n _, _, _, loss = self._shared_step(batch)\n self.log(\"train_loss\", loss)\n return loss\n\n def validation_step(self, batch, batch_idx):\n \"\"\"\n Per validation step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#validation-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the validation loss and metric.\n The validation metric is used for top k model selection and early stopping.\n batch_idx\n Index of mini-batch.\n \"\"\"\n query_embeddings, response_embeddings, logit_scale, loss = self._shared_step(batch)\n # By default, on_step=False and on_epoch=True\n self.log(\"val_loss\", loss)\n\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n query_embeddings=query_embeddings,\n response_embeddings=response_embeddings,\n label=self._get_label(batch),\n logit_scale=logit_scale,\n reverse_prob=self.reverse_prob,\n )\n\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n\n def predict_step(self, batch, batch_idx, dataloader_idx=0):\n \"\"\"\n Per prediction step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#prediction-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data.\n The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix.\n Ground-truth labels are not needed for prediction.\n batch_idx\n Index of mini-batch.\n dataloader_idx\n Index of dataloader.\n Returns\n -------\n A dictionary with the mini-batch's logits and features.\n \"\"\"\n if self.signature == QUERY:\n embeddings = run_model(self.query_model, batch)[self.query_model.prefix][FEATURES]\n return {FEATURES: embeddings}\n elif self.signature == RESPONSE:\n embeddings = run_model(self.response_model, batch)[self.response_model.prefix][FEATURES]\n return {FEATURES: embeddings}\n else:\n query_embeddings = run_model(self.query_model, batch)[self.query_model.prefix][FEATURES]\n response_embeddings = run_model(self.response_model, batch)[self.response_model.prefix][FEATURES]\n\n match_prob = compute_matching_probability(\n embeddings1=query_embeddings,\n embeddings2=response_embeddings,\n )\n if self.match_label == 0:\n probability = torch.stack([match_prob, 1 - match_prob]).t()\n else:\n probability = torch.stack([1 - match_prob, match_prob]).t()\n\n return {PROBABILITY: probability}\n\n def configure_optimizers(self):\n \"\"\"\n Configure optimizer. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#configure-optimizers\n Returns\n -------\n [optimizer]\n Optimizer.\n [sched]\n Learning rate scheduler.\n \"\"\"\n # TODO: need to consider query_model and response_model in the optimizer\n # TODO: how to avoid pass one parameter multiple times in the optimizer?\n # TODO: in the late-fusion siamese setting, one shared parameter may have different layer ids in the query and response models.\n kwargs = dict(\n model=self.query_model,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n if self.hparams.lr_choice == \"two_stages\":\n logger.debug(\"applying 2-stage learning rate...\")\n grouped_parameters = apply_two_stages_lr(\n lr_mult=self.hparams.lr_mult,\n return_params=True,\n **kwargs,\n )\n elif self.hparams.lr_choice == \"layerwise_decay\":\n logger.debug(\"applying layerwise learning rate decay...\")\n grouped_parameters = apply_layerwise_lr_decay(\n lr_decay=self.hparams.lr_decay,\n **kwargs,\n )\n else:\n logger.debug(\"applying single learning rate...\")\n grouped_parameters = apply_single_lr(\n **kwargs,\n )\n\n optimizer = get_optimizer(\n optim_type=self.hparams.optim_type,\n optimizer_grouped_parameters=grouped_parameters,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n\n logger.debug(f\"trainer.max_steps: {self.trainer.max_steps}\")\n if self.trainer.max_steps is None or -1:\n max_steps = (\n len(self.trainer.datamodule.train_dataloader())\n * self.trainer.max_epochs\n // self.trainer.accumulate_grad_batches\n )\n logger.debug(\n f\"len(trainer.datamodule.train_dataloader()): {len(self.trainer.datamodule.train_dataloader())}\"\n )\n logger.debug(f\"trainer.max_epochs: {self.trainer.max_epochs}\")\n logger.debug(f\"trainer.accumulate_grad_batches: {self.trainer.accumulate_grad_batches}\")\n else:\n max_steps = self.trainer.max_steps\n\n logger.debug(f\"max steps: {max_steps}\")\n\n warmup_steps = self.hparams.warmup_steps\n if isinstance(warmup_steps, float):\n warmup_steps = int(max_steps * warmup_steps)\n\n logger.debug(f\"warmup steps: {warmup_steps}\")\n logger.debug(f\"lr_schedule: {self.hparams.lr_schedule}\")\n scheduler = get_lr_scheduler(\n optimizer=optimizer,\n num_max_steps=max_steps,\n num_warmup_steps=warmup_steps,\n lr_schedule=self.hparams.lr_schedule,\n end_lr=self.hparams.end_lr,\n )\n\n sched = {\"scheduler\": scheduler, \"interval\": \"step\"}\n logger.debug(\"done configuring optimizer and scheduler\")\n return [optimizer], [sched]\n\n def on_before_optimizer_step(self, optimizer):\n # If using mixed precision, the gradients are already unscaled here\n if self.track_grad_norm != -1:\n self.log_dict(grad_norm(self, norm_type=self.track_grad_norm))\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_mmdet.py",
"content": "import logging\nfrom typing import Callable, Optional, Union\n\nimport lightning.pytorch as pl\nimport torchmetrics\nfrom lightning.pytorch.utilities import grad_norm\nfrom torchmetrics.aggregation import BaseAggregator\n\nfrom ..constants import BBOX, IMAGE, LABEL\nfrom .lr import apply_layerwise_lr_decay, apply_single_lr, apply_two_stages_lr, get_lr_scheduler\nfrom .utils import get_optimizer, remove_parameters_without_grad\n\ntry:\n import mmdet\n from mmcv import ConfigDict\nexcept ImportError as e:\n mmdet = None\n ConfigDict = None\n\nlogger = logging.getLogger(__name__)\n\n\nclass MMDetLitModule(pl.LightningModule):\n def __init__(\n self,\n model,\n optim_type: Optional[str] = None,\n lr_choice: Optional[str] = None,\n lr_schedule: Optional[str] = None,\n lr: Optional[float] = None,\n lr_decay: Optional[float] = None,\n end_lr: Optional[Union[float, int]] = None,\n lr_mult: Optional[Union[float, int]] = None,\n weight_decay: Optional[float] = None,\n warmup_steps: Optional[int] = None,\n validation_metric: Optional[torchmetrics.Metric] = None,\n validation_metric_name: Optional[str] = None,\n custom_metric_func: Callable = None,\n test_metric: Optional[torchmetrics.Metric] = None,\n track_grad_norm: Optional[Union[int, str]] = -1,\n ):\n super().__init__() # TODO: inherit LitModule\n self.save_hyperparameters(\n ignore=[\n \"model\",\n \"validation_metric\",\n \"test_metric\",\n ]\n )\n self.model = model\n self.validation_metric = validation_metric\n self.validation_metric_name = f\"val_{validation_metric_name}\"\n self.use_loss = isinstance(validation_metric, BaseAggregator)\n self.id2label = self.model.id2label\n self.input_data_key = self.model.prefix + \"_\" + IMAGE\n self.input_label_key = self.model.prefix + \"_\" + LABEL\n self.track_grad_norm = track_grad_norm\n\n def _base_step(self, batch, mode):\n ret = self.model(batch=batch[self.input_data_key], mode=mode)\n\n return ret\n\n def _predict_step(self, batch):\n return self._base_step(batch=batch, mode=\"predict\")\n\n def _loss_step(self, batch):\n return self._base_step(batch=batch, mode=\"loss\")\n\n def _get_map_input(self, pred_results):\n preds = []\n target = []\n\n batch_size = len(pred_results)\n\n for i in range(batch_size):\n if hasattr(pred_results[i][BBOX], \"masks\"):\n # has one additional dimension with 2 outputs: img_result=img_result[0], mask_result=img_result[1]\n raise NotImplementedError(\n \"Do not support training for models with masks like mask r-cnn, \"\n \"because most custom datasets do not have a ground truth mask.\"\n \" However, you can still inference with this model.\"\n )\n preds.append(\n dict(\n boxes=pred_results[i][BBOX].bboxes, # .float().to(self.device)?\n scores=pred_results[i][BBOX].scores, # .float().to(self.device)?\n labels=pred_results[i][BBOX].labels, # .long().to(self.device)?\n )\n )\n target.append(\n dict(\n boxes=pred_results[i][LABEL].bboxes,\n labels=pred_results[i][LABEL].labels,\n )\n )\n\n return preds, target\n\n def evaluate(self, sample, stage=None):\n \"\"\"\n sample: dict\n Single data sample.\n \"\"\"\n pred_results = self._predict_step(sample)\n\n preds, target = self._get_map_input(pred_results)\n\n # use MeanAveragePrecision, example code: https://github.com/Lightning-AI/metrics/blob/master/examples/detection_map.py\n self.validation_metric.update(preds, target)\n\n return pred_results\n\n def sum_and_log_step_results(self, losses, logging=True):\n # losses is a dict of several type of losses, e.g. ['loss_cls', 'loss_conf', 'loss_xy', 'loss_wh']\n # each type of losses may have multiple channels due to multiple resolution settings\n total_loss = 0.0\n for loss_key, loss_values in losses.items():\n curr_loss = 0.0\n if isinstance(loss_values, list) or isinstance(loss_values, tuple): # is a collection of shape 0 tensors\n for loss_chanel_idx, loss_val in enumerate(loss_values):\n if logging:\n self.log(f\"step/{loss_key}_{loss_chanel_idx}\", loss_val)\n curr_loss += loss_val\n else: # is a tensor\n curr_loss += loss_values.sum()\n\n if logging:\n self.log(f\"step/{loss_key}\", curr_loss)\n total_loss += curr_loss\n\n return total_loss\n\n def training_step(self, batch, batch_idx):\n losses = self._loss_step(batch=batch)\n # sum and log step losses\n total_loss = self.sum_and_log_step_results(losses)\n return total_loss\n\n def validation_step(self, batch, batch_idx, dataloader_idx=0):\n if self.use_loss:\n losses = self._loss_step(batch=batch)\n total_loss = self.sum_and_log_step_results(losses, logging=False)\n self.validation_metric.update(total_loss)\n else:\n self.evaluate(batch, \"val\")\n\n def on_validation_epoch_end(self):\n val_result = self.validation_metric.compute()\n if self.use_loss:\n self.log_dict({\"val_direct_loss\": val_result}, sync_dist=True)\n else:\n # TODO: add mAP/mAR_per_class\n val_result.pop(\"classes\", None) # introduced in torchmetrics v1.0.0\n mAPs = {\"val_\" + k: v for k, v in val_result.items()}\n mAPs[\"val_mAP\"] = mAPs[\"val_map\"]\n self.log_dict(mAPs, sync_dist=True)\n self.validation_metric.reset()\n\n def test_step(self, batch, batch_idx, dataloader_idx=0):\n raise NotImplementedError(\"test with lit_mmdet is not implemented yet.\")\n\n def predict_step(self, batch, batch_idx, dataloader_idx=0):\n pred = self._predict_step(batch)\n\n return pred\n\n def configure_optimizers(self):\n \"\"\"\n Configure optimizer. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#configure-optimizers\n Returns\n -------\n [optimizer]\n Optimizer.\n [sched]\n Learning rate scheduler.\n \"\"\"\n kwargs = dict(\n model=self.model,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n if self.hparams.lr_choice == \"two_stages\":\n logger.debug(\"applying 2-stage learning rate...\")\n grouped_parameters = apply_two_stages_lr(\n lr_mult=self.hparams.lr_mult,\n return_params=True,\n **kwargs,\n )\n elif self.hparams.lr_choice == \"layerwise_decay\":\n logger.debug(\"applying layerwise learning rate decay...\")\n grouped_parameters = apply_layerwise_lr_decay(\n lr_decay=self.hparams.lr_decay,\n **kwargs,\n )\n else:\n logger.debug(\"applying single learning rate...\")\n grouped_parameters = apply_single_lr(\n **kwargs,\n )\n\n grouped_parameters = remove_parameters_without_grad(grouped_parameters=grouped_parameters)\n\n optimizer = get_optimizer(\n optim_type=self.hparams.optim_type,\n optimizer_grouped_parameters=grouped_parameters,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n\n logger.debug(f\"trainer.max_steps: {self.trainer.max_steps}\")\n if self.trainer.max_steps is None or -1:\n max_steps = (\n len(self.trainer.datamodule.train_dataloader())\n * self.trainer.max_epochs\n // self.trainer.accumulate_grad_batches\n )\n logger.debug(\n f\"len(trainer.datamodule.train_dataloader()): {len(self.trainer.datamodule.train_dataloader())}\"\n )\n logger.debug(f\"trainer.max_epochs: {self.trainer.max_epochs}\")\n logger.debug(f\"trainer.accumulate_grad_batches: {self.trainer.accumulate_grad_batches}\")\n else:\n max_steps = self.trainer.max_steps\n\n logger.debug(f\"max steps: {max_steps}\")\n\n warmup_steps = self.hparams.warmup_steps\n if isinstance(warmup_steps, float):\n warmup_steps = int(max_steps * warmup_steps)\n\n logger.debug(f\"warmup steps: {warmup_steps}\")\n logger.debug(f\"lr_schedule: {self.hparams.lr_schedule}\")\n\n scheduler = get_lr_scheduler(\n optimizer=optimizer,\n num_max_steps=max_steps,\n num_warmup_steps=warmup_steps,\n lr_schedule=self.hparams.lr_schedule,\n end_lr=self.hparams.end_lr,\n )\n\n # TODO: add lr_interval into hyperparameters?\n if self.hparams.lr_schedule == \"multi_step\":\n lr_interval = \"epoch\"\n else:\n lr_interval = \"step\"\n\n sched = {\"scheduler\": scheduler, \"interval\": lr_interval}\n logger.debug(\"done configuring optimizer and scheduler\")\n return [optimizer], [sched]\n\n def on_before_optimizer_step(self, optimizer):\n # If using mixed precision, the gradients are already unscaled here\n if self.track_grad_norm != -1:\n self.log_dict(grad_norm(self, norm_type=self.track_grad_norm))\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_module.py",
"content": "import logging\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport lightning.pytorch as pl\nimport torch\nimport torch.nn.functional as F\nimport torchmetrics\nfrom lightning.pytorch.strategies import DeepSpeedStrategy\nfrom lightning.pytorch.utilities import grad_norm\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\nfrom torchmetrics.aggregation import BaseAggregator\n\nfrom ..constants import (\n AUG_LOGITS,\n LM_TARGET,\n LOGITS,\n MULTIMODAL_FEATURES,\n MULTIMODAL_FEATURES_POST_AUG,\n MULTIMODAL_FEATURES_PRE_AUG,\n ORI_LOGITS,\n T_FEW,\n TEMPLATE_LOGITS,\n VAE_MEAN,\n VAE_VAR,\n WEIGHT,\n)\nfrom ..data.mixup import MixupModule, multimodel_mixup\nfrom ..models.utils import run_model\nfrom .lr import apply_layerwise_lr_decay, apply_single_lr, apply_two_stages_lr, get_lr_scheduler\nfrom .metrics import Coverage\nfrom .utils import get_optimizer\n\nlogger = logging.getLogger(__name__)\n\n\nclass LitModule(pl.LightningModule):\n \"\"\"\n Control the loops for training, evaluation, and prediction. This module is independent of\n the model definition. This class inherits from the Pytorch Lightning's LightningModule:\n https://lightning.ai/docs/pytorch/stable/common/lightning_module.html\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n optim_type: Optional[str] = None,\n lr_choice: Optional[str] = None,\n lr_schedule: Optional[str] = None,\n lr: Optional[float] = None,\n lr_decay: Optional[float] = None,\n end_lr: Optional[Union[float, int]] = None,\n lr_mult: Optional[Union[float, int]] = None,\n weight_decay: Optional[float] = None,\n warmup_steps: Optional[int] = None,\n loss_func: Optional[_Loss] = None,\n validation_metric: Optional[torchmetrics.Metric] = None,\n validation_metric_name: Optional[str] = None,\n custom_metric_func: Callable = None,\n test_metric: Optional[torchmetrics.Metric] = None,\n peft: Optional[str] = None,\n trainable_param_names: Optional[List] = None,\n mixup_fn: Optional[MixupModule] = None,\n mixup_off_epoch: Optional[int] = 0,\n model_postprocess_fn: Callable = None,\n skip_final_val: Optional[bool] = False,\n track_grad_norm: Optional[Union[int, str]] = -1,\n cross_modal_align: Optional[str] = None,\n cross_modal_align_weight: Optional[float] = 0,\n automatic_optimization: Optional[bool] = True,\n accumulate_grad_batches: Optional[int] = None,\n gradient_clip_val: Optional[float] = None,\n gradient_clip_algorithm: Optional[str] = None,\n use_aug_optim: Optional[bool] = False,\n aug_loss_func: Optional[_Loss] = None,\n aug_lr: Optional[float] = None,\n aug_weight_decay: Optional[float] = None,\n aug_optim_type: Optional[str] = None,\n ):\n \"\"\"\n Parameters\n ----------\n model\n A Pytorch model\n optim_type\n Optimizer type. We now support:\n - adamw\n - adam\n - sgd\n lr_choice\n How to set each layer's learning rate. If not specified, the default is a single\n learnng rate for all layers. Otherwise, we now support two choices:\n - two_stages\n The layers in the pretrained models have a small learning rate (lr * lr_mult),\n while the newly added head layers use the provided learning rate.\n - layerwise_decay\n The layers have decreasing learning rate from the output end to the input end.\n The intuition is that later layers are more task-related, hence larger learning rates.\n lr_schedule\n Learning rate schedule. We now support:\n - cosine_decay\n Linear warmup followed by cosine decay\n - polynomial_decay\n Linear warmup followed by polynomial decay\n lr\n Learning rate.\n lr_decay\n The learning rate decay factor (0, 1). It is used only when lr_choice is \"layerwise_decay\".\n end_lr\n The final learning rate after decay.\n lr_mult\n The learning rate multiplier (0, 1). It is used only when lr_choice is \"two_stages\".\n weight_decay\n The weight decay to regularize layer weights' l2 norm.\n warmup_steps\n How many steps to warmup learning rate. If a float (0, 1), it would represent the\n percentage of steps over all the training steps. The actual number is calculated as\n \"int(warmup_steps * max_steps)\". If an integer, it would be the exact step number.\n loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss().\n validation_metric\n A torchmetrics module used in the validation stage, e.g., torchmetrics.Accuracy().\n validation_metric_name\n Name of validation metric in case that validation_metric is a aggregation metric,\n e.g., torchmetrics.MeanMetric, whose name can't reflect the real metric name.\n custom_metric_func\n A customized metric function in case that torchmetrics doesn't have the metric.\n It is generally used together with torchmetrics' aggregators, e.g., torchmetrics.MeanMetric.\n Refer to https://github.com/PyTorchLightning/metrics/blob/master/torchmetrics/aggregation.py\n test_metric\n A torchmetrics module used in the test stage, e.g., torchmetrics.Accuracy().\n peft\n Whether to use efficient finetuning strategies. This will be helpful for fast finetuning of large backbones.\n We support options such as:\n\n - bit_fit (only finetune the bias terms)\n - norm_fit (only finetune the weights in norm layers / bias layer)\n - lora, lora_bias, lora_norm (only finetunes decomposition matrices inserted into model, in combination with either bit_fit or norm_fit)\n - ia3, ia3_bias, ia3_norm (adds vector that scales activations by learned vectors, in combination with either bit_fit or norm_fit)\n - None (do not use efficient finetuning strategies)\n track_grad_norm\n Track the p-norm of gradients during training. May be set to 'inf' infinity-norm.\n If using Automatic Mixed Precision (AMP), the gradients will be unscaled before logging them.\n\n \"\"\"\n super().__init__()\n self.save_hyperparameters(\n ignore=[\n \"model\",\n \"validation_metric\",\n \"test_metric\",\n \"loss_func\",\n \"model_postprocess_fn\",\n \"mixup_fn\",\n \"trainable_param_names\",\n \"custom_metric_func\",\n \"aug_loss_func\",\n ]\n )\n self.model = model\n self.validation_metric = validation_metric\n self.validation_metric_name = f\"val_{validation_metric_name}\"\n self.loss_func = loss_func\n self.mixup_fn = mixup_fn\n if isinstance(validation_metric, BaseAggregator) and custom_metric_func is None:\n raise ValueError(\n f\"validation_metric {validation_metric} is an aggregation metric,\"\n \"which must be used with a customized metric function.\"\n )\n self.custom_metric_func = custom_metric_func\n self.model_postprocess_fn = model_postprocess_fn\n self.trainable_param_names = trainable_param_names if trainable_param_names else []\n self.skip_final_val = skip_final_val\n self.automatic_optimization = automatic_optimization\n self.aug_loss_func = aug_loss_func\n if self.hparams.cross_modal_align:\n assert self.hparams.cross_modal_align_weight > 0\n logger.debug(f\"Cross modal alignment mode: {self.hparams.cross_modal_align}\")\n logger.debug(f\"Cross modal alignment loss weight: {self.hparams.cross_modal_align_weight}\")\n\n def _compute_template_loss(\n self,\n per_output: Dict,\n label: torch.Tensor,\n ):\n logits = per_output[TEMPLATE_LOGITS]\n choices_scores = per_output[LOGITS]\n lm_target = per_output[LM_TARGET]\n\n bs = lm_target.size(0)\n num_choices = lm_target.size(1)\n\n lm_loss = F.cross_entropy(\n logits[range(bs), label].flatten(0, 1),\n lm_target[range(bs), label].flatten(0, 1),\n )\n if self.model.mc_loss > 0:\n mc_loss = F.cross_entropy(choices_scores, label)\n else:\n mc_loss = 0.0\n\n if self.model.unlikely_loss > 0:\n cand_loglikely = -F.cross_entropy(logits.flatten(0, 2), lm_target.flatten(0, 2), reduction=\"none\").view(\n bs, num_choices, -1\n )\n cand_loglikely += (lm_target < 0) * -100\n cand_loglikely[range(bs), label] = -100\n unlikely_loss = -torch.log(1 - torch.exp(cand_loglikely) + 1e-2).sum() / (cand_loglikely != -100).sum()\n else:\n unlikely_loss = 0.0\n\n return lm_loss + mc_loss * self.model.mc_loss + unlikely_loss * self.model.unlikely_loss\n\n def _compute_cross_modal_align_loss(self, multimodal_features):\n if self.hparams.cross_modal_align == \"positive_only\":\n kl_loss = nn.KLDivLoss(reduction=\"batchmean\", log_target=True)\n loss = 0\n num = 0\n for i in range(len(multimodal_features)):\n # input should be a distribution in the log space\n a = F.log_softmax(multimodal_features[i], dim=1)\n # kl divergence is not symmetric, so need to compute both (i, j) and (j, i)\n for j in range(len(multimodal_features)):\n if i == j:\n continue\n # input should be a distribution in the log space\n b = F.log_softmax(multimodal_features[j], dim=1)\n loss += kl_loss(a, b)\n num += 1\n return self.hparams.cross_modal_align_weight * loss / num\n else:\n raise ValueError(f\"Unsupported cross modal alignment loss: {self.hparams.cross_modal_align}.\")\n\n def _compute_loss(\n self,\n output: Dict,\n label: torch.Tensor,\n ):\n loss = 0\n for per_prefix, per_output in output.items():\n weight = per_output[WEIGHT] if WEIGHT in per_output else 1\n if (\n TEMPLATE_LOGITS in per_output and self.model.prefix == T_FEW\n ): # Do only add template loss if T-Few. #TODO Add compatibility to Fusion models.\n loss += self._compute_template_loss(per_output, label) * weight\n else:\n if self.training and self.hparams.use_aug_optim and per_prefix.startswith(\"fusion\"):\n label = label.tile((2,))\n loss += (\n self.loss_func(\n input=per_output[LOGITS].squeeze(dim=1),\n target=label,\n )\n * weight\n )\n\n if self.hparams.cross_modal_align:\n loss += self._compute_cross_modal_align_loss(\n multimodal_features=output[self.model.prefix][MULTIMODAL_FEATURES]\n )\n\n if self.training and self.hparams.use_aug_optim:\n loss += self.aug_loss_func(\n pre_aug=output[self.model.prefix][MULTIMODAL_FEATURES_PRE_AUG],\n post_aug=output[self.model.prefix][MULTIMODAL_FEATURES_POST_AUG],\n vae_mean=output[self.model.prefix][VAE_MEAN],\n vae_var=output[self.model.prefix][VAE_VAR],\n ori_logits=output[self.model.prefix][ORI_LOGITS],\n aug_logits=output[self.model.prefix][AUG_LOGITS],\n )\n\n return loss\n\n def _compute_metric_score(\n self,\n metric: torchmetrics.Metric,\n custom_metric_func: Callable,\n logits: torch.Tensor,\n label: torch.Tensor,\n ):\n if isinstance(\n metric,\n (\n torchmetrics.classification.BinaryAUROC,\n torchmetrics.classification.BinaryAveragePrecision,\n torchmetrics.classification.BinaryF1Score,\n Coverage,\n ),\n ):\n prob = F.softmax(logits.float(), dim=1)\n metric.update(preds=prob[:, 1], target=label) # only for binary classification\n elif isinstance(metric, BaseAggregator):\n metric.update(custom_metric_func(logits, label))\n else:\n metric.update(logits.squeeze(dim=1).float(), label)\n\n def _shared_step(\n self,\n batch: Dict,\n ):\n label = batch[self.model.label_key]\n if self.mixup_fn is not None:\n self.mixup_fn.mixup_enabled = self.training & (self.current_epoch < self.hparams.mixup_off_epoch)\n batch, label = multimodel_mixup(batch=batch, model=self.model, mixup_fn=self.mixup_fn)\n output = run_model(self.model, batch)\n loss = self._compute_loss(output=output, label=label)\n return output, loss\n\n def training_step(self, batch, batch_idx):\n \"\"\"\n Per training step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#training-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the training loss.\n batch_idx\n Index of mini-batch.\n\n Returns\n -------\n Average loss of the mini-batch data.\n \"\"\"\n output, loss = self._shared_step(batch)\n\n if not self.automatic_optimization:\n if self.hparams.use_aug_optim:\n optimizer, aug_optimizer = self.optimizers()\n else:\n optimizer = self.optimizers()\n aug_optimizer = None\n\n lr_scheduler = self.lr_schedulers()\n loss = loss / self.hparams.accumulate_grad_batches\n self.manual_backward(loss)\n\n if (batch_idx + 1) % self.hparams.accumulate_grad_batches == 0 or self.trainer.is_last_batch:\n optimizer.step()\n optimizer.zero_grad()\n lr_scheduler.step()\n\n if aug_optimizer is not None:\n aug_optimizer.step()\n aug_optimizer.zero_grad()\n\n self.log(\"train_loss\", loss)\n return loss\n\n def on_validation_start(self) -> None:\n if self.skip_final_val and self.trainer.should_stop:\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n return None\n else:\n return super().on_validation_start()\n\n def validation_step(self, batch, batch_idx):\n \"\"\"\n Per validation step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#validation-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the validation loss and metric.\n The validation metric is used for top k model selection and early stopping.\n batch_idx\n Index of mini-batch.\n \"\"\"\n output, loss = self._shared_step(batch)\n if self.model_postprocess_fn:\n output = self.model_postprocess_fn(output)\n # By default, on_step=False and on_epoch=True\n self.log(\"val_loss\", loss)\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n logits=output[self.model.prefix][LOGITS],\n label=batch[self.model.label_key],\n )\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n\n def predict_step(self, batch, batch_idx, dataloader_idx=0):\n \"\"\"\n Per prediction step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#prediction-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data.\n The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix.\n Ground-truth labels are not needed for prediction.\n batch_idx\n Index of mini-batch.\n dataloader_idx\n Index of dataloader.\n Returns\n -------\n A dictionary with the mini-batch's logits and features.\n \"\"\"\n output = run_model(self.model, batch)\n if self.model_postprocess_fn:\n output = self.model_postprocess_fn(output)\n\n return output[self.model.prefix]\n\n def configure_optimizers(self):\n \"\"\"\n Configure optimizer. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#configure-optimizers\n Returns\n -------\n [optimizer]\n Optimizer.\n [sched]\n Learning rate scheduler.\n \"\"\"\n kwargs = dict(\n model=self.model,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n exclude_keys=[\n \"augmenter\"\n ], # exclude augmenter parameters from the optimizer as they would use an independent optimizer\n )\n if self.hparams.lr_choice == \"two_stages\":\n logger.debug(\"applying 2-stage learning rate...\")\n grouped_parameters = apply_two_stages_lr(\n lr_mult=self.hparams.lr_mult,\n return_params=True,\n **kwargs,\n )\n elif self.hparams.lr_choice == \"layerwise_decay\":\n logger.debug(\"applying layerwise learning rate decay...\")\n grouped_parameters = apply_layerwise_lr_decay(\n lr_decay=self.hparams.lr_decay,\n peft=self.hparams.peft,\n trainable_param_names=self.trainable_param_names,\n **kwargs,\n )\n else:\n logger.debug(\"applying single learning rate...\")\n grouped_parameters = apply_single_lr(\n peft=self.hparams.peft,\n trainable_param_names=self.trainable_param_names,\n **kwargs,\n )\n\n optimizer = get_optimizer(\n optim_type=self.hparams.optim_type,\n optimizer_grouped_parameters=grouped_parameters,\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay,\n )\n\n logger.debug(f\"trainer.max_steps: {self.trainer.max_steps}\")\n if self.trainer.max_steps is None or -1:\n if isinstance(self.trainer.strategy, DeepSpeedStrategy):\n max_steps = 1\n else:\n accumulate_grad_batches = (\n self.trainer.accumulate_grad_batches\n if self.automatic_optimization\n else self.hparams.accumulate_grad_batches\n )\n max_steps = (\n len(self.trainer.datamodule.train_dataloader())\n * self.trainer.max_epochs\n // accumulate_grad_batches\n // self.trainer.num_devices # Account for distributed training\n )\n logger.debug(\n f\"len(trainer.datamodule.train_dataloader()): {len(self.trainer.datamodule.train_dataloader())}\"\n )\n logger.debug(f\"trainer.max_epochs: {self.trainer.max_epochs}\")\n logger.debug(f\"accumulate_grad_batches: {accumulate_grad_batches}\")\n logger.debug(f\"num_devices: {self.trainer.num_devices}\")\n else:\n max_steps = self.trainer.max_steps\n\n logger.debug(f\"max steps: {max_steps}\")\n\n warmup_steps = self.hparams.warmup_steps\n if isinstance(warmup_steps, float):\n warmup_steps = int(max_steps * warmup_steps)\n\n logger.debug(f\"warmup steps: {warmup_steps}\")\n logger.debug(f\"lr_schedule: {self.hparams.lr_schedule}\")\n scheduler = get_lr_scheduler(\n optimizer=optimizer,\n num_max_steps=max_steps,\n num_warmup_steps=warmup_steps,\n lr_schedule=self.hparams.lr_schedule,\n end_lr=self.hparams.end_lr,\n )\n\n sched = {\"scheduler\": scheduler, \"interval\": \"step\"}\n ret_optimizers = [optimizer]\n ret_schedulers = [sched]\n if self.hparams.use_aug_optim:\n logger.debug(\"initializing augment optimizer\")\n # augmenter's optimizer\n aug_grouped_parameters = apply_single_lr(\n model=self.model.augmenter,\n lr=self.hparams.aug_lr,\n weight_decay=self.hparams.aug_weight_decay,\n )\n aug_optimizer = get_optimizer(\n optim_type=self.hparams.aug_optim_type,\n optimizer_grouped_parameters=aug_grouped_parameters,\n lr=self.hparams.aug_lr,\n weight_decay=self.hparams.aug_weight_decay,\n )\n ret_optimizers.append(aug_optimizer)\n\n logger.debug(\"done configuring optimizer and scheduler\")\n return ret_optimizers, ret_schedulers\n\n def on_before_optimizer_step(self, optimizer):\n # If using mixed precision, the gradients are already unscaled here\n # TODO: apply gradient clip only to the target optimizer\n if not self.automatic_optimization and self.hparams.gradient_clip_val > 0:\n self.clip_gradients(\n optimizer,\n gradient_clip_val=self.hparams.gradient_clip_val,\n gradient_clip_algorithm=self.hparams.gradient_clip_algorithm,\n )\n if self.hparams.track_grad_norm != -1:\n self.log_dict(grad_norm(self, norm_type=self.hparams.track_grad_norm))\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_ner.py",
"content": "import logging\nfrom typing import Callable, Dict, List, Optional, Union\n\nimport torch\nimport torchmetrics\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\n\nfrom ..constants import FUSION_NER, LOGITS, NER_TEXT, WEIGHT\nfrom ..data.mixup import MixupModule\nfrom .lit_module import LitModule\n\nlogger = logging.getLogger(__name__)\n\n\nclass NerLitModule(LitModule):\n \"\"\"\n Control the loops for training, evaluation, and prediction of Named Entity Recognition. This module is independent of\n the model definition. This class inherits from Lightning's LightningModule:\n https://lightning.ai/docs/pytorch/stable/common/lightning_module.html\n \"\"\"\n\n def __init__(\n self,\n model: nn.Module,\n optim_type: Optional[str] = None,\n lr_choice: Optional[str] = None,\n lr_schedule: Optional[str] = None,\n lr: Optional[float] = None,\n lr_decay: Optional[float] = None,\n end_lr: Optional[Union[float, int]] = None,\n lr_mult: Optional[Union[float, int]] = None,\n weight_decay: Optional[float] = None,\n warmup_steps: Optional[int] = None,\n loss_func: Optional[_Loss] = None,\n validation_metric: Optional[torchmetrics.Metric] = None,\n validation_metric_name: Optional[str] = None,\n custom_metric_func: Callable = None,\n test_metric: Optional[torchmetrics.Metric] = None,\n peft: Optional[str] = None,\n trainable_param_names: Optional[List] = None,\n mixup_fn: Optional[MixupModule] = None,\n mixup_off_epoch: Optional[int] = 0,\n model_postprocess_fn: Callable = None,\n skip_final_val: Optional[bool] = False,\n track_grad_norm: Optional[Union[int, str]] = -1,\n ):\n \"\"\"\n Parameters\n ----------\n model\n A Pytorch model\n optim_type\n Optimizer type. We now support:\n - adamw\n - adam\n - sgd\n lr_choice\n How to set each layer's learning rate. If not specified, the default is a single\n learnng rate for all layers. Otherwise, we now support two choices:\n - two_stages\n The layers in the pretrained models have a small learning rate (lr * lr_mult),\n while the newly added head layers use the provided learning rate.\n - layerwise_decay\n The layers have decreasing learning rate from the output end to the input end.\n The intuition is that later layers are more task-related, hence larger learning rates.\n lr_schedule\n Learning rate schedule. We now support:\n - cosine_decay\n Linear warmup followed by cosine decay\n - polynomial_decay\n Linear warmup followed by polynomial decay\n lr\n Learning rate.\n lr_decay\n The learning rate decay factor (0, 1). It is used only when lr_choice is \"layerwise_decay\".\n end_lr\n The final learning rate after decay.\n lr_mult\n The learning rate multiplier (0, 1). It is used only when lr_choice is \"two_stages\".\n weight_decay\n The weight decay to regularize layer weights' l2 norm.\n warmup_steps\n How many steps to warmup learning rate. If a float (0, 1), it would represent the\n percentage of steps over all the training steps. The actual number is calculated as\n \"int(warmup_steps * max_steps)\". If an integer, it would be the exact step number.\n loss_func\n A Pytorch loss module, e.g., nn.CrossEntropyLoss().\n validation_metric\n A torchmetrics module used in the validation stage, e.g., torchmetrics.Accuracy().\n validation_metric_name\n Name of validation metric in case that validation_metric is a aggregation metric,\n e.g., torchmetrics.MeanMetric, whose name can't reflect the real metric name.\n custom_metric_func\n A customized metric function in case that torchmetrics doesn't have the metric.\n It is generally used together with torchmetrics' aggregators, e.g., torchmetrics.MeanMetric.\n Refer to https://github.com/PyTorchLightning/metrics/blob/master/torchmetrics/aggregation.py\n test_metric\n A torchmetrics module used in the test stage, e.g., torchmetrics.Accuracy().\n peft\n Whether to use efficient finetuning strategies. This will be helpful for fast finetuning of large backbones.\n We support options such as:\n\n - bit_fit (only finetune the bias terms)\n - norm_fit (only finetune the weights in norm layers / bias layer)\n - lora, lora_bias, lora_norm (only finetunes decomposition matrices inserted into model, in combination with either bit_fit or norm_fit)\n - ia3, ia3_bias, ia3_norm (adds vector that scales activations by learned vectors, in combination with either bit_fit or norm_fit)\n - None (do not use efficient finetuning strategies)\n track_grad_norm\n Track the p-norm of gradients during training. May be set to âinfâ infinity-norm.\n If using Automatic Mixed Precision (AMP), the gradients will be unscaled before logging them.\n\n \"\"\"\n super().__init__(\n model=model,\n optim_type=optim_type,\n lr_choice=lr_choice,\n lr_schedule=lr_schedule,\n lr=lr,\n lr_decay=lr_decay,\n end_lr=end_lr,\n lr_mult=lr_mult,\n weight_decay=weight_decay,\n warmup_steps=warmup_steps,\n loss_func=loss_func,\n validation_metric=validation_metric,\n validation_metric_name=validation_metric_name,\n custom_metric_func=custom_metric_func,\n test_metric=test_metric,\n peft=peft,\n trainable_param_names=trainable_param_names,\n mixup_fn=mixup_fn,\n mixup_off_epoch=mixup_off_epoch,\n model_postprocess_fn=model_postprocess_fn,\n skip_final_val=skip_final_val,\n track_grad_norm=track_grad_norm,\n )\n\n def _compute_loss(\n self,\n output: Dict,\n label: torch.Tensor,\n ):\n loss = 0\n for prefix, per_output in output.items():\n if prefix == NER_TEXT or prefix == FUSION_NER:\n weight = per_output[WEIGHT] if WEIGHT in per_output else 1\n active_loss = label.view(-1) != 0\n active_logits = per_output[LOGITS].view(-1, self.model.num_classes)[active_loss]\n active_labels = label.view(-1)[active_loss]\n loss += (\n self.loss_func(\n input=active_logits,\n target=active_labels,\n )\n * weight\n )\n return loss\n\n def validation_step(self, batch, batch_idx):\n \"\"\"\n Per validation step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#validation\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the validation loss and metric.\n The validation metric is used for top k model selection and early stopping.\n batch_idx\n Index of mini-batch.\n \"\"\"\n output, loss = self._shared_step(batch)\n if self.model_postprocess_fn:\n output = self.model_postprocess_fn(output)\n\n # By default, on_step=False and on_epoch=True\n self.log(\"val_loss\", loss)\n label = batch[self.model.label_key]\n logits = output[self.model.prefix][LOGITS]\n active_loss = label.view(-1) != 0\n active_logits = logits.view(-1, self.model.num_classes)[active_loss]\n active_labels = label.view(-1)[active_loss]\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n logits=active_logits,\n label=active_labels,\n )\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lit_semantic_seg.py",
"content": "import logging\nfrom typing import Callable, Dict\n\nimport torch\nimport torchmetrics\nfrom transformers.models.mask2former.modeling_mask2former import Mask2FormerLoss\n\nfrom ..constants import CLASS_LOGITS, LOGITS, MOE_LOSS, SEMANTIC_MASK, WEIGHT\nfrom ..models.utils import run_model\nfrom .lit_module import LitModule\nfrom .metrics.semantic_seg_metrics import Multiclass_IoU\n\nlogger = logging.getLogger(__name__)\n\n\nclass SemanticSegmentationLitModule(LitModule):\n \"\"\"\n Control the loops for training, evaluation, and prediction. This module is independent of\n the model definition. This class inherits from the Pytorch Lightning's LightningModule:\n https://lightning.ai/docs/pytorch/stable/common/lightning_module.html\n \"\"\"\n\n def _compute_loss(self, output: Dict, label: torch.Tensor, **kwargs):\n loss = 0\n for _, per_output in output.items():\n weight = per_output[WEIGHT] if WEIGHT in per_output else 1\n if isinstance(self.loss_func, Mask2FormerLoss):\n mask_labels = [mask_labels.to(per_output[LOGITS]) for mask_labels in kwargs[\"mask_labels\"]]\n dict_loss = self.loss_func(\n masks_queries_logits=per_output[LOGITS], # bs, num_mask_tokens, height, width\n class_queries_logits=per_output[CLASS_LOGITS], # bs, num_mask_tokens, num_classes\n mask_labels=mask_labels,\n class_labels=kwargs[\"class_labels\"],\n )\n for v in dict_loss.values():\n loss += v\n else:\n loss += (\n self.loss_func(\n input=per_output[LOGITS],\n target=label,\n )\n * weight\n )\n # MoE loss\n if MOE_LOSS in per_output:\n loss += per_output[MOE_LOSS]\n\n return loss\n\n def _compute_metric_score(\n self,\n metric: torchmetrics.Metric,\n custom_metric_func: Callable,\n logits: torch.Tensor,\n label: torch.Tensor,\n **kwargs,\n ):\n if isinstance(metric, Multiclass_IoU):\n metric.update(kwargs[\"semantic_masks\"], label)\n else:\n metric.update(logits.float(), label)\n\n def _shared_step(\n self,\n batch: Dict,\n ):\n label = batch[self.model.label_key]\n # prepare_targets\n output = run_model(self.model, batch)\n if isinstance(self.loss_func, Mask2FormerLoss):\n loss = self._compute_loss(\n output=output,\n label=label,\n mask_labels=batch[self.model.mask_label_key],\n class_labels=batch[self.model.class_label_key],\n )\n else:\n loss = self._compute_loss(\n output=output,\n label=label,\n )\n\n return output, loss\n\n def validation_step(self, batch, batch_idx, **kwargs):\n \"\"\"\n Per validation step. This function is registered by LightningModule.\n Refer to https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#validation-loop\n\n Parameters\n ----------\n batch\n A dictionary containing the mini-batch data, including both input data and\n ground-truth labels. The mini-batch data are passed to each individual model,\n which indexes its required input data by keys with its model prefix. The\n ground-truth labels are used here to compute the validation loss and metric.\n The validation metric is used for top k model selection and early stopping.\n batch_idx\n Index of mini-batch.\n \"\"\"\n output, loss = self._shared_step(batch)\n if self.model_postprocess_fn:\n output = self.model_postprocess_fn(output)\n # By default, on_step=False and on_epoch=True\n self.log(\"val_loss\", loss)\n if isinstance(self.loss_func, Mask2FormerLoss):\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n logits=output[self.model.prefix][LOGITS],\n label=batch[self.model.label_key],\n semantic_masks=output[self.model.prefix][SEMANTIC_MASK],\n )\n else:\n self._compute_metric_score(\n metric=self.validation_metric,\n custom_metric_func=self.custom_metric_func,\n logits=output[self.model.prefix][LOGITS],\n label=batch[self.model.label_key],\n )\n\n self.log(\n self.validation_metric_name,\n self.validation_metric,\n on_step=False,\n on_epoch=True,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/__init__.py",
"content": "from .bce_loss import BBCEWithLogitLoss\nfrom .focal_loss import FocalLoss\nfrom .lemda_loss import LemdaLoss\nfrom .rkd_loss import RKDLoss\nfrom .softmax_losses import MultiNegativesSoftmaxLoss, SoftTargetCrossEntropy\nfrom .structure_loss import StructureLoss\nfrom .utils import (\n generate_metric_learning_labels,\n get_aug_loss_func,\n get_loss_func,\n get_matcher_loss_func,\n get_matcher_miner_func,\n get_metric_learning_distance_func,\n)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/bce_loss.py",
"content": "import torch\nimport torch.nn as nn\n\n\nclass BBCEWithLogitLoss(nn.Module):\n \"\"\"\n Balanced BCEWithLogitLoss based on https://github.com/NiFangBaAGe/Explicit-Visual-Prompt/blob/latest_branch/models/segformer.py\n \"\"\"\n\n def __init__(self):\n super(BBCEWithLogitLoss, self).__init__()\n\n def forward(self, input: torch.Tensor, target: torch.Tensor):\n if input.dim() == 3:\n input = input.unsqueeze(1)\n eps = 1e-10\n count_pos = torch.sum(target) + eps\n count_neg = torch.sum(1.0 - target)\n ratio = count_neg / count_pos\n w_neg = count_pos / (count_pos + count_neg)\n\n bce1 = nn.BCEWithLogitsLoss(pos_weight=ratio)\n loss = w_neg * bce1(input, target)\n\n return loss\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/focal_loss.py",
"content": "from typing import Optional\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass FocalLoss(nn.Module):\n \"\"\"\n Focal loss based on https://github.com/AdeelH/pytorch-multi-class-focal-loss/blob/master/focal_loss.py\n\n References:\n [1] https://arxiv.org/abs/1708.02002\n \"\"\"\n\n def __init__(\n self,\n alpha: Optional[torch.Tensor] = None,\n gamma: Optional[float] = 2.0,\n reduction: Optional[str] = \"mean\",\n eps: Optional[float] = 1e-6,\n ):\n \"\"\"\n\n Parameters\n ----------\n alpha\n weighting factor for each class. Should be of shape (num_classes)\n gamma\n the focal parameter for calculating weights on easy/hard samples\n reduction\n the reduction to apply to the final loss output. Default: \"mean\". Options:\n \"mean\", \"sum\"\n eps\n epsilon for numerical stability\n \"\"\"\n super(FocalLoss, self).__init__()\n\n self.gamma = float(gamma) if gamma is not None else 2.0\n self.reduction = reduction\n self.eps = float(eps) if eps is not None else 1e-6\n\n alpha = self._parse_alpha(alpha)\n self.nll_loss = nn.NLLLoss(weight=alpha, reduction=\"none\")\n\n def _parse_alpha(self, alpha) -> Optional[torch.Tensor]:\n \"\"\"Parse and convert alpha to a torch.Tensor with proper dtype.\"\"\"\n if alpha is None:\n return None\n\n if torch.is_tensor(alpha):\n return alpha.float()\n\n if isinstance(alpha, str):\n # Handles Ray Tune HPO sampled hyperparameter\n try:\n numbers = alpha.strip(\"()\").split(\",\")\n alpha = [float(num) for num in numbers]\n except Exception:\n raise ValueError(f\"{type(alpha)} {alpha} is not in a supported format.\")\n\n return torch.tensor(alpha, dtype=torch.float32)\n\n def forward(self, input: torch.Tensor, target: torch.Tensor):\n if not torch.is_tensor(input):\n raise TypeError(\"input type is not a torch.Tensor. Got {}\".format(type(input)))\n if input.ndim > 2:\n # (N, C, d1, d2, ..., dK) --> (N * d1 * ... * dK, C)\n num_class = input.shape[1]\n input = input.permute(0, *range(2, input.ndim), 1).reshape(-1, num_class)\n # (N, d1, d2, ..., dK) --> (N * d1 * ... * dK,)\n target = target.view(-1)\n\n pt = F.softmax(input, dim=-1)\n\n # -alpha_t * log(pt) term\n log_p = torch.log_softmax(input, dim=-1)\n ce = self.nll_loss(log_p, target)\n\n # (1 - pt)^gamma term\n all_rows = torch.arange(input.shape[0])\n pt = pt[all_rows, target]\n focal_term = (1 - pt) ** self.gamma\n\n loss = focal_term * ce\n\n if self.reduction == \"mean\":\n loss = loss.mean()\n elif self.reduction == \"sum\":\n loss = loss.sum()\n\n return loss\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/lemda_loss.py",
"content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom ...constants import BINARY, REGRESSION\n\n\nclass LemdaLoss(nn.Module):\n def __init__(self, mse_weight, kld_weight, consist_weight, consist_threshold, problem_type):\n super().__init__()\n self.mse_loss = nn.MSELoss(reduction=\"mean\")\n self.mse_weight = mse_weight\n self.kld_weight = kld_weight\n self.consist_weight = consist_weight\n self.consist_threshold = consist_threshold\n self.problem_type = problem_type\n\n def consist_loss(self, p_logits, q_logits):\n p = F.softmax(p_logits, dim=1)\n logp = F.log_softmax(p_logits, dim=1)\n logq = F.log_softmax(q_logits, dim=1)\n loss = torch.sum(p * (logp - logq), dim=-1)\n q = F.softmax(q_logits, dim=1)\n q_largest = torch.max(q, dim=1)[0]\n loss_mask = torch.gt(q_largest, self.consist_threshold).float()\n loss = loss * loss_mask\n return torch.mean(loss)\n\n def forward(self, pre_aug, post_aug, vae_mean, vae_var, ori_logits, aug_logits):\n mse_loss = self.mse_loss(pre_aug, post_aug) * self.mse_weight\n # see Appendix B from VAE paper: https://arxiv.org/abs/1312.6114\n # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n kld_loss = -0.5 * torch.mean(1 + vae_var - vae_mean.pow(2) - vae_var.exp()) * self.kld_weight\n if self.problem_type in [REGRESSION, BINARY]:\n consist_loss = self.mse_loss(ori_logits, aug_logits) * self.consist_weight\n else:\n consist_loss = self.consist_loss(ori_logits, aug_logits) * self.consist_weight\n\n return mse_loss + kld_loss + consist_loss\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/rkd_loss.py",
"content": "from typing import Optional\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass RKDLoss(nn.Module):\n \"\"\"\n Compute RKD Distance Loss.\n Paper Refer to: Relational Knowledge Disitllation, CVPR2019. https://arxiv.org/abs/1904.05068\n Code Refer to: https://github.com/HobbitLong/RepDistiller/blob/master/distiller_zoo/RKD.py\n and https://github.com/lenscloth/RKD/blob/master/metric/loss.py\n \"\"\"\n\n def __init__(self, distance_loss_weight: Optional[float] = 25.0, angle_loss_weight: Optional[float] = 50.0):\n \"\"\"\n Parameters\n ----------\n distance_loss_weight\n Weight of RKD distance loss\n angle_loss_weight\n Weight of RKD angle loss\n Returns\n -------\n \"\"\"\n super(RKDLoss, self).__init__()\n self.distance_loss_weight = distance_loss_weight\n self.angle_loss_weight = angle_loss_weight\n\n def forward(self, feature_student: Optional[torch.Tensor], feature_teacher: Optional[torch.Tensor]):\n \"\"\"\n Parameters\n ----------\n feature_student\n Output feature of student model, shape: (N, D)\n feature_teacher\n Output feature of teacher model, shape: (N, D)\n Returns\n -------\n The RKD Loss between teacher and student\n \"\"\"\n # RKD loss\n if self.distance_loss_weight > 0:\n with torch.no_grad():\n t_dist = self.pdist(feature_teacher, squared=False)\n mean_td = t_dist[t_dist > 0].mean()\n t_dist = t_dist / mean_td\n\n s_dist = self.pdist(feature_student, squared=False)\n mean_d = s_dist[s_dist > 0].mean()\n s_dist = s_dist / mean_d\n\n loss_distance = F.smooth_l1_loss(s_dist, t_dist)\n\n # RKD Angle loss\n if self.angle_loss_weight > 0:\n with torch.no_grad():\n td = feature_teacher.unsqueeze(0) - feature_teacher.unsqueeze(1)\n norm_td = F.normalize(td, p=2, dim=2)\n t_angle = torch.bmm(norm_td, norm_td.transpose(1, 2)).view(-1)\n\n sd = feature_student.unsqueeze(0) - feature_student.unsqueeze(1)\n norm_sd = F.normalize(sd, p=2, dim=2)\n s_angle = torch.bmm(norm_sd, norm_sd.transpose(1, 2)).view(-1)\n\n loss_angle = F.smooth_l1_loss(s_angle, t_angle)\n\n loss = ((self.distance_loss_weight * loss_distance) if self.distance_loss_weight > 0 else 0) + (\n (self.angle_loss_weight * loss_angle) if self.angle_loss_weight > 0 else 0\n )\n\n return loss\n\n @staticmethod\n def pdist(embeddings: Optional[torch.Tensor], squared: Optional[bool] = False, eps: Optional[float] = 1e-12):\n \"\"\"\n Compute pairwise Euclidean distances between embeddings in n-dimensional space.\n\n Parameters\n ----------\n embeddings\n The embeddings to compute pairwise distance between. Shape: (N,D)\n squared\n If the result is square of Euclidean distance.\n eps\n Min value of each entry.\n\n Returns\n -------\n Pairwise Euclidean distances. Shape: (N,N)\n \"\"\"\n e_square = embeddings.pow(2).sum(dim=1)\n prod = embeddings @ embeddings.t()\n res = (e_square.unsqueeze(1) + e_square.unsqueeze(0) - 2 * prod).clamp(min=eps)\n\n if not squared:\n res = res.sqrt()\n\n res = res.clone()\n res[range(len(embeddings)), range(len(embeddings))] = 0\n\n return res\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/softmax_losses.py",
"content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ntry:\n import torch.distributed.nn\n from torch import distributed as dist\n\n has_distributed = True\nexcept ImportError:\n has_distributed = False\n\ntry:\n import horovod.torch as hvd\nexcept ImportError:\n hvd = None\n\n\nclass SoftTargetCrossEntropy(nn.Module):\n \"\"\"\n The soft target CrossEntropy from timm.\n https://github.com/rwightman/pytorch-image-models/blob/e4360e6125bb0bb4279785810c8eb33b40af3ebd/timm/loss/cross_entropy.py\n It works under the mixup.\n It can calculate the crossentropy of input and label with one-hot.\n \"\"\"\n\n def __init__(self):\n super(SoftTargetCrossEntropy, self).__init__()\n\n def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n loss = torch.sum(-target * F.log_softmax(input, dim=-1), dim=-1)\n return loss.mean()\n\n\nclass MultiNegativesSoftmaxLoss(nn.Module):\n \"\"\"\n This loss expects as input a batch consisting of pairs (a_1, p_1), (a_2, p_2)âĻ, (a_n, p_n) where\n we assume that (a_i, p_i) are a positive pair and (a_i, p_j) for i!=j a negative pair.\n For each a_i, it uses all other p_j as negative samples, i.e., for a_i,\n we have 1 positive example (p_i) and n-1 negative examples (p_j).\n It then minimizes the negative log-likehood for softmax normalized scores.\n It can also support gather negatives across processes.\n \"\"\"\n\n def __init__(\n self,\n local_loss=False,\n gather_with_grad=False,\n cache_labels=False,\n use_horovod=False,\n ):\n \"\"\"\n Parameters\n ----------\n local_loss\n Whether to compute the loss only for the current process's samples.\n gather_with_grad\n Whether to gather all features with gradients enabled.\n cache_labels\n Whether to cache labels for loss in next iterations.\n use_horovod\n Whether to use horovod.\n \"\"\"\n super().__init__()\n self.local_loss = local_loss\n self.gather_with_grad = gather_with_grad\n self.cache_labels = cache_labels\n self.use_horovod = use_horovod\n\n # cache state\n self.prev_num_logits = 0\n self.labels = {}\n\n def forward(self, features_a, features_b, logit_scale, rank=0, world_size=1):\n device = features_a.device\n if world_size > 1:\n all_features_a, all_features_b = self.gather_features(\n features_a, features_b, self.local_loss, self.gather_with_grad, rank, world_size, self.use_horovod\n )\n\n if self.local_loss:\n logits_per_a = logit_scale * features_a @ all_features_b.T\n logits_per_b = logit_scale * features_b @ all_features_a.T\n else:\n logits_per_a = logit_scale * all_features_a @ all_features_b.T\n logits_per_b = logits_per_a.T\n else:\n logits_per_a = logit_scale * features_a @ features_b.T\n logits_per_b = logit_scale * features_b @ features_a.T\n\n # calculated ground-truth and cache if enabled\n num_logits = logits_per_a.shape[0]\n if self.prev_num_logits != num_logits or device not in self.labels:\n labels = torch.arange(num_logits, device=device, dtype=torch.long)\n if world_size > 1 and self.local_loss:\n labels = labels + num_logits * rank\n if self.cache_labels:\n self.labels[device] = labels\n self.prev_num_logits = num_logits\n else:\n labels = self.labels[device]\n\n total_loss = (F.cross_entropy(logits_per_a, labels) + F.cross_entropy(logits_per_b, labels)) / 2\n return total_loss\n\n @staticmethod\n def gather_features(\n image_features,\n text_features,\n local_loss=False,\n gather_with_grad=False,\n rank=0,\n world_size=1,\n use_horovod=False,\n ):\n \"\"\"\n Gather features across GPUs.\n\n Parameters\n ----------\n image_features\n image features of the current process.\n text_features\n text features of the current process.\n local_loss\n If False, make sure the features on the current GPU have gradients.\n gather_with_grad\n Whether to gather all features with gradients enabled.\n rank\n Rank of the current process (it should be a number between 0 and world_size-1).\n world_size\n Number of processes participating in the job.\n use_horovod\n Whether to use horovod.\n\n Returns\n -------\n Gathered image and text features from all processes.\n \"\"\"\n assert has_distributed, (\n \"torch.distributed did not import correctly, please use a PyTorch version with support.\"\n )\n if use_horovod:\n assert hvd is not None, \"Please install horovod\"\n if gather_with_grad:\n all_image_features = hvd.allgather(image_features)\n all_text_features = hvd.allgather(text_features)\n else:\n with torch.no_grad():\n all_image_features = hvd.allgather(image_features)\n all_text_features = hvd.allgather(text_features)\n if not local_loss:\n # ensure grads for local rank when all_* features don't have a gradient\n gathered_image_features = list(all_image_features.chunk(world_size, dim=0))\n gathered_text_features = list(all_text_features.chunk(world_size, dim=0))\n gathered_image_features[rank] = image_features\n gathered_text_features[rank] = text_features\n all_image_features = torch.cat(gathered_image_features, dim=0)\n all_text_features = torch.cat(gathered_text_features, dim=0)\n else:\n # We gather tensors from all gpus\n if gather_with_grad:\n all_image_features = torch.cat(torch.distributed.nn.all_gather(image_features), dim=0)\n all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features), dim=0)\n else:\n gathered_image_features = [torch.zeros_like(image_features) for _ in range(world_size)]\n gathered_text_features = [torch.zeros_like(text_features) for _ in range(world_size)]\n dist.all_gather(gathered_image_features, image_features)\n dist.all_gather(gathered_text_features, text_features)\n if not local_loss:\n # ensure grads for local rank when all_* features don't have a gradient\n gathered_image_features[rank] = image_features\n gathered_text_features[rank] = text_features\n all_image_features = torch.cat(gathered_image_features, dim=0)\n all_text_features = torch.cat(gathered_text_features, dim=0)\n\n return all_image_features, all_text_features\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/structure_loss.py",
"content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass StructureLoss(nn.Module):\n \"\"\"\n Structure Loss based on https://github.com/DengPingFan/PraNet/blob/master/MyTrain.py\n The loss represent the weighted IoU loss and binary cross entropy (BCE) loss for the global restriction and local (pixel-level) restriction.\n\n References:\n [1] https://arxiv.org/abs/2006.11392\n \"\"\"\n\n def forward(self, input: torch.Tensor, target: torch.Tensor):\n if input.dim() == 3:\n input = input.unsqueeze(1)\n weit = 1 + 5 * torch.abs(F.avg_pool2d(target, kernel_size=31, stride=1, padding=15) - target)\n wbce = F.binary_cross_entropy_with_logits(input, target, reduce=\"none\")\n wbce = (weit * wbce).sum(dim=(2, 3)) / weit.sum(dim=(2, 3))\n\n input = torch.sigmoid(input)\n inter = ((input * target) * weit).sum(dim=(2, 3))\n union = ((input + target) * weit).sum(dim=(2, 3))\n wiou = 1 - (inter + 1) / (union - inter + 1)\n return (wbce + wiou).mean()\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/losses/utils.py",
"content": "import logging\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nimport torch\nfrom omegaconf import DictConfig, OmegaConf\nfrom pytorch_metric_learning import distances, losses, miners\nfrom torch import nn\nfrom transformers.models.mask2former.modeling_mask2former import Mask2FormerConfig, Mask2FormerLoss\n\nfrom ...constants import (\n BINARY,\n CONTRASTIVE_LOSS,\n COSINE_SIMILARITY,\n FEW_SHOT_CLASSIFICATION,\n MULTI_NEGATIVES_SOFTMAX_LOSS,\n MULTICLASS,\n NER,\n OBJECT_DETECTION,\n PAIR_MARGIN_MINER,\n REGRESSION,\n SEMANTIC_SEGMENTATION,\n)\nfrom .bce_loss import BBCEWithLogitLoss\nfrom .focal_loss import FocalLoss\nfrom .lemda_loss import LemdaLoss\nfrom .softmax_losses import MultiNegativesSoftmaxLoss, SoftTargetCrossEntropy\nfrom .structure_loss import StructureLoss\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_loss_func(\n problem_type: str,\n mixup_active: Optional[bool] = None,\n loss_func_name: Optional[str] = None,\n config: Optional[DictConfig] = None,\n **kwargs,\n):\n \"\"\"\n Choose a suitable Pytorch loss module based on the provided problem type.\n\n Parameters\n ----------\n problem_type\n Type of problem.\n mixup_active\n The activation determining whether to use mixup.\n loss_func_name\n The name of the function the user wants to use.\n config\n The optimization configs containing values such as i.e. optim.loss_func\n An example purpose of this config here is to pass through the parameters for focal loss, i.e.:\n alpha = optim.focal_loss.alpha\n Returns\n -------\n A Pytorch loss module.\n \"\"\"\n if problem_type in [BINARY, MULTICLASS]:\n if mixup_active:\n loss_func = SoftTargetCrossEntropy()\n else:\n if loss_func_name is not None and loss_func_name.lower() == \"focal_loss\":\n loss_func = FocalLoss(\n alpha=config.focal_loss.alpha,\n gamma=config.focal_loss.gamma,\n reduction=config.focal_loss.reduction,\n )\n else:\n loss_func = nn.CrossEntropyLoss(label_smoothing=config.label_smoothing)\n logger.debug(f\"loss_func.label_smoothing: {loss_func.label_smoothing}\")\n elif problem_type == REGRESSION:\n if loss_func_name is not None:\n if \"bcewithlogitsloss\" in loss_func_name.lower():\n loss_func = nn.BCEWithLogitsLoss()\n else:\n loss_func = nn.MSELoss()\n else:\n loss_func = nn.MSELoss()\n elif problem_type == NER:\n loss_func = nn.CrossEntropyLoss(ignore_index=0)\n elif problem_type in [None, OBJECT_DETECTION, FEW_SHOT_CLASSIFICATION]:\n return None\n elif problem_type == SEMANTIC_SEGMENTATION:\n if \"structure_loss\" in loss_func_name.lower():\n loss_func = StructureLoss()\n elif \"balanced_bce\" in loss_func_name.lower():\n loss_func = BBCEWithLogitLoss()\n elif \"mask2former_loss\" in loss_func_name.lower():\n weight_dict = {\n \"loss_cross_entropy\": config.mask2former_loss.loss_cross_entropy_weight,\n \"loss_mask\": config.mask2former_loss.loss_mask_weight,\n \"loss_dice\": config.mask2former_loss.loss_dice_weight,\n }\n loss_func = Mask2FormerLoss(\n config=Mask2FormerConfig(num_labels=kwargs[\"num_classes\"]), weight_dict=weight_dict\n )\n else:\n loss_func = nn.BCEWithLogitsLoss()\n else:\n raise NotImplementedError\n\n return loss_func\n\n\ndef get_metric_learning_distance_func(\n name: str,\n):\n \"\"\"\n Return one pytorch metric learning's distance function based on its name.\n\n Parameters\n ----------\n name\n distance function name\n\n Returns\n -------\n A distance function from the pytorch metric learning package.\n \"\"\"\n if name.lower() == COSINE_SIMILARITY:\n return distances.CosineSimilarity()\n else:\n raise ValueError(f\"Unknown distance measure: {name}\")\n\n\ndef infer_matcher_loss(data_format: str, problem_type: str):\n \"\"\"\n Infer the loss type to train the matcher.\n\n Parameters\n ----------\n data_format\n The training data format, e.g., pair or triplet.\n problem_type\n Type of problem.\n\n Returns\n -------\n The loss name.\n \"\"\"\n if data_format == \"pair\":\n if problem_type is None:\n return [MULTI_NEGATIVES_SOFTMAX_LOSS]\n elif problem_type == BINARY:\n return [CONTRASTIVE_LOSS]\n elif problem_type == REGRESSION:\n return [\"cosine_similarity_loss\"]\n else:\n raise ValueError(f\"Unsupported data format {data_format} with problem type {problem_type}\")\n elif data_format == \"triplet\":\n if problem_type is None:\n return [MULTI_NEGATIVES_SOFTMAX_LOSS]\n else:\n raise ValueError(f\"Unsupported data format {data_format} with problem type {problem_type}\")\n else:\n raise ValueError(f\"Unsupported data format: {data_format}\")\n\n\ndef get_matcher_loss_func(\n data_format: str,\n problem_type: str,\n loss_type: Optional[str] = None,\n pos_margin: Optional[float] = None,\n neg_margin: Optional[float] = None,\n distance_type: Optional[str] = None,\n):\n \"\"\"\n Return a list of pytorch metric learning's loss functions based on their names.\n\n Parameters\n ----------\n data_format\n The training data format, e.g., pair or triplet.\n problem_type\n Type of problem.\n loss_type\n The provided loss type.\n pos_margin\n The positive margin in computing the metric learning loss.\n neg_margin\n The negative margin in computing the metric learning loss.\n distance_type\n The distance function type.\n\n Returns\n -------\n A loss function of metric learning.\n \"\"\"\n\n allowable_loss_types = infer_matcher_loss(data_format=data_format, problem_type=problem_type)\n if loss_type is not None:\n assert loss_type in allowable_loss_types, f\"data format {data_format} can't use loss {loss_type}.\"\n else:\n loss_type = allowable_loss_types[0]\n\n if loss_type.lower() == CONTRASTIVE_LOSS:\n return losses.ContrastiveLoss(\n pos_margin=pos_margin,\n neg_margin=neg_margin,\n distance=get_metric_learning_distance_func(distance_type),\n )\n elif loss_type.lower() == MULTI_NEGATIVES_SOFTMAX_LOSS:\n return MultiNegativesSoftmaxLoss(\n local_loss=True,\n gather_with_grad=True,\n cache_labels=False,\n )\n else:\n raise ValueError(f\"Unknown metric learning loss: {loss_type}\")\n\n\ndef get_matcher_miner_func(\n miner_type: str,\n pos_margin: float,\n neg_margin: float,\n distance_type: str,\n):\n \"\"\"\n Return a pytorch metric learning's miner functions based on their names.\n The miners are used to mine the positive and negative examples.\n\n Parameters\n ----------\n miner_type\n The miner function type.\n pos_margin\n The positive margin used by the miner function.\n neg_margin\n The negative margin used by the miner function.\n distance_type\n The distance function type.\n\n Returns\n -------\n A miner function to mine positive and negative samples.\n \"\"\"\n if miner_type.lower() == PAIR_MARGIN_MINER:\n return miners.PairMarginMiner(\n pos_margin=pos_margin,\n neg_margin=neg_margin,\n distance=get_metric_learning_distance_func(distance_type),\n )\n else:\n raise ValueError(f\"Unknown metric learning miner: {miner_type}\")\n\n\ndef generate_metric_learning_labels(\n num_samples: int,\n match_label: int,\n labels: torch.Tensor,\n):\n \"\"\"\n Generate labels to compute the metric learning loss of one mini-batch.\n For n samples, it generates 2*n labels since each match has two sides, each of which\n has one label. If we know the matching label, then it determines the two sides' labels\n according to whether their label is the matching label. If the matching label is None,\n it assigns a unique label for each side.\n\n Parameters\n ----------\n num_samples\n number of samples.\n match_label\n The matching label, which can be None.\n labels\n The sample labels used in the supervised setting. It's required only when match_label is not None.\n\n Returns\n -------\n The labels used in computing the metric learning loss.\n \"\"\"\n device = labels.device\n labels_1 = torch.arange(num_samples, device=device)\n\n if match_label is not None:\n labels_2 = torch.arange(num_samples, num_samples * 2, device=device)\n # users need to specify the match_label based on the raw label's semantic meaning.\n mask = labels == match_label\n labels_2[mask] = labels_1[mask]\n else:\n labels_2 = torch.arange(num_samples, device=device)\n\n metric_learning_labels = torch.cat([labels_1, labels_2], dim=0)\n\n return metric_learning_labels\n\n\ndef get_aug_loss_func(config: Optional[DictConfig] = None, problem_type: Optional[str] = None):\n \"\"\"\n Return the loss function for lemda augmentation\n\n Parameters\n ----------\n config\n The optimization configuration.\n problem_type\n Problem type (binary, multimclass, or regression)\n\n Returns\n -------\n Augmentation loss function.\n \"\"\"\n loss_func = None\n if config.lemda.turn_on:\n loss_func = LemdaLoss(\n mse_weight=config.lemda.mse_weight,\n kld_weight=config.lemda.kld_weight,\n consist_weight=config.lemda.consist_weight,\n consist_threshold=config.lemda.consist_threshold,\n problem_type=problem_type,\n )\n\n return loss_func\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lr/__init__.py",
"content": "from .utils import apply_layerwise_lr_decay, apply_single_lr, apply_two_stages_lr, get_lr_scheduler\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lr/lr_schedulers.py",
"content": "\"\"\"\nThis file is largely borrowed from `transformers/optimization.py`.\nThis is to make lr schedulers pickle-able so that we can use the training strategy \"ddp_spawn\" in Pytorch Lightning.\n\"\"\"\n\nimport functools\nimport math\n\nfrom torch.optim.lr_scheduler import LambdaLR\n\n\ndef _cosine_decay_lr_lambda(current_step, num_warmup_steps, num_training_steps, num_cycles):\n if current_step < num_warmup_steps:\n return float(current_step) / float(max(1, num_warmup_steps))\n progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress)))\n\n\ndef get_cosine_schedule_with_warmup(\n optimizer, num_warmup_steps: int, num_training_steps: int, num_cycles: float = 0.5, last_epoch: int = -1\n):\n \"\"\"\n Create a schedule with a learning rate that decreases following the values of the cosine function between the\n initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the\n initial lr set in the optimizer.\n Args:\n optimizer ([`~torch.optim.Optimizer`]):\n The optimizer for which to schedule the learning rate.\n num_warmup_steps (`int`):\n The number of steps for the warmup phase.\n num_training_steps (`int`):\n The total number of training steps.\n num_cycles (`float`, *optional*, defaults to 0.5):\n The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0\n following a half-cosine).\n last_epoch (`int`, *optional*, defaults to -1):\n The index of the last epoch when resuming training.\n Return:\n `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.\n \"\"\"\n lr_lambda = functools.partial(\n _cosine_decay_lr_lambda,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_training_steps,\n num_cycles=num_cycles,\n )\n return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef _poly_decay_lr_lambda(\n current_step: int, num_warmup_steps: int, num_training_steps: int, lr_init: float, lr_end: float, power: float\n):\n if current_step < num_warmup_steps:\n return float(current_step) / float(max(1, num_warmup_steps))\n elif current_step > num_training_steps:\n return lr_end / lr_init # as LambdaLR multiplies by lr_init\n else:\n lr_range = lr_init - lr_end\n decay_steps = num_training_steps - num_warmup_steps\n pct_remaining = 1 - (current_step - num_warmup_steps) / decay_steps\n decay = lr_range * pct_remaining**power + lr_end\n return decay / lr_init # as LambdaLR multiplies by lr_init\n\n\ndef get_polynomial_decay_schedule_with_warmup(\n optimizer, num_warmup_steps, num_training_steps, lr_end=1e-7, power=1.0, last_epoch=-1\n):\n \"\"\"Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the\n optimizer to end lr defined by *lr_end*, after a warmup period during which it increases linearly from 0 to the\n initial lr set in the optimizer.\n Note: *power* defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT\n implementation at\n https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37\n\n This function is borrowed from transformers/optimization.py. We make the function pickleble.\n\n Parameters\n ----------\n optimizer\n The optimizer for which to schedule the learning rate.\n num_warmup_steps\n The number of steps for the warmup phase.\n num_training_steps\n The total number of training steps.\n lr_end\n The end LR.\n power\n Power factor.\n last_epoch\n The index of the last epoch when resuming training.\n\n Returns\n -------\n lr_schedule\n `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.\n \"\"\"\n\n lr_init = optimizer.defaults[\"lr\"]\n if not (lr_init >= lr_end):\n raise ValueError(f\"lr_end ({lr_end}) must not be larger than initial lr ({lr_init})\")\n lr_lambda = functools.partial(\n _poly_decay_lr_lambda,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_training_steps,\n lr_init=lr_init,\n lr_end=lr_end,\n power=power,\n )\n return LambdaLR(optimizer, lr_lambda, last_epoch)\n\n\ndef _linear_warmup_lr_lambda(current_step: int, num_warmup_steps: int, num_training_steps: int):\n if current_step < num_warmup_steps:\n return float(current_step) / float(max(1, num_warmup_steps))\n return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)))\n\n\ndef get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):\n \"\"\"\n Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after\n a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.\n Args:\n optimizer ([`~torch.optim.Optimizer`]):\n The optimizer for which to schedule the learning rate.\n num_warmup_steps (`int`):\n The number of steps for the warmup phase.\n num_training_steps (`int`):\n The total number of training steps.\n last_epoch (`int`, *optional*, defaults to -1):\n The index of the last epoch when resuming training.\n Return:\n `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.\n \"\"\"\n lr_lambda = functools.partial(\n _linear_warmup_lr_lambda, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps\n )\n return LambdaLR(optimizer, lr_lambda, last_epoch)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/lr/utils.py",
"content": "import logging\nimport re\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import nn, optim\n\nfrom ..utils import get_weight_decay_param_names\nfrom .lr_schedulers import (\n get_cosine_schedule_with_warmup,\n get_linear_schedule_with_warmup,\n get_polynomial_decay_schedule_with_warmup,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_lr_scheduler(\n optimizer: optim.Optimizer,\n num_max_steps: int,\n num_warmup_steps: int,\n lr_schedule: str,\n end_lr: Union[float, int],\n):\n \"\"\"\n Get the learning rate scheduler from its name. Here we use our defined learning rate\n scheduler instead of those imported from \"transformers\" because we want to support\n Pytorch lightning's \"ddp_spawn\" training strategy.\n\n Parameters\n ----------\n optimizer\n A Pytorch optimizer.\n num_max_steps\n Number of maximum training steps.\n num_warmup_steps\n Number of steps to do learning rate warmup.\n lr_schedule\n Name of the learning rate scheduler.\n end_lr\n The final learning rate after decay.\n\n Returns\n -------\n A learning rate scheduler.\n \"\"\"\n if lr_schedule == \"cosine_decay\":\n scheduler = get_cosine_schedule_with_warmup(\n optimizer=optimizer,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_max_steps,\n )\n elif lr_schedule == \"polynomial_decay\":\n scheduler = get_polynomial_decay_schedule_with_warmup(\n optimizer=optimizer,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_max_steps,\n lr_end=end_lr,\n power=1,\n )\n elif lr_schedule == \"linear_decay\":\n scheduler = get_linear_schedule_with_warmup(\n optimizer=optimizer,\n num_warmup_steps=num_warmup_steps,\n num_training_steps=num_max_steps,\n )\n elif lr_schedule == \"multi_step\":\n # TODO: add milestones, gamma into hyperparameters\n scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[30, 55], gamma=0.1)\n else:\n raise ValueError(f\"unknown lr schedule: {lr_schedule}\")\n\n return scheduler\n\n\ndef apply_single_lr(\n model: nn.Module,\n lr: float,\n weight_decay: float,\n return_params: Optional[bool] = True,\n peft: Optional[str] = None,\n trainable_param_names: Optional[List] = None,\n exclude_keys: Optional[List] = None,\n):\n \"\"\"\n Set to use a single learning rate for all parameters. Layer normalization parameters and other\n layers' bias parameters don't use weight decay.\n\n Parameters\n ----------\n model\n A Pytorch model.\n lr\n Learning rate.\n weight_decay\n Weight decay.\n return_params\n Whether to return parameters or their names. If you want to double-check\n whether the learning rate setup is as expected, you can set \"return_params=False\",\n and print the layer names along with their learning rates through\n \"print(\"Param groups = %s\" % json.dumps(optimizer_grouped_parameters, indent=2))\".\n peft\n Efficient finetuning strategy. It will only finetune part of the parameters\n trainable_param_names\n A list of trainable parameters. (Optional)\n exclude_keys\n A list of keys to be excluded.\n\n Returns\n -------\n The grouped parameters or their names.\n \"\"\"\n decay_param_names = get_weight_decay_param_names(model)\n decay_grad_param_names = []\n no_decay_grad_param_names = []\n\n for name, param in model.named_parameters():\n if exclude_keys and any([exc in name for exc in exclude_keys]):\n continue\n\n if (\n peft is not None\n and trainable_param_names\n and not any([re.match(trainable_param_name, name) for trainable_param_name in trainable_param_names])\n ):\n param.requires_grad = False\n\n if not param.requires_grad:\n continue # frozen weights\n\n if name in decay_param_names:\n if return_params:\n decay_grad_param_names.append(param)\n else:\n decay_grad_param_names.append(name)\n\n else:\n if return_params:\n no_decay_grad_param_names.append(param)\n else:\n no_decay_grad_param_names.append(name)\n\n optimizer_grouped_parameters = [\n {\n \"params\": decay_grad_param_names,\n \"weight_decay\": weight_decay,\n \"lr\": lr,\n },\n {\n \"params\": no_decay_grad_param_names,\n \"weight_decay\": 0.0,\n \"lr\": lr,\n },\n ]\n return optimizer_grouped_parameters\n\n\ndef apply_two_stages_lr(\n model: nn.Module,\n lr: float,\n lr_mult: Union[float, int],\n weight_decay: float,\n return_params: Optional[bool] = True,\n exclude_keys: Optional[List] = None,\n):\n \"\"\"\n Set up the pretrained backbone to use a smaller learning rate (lr * lr_mult).\n The newly added head layers use the normal learning rate (lr).\n Layer normalization parameters and other layers' bias parameters don't use weight decay.\n\n Parameters\n ----------\n model\n A Pytorch model.\n lr\n The learning rate.\n lr_mult\n The multiplier (0, 1) to scale down the learning rate.\n weight_decay\n Weight decay.\n return_params\n return_params\n Whether to return parameters or their names. If you want to double-check\n whether the learning rate setup is as expected, you can set \"return_params=False\",\n and print the layer names along with their learning rates through\n \"print(\"Param groups = %s\" % json.dumps(optimizer_grouped_parameters, indent=2))\".\n exclude_keys\n A list of keys to be excluded.\n\n Returns\n -------\n The grouped parameters or their names.\n \"\"\"\n decay_param_names = get_weight_decay_param_names(model)\n\n optimizer_grouped_parameters = [\n {\n \"params\": [\n p if return_params else n\n for n, p in model.named_parameters()\n if n in decay_param_names\n and not any(bb in n for bb in model.head_layer_names)\n and (not exclude_keys or not any([exc in n for exc in exclude_keys]))\n ],\n \"weight_decay\": weight_decay,\n \"lr\": lr,\n },\n {\n \"params\": [\n p if return_params else n\n for n, p in model.named_parameters()\n if n not in decay_param_names\n and not any(bb in n for bb in model.head_layer_names)\n and (not exclude_keys or not any([exc in n for exc in exclude_keys]))\n ],\n \"weight_decay\": 0.0,\n \"lr\": lr,\n },\n {\n \"params\": [\n p if return_params else n\n for n, p in model.named_parameters()\n if n in decay_param_names\n and any(bb in n for bb in model.head_layer_names)\n and (not exclude_keys or not any([exc in n for exc in exclude_keys]))\n ],\n \"weight_decay\": weight_decay,\n \"lr\": lr * lr_mult,\n },\n {\n \"params\": [\n p if return_params else n\n for n, p in model.named_parameters()\n if n not in decay_param_names\n and any(bb in n for bb in model.head_layer_names)\n and (not exclude_keys or not any([exc in n for exc in exclude_keys]))\n ],\n \"weight_decay\": 0.0,\n \"lr\": lr * lr_mult,\n },\n ]\n\n return optimizer_grouped_parameters\n\n\ndef apply_layerwise_lr_decay(\n model: nn.Module,\n lr: float,\n lr_decay: float,\n weight_decay: float,\n peft: Optional[str] = None,\n trainable_param_names: Optional[List] = None,\n exclude_keys: Optional[List] = None,\n):\n \"\"\"\n Assign monotonically decreasing learning rates for layers from the output end to the input end.\n The intuition behind is that later layers are more task-related compared to the early layers.\n Layer normalization parameters and other layers' bias parameters don't use weight decay.\n If you want to double-check whether the learning rate setup is as expected,\n you can print the layer names along with their learning rates through\n \"print(\"Param groups = %s\" % json.dumps(parameter_group_names, indent=2))\".\n\n Parameters\n ----------\n model\n A Pytorch model.\n lr\n The learning rate.\n lr_decay\n The learning rate decay factor (0, 1).\n weight_decay\n Weight decay.\n peft\n Efficient finetuning strategy. It will only finetune part of the parameters\n trainable_param_names\n A list of trainable parameters. (Optional)\n exclude_keys\n A list of keys to be excluded.\n\n Returns\n -------\n The grouped parameters based on their layer ids and whether using weight decay.\n \"\"\"\n parameter_group_names = {}\n parameter_group_vars = {}\n decay_param_names = get_weight_decay_param_names(model)\n\n for name, param in model.named_parameters():\n if name.startswith(\"_orig_mod.\"):\n name = \"\".join(name.split(\"_orig_mod.\"))\n if exclude_keys and any([exc in name for exc in exclude_keys]):\n continue\n layer_id = model.name_to_id[name]\n if layer_id == 0: # Set top layer (e.g. head, fusion_mlp, adapter) as being trainable.\n param.requires_grad = True\n elif (\n peft is not None\n and trainable_param_names\n and not any([re.match(trainable_param_name, name) for trainable_param_name in trainable_param_names])\n ):\n param.requires_grad = False\n\n if not param.requires_grad:\n continue # frozen weights\n\n if name in decay_param_names:\n group_name = \"decay\"\n this_weight_decay = weight_decay\n else:\n group_name = \"no_decay\"\n this_weight_decay = 0.0\n\n layer_id = model.name_to_id[name]\n group_name = \"layer_%d_%s\" % (layer_id, group_name)\n\n if group_name not in parameter_group_names:\n scale = lr_decay**layer_id\n parameter_group_names[group_name] = {\n \"weight_decay\": this_weight_decay,\n \"params\": [],\n \"lr\": scale * lr,\n }\n parameter_group_vars[group_name] = {\n \"weight_decay\": this_weight_decay,\n \"params\": [],\n \"lr\": scale * lr,\n }\n\n parameter_group_vars[group_name][\"params\"].append(param)\n parameter_group_names[group_name][\"params\"].append(name)\n\n return list(parameter_group_vars.values())\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/__init__.py",
"content": "from .coverage_metrics import Coverage\nfrom .hit_rate_metrics import CustomHitRate\nfrom .ranking_metrics import compute_ranking_score\nfrom .utils import compute_score, get_minmax_mode, get_stopping_threshold, get_torchmetric, infer_metrics\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/coverage_metrics.py",
"content": "import torch\nfrom torchmetrics import Metric\nfrom torchmetrics.utilities import dim_zero_cat\n\n\nclass Coverage(Metric):\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.add_state(\"pos_probs\", default=[], dist_reduce_fx=\"cat\")\n self.add_state(\"targets\", default=[], dist_reduce_fx=\"cat\")\n self.tp_threshold = 0.97\n self.tn_threshold = 0.99\n higher_is_better = True\n\n def update(self, preds: torch.Tensor, target: torch.Tensor) -> None:\n assert preds.dim() == 1\n assert target.dim() == 1\n self.pos_probs.append(preds)\n self.targets.append(target)\n\n def compute(self):\n # parse inputs\n pos_probs = dim_zero_cat(self.pos_probs)\n targets = dim_zero_cat(self.targets)\n y_pos = targets[torch.where(pos_probs >= self.tp_threshold)]\n y_neg = targets[torch.where(pos_probs <= 1 - self.tn_threshold)]\n tp = sum(y_pos == 1)\n tn = sum(y_neg == 0)\n if len(y_pos) == 0 or len(targets) == 0:\n tp_precision, tp_coverage = 0, 0\n else:\n tp_precision, tp_coverage = tp / len(y_pos), len(y_pos) / len(targets)\n if tp_precision < self.tp_threshold:\n tp_coverage = 0\n if len(y_neg) == 0 or len(targets) == 0:\n tn_precision, tn_coverage = 0, 0\n else:\n tn_precision, tn_coverage = tn / len(y_neg), len(y_neg) / len(targets)\n if tn_precision < self.tn_threshold:\n tn_coverage = 0\n\n return torch.tensor(tp_coverage) + torch.tensor(tn_coverage)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/hit_rate_metrics.py",
"content": "from typing import Any, Dict, List, Optional, Tuple, Union\n\nimport torch\nimport torchmetrics\n\n\nclass CustomHitRate(torchmetrics.Metric):\n \"\"\"\n Compute the hit rate when doing semantic search between two group of embeddings.\n We assume that (a_i, p_i) are a positive pair and (a_i, p_j) for i!=j a negative pair.\n \"\"\"\n\n def __init__(\n self,\n ):\n super().__init__()\n self.add_state(\"query_embeddings\", default=[], dist_reduce_fx=None)\n self.add_state(\"response_embeddings\", default=[], dist_reduce_fx=None)\n self.add_state(\"logit_scale\", default=[], dist_reduce_fx=None)\n\n def update(\n self,\n batch_query_embeds: torch.Tensor,\n batch_response_embeds: torch.Tensor,\n logit_scale: Optional[torch.Tensor] = None,\n ):\n self.query_embeddings.append(batch_query_embeds)\n self.response_embeddings.append(batch_response_embeds)\n if logit_scale is not None:\n self.logit_scale.append(logit_scale)\n\n def compute(self):\n query_embeddings = torch.cat(self.query_embeddings)\n response_embeddings = torch.cat(self.response_embeddings)\n if self.logit_scale:\n logit_scale = torch.mean(torch.stack(self.logit_scale))\n else:\n logit_scale = 1\n\n return self.compute_hit_rate(query_embeddings, response_embeddings, logit_scale)\n\n @staticmethod\n def compute_hit_rate(features_a, features_b, logit_scale, top_ks=[1, 5, 10]):\n \"\"\"\n Compute symmetric hit rates between two groups of features.\n\n Parameters\n ----------\n features_a\n One group of features.\n features_b\n The other group of features.\n logit_scale\n The scale of logit (Used in CLIP).\n top_ks\n Consider only the top k elements for each query.\n\n Returns\n -------\n The accumulated hit rate.\n \"\"\"\n assert len(features_a) == len(features_b)\n hit_rate = 0\n logits_per_a = (logit_scale * features_a @ features_b.t()).detach().cpu()\n logits_per_b = logits_per_a.t().detach().cpu()\n\n logits = {\"logits_per_a\": logits_per_a, \"logits_per_b\": logits_per_b}\n ground_truth = torch.arange(len(features_b)).view(-1, 1)\n\n for name, logit in logits.items():\n ranking = torch.argsort(logit, descending=True)\n preds = torch.where(ranking == ground_truth)[1]\n\n for k in top_ks:\n hit_rate += (preds < k).float().mean()\n\n hit_rate /= len(top_ks) * len(logits)\n return hit_rate\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/ranking_metrics.py",
"content": "import logging\nimport math\nimport operator\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport numpy as np\n\nfrom ...constants import MAP, NDCG, PRECISION, RECALL\n\nlogger = logging.getLogger(__name__)\n\n\nclass RankingMetrics:\n def __init__(\n self,\n pred: Dict[str, Dict],\n target: Dict[str, Dict],\n is_higher_better=True,\n ):\n \"\"\"\n Evaluation Metrics for information retrieval tasks such as document retrieval, image retrieval, etc.\n Reference: https://www.cs.cornell.edu/courses/cs4300/2013fa/lectures/metrics-2-4pp.pdf\n\n Parameters\n ----------\n pred:\n the prediction of the ranking model. It has the following form.\n pred = {\n 'q1': {\n 'd1': 1,\n 'd3': 0,\n },\n 'q2': {\n 'd2': 1,\n 'd3': 1,\n },\n }\n where q refers to queries, and d refers to documents, each query has a few relevant documents.\n 0s and 1s are model predicted scores (does not need to be binary).\n target:\n the ground truth query and response relevance which has the same form as pred.\n is_higher_better:\n if higher relevance score means higher ranking.\n if the relevance score is cosine similarity / dot product, it should be set to True;\n if it is Eulidean distance, it should be False.\n \"\"\"\n self.pred = pred\n self.target = target\n self.is_higher_better = is_higher_better\n # the supported metrics in this script\n self.supported_metrics = {\n \"precision\": 0,\n \"recall\": 1,\n \"mrr\": 2,\n \"map\": 3,\n \"ndcg\": 4,\n }\n\n assert len(pred) == len(target), (\n f\"The prediction and groudtruth target should have the same number of queries, \\\n while there are {len(pred)} queries in prediction and {len(target)} in the target.\"\n )\n\n self.results = {}\n for key in target.keys():\n self.results.update({key: [target[key], pred[key]]})\n\n def compute(self, metrics: Union[str, list] = None, k: Optional[int] = 10):\n \"\"\"\n compute and return ranking scores.\n\n Parameters\n ----------\n metrics:\n user provided metrics\n k:\n the cutoff value for NDCG, MAP, Recall, MRR, and Precision\n\n Returns\n -------\n Computed score.\n\n \"\"\"\n if isinstance(metrics, str):\n metrics = [metrics]\n if not metrics: # no metric is provided\n metrics = self.supported_metrics.keys()\n\n return_res = {}\n\n eval_res = np.mean(\n [list(self._compute_one(idx, k)) for idx in self.results.keys()],\n axis=0,\n )\n\n for metric in metrics:\n metric = metric.lower()\n if metric in self.supported_metrics:\n return_res.update({f\"{metric}@{k}\": eval_res[self.supported_metrics[metric]]})\n\n return return_res\n\n def _compute_one(self, idx, k):\n \"\"\"\n compute and return the ranking scores for one query.\n for definition of these metrics, please refer to\n https://www.cs.cornell.edu/courses/cs4300/2013fa/lectures/metrics-2-4pp.pdf\n\n Parameters\n ----------\n idx:\n the index of the query\n k:\n the cutoff value for NDCG, MAP, Recall, MRR, and Precision\n\n Returns\n -------\n Computed score.\n \"\"\"\n precision, recall, mrr, mAP, ndcg = 0, 0, 0, 0, 0\n target, pred = self.results[idx][0], self.results[idx][1]\n\n # sort the ground truth and predictions in descending order\n sorted_target = dict(\n sorted(\n target.items(),\n key=operator.itemgetter(1),\n reverse=self.is_higher_better,\n )\n )\n sorted_pred = dict(\n sorted(\n pred.items(),\n key=operator.itemgetter(1),\n reverse=self.is_higher_better,\n )\n )\n sorted_target_values = list(sorted_target.values())\n sorted_pred_values = list(sorted_pred.values())\n\n # number of positive relevance in target\n # negative numbers and zero are considered as negative response\n num_pos_target = len([val for val in sorted_target_values if val > 0])\n\n at_k = k if num_pos_target > k else num_pos_target\n\n first_k_items_list = list(sorted_pred.items())[0:k]\n\n rank = 0\n hit_rank = [] # correctly retrieved items\n for key, value in first_k_items_list:\n if key in sorted_target and sorted_target[key] > 0:\n hit_rank.append(rank)\n rank += 1\n count = len(hit_rank)\n # compute the precision and recall\n precision = count / k\n recall = count / num_pos_target\n\n dcg = 0\n if hit_rank: # not empty\n # compute the mean reciprocal rank\n mrr = 1 / (hit_rank[0] + 1)\n # compute the mean average precision\n mAP = np.sum([sorted_pred_values[rank] * (i + 1) / (rank + 1) for i, rank in enumerate(hit_rank)])\n # compute the discounted cumulative gain\n dcg = np.sum([1 / math.log(rank + 2, 2) for rank in hit_rank])\n\n # compute the ideal discounted cumulative gain\n idcg = np.sum([1 / math.log(i + 2, 2) for i in range(at_k)])\n # compute the normalized discounted cumulative gain\n ndcg = dcg / idcg\n mAP /= at_k\n\n return precision, recall, mrr, mAP, ndcg\n\n\ndef compute_ranking_score(\n results: Dict[str, Dict],\n qrel_dict: Dict[str, Dict],\n metrics: List[str],\n cutoffs: Optional[List[int]] = [5, 10, 20],\n):\n \"\"\"\n Compute the ranking metrics, e.g., NDCG, MAP, Recall, and Precision.\n TODO: Consider MRR.\n\n Parameters\n ----------\n results:\n The query/document ranking list by the model.\n qrel_dict:\n The groundtruth query and document relevance.\n metrics\n A list of metrics to compute.\n cutoffs:\n The cutoff values for NDCG, MAP, Recall, and Precision.\n\n Returns\n -------\n A dict of metric scores.\n \"\"\"\n scores = {}\n evaluator = RankingMetrics(pred=results, target=qrel_dict)\n for k in cutoffs:\n scores.update(evaluator.compute(k=k))\n\n metric_results = dict()\n for k in cutoffs:\n for per_metric in metrics:\n if per_metric.lower() == NDCG:\n metric_results[f\"{NDCG}@{k}\"] = 0.0\n elif per_metric.lower() == MAP:\n metric_results[f\"{MAP}@{k}\"] = 0.0\n elif per_metric.lower() == RECALL:\n metric_results[f\"{RECALL}@{k}\"] = 0.0\n elif per_metric.lower() == PRECISION:\n metric_results[f\"{PRECISION}@{k}\"] = 0.0\n\n for k in cutoffs:\n for per_metric in metrics:\n if per_metric.lower() == NDCG:\n metric_results[f\"{NDCG}@{k}\"] = round(scores[f\"{NDCG}@{k}\"], 5)\n elif per_metric.lower() == MAP:\n metric_results[f\"{MAP}@{k}\"] = round(scores[f\"{MAP}@{k}\"], 5)\n elif per_metric.lower() == RECALL:\n metric_results[f\"{RECALL}@{k}\"] = round(scores[f\"{RECALL}@{k}\"], 5)\n elif per_metric.lower() == PRECISION:\n metric_results[f\"{PRECISION}@{k}\"] = round(scores[f\"{PRECISION}@{k}\"], 5)\n\n return metric_results\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/semantic_seg_metrics.py",
"content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport torch\nimport torchmetrics\nfrom scipy.ndimage import convolve\nfrom scipy.ndimage import distance_transform_edt as bwdist\n\n_EPS = np.spacing(1) # the different implementation of epsilon (extreme min value) between numpy and matlab\n_TYPE = np.float64\n\n\ndef _prepare_data(pred: np.ndarray, gt: np.ndarray) -> tuple:\n \"\"\"\n A numpy-based function for preparing ``pred`` and ``gt``.\n - for ``pred``, it looks like ``mapminmax(im2double(...))`` of matlab;\n - ``gt`` will be binarized by 128.\n :param pred: prediction\n :param gt: mask\n :return: pred, gt\n \"\"\"\n gt = gt > 128\n # im2double, mapminmax\n pred = pred / 255\n if pred.max() != pred.min():\n pred = (pred - pred.min()) / (pred.max() - pred.min())\n return pred, gt\n\n\ndef _get_adaptive_threshold(matrix: np.ndarray, max_value: float = 1) -> float:\n \"\"\"\n Return an adaptive threshold, which is equal to twice the mean of ``matrix``.\n :param matrix: a data array\n :param max_value: the upper limit of the threshold\n :return: min(2 * matrix.mean(), max_value)\n \"\"\"\n return min(2 * matrix.mean(), max_value)\n\n\nclass Fmeasure(object):\n def __init__(self, beta: float = 1.0):\n \"\"\"\n F-measure for SOD.\n ::\n @inproceedings{Fmeasure,\n title={Frequency-tuned salient region detection},\n author={Achanta, Radhakrishna and Hemami, Sheila and Estrada, Francisco and S{\\\"u}sstrunk, Sabine},\n booktitle=CVPR,\n number={CONF},\n pages={1597--1604},\n year={2009}\n }\n :param beta: the weight of the precision\n \"\"\"\n self.beta = beta\n self.precisions = []\n self.recalls = []\n self.adaptive_fms = []\n self.changeable_fms = []\n\n def step(self, pred: np.ndarray, gt: np.ndarray):\n pred, gt = _prepare_data(pred, gt)\n\n adaptive_fm = self.cal_adaptive_fm(pred=pred, gt=gt)\n self.adaptive_fms.append(adaptive_fm)\n\n precisions, recalls, changeable_fms = self.cal_pr(pred=pred, gt=gt)\n self.precisions.append(precisions)\n self.recalls.append(recalls)\n self.changeable_fms.append(changeable_fms)\n\n def cal_adaptive_fm(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the adaptive F-measure.\n :return: adaptive_fm\n \"\"\"\n # ``np.count_nonzero`` is faster and better\n adaptive_threshold = _get_adaptive_threshold(pred, max_value=1)\n binary_predcition = pred >= adaptive_threshold\n area_intersection = binary_predcition[gt].sum()\n if area_intersection == 0:\n adaptive_fm = 0\n else:\n pre = area_intersection / np.count_nonzero(binary_predcition)\n rec = area_intersection / np.count_nonzero(gt)\n adaptive_fm = (1 + self.beta) * pre * rec / (self.beta * pre + rec)\n return adaptive_fm\n\n def cal_pr(self, pred: np.ndarray, gt: np.ndarray) -> tuple:\n \"\"\"\n Calculate the corresponding precision and recall when the threshold changes from 0 to 255.\n These precisions and recalls can be used to obtain the mean F-measure, maximum F-measure,\n precision-recall curve and F-measure-threshold curve.\n For convenience, ``changeable_fms`` is provided here, which can be used directly to obtain\n the mean F-measure, maximum F-measure and F-measure-threshold curve.\n :return: precisions, recalls, changeable_fms\n \"\"\"\n pred = (pred * 255).astype(np.uint8)\n bins = np.linspace(0, 256, 257)\n fg_hist, _ = np.histogram(pred[gt], bins=bins)\n bg_hist, _ = np.histogram(pred[~gt], bins=bins)\n\n fg_w_thrs = np.cumsum(np.flip(fg_hist), axis=0)\n bg_w_thrs = np.cumsum(np.flip(bg_hist), axis=0)\n\n TPs = fg_w_thrs\n Ps = fg_w_thrs + bg_w_thrs\n\n Ps[Ps == 0] = 1\n T = max(np.count_nonzero(gt), 1)\n\n precisions = TPs / Ps\n recalls = TPs / T\n\n numerator = (1 + self.beta) * precisions * recalls\n denominator = np.where(numerator == 0, 1, self.beta * precisions + recalls)\n changeable_fms = numerator / denominator\n return precisions, recalls, changeable_fms\n\n def get_results(self) -> dict:\n \"\"\"\n Return the results about F-measure.\n :return: dict(fm=dict(adp=adaptive_fm, curve=changeable_fm), pr=dict(p=precision, r=recall))\n \"\"\"\n adaptive_fm = np.mean(np.array(self.adaptive_fms, _TYPE))\n changeable_fm = np.mean(np.array(self.changeable_fms, dtype=_TYPE), axis=0)\n precision = np.mean(np.array(self.precisions, dtype=_TYPE), axis=0) # N, 256\n recall = np.mean(np.array(self.recalls, dtype=_TYPE), axis=0) # N, 256\n return dict(fm=dict(adp=adaptive_fm, curve=changeable_fm), pr=dict(p=precision, r=recall))\n\n\nclass MAE_SOD(object):\n def __init__(self):\n \"\"\"\n MAE(mean absolute error) for SOD.\n ::\n @inproceedings{MAE,\n title={Saliency filters: Contrast based filtering for salient region detection},\n author={Perazzi, Federico and Kr{\\\"a}henb{\\\"u}hl, Philipp and Pritch, Yael and Hornung, Alexander},\n booktitle=CVPR,\n pages={733--740},\n year={2012}\n }\n \"\"\"\n self.maes = []\n\n def step(self, pred: np.ndarray, gt: np.ndarray):\n pred, gt = _prepare_data(pred, gt)\n\n mae = self.cal_mae(pred, gt)\n # mae = np.sum(cv2.absdiff(gt.astype(float), pred.astype(float))) / (pred.shape[1] * pred.shape[0])\n self.maes.append(mae)\n\n def cal_mae(self, pred: np.ndarray, gt: np.ndarray) -> np.ndarray:\n \"\"\"\n Calculate the mean absolute error.\n :return: mae\n \"\"\"\n mae = np.mean(np.abs(pred - gt))\n return mae\n\n def get_results(self) -> dict:\n \"\"\"\n Return the results about MAE.\n :return: dict(mae=mae)\n \"\"\"\n mae = np.mean(np.array(self.maes, _TYPE))\n return dict(mae=mae)\n\n\nclass Smeasure(object):\n def __init__(self, alpha: float = 0.5):\n \"\"\"\n S-measure(Structure-measure) of SOD.\n ::\n @inproceedings{Smeasure,\n title={Structure-measure: A new way to eval foreground maps},\n author={Fan, Deng-Ping and Cheng, Ming-Ming and Liu, Yun and Li, Tao and Borji, Ali},\n booktitle=ICCV,\n pages={4548--4557},\n year={2017}\n }\n :param alpha: the weight for balancing the object score and the region score\n \"\"\"\n self.sms = []\n self.alpha = alpha\n\n def step(self, pred: np.ndarray, gt: np.ndarray):\n pred, gt = _prepare_data(pred=pred, gt=gt)\n\n sm = self.cal_sm(pred, gt)\n self.sms.append(sm)\n\n def cal_sm(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the S-measure.\n :return: s-measure\n \"\"\"\n y = np.mean(gt)\n if y == 0:\n sm = 1 - np.mean(pred)\n elif y == 1:\n sm = np.mean(pred)\n else:\n sm = self.alpha * self.object(pred, gt) + (1 - self.alpha) * self.region(pred, gt)\n sm = max(0, sm)\n return sm\n\n def object(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the object score.\n \"\"\"\n fg = pred * gt\n bg = (1 - pred) * (1 - gt)\n u = np.mean(gt)\n object_score = u * self.s_object(fg, gt) + (1 - u) * self.s_object(bg, 1 - gt)\n return object_score\n\n def s_object(self, pred: np.ndarray, gt: np.ndarray) -> float:\n x = np.mean(pred[gt == 1])\n sigma_x = np.std(pred[gt == 1], ddof=1)\n score = 2 * x / (np.power(x, 2) + 1 + sigma_x + _EPS)\n return score\n\n def region(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the region score.\n \"\"\"\n x, y = self.centroid(gt)\n part_info = self.divide_with_xy(pred, gt, x, y)\n w1, w2, w3, w4 = part_info[\"weight\"]\n # assert np.isclose(w1 + w2 + w3 + w4, 1), (w1 + w2 + w3 + w4, pred.mean(), gt.mean())\n\n pred1, pred2, pred3, pred4 = part_info[\"pred\"]\n gt1, gt2, gt3, gt4 = part_info[\"gt\"]\n score1 = self.ssim(pred1, gt1)\n score2 = self.ssim(pred2, gt2)\n score3 = self.ssim(pred3, gt3)\n score4 = self.ssim(pred4, gt4)\n\n return w1 * score1 + w2 * score2 + w3 * score3 + w4 * score4\n\n def centroid(self, matrix: np.ndarray) -> tuple:\n \"\"\"\n To ensure consistency with the matlab code, one is added to the centroid coordinate,\n so there is no need to use the redundant addition operation when dividing the region later,\n because the sequence generated by ``1:X`` in matlab will contain ``X``.\n :param matrix: a bool data array\n :return: the centroid coordinate\n \"\"\"\n h, w = matrix.shape\n area_object = np.count_nonzero(matrix)\n if area_object == 0:\n x = np.round(w / 2)\n y = np.round(h / 2)\n else:\n # More details can be found at: https://www.yuque.com/lart/blog/gpbigm\n y, x = np.argwhere(matrix).mean(axis=0).round()\n return int(x) + 1, int(y) + 1\n\n def divide_with_xy(self, pred: np.ndarray, gt: np.ndarray, x: int, y: int) -> dict:\n \"\"\"\n Use (x,y) to divide the ``pred`` and the ``gt`` into four submatrices, respectively.\n \"\"\"\n h, w = gt.shape\n area = h * w\n\n gt_LT = gt[0:y, 0:x]\n gt_RT = gt[0:y, x:w]\n gt_LB = gt[y:h, 0:x]\n gt_RB = gt[y:h, x:w]\n\n pred_LT = pred[0:y, 0:x]\n pred_RT = pred[0:y, x:w]\n pred_LB = pred[y:h, 0:x]\n pred_RB = pred[y:h, x:w]\n\n w1 = x * y / area\n w2 = y * (w - x) / area\n w3 = (h - y) * x / area\n w4 = 1 - w1 - w2 - w3\n\n return dict(\n gt=(gt_LT, gt_RT, gt_LB, gt_RB),\n pred=(pred_LT, pred_RT, pred_LB, pred_RB),\n weight=(w1, w2, w3, w4),\n )\n\n def ssim(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the ssim score.\n \"\"\"\n h, w = pred.shape\n N = h * w\n\n x = np.mean(pred)\n y = np.mean(gt)\n\n sigma_x = np.sum((pred - x) ** 2) / (N - 1)\n sigma_y = np.sum((gt - y) ** 2) / (N - 1)\n sigma_xy = np.sum((pred - x) * (gt - y)) / (N - 1)\n\n alpha = 4 * x * y * sigma_xy\n beta = (x**2 + y**2) * (sigma_x + sigma_y)\n\n if alpha != 0:\n score = alpha / (beta + _EPS)\n elif alpha == 0 and beta == 0:\n score = 1\n else:\n score = 0\n return score\n\n def get_results(self) -> dict:\n \"\"\"\n Return the results about S-measure.\n :return: dict(sm=sm)\n \"\"\"\n sm = np.mean(np.array(self.sms, dtype=_TYPE))\n return dict(sm=sm)\n\n\nclass Emeasure(object):\n def __init__(self):\n \"\"\"\n E-measure(Enhanced-alignment Measure) for SOD.\n More details about the implementation can be found in https://www.yuque.com/lart/blog/lwgt38\n ::\n @inproceedings{Emeasure,\n title=\"Enhanced-alignment Measure for Binary Foreground Map Evaluation\",\n author=\"Deng-Ping {Fan} and Cheng {Gong} and Yang {Cao} and Bo {Ren} and Ming-Ming {Cheng} and Ali {Borji}\",\n booktitle=IJCAI,\n pages=\"698--704\",\n year={2018}\n }\n \"\"\"\n self.adaptive_ems = []\n self.changeable_ems = []\n\n def step(self, pred: np.ndarray, gt: np.ndarray):\n pred, gt = _prepare_data(pred=pred, gt=gt)\n\n self.gt_fg_numel = np.count_nonzero(gt)\n self.gt_size = gt.shape[0] * gt.shape[1]\n\n changeable_ems = self.cal_changeable_em(pred, gt)\n self.changeable_ems.append(changeable_ems)\n adaptive_em = self.cal_adaptive_em(pred, gt)\n self.adaptive_ems.append(adaptive_em)\n\n def cal_adaptive_em(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the adaptive E-measure.\n :return: adaptive_em\n \"\"\"\n adaptive_threshold = _get_adaptive_threshold(pred, max_value=1)\n adaptive_em = self.cal_em_with_threshold(pred, gt, threshold=adaptive_threshold)\n return adaptive_em\n\n def cal_changeable_em(self, pred: np.ndarray, gt: np.ndarray) -> np.ndarray:\n \"\"\"\n Calculate the changeable E-measure, which can be used to obtain the mean E-measure,\n the maximum E-measure and the E-measure-threshold curve.\n :return: changeable_ems\n \"\"\"\n changeable_ems = self.cal_em_with_cumsumhistogram(pred, gt)\n return changeable_ems\n\n def cal_em_with_threshold(self, pred: np.ndarray, gt: np.ndarray, threshold: float) -> float:\n \"\"\"\n Calculate the E-measure corresponding to the specific threshold.\n Variable naming rules within the function:\n ``[pred attribute(foreground fg, background bg)]_[gt attribute(foreground fg, background bg)]_[meaning]``\n If only ``pred`` or ``gt`` is considered, another corresponding attribute location is replaced with '``_``'.\n \"\"\"\n binarized_pred = pred >= threshold\n fg_fg_numel = np.count_nonzero(binarized_pred & gt)\n fg_bg_numel = np.count_nonzero(binarized_pred & ~gt)\n\n fg___numel = fg_fg_numel + fg_bg_numel\n bg___numel = self.gt_size - fg___numel\n\n if self.gt_fg_numel == 0:\n enhanced_matrix_sum = bg___numel\n elif self.gt_fg_numel == self.gt_size:\n enhanced_matrix_sum = fg___numel\n else:\n parts_numel, combinations = self.generate_parts_numel_combinations(\n fg_fg_numel=fg_fg_numel,\n fg_bg_numel=fg_bg_numel,\n pred_fg_numel=fg___numel,\n pred_bg_numel=bg___numel,\n )\n\n results_parts = []\n for i, (part_numel, combination) in enumerate(zip(parts_numel, combinations)):\n align_matrix_value = (\n 2 * (combination[0] * combination[1]) / (combination[0] ** 2 + combination[1] ** 2 + _EPS)\n )\n enhanced_matrix_value = (align_matrix_value + 1) ** 2 / 4\n results_parts.append(enhanced_matrix_value * part_numel)\n enhanced_matrix_sum = sum(results_parts)\n\n em = enhanced_matrix_sum / (self.gt_size - 1 + _EPS)\n return em\n\n def cal_em_with_cumsumhistogram(self, pred: np.ndarray, gt: np.ndarray) -> np.ndarray:\n \"\"\"\n Calculate the E-measure corresponding to the threshold that varies from 0 to 255..\n Variable naming rules within the function:\n ``[pred attribute(foreground fg, background bg)]_[gt attribute(foreground fg, background bg)]_[meaning]``\n If only ``pred`` or ``gt`` is considered, another corresponding attribute location is replaced with '``_``'.\n \"\"\"\n pred = (pred * 255).astype(np.uint8)\n bins = np.linspace(0, 256, 257)\n fg_fg_hist, _ = np.histogram(pred[gt], bins=bins)\n fg_bg_hist, _ = np.histogram(pred[~gt], bins=bins)\n fg_fg_numel_w_thrs = np.cumsum(np.flip(fg_fg_hist), axis=0)\n fg_bg_numel_w_thrs = np.cumsum(np.flip(fg_bg_hist), axis=0)\n\n fg___numel_w_thrs = fg_fg_numel_w_thrs + fg_bg_numel_w_thrs\n bg___numel_w_thrs = self.gt_size - fg___numel_w_thrs\n\n if self.gt_fg_numel == 0:\n enhanced_matrix_sum = bg___numel_w_thrs\n elif self.gt_fg_numel == self.gt_size:\n enhanced_matrix_sum = fg___numel_w_thrs\n else:\n parts_numel_w_thrs, combinations = self.generate_parts_numel_combinations(\n fg_fg_numel=fg_fg_numel_w_thrs,\n fg_bg_numel=fg_bg_numel_w_thrs,\n pred_fg_numel=fg___numel_w_thrs,\n pred_bg_numel=bg___numel_w_thrs,\n )\n\n results_parts = np.empty(shape=(4, 256), dtype=np.float64)\n for i, (part_numel, combination) in enumerate(zip(parts_numel_w_thrs, combinations)):\n align_matrix_value = (\n 2 * (combination[0] * combination[1]) / (combination[0] ** 2 + combination[1] ** 2 + _EPS)\n )\n enhanced_matrix_value = (align_matrix_value + 1) ** 2 / 4\n results_parts[i] = enhanced_matrix_value * part_numel\n enhanced_matrix_sum = results_parts.sum(axis=0)\n\n em = enhanced_matrix_sum / (self.gt_size - 1 + _EPS)\n return em\n\n def generate_parts_numel_combinations(self, fg_fg_numel, fg_bg_numel, pred_fg_numel, pred_bg_numel):\n bg_fg_numel = self.gt_fg_numel - fg_fg_numel\n bg_bg_numel = pred_bg_numel - bg_fg_numel\n\n parts_numel = [fg_fg_numel, fg_bg_numel, bg_fg_numel, bg_bg_numel]\n\n mean_pred_value = pred_fg_numel / self.gt_size\n mean_gt_value = self.gt_fg_numel / self.gt_size\n\n demeaned_pred_fg_value = 1 - mean_pred_value\n demeaned_pred_bg_value = 0 - mean_pred_value\n demeaned_gt_fg_value = 1 - mean_gt_value\n demeaned_gt_bg_value = 0 - mean_gt_value\n\n combinations = [\n (demeaned_pred_fg_value, demeaned_gt_fg_value),\n (demeaned_pred_fg_value, demeaned_gt_bg_value),\n (demeaned_pred_bg_value, demeaned_gt_fg_value),\n (demeaned_pred_bg_value, demeaned_gt_bg_value),\n ]\n return parts_numel, combinations\n\n def get_results(self) -> dict:\n \"\"\"\n Return the results about E-measure.\n :return: dict(em=dict(adp=adaptive_em, curve=changeable_em))\n \"\"\"\n adaptive_em = np.mean(np.array(self.adaptive_ems, dtype=_TYPE))\n changeable_em = np.mean(np.array(self.changeable_ems, dtype=_TYPE), axis=0)\n return dict(em=dict(adp=adaptive_em, curve=changeable_em))\n\n\nclass WeightedFmeasure(object):\n def __init__(self, beta: float = 0.3):\n \"\"\"\n Weighted F-measure for SOD.\n ::\n @inproceedings{wFmeasure,\n title={How to eval foreground maps?},\n author={Margolin, Ran and Zelnik-Manor, Lihi and Tal, Ayellet},\n booktitle=CVPR,\n pages={248--255},\n year={2014}\n }\n :param beta: the weight of the precision\n \"\"\"\n self.beta = beta\n self.weighted_fms = []\n\n def step(self, pred: np.ndarray, gt: np.ndarray):\n pred, gt = _prepare_data(pred=pred, gt=gt)\n\n if np.all(~gt):\n wfm = 0\n else:\n wfm = self.cal_wfm(pred, gt)\n self.weighted_fms.append(wfm)\n\n def cal_wfm(self, pred: np.ndarray, gt: np.ndarray) -> float:\n \"\"\"\n Calculate the weighted F-measure.\n \"\"\"\n # [Dst,IDXT] = bwdist(dGT);\n Dst, Idxt = bwdist(gt == 0, return_indices=True)\n\n # %Pixel dependency\n # E = abs(FG-dGT);\n E = np.abs(pred - gt)\n # Et = E;\n # Et(~GT)=Et(IDXT(~GT)); %To deal correctly with the edges of the foreground region\n Et = np.copy(E)\n Et[gt == 0] = Et[Idxt[0][gt == 0], Idxt[1][gt == 0]]\n\n # K = fspecial('gaussian',7,5);\n # EA = imfilter(Et,K);\n K = self.matlab_style_gauss2D((7, 7), sigma=5)\n EA = convolve(Et, weights=K, mode=\"constant\", cval=0)\n # MIN_E_EA = E;\n # MIN_E_EA(GT & EA np.ndarray:\n \"\"\"\n 2D gaussian mask - should give the same result as MATLAB's\n fspecial('gaussian',[shape],[sigma])\n \"\"\"\n m, n = [(ss - 1) / 2 for ss in shape]\n y, x = np.ogrid[-m : m + 1, -n : n + 1]\n h = np.exp(-(x * x + y * y) / (2 * sigma * sigma))\n h[h < np.finfo(h.dtype).eps * h.max()] = 0\n sumh = h.sum()\n if sumh != 0:\n h /= sumh\n return h\n\n def get_results(self) -> dict:\n \"\"\"\n Return the results about weighted F-measure.\n :return: dict(wfm=weighted_fm)\n \"\"\"\n weighted_fm = np.mean(np.array(self.weighted_fms, dtype=_TYPE))\n return dict(wfm=weighted_fm)\n\n\nclass Multiclass_IoU(torchmetrics.Metric):\n \"\"\"\n Compute the IoU for multi-class semantic segmentation based on https://github.com/xieenze/Trans2Seg/blob/master/segmentron/utils/score.py.\n The direct use of torchmetrics for large dataset will lead to issues such as high CPU usage or insufficient memory.\n \"\"\"\n\n def __init__(self, num_classes):\n super().__init__()\n self.add_state(\"total_inter\", default=torch.zeros(num_classes), dist_reduce_fx=None)\n self.add_state(\"total_union\", default=torch.zeros(num_classes), dist_reduce_fx=None)\n self.num_classes = num_classes\n\n def update(self, logits, labels):\n inter, union = self.batch_intersection_union(logits, labels)\n self.total_inter += inter\n self.total_union += union\n\n def compute(self):\n IoU = 1.0 * self.total_inter / (2.220446049250313e-16 + self.total_union)\n return torch.tensor(IoU.mean().item())\n\n def batch_intersection_union(self, output, target):\n mini = 1\n maxi = self.num_classes\n nbins = self.num_classes\n predict = torch.argmax(output, 1) + 1\n target = target.float() + 1\n\n predict = predict.float() * (target > 0).float()\n intersection = predict * (predict == target).float()\n # areas of intersection and union\n area_inter = torch.histc(intersection, bins=nbins, min=mini, max=maxi)\n area_pred = torch.histc(predict, bins=nbins, min=mini, max=maxi)\n area_lab = torch.histc(target, bins=nbins, min=mini, max=maxi)\n area_union = area_pred + area_lab - area_inter\n assert torch.sum(area_inter > area_union).item() == 0, \"Intersection area should be smaller than Union area\"\n return area_inter.float(), area_union.float()\n\n\nclass Binary_IoU(torchmetrics.Metric):\n \"\"\"\n Compute the IoU for binary semantic segmentation. The direct use of torchmetrics to calculate IoU for multiple samples does not yield accurate results.\n So we iteratively calculate metric values and then take the average.\n \"\"\"\n\n def __init__(\n self,\n ):\n super().__init__()\n self.add_state(\"logits\", default=[], dist_reduce_fx=None)\n self.add_state(\"labels\", default=[], dist_reduce_fx=None)\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n logits = torch.cat(self.logits).cpu()\n labels = torch.cat(self.labels).cpu()\n\n res_list = []\n metric = torchmetrics.JaccardIndex(task=\"binary\")\n for logit, label in zip(logits, labels):\n res_list.append(metric(logit, label))\n return torch.mean(torch.tensor(res_list))\n\n\nclass Balanced_Error_Rate(torchmetrics.Metric):\n \"\"\"\n Compute the balanced error rate.\n \"\"\"\n\n def __init__(\n self,\n ):\n super().__init__()\n self.add_state(\"logits\", default=[], dist_reduce_fx=None)\n self.add_state(\"labels\", default=[], dist_reduce_fx=None)\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n logits = torch.cat(self.logits).cpu()\n labels = torch.cat(self.labels).cpu()\n\n labels = (labels * 255) > 125\n logits = (logits * 255) > 125\n ber = 1 - torchmetrics.Accuracy(\n task=\"multiclass\", num_classes=2, average=\"macro\", multidim_average=\"samplewise\"\n )(logits, labels)\n return torch.mean(ber)\n\n\nclass COD(torchmetrics.Metric):\n def __init__(self):\n super().__init__()\n self.add_state(\"logits\", default=[], dist_reduce_fx=None)\n self.add_state(\"labels\", default=[], dist_reduce_fx=None)\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n pass\n\n\nclass SM(COD):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_SM = Smeasure()\n\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n metric_SM.step(pred=pred, gt=true)\n\n return torch.tensor(metric_SM.get_results()[\"sm\"])\n\n\nclass FM(COD):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_WFM = WeightedFmeasure()\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_WFM.step(pred=pred, gt=true)\n\n return torch.tensor(metric_WFM.get_results()[\"wfm\"])\n\n\nclass EM(COD):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_EM = Emeasure()\n\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_EM.step(pred=pred, gt=true)\n\n return torch.tensor(metric_EM.get_results()[\"em\"][\"curve\"].mean())\n\n\nclass MAE(COD):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_MAE = MAE_SOD()\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_MAE.step(pred=pred, gt=true)\n\n return torch.tensor(metric_MAE.get_results()[\"mae\"])\n\n\nCOD_METRICS_NAMES = {\"sm\": SM(), \"fm\": FM(), \"em\": EM(), \"mae\": MAE()}\n\n\n# TODO: Modify multi-gpu evaluation error. Maybe there will be a more elegant way.\nclass Multiclass_IoU_Pred:\n \"\"\"\n Compute the IoU for multi-class semantic segmentation based on https://github.com/xieenze/Trans2Seg/blob/master/segmentron/utils/score.py.\n The direct use of torchmetrics for large dataset will lead to issues such as high CPU usage or insufficient memory.\n \"\"\"\n\n def __init__(self, num_classes):\n super().__init__()\n self.total_inter = torch.zeros(num_classes)\n self.total_union = torch.zeros(num_classes)\n self.num_classes = num_classes\n\n def update(self, logits, labels):\n inter, union = self.batch_intersection_union(logits, labels)\n self.total_inter += inter\n self.total_union += union\n\n def compute(self):\n IoU = 1.0 * self.total_inter / (2.220446049250313e-16 + self.total_union)\n return torch.tensor(IoU.mean().item())\n\n def batch_intersection_union(self, output, target):\n mini = 1\n maxi = self.num_classes\n nbins = self.num_classes\n predict = torch.argmax(output, 1) + 1\n target = target.float() + 1\n\n predict = predict.float() * (target > 0).float()\n intersection = predict * (predict == target).float()\n # areas of intersection and union\n area_inter = torch.histc(intersection, bins=nbins, min=mini, max=maxi)\n area_pred = torch.histc(predict, bins=nbins, min=mini, max=maxi)\n area_lab = torch.histc(target, bins=nbins, min=mini, max=maxi)\n area_union = area_pred + area_lab - area_inter\n assert torch.sum(area_inter > area_union).item() == 0, \"Intersection area should be smaller than Union area\"\n return area_inter.float(), area_union.float()\n\n\nclass Binary_IoU_Pred:\n \"\"\"\n Compute the IoU for binary semantic segmentation. The direct use of torchmetrics to calculate IoU for multiple samples does not yield accurate results.\n So we iteratively calculate metric values and then take the average.\n \"\"\"\n\n def __init__(\n self,\n ):\n super().__init__()\n self.logits = []\n self.labels = []\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n logits = torch.cat(self.logits).cpu()\n labels = torch.cat(self.labels).cpu()\n\n res_list = []\n metric = torchmetrics.JaccardIndex(task=\"binary\")\n for logit, label in zip(logits, labels):\n res_list.append(metric(logit, label))\n return torch.mean(torch.tensor(res_list))\n\n\nclass Balanced_Error_Rate_Pred:\n \"\"\"\n Compute the balanced error rate.\n \"\"\"\n\n def __init__(\n self,\n ):\n super().__init__()\n self.logits = []\n self.labels = []\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n logits = torch.cat(self.logits).cpu()\n labels = torch.cat(self.labels).cpu()\n\n labels = (labels * 255) > 125\n logits = (logits * 255) > 125\n ber = 1 - torchmetrics.Accuracy(\n task=\"multiclass\", num_classes=2, average=\"macro\", multidim_average=\"samplewise\"\n )(logits, labels)\n return torch.mean(ber)\n\n\nclass COD_Pred:\n def __init__(self):\n super().__init__()\n self.logits = []\n self.labels = []\n\n def update(self, logits, labels):\n self.logits.append(logits)\n self.labels.append(labels)\n\n def compute(self):\n pass\n\n def reset(self):\n self.logits = []\n self.labels = []\n\n\nclass SM_Pred(COD_Pred):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n\n batchsize = labels.shape[0]\n\n metric_SM = Smeasure()\n\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n metric_SM.step(pred=pred, gt=true)\n self.reset()\n return torch.tensor(metric_SM.get_results()[\"sm\"])\n\n\nclass FM_Pred(COD_Pred):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_WFM = WeightedFmeasure()\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_WFM.step(pred=pred, gt=true)\n self.reset()\n return torch.tensor(metric_WFM.get_results()[\"wfm\"])\n\n\nclass EM_Pred(COD_Pred):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_EM = Emeasure()\n\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_EM.step(pred=pred, gt=true)\n self.reset()\n return torch.tensor(metric_EM.get_results()[\"em\"][\"curve\"].mean())\n\n\nclass MAE_Pred(COD_Pred):\n def compute(self):\n logits = torch.cat(self.logits)\n labels = torch.cat(self.labels)\n assert logits.shape == labels.shape\n batchsize = labels.shape[0]\n\n metric_MAE = MAE_SOD()\n for i in range(batchsize):\n true, pred = labels[i, 0].cpu().data.numpy() * 255, logits[i, 0].cpu().data.numpy() * 255\n\n metric_MAE.step(pred=pred, gt=true)\n self.reset()\n return torch.tensor(metric_MAE.get_results()[\"mae\"])\n\n\nCOD_METRICS_NAMES_Pred = {\"sm\": SM_Pred(), \"fm\": FM_Pred(), \"em\": EM_Pred(), \"mae\": MAE_Pred()}\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/metrics/utils.py",
"content": "import functools\nimport logging\nimport warnings\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport evaluate\nimport torchmetrics\nfrom torch.nn import functional as F\nfrom torchmetrics.detection.mean_ap import MeanAveragePrecision\n\nfrom autogluon.core.metrics import Scorer, compute_metric, get_metric\n\nfrom ...constants import (\n ACC,\n ACCURACY,\n AVERAGE_PRECISION,\n BER,\n COSINE_EMBEDDING_LOSS,\n COVERAGE,\n CROSS_ENTROPY,\n DETECTION_METRICS,\n DIRECT_LOSS,\n EM,\n F1,\n F1_MACRO,\n F1_MICRO,\n F1_WEIGHTED,\n FM,\n HIT_RATE,\n IOU,\n LOG_LOSS,\n MAE,\n MATCHING_METRICS,\n MATCHING_METRICS_WITHOUT_PROBLEM_TYPE,\n MAX,\n METRIC_MODE_MAP,\n MIN,\n MULTICLASS,\n NER_TOKEN_F1,\n OVERALL_ACCURACY,\n OVERALL_F1,\n PEARSONR,\n QUADRATIC_KAPPA,\n R2,\n RECALL,\n RETRIEVAL_METRICS,\n RMSE,\n ROC_AUC,\n ROOT_MEAN_SQUARED_ERROR,\n SM,\n SPEARMANR,\n Y_PRED,\n Y_PRED_PROB,\n Y_TRUE,\n)\nfrom .coverage_metrics import Coverage\nfrom .hit_rate_metrics import CustomHitRate\nfrom .semantic_seg_metrics import COD_METRICS_NAMES, Balanced_Error_Rate, Binary_IoU, Multiclass_IoU\n\nlogger = logging.getLogger(__name__)\n\n\ndef compute_score(\n metric_data: dict,\n metric: Union[str, Scorer],\n pos_label: Optional[int] = 1,\n) -> float:\n \"\"\"\n Use sklearn to compute the score of one metric.\n\n Parameters\n ----------\n metric_data\n A dictionary with the groundtruth (Y_TRUE) and predicted values (Y_PRED, Y_PRED_PROB).\n The predicted class probabilities are required to compute the roc_auc score.\n metric\n The name of metric or the function of metric to compute.\n pos_label\n The encoded label (0 or 1) of binary classification's positive class.\n\n Returns\n -------\n Computed score.\n \"\"\"\n if isinstance(metric, str) and metric in [OVERALL_ACCURACY, OVERALL_F1]:\n metric = evaluate.load(\"seqeval\")\n warnings.filterwarnings(\"ignore\")\n for p in metric_data[Y_TRUE]:\n if \"_\" in p:\n print(p)\n for p in metric_data[Y_PRED]:\n if \"_\" in p:\n print(p)\n return metric.compute(references=metric_data[Y_TRUE], predictions=metric_data[Y_PRED])\n\n metric = get_metric(metric)\n\n y = metric_data[Y_TRUE]\n if metric.needs_proba or metric.needs_threshold:\n y_pred_proba = metric_data[Y_PRED_PROB]\n y_pred_proba = (\n y_pred_proba if y_pred_proba.shape[1] > 2 else y_pred_proba[:, pos_label]\n ) # only use pos_label for binary classification\n return metric.convert_score_to_original(\n compute_metric(y=y, y_pred_proba=y_pred_proba, metric=metric, weights=None)\n )\n else:\n y_pred = metric_data[Y_PRED]\n\n # TODO: This is a hack. Doesn't support `f1_macro`, `f1_micro`, `f1_weighted`, or custom `f1` metrics with different names.\n # TODO: Longterm the solution should be to have the input data to this function use the internal representation without the original class names. This way `pos_label` would not need to be specified.\n if metric.name == F1: # only for binary classification\n y = (y == pos_label).astype(int)\n y_pred = (y_pred == pos_label).astype(int)\n\n return metric.convert_score_to_original(compute_metric(y=y, y_pred=y_pred, metric=metric, weights=None))\n\n\ndef infer_metrics(\n problem_type: Optional[str] = None,\n eval_metric: Optional[Union[str, Scorer]] = None,\n validation_metric_name: Optional[str] = None,\n is_matching: Optional[bool] = False,\n):\n \"\"\"\n Infer the validation metric and the evaluation metric if not provided.\n Validation metric is for early-stopping and selecting the best model checkpoints.\n Evaluation metric is to report performance to users.\n\n Parameters\n ----------\n problem_type\n Type of problem.\n eval_metric\n Name of evaluation metric provided by users.\n validation_metric_name\n The provided validation metric name\n is_matching\n Whether is matching.\n\n Returns\n -------\n validation_metric_name\n Name of validation metric.\n eval_metric_name\n Name of evaluation metric.\n \"\"\"\n is_customized = False\n if eval_metric is None:\n eval_metric_name = None\n elif isinstance(eval_metric, str):\n eval_metric_name = eval_metric\n elif isinstance(eval_metric, Scorer):\n eval_metric_name = eval_metric.name\n is_customized = True\n else:\n raise TypeError(f\"eval_metric can be a str, a Scorer, or None, but is type: {type(eval_metric)}\")\n\n if problem_type is not None:\n from ...utils.problem_types import PROBLEM_TYPES_REG\n\n problem_property = PROBLEM_TYPES_REG.get(problem_type)\n\n if is_matching:\n if eval_metric_name is not None:\n # if eval_metric_name is a valid metric\n if eval_metric_name.lower() in METRIC_MODE_MAP.keys():\n validation_metric_name = eval_metric_name\n return validation_metric_name, eval_metric_name\n elif eval_metric_name.lower() in RETRIEVAL_METRICS:\n # Currently only support recall as validation metric in retrieval tasks.\n validation_metric_name = RECALL\n return validation_metric_name, eval_metric_name\n\n # When eval_metric_name is either None or not supported:\n # Fallback based on problem type unless it's a customized metric\n if problem_type is None:\n validation_metric_name, fallback_evaluation_metric = MATCHING_METRICS_WITHOUT_PROBLEM_TYPE\n elif problem_type in MATCHING_METRICS:\n validation_metric_name, fallback_evaluation_metric = MATCHING_METRICS[problem_type]\n else:\n raise NotImplementedError(f\"Problem type: {problem_type} is not yet supported for matching!\")\n if not is_customized:\n if eval_metric_name is not None:\n warnings.warn(\n f\"Metric {eval_metric_name} is not supported as the evaluation metric for {problem_type} in matching tasks.\"\n f\"The evaluation metric is changed to {fallback_evaluation_metric} by default.\"\n )\n eval_metric_name = fallback_evaluation_metric\n return validation_metric_name, eval_metric_name\n\n if eval_metric_name is not None:\n # Infer evaluation metric\n if eval_metric_name.lower() not in problem_property.supported_evaluation_metrics and not is_customized:\n warnings.warn(\n f\"Metric {eval_metric_name} is not supported as the evaluation metric for {problem_type}. \"\n f\"The evaluation metric is changed to {problem_property.fallback_evaluation_metric} by default.\"\n )\n if problem_property.fallback_evaluation_metric is not None:\n eval_metric_name = problem_property.fallback_evaluation_metric\n else:\n # Problem types like extract_embedding does not need a eval/val metric\n return None, None\n\n # Infer validation metric\n if eval_metric_name.lower() in problem_property.supported_validation_metrics:\n validation_metric_name = eval_metric_name\n else:\n if problem_property.fallback_validation_metric is not None:\n validation_metric_name = problem_property.fallback_validation_metric\n else:\n eval_metric_name = problem_property.fallback_evaluation_metric\n validation_metric_name = problem_property.fallback_validation_metric\n\n return validation_metric_name, eval_metric_name\n\n\ndef get_minmax_mode(\n metric_name: Union[str, Scorer],\n):\n \"\"\"\n Get minmax mode based on metric name\n\n Parameters\n ----------\n metric_name\n A string representing metric\n\n Returns\n -------\n mode\n The min/max mode used in selecting model checkpoints.\n - min\n Its means that smaller metric is better.\n - max\n It means that larger metric is better.\n \"\"\"\n if isinstance(metric_name, str):\n assert metric_name in METRIC_MODE_MAP, (\n f\"{metric_name} is not a supported metric. Options are: {METRIC_MODE_MAP.keys()}\"\n )\n return METRIC_MODE_MAP.get(metric_name)\n else:\n return MAX if metric_name.greater_is_better else MIN\n\n\ndef get_stopping_threshold(metric_name: str):\n \"\"\"\n Get the metric threshold for early stopping.\n\n Parameters\n ----------\n metric_name\n Name of validation metric.\n\n Returns\n -------\n The stopping threshold.\n \"\"\"\n try:\n metric = get_metric(metric_name)\n stopping_threshold = metric.optimum - metric._sign * 1e-7\n except:\n stopping_threshold = None\n\n return stopping_threshold\n\n\ndef get_torchmetric(\n metric_name: str,\n num_classes: Optional[int] = None,\n is_matching: Optional[bool] = False,\n problem_type: Optional[str] = None,\n):\n \"\"\"\n Obtain a torchmerics.Metric from its name.\n Define a customized metric function in case that torchmetrics doesn't support some metric.\n\n Parameters\n ----------\n metric_name\n Name of metric.\n num_classes\n Number of classes.\n is_matching\n Whether is matching.\n problem_type\n Type of problem, e.g., binary and multiclass.\n\n Returns\n -------\n torchmetrics.Metric\n A torchmetrics.Metric object.\n custom_metric_func\n A customized metric function.\n \"\"\"\n metric_name = metric_name.lower()\n if metric_name in [ACC, ACCURACY, OVERALL_ACCURACY]:\n # use MULTICLASS since the head output dim is 2 for the binary problem type.\n return torchmetrics.Accuracy(task=MULTICLASS, num_classes=num_classes), None\n elif metric_name == NER_TOKEN_F1:\n return torchmetrics.F1Score(task=MULTICLASS, num_classes=num_classes, ignore_index=1), None\n elif metric_name in [RMSE, ROOT_MEAN_SQUARED_ERROR]:\n return torchmetrics.MeanSquaredError(squared=False), None\n elif metric_name == R2:\n return torchmetrics.R2Score(), None\n elif metric_name == QUADRATIC_KAPPA:\n return (\n torchmetrics.CohenKappa(task=problem_type, num_classes=num_classes, weights=\"quadratic\"),\n None,\n )\n elif metric_name == ROC_AUC:\n return torchmetrics.AUROC(task=problem_type, num_classes=num_classes), None\n elif metric_name == AVERAGE_PRECISION:\n return torchmetrics.AveragePrecision(task=problem_type, num_classes=num_classes)\n elif metric_name in [LOG_LOSS, CROSS_ENTROPY]:\n return torchmetrics.MeanMetric(), functools.partial(F.cross_entropy, reduction=\"none\")\n elif metric_name == COSINE_EMBEDDING_LOSS:\n return torchmetrics.MeanMetric(), functools.partial(F.cosine_embedding_loss, reduction=\"none\")\n elif metric_name == PEARSONR:\n return torchmetrics.PearsonCorrCoef(), None\n elif metric_name == SPEARMANR:\n if is_matching: # TODO: add support for matching.\n raise ValueError(\"spearman relation is not supported for matching yet.\")\n else:\n return torchmetrics.SpearmanCorrCoef(), None\n elif metric_name == F1:\n return torchmetrics.F1Score(task=problem_type, num_classes=num_classes), None\n elif metric_name in [F1_MACRO, F1_MICRO, F1_WEIGHTED]:\n average = metric_name.split(\"_\")[1]\n return torchmetrics.F1Score(task=problem_type, num_classes=num_classes, average=average), None\n elif metric_name in DETECTION_METRICS:\n return (\n MeanAveragePrecision(box_format=\"xyxy\", iou_type=\"bbox\", class_metrics=False),\n None,\n ) # TODO: remove parameter hardcodings here, and add class_metrics\n elif metric_name == DIRECT_LOSS:\n return (\n torchmetrics.MeanMetric(nan_strategy=\"warn\"),\n None,\n ) # This only works for detection where custom_metric is not required for BaseAggregator\n elif metric_name in [RECALL, HIT_RATE]:\n if is_matching:\n return CustomHitRate(), None\n else: # TODO: support recall for general classification tasks.\n raise ValueError(\"Recall is not supported yet.\")\n elif metric_name == BER:\n return Balanced_Error_Rate(), None\n elif metric_name in [SM, EM, FM, MAE]:\n return COD_METRICS_NAMES[metric_name], None\n elif metric_name == IOU:\n if num_classes == 1:\n return Binary_IoU(), None\n else:\n return Multiclass_IoU(num_classes=num_classes), None\n elif metric_name == COVERAGE:\n return Coverage(), None\n else:\n raise ValueError(f\"Unknown metric {metric_name}\")\n"
},
{
"path": "multimodal/src/autogluon/multimodal/optim/utils.py",
"content": "import logging\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import nn, optim\nfrom transformers import Adafactor\nfrom transformers.trainer_pt_utils import get_parameter_names\n\nfrom ..constants import (\n BIT_FIT,\n COLUMN_FEATURES,\n CONV_LORA,\n FEATURES,\n IA3,\n IA3_BIAS,\n IA3_LORA,\n IA3_LORA_BIAS,\n IA3_LORA_NORM,\n IA3_NORM,\n LORA,\n LORA_BIAS,\n LORA_NORM,\n NORM_FIT,\n PEFT_STRATEGIES,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_optimizer(\n optim_type: str,\n optimizer_grouped_parameters,\n lr: float,\n weight_decay: float,\n eps: Optional[float] = 1e-6,\n betas: Optional[Tuple[float, float]] = (0.9, 0.999),\n momentum: Optional[float] = 0.9,\n):\n \"\"\"\n Choose a Pytorch optimizer based on its name.\n\n Parameters\n ----------\n optim_type\n Name of optimizer.\n optimizer_grouped_parameters\n The model parameters to be optimized.\n lr\n Learning rate.\n weight_decay\n Optimizer weight decay.\n eps\n Optimizer eps.\n betas\n Optimizer betas.\n momentum\n Momentum used in the SGD optimizer.\n\n Returns\n -------\n A Pytorch optimizer.\n \"\"\"\n if optim_type == \"adamw\":\n optimizer = optim.AdamW(\n optimizer_grouped_parameters,\n lr=lr,\n weight_decay=weight_decay,\n eps=eps,\n betas=betas,\n )\n elif optim_type == \"adam\":\n optimizer = optim.Adam(\n optimizer_grouped_parameters,\n lr=lr,\n weight_decay=weight_decay,\n )\n elif optim_type == \"sgd\":\n optimizer = optim.SGD(\n optimizer_grouped_parameters,\n lr=lr,\n weight_decay=weight_decay,\n momentum=momentum,\n )\n elif optim_type == \"adafactor\":\n optimizer = Adafactor(\n optimizer_grouped_parameters,\n lr=lr,\n weight_decay=weight_decay,\n scale_parameter=True, # Generally recommended to enable scaling\n relative_step=False,\n warmup_init=False,\n )\n else:\n raise ValueError(f\"unknown optimizer: {optim_type}\")\n\n return optimizer\n\n\ndef get_weight_decay_param_names(model: nn.Module):\n \"\"\"\n Set the layer normalization parameters and other layers' bias parameters not to use weight decay.\n\n Parameters\n ----------\n model\n A Pytorch model.\n\n Returns\n -------\n A list of parameter names not using weight decay.\n \"\"\"\n # By default, we should not apply weight decay for all the norm layers\n decay_param_names = get_parameter_names(\n model,\n [nn.LayerNorm, nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.GroupNorm],\n )\n decay_param_names = [\n name\n for name in decay_param_names\n if (\n \"bias\" not in name\n and \"cls_token\" not in name\n and \"categorical_feature_tokenizer\" not in name\n and \"numerical_feature_tokenizer\" not in name\n )\n ]\n return decay_param_names\n\n\ndef get_norm_layer_param_names(model: nn.Module):\n \"\"\"\n Get parameters associated with the normalization layers\n\n Parameters\n ----------\n model\n A Pytorch model\n\n Returns\n -------\n norm_param_names\n A list of normalization parameter names\n \"\"\"\n all_param_names = [name for name, _ in model.named_parameters()]\n all_param_names_except_norm_names = get_parameter_names(\n model,\n [nn.LayerNorm, nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.GroupNorm],\n )\n norm_param_names = [name for name in all_param_names if name not in all_param_names_except_norm_names]\n return norm_param_names\n\n\ndef get_peft_param_names(norm_param_names: List[str], peft: Optional[str] = None, extra_params: Optional[List] = None):\n \"\"\"\n Get the list of trainable parameters according to the provided efficient finetuning method.\n\n Parameters\n ----------\n norm_param_names\n The parameters associated with the normalization layers\n peft\n Efficient finetuning strategy. Trainable parameters will be adjusted according to the method.\n extra_params\n Extra parameters to train.\n\n Returns\n -------\n Get list of trainable parameter names according to the provided efficient finetuning method.\n \"\"\"\n peft_param_names = []\n\n if peft == BIT_FIT:\n peft_param_names.append(\".*bias*.\")\n elif peft == NORM_FIT:\n peft_param_names.append(\".*bias*.\")\n peft_param_names += norm_param_names\n elif peft in [LORA, IA3, IA3_LORA, CONV_LORA]:\n peft_param_names.append(\".*lora_*.\")\n elif peft in [LORA_BIAS, IA3_BIAS, IA3_LORA_BIAS]:\n peft_param_names.append(\".*lora_*.\")\n peft_param_names.append(\".*bias*.\")\n elif peft in [LORA_NORM, IA3_NORM, IA3_LORA_NORM]:\n peft_param_names.append(\".*lora_*.\")\n peft_param_names.append(\".*bias*.\")\n peft_param_names += norm_param_names\n elif peft is not None:\n raise NotImplementedError(\n f\"The efficient finetuning strategy '{peft}'\"\n f\" is not supported. We only support\"\n f\" {', '.join(PEFT_STRATEGIES)}.\"\n )\n\n if extra_params:\n peft_param_names.extend(extra_params)\n\n return peft_param_names\n\n\ndef remove_parameters_without_grad(\n grouped_parameters: List[Dict],\n):\n \"\"\"\n Remove layers\n\n Parameters\n ----------\n grouped_parameters\n The grouped parameters or their names output from lr_choice.\n\n Returns\n -------\n The updated grouped parameters or their names.\n \"\"\"\n for group_idx, group_param in enumerate(grouped_parameters):\n updated_params = []\n for p in group_param[\"params\"]:\n if p.requires_grad:\n updated_params.append(p)\n grouped_parameters[group_idx][\"params\"] = updated_params\n\n return grouped_parameters\n\n\ndef gather_column_features(\n output: Dict[str, Dict],\n column_names: Union[str, List[str]],\n):\n \"\"\"\n Gather column features from models' outputs.\n For each feature name in one model's output, we enumerate the provided column names to see\n whether (partial) the provided columns share one cls feature or they have independent features.\n\n TODO: return features' masks and use them to filter the losses.\n\n Parameters\n ----------\n output\n The models' outputs.\n column_names\n The columns whose features we want to get.\n\n Returns\n -------\n The gathered feature vectors. Each sample should only have one feature vector.\n \"\"\"\n if isinstance(column_names, str):\n column_names = [column_names]\n\n gathered_features = []\n # logger.debug(f\"gather features for columns: {column_names}\")\n for per_model_name, per_model_output in output.items():\n # logger.debug(f\"gather column features from model: {per_model_name}\")\n for feature_name in per_model_output[COLUMN_FEATURES][FEATURES]:\n # logger.debug(f\"processing feature: {feature_name}\")\n columns_share_one_feature = []\n for col_name in column_names:\n if col_name in feature_name:\n # this column feature is part of the cls feature\n if not (feature_name.startswith(col_name) and feature_name.endswith(col_name)):\n columns_share_one_feature.append(col_name)\n # logger.debug(f\"column {col_name} is included in feature {feature_name}\")\n else: # this column's feature is independent of other columns'\n gathered_features.append(per_model_output[COLUMN_FEATURES][FEATURES][col_name])\n # logger.debug(f\"col_name {col_name} has an independent feature in model: {per_model_name}\")\n\n # two or more columns share one cls feature, and no other columns share it.\n if len(columns_share_one_feature) > 0:\n assert len(\"_\".join(columns_share_one_feature)) == len(feature_name), (\n f\"model `{per_model_name}`'s cls feature name `{feature_name}` doesn't match `{columns_share_one_feature}`\"\n )\n gathered_features.append(per_model_output[COLUMN_FEATURES][FEATURES][feature_name])\n\n if len(gathered_features) > 1:\n # currently only support features of the same shape\n assert all(per_features.shape == gathered_features[0].shape for per_features in gathered_features), (\n \"Currently we only support gathering features of the same dimension.\"\n )\n\n if len(gathered_features) == 0:\n raise ValueError(f\"No features are found for columns names {column_names}.\")\n\n gathered_features = torch.stack(gathered_features, dim=0).mean(dim=0) # (b, d)\n\n return gathered_features\n"
},
{
"path": "multimodal/src/autogluon/multimodal/predictor.py",
"content": "\"\"\"Implementation of the multimodal predictor\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport logging\nimport os\nimport warnings\nfrom typing import Dict, List, Optional, Union\n\nimport numpy as np\nimport pandas as pd\nimport transformers\n\nfrom autogluon.common.utils.log_utils import set_logger_verbosity, verbosity2loglevel\nfrom autogluon.core.metrics import Scorer\n\nfrom .constants import AUTOMM_TUTORIAL_MODE, FEW_SHOT_CLASSIFICATION, NER, OBJECT_DETECTION, SEMANTIC_SEGMENTATION\nfrom .learners import (\n BaseLearner,\n EnsembleLearner,\n FewShotSVMLearner,\n MatchingLearner,\n NERLearner,\n ObjectDetectionLearner,\n SemanticSegmentationLearner,\n)\nfrom .utils import get_dir_ckpt_paths\nfrom .utils.problem_types import PROBLEM_TYPES_REG\n\npl_logger = logging.getLogger(\"lightning\")\npl_logger.propagate = False # https://github.com/Lightning-AI/lightning/issues/4621\nlogger = logging.getLogger(__name__)\n\n\nclass MultiModalPredictor:\n \"\"\"\n AutoMM is designed to simplify the fine-tuning of foundation models\n for downstream applications with just three lines of code.\n AutoMM seamlessly integrates with popular model zoos such as\n `HuggingFace Transformers `_,\n `TIMM `_,\n and `MMDetection `_,\n accommodating a diverse range of data modalities,\n including image, text, tabular, and document data, whether used individually or in combination.\n It offers support for an array of tasks, encompassing classification, regression,\n object detection, named entity recognition, semantic matching, and image segmentation.\n \"\"\"\n\n def __init__(\n self,\n label: Optional[str] = None,\n problem_type: Optional[str] = None,\n query: Optional[Union[str, List[str]]] = None,\n response: Optional[Union[str, List[str]]] = None,\n match_label: Optional[Union[int, str]] = None,\n presets: Optional[str] = None,\n eval_metric: Optional[Union[str, Scorer]] = None,\n hyperparameters: Optional[dict] = None,\n path: Optional[str] = None,\n verbosity: Optional[int] = 2,\n num_classes: Optional[int] = None, # TODO: can we infer this from data?\n classes: Optional[list] = None,\n warn_if_exist: Optional[bool] = True,\n enable_progress_bar: Optional[bool] = None,\n pretrained: Optional[bool] = True,\n validation_metric: Optional[str] = None,\n sample_data_path: Optional[str] = None,\n use_ensemble: Optional[bool] = False,\n ensemble_size: Optional[int] = 2,\n ensemble_mode: Optional[str] = \"one_shot\",\n ):\n \"\"\"\n Parameters\n ----------\n label\n Name of one pd.DataFrame column that contains the target variable to predict.\n problem_type\n Type of problem. We support standard problems like\n\n - 'binary': Binary classification\n - 'multiclass': Multi-class classification\n - 'regression': Regression\n - 'classification': Classification problems include 'binary' and 'multiclass' classification.\n\n In addition, we support advanced problems such as\n\n - 'object_detection': Object detection\n - 'ner' or 'named_entity_recognition': Named entity extraction\n - 'text_similarity': Text-text semantic matching\n - 'image_similarity': Image-image semantic matching\n - 'image_text_similarity': Text-image semantic matching\n - 'feature_extraction': Extracting feature (only support inference)\n - 'zero_shot_image_classification': Zero-shot image classification (only support inference)\n - 'few_shot_classification': Few-shot classification for image or text data.\n - 'semantic_segmentation': Semantic segmentation with Segment Anything Model.\n\n For certain problem types, the default behavior is to load a pretrained model based on\n the presets / hyperparameters and the predictor can do zero-shot inference\n (running inference without .fit()). Those include the following\n problem types:\n\n - 'object_detection'\n - 'text_similarity'\n - 'image_similarity'\n - 'image_text_similarity'\n - 'feature_extraction'\n - 'zero_shot_image_classification'\n\n query\n Name of one pd.DataFrame column that has the query data in semantic matching tasks.\n response\n Name of one pd.DataFrame column that contains the response data in semantic matching tasks.\n If no label column is provided, the query and response pairs in\n one pd.DataFrame row are assumed to be positive pairs.\n match_label\n The label class that indicates the pair is counted as a \"match\".\n This is used when the task belongs to semantic matching, and the labels are binary.\n For example, the label column can contain [\"duplicate\", \"not duplicate\"] in a duplicate detection task.\n The match_label should be \"duplicate\" since it means that two items match.\n presets\n Presets regarding model quality, e.g., 'best_quality', 'high_quality' (default), and 'medium_quality'.\n Each quality has its corresponding HPO presets: 'best_quality_hpo', 'high_quality_hpo', and 'medium_quality_hpo'.\n eval_metric\n Evaluation metric name. If `eval_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'accuracy' for multiclass classification, `roc_auc` for binary classification,\n and 'root_mean_squared_error' for regression.\n hyperparameters\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n hyperparameters = \"model.hf_text.checkpoint_name=google/electra-small-discriminator model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n hyperparameters = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\", \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n path\n Path to directory where models and related artifacts should be saved.\n If unspecified, a time-stamped folder called \"AutogluonAutoMM/ag-[TIMESTAMP]\"\n will be created in the working directory.\n Note: To call `fit()` twice and save all results of each fit,\n you must specify different `path` locations or don't specify `path` at all.\n verbosity\n Verbosity levels range from 0 to 4, controlling how much logging information is printed.\n Higher levels correspond to more detailed print statements.\n You can set verbosity = 0 to suppress warnings.\n num_classes\n Number of classes (used for object detection).\n If this is specified and is different from the pretrained model's output shape,\n the model's head will be changed to have output.\n classes\n All the classes (used for object detection).\n warn_if_exist\n Whether to raise warning if the specified path already exists (Default True).\n enable_progress_bar\n Whether to show progress bar (default True). It would be\n disabled if the environment variable os.environ[\"AUTOMM_DISABLE_PROGRESS_BAR\"] is set.\n pretrained\n Whether to initialize the model with pretrained weights (default True).\n If False, it creates a model with random initialization.\n validation_metric\n Validation metric for selecting the best model and early-stopping during training.\n If not provided, it would be automatically chosen based on the problem type.\n sample_data_path\n The path to sample data from which we can infer num_classes or classes used for object detection.\n use_ensemble\n Whether to use ensembling when fitting the predictor (Default False).\n Currently, it works only on multimodal data (image+text, image+tabular, text+tabular, image+text+tabular) with classification or regression tasks.\n ensemble_size\n A multiple of number of models in the ensembling pool (Default 2). The actual ensemble size = ensemble_size * the model number\n ensemble_mode\n The mode of conducting ensembling:\n - `one_shot`: the classic ensemble selection\n - `sequential`: iteratively calling the classic ensemble selection with each time growing the model zoo by the best next model.\n \"\"\"\n if problem_type is not None:\n problem_type = problem_type.lower()\n assert problem_type in PROBLEM_TYPES_REG, (\n f\"problem_type='{problem_type}' is not supported yet. You may pick a problem type from\"\n f\" {PROBLEM_TYPES_REG.list_keys()}.\"\n )\n problem_property = PROBLEM_TYPES_REG.get(problem_type)\n if problem_property.experimental:\n warnings.warn(\n f\"problem_type='{problem_type}' is currently experimental.\",\n UserWarning,\n )\n problem_type = problem_property.name\n else:\n problem_property = None\n\n if os.environ.get(AUTOMM_TUTORIAL_MODE):\n enable_progress_bar = False\n # Also disable progress bar of transformers package\n transformers.logging.disable_progress_bar()\n\n if verbosity is not None:\n set_logger_verbosity(verbosity)\n\n self._verbosity = verbosity\n\n if problem_property and problem_property.is_matching:\n learner_class = MatchingLearner\n elif problem_type == OBJECT_DETECTION:\n learner_class = ObjectDetectionLearner\n elif problem_type == NER:\n learner_class = NERLearner\n elif problem_type == FEW_SHOT_CLASSIFICATION:\n learner_class = FewShotSVMLearner\n elif problem_type == SEMANTIC_SEGMENTATION:\n learner_class = SemanticSegmentationLearner\n else:\n learner_class = BaseLearner\n\n if use_ensemble:\n learner_class = EnsembleLearner\n\n self._learner = learner_class(\n label=label,\n problem_type=problem_type,\n presets=presets,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n path=path,\n verbosity=verbosity,\n num_classes=num_classes,\n classes=classes,\n warn_if_exist=warn_if_exist,\n enable_progress_bar=enable_progress_bar,\n pretrained=pretrained,\n sample_data_path=sample_data_path,\n validation_metric=validation_metric,\n query=query,\n response=response,\n match_label=match_label,\n ensemble_size=ensemble_size,\n ensemble_mode=ensemble_mode,\n )\n\n @property\n def path(self):\n \"\"\"\n Path to directory where the model and related artifacts are stored.\n \"\"\"\n return self._learner.path\n\n @property\n def label(self):\n \"\"\"\n Name of one pd.DataFrame column that contains the target variable to predict.\n \"\"\"\n return self._learner.label\n\n @property\n def query(self):\n \"\"\"\n Name of one pd.DataFrame column that has the query data in semantic matching tasks.\n \"\"\"\n return self._learner.query\n\n @property\n def response(self):\n \"\"\"\n Name of one pd.DataFrame column that contains the response data in semantic matching tasks.\n \"\"\"\n return self._learner.response\n\n @property\n def match_label(self):\n \"\"\"\n The label class that indicates the pair is counted as \"match\" in the semantic matching tasks.\n \"\"\"\n return self._learner.match_label\n\n @property\n def problem_type(self):\n \"\"\"\n What type of prediction problem this predictor has been trained for.\n \"\"\"\n return self._learner.problem_type\n\n @property\n def problem_property(self):\n \"\"\"\n Property of the problem, storing the problem type and its related properties.\n \"\"\"\n return self._learner.problem_property\n\n @property\n def column_types(self):\n \"\"\"\n Column types in the pd.DataFrame.\n \"\"\"\n return self._learner.column_types\n\n @property\n def eval_metric(self):\n \"\"\"\n What metric is used to evaluate predictive performance.\n \"\"\"\n return self._learner.eval_metric\n\n @property\n def validation_metric(self):\n \"\"\"\n Validation metric for selecting the best model and early-stopping during training.\n Note that the validation metric may be different from the evaluation metric.\n \"\"\"\n return self._learner.validation_metric\n\n @property\n def verbosity(self):\n \"\"\"\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements.\n \"\"\"\n return self._verbosity\n\n @property\n def total_parameters(self) -> int:\n \"\"\"\n The number of model parameters.\n \"\"\"\n return self._learner.total_parameters\n\n @property\n def trainable_parameters(self) -> int:\n \"\"\"\n The number of trainable model parameters, usually those with requires_grad=True.\n \"\"\"\n return self._learner.trainable_parameters\n\n @property\n def model_size(self) -> float:\n \"\"\"\n Returns the model size in Megabyte.\n \"\"\"\n return self._learner.model_size\n\n @property\n def classes(self):\n \"\"\"\n Object classes for the object detection problem type.\n \"\"\"\n return self._learner.classes\n\n @property\n def class_labels(self):\n \"\"\"\n The original name of the class labels.\n For example, the tabular data may contain classes equal to\n \"entailment\", \"contradiction\", \"neutral\". Internally, these will be converted to\n 0, 1, 2, ...\n This function returns the original names of these raw labels.\n\n Returns\n -------\n List that contain the class names. It will be None if it's not a classification problem.\n \"\"\"\n return self._learner.class_labels\n\n @property\n def positive_class(self):\n \"\"\"\n Name of the class label that will be mapped to 1.\n This is only meaningful for binary classification problems.\n\n It is useful for computing metrics such as F1 which require a positive and negative class.\n You may refer to https://en.wikipedia.org/wiki/F-score for more details.\n In binary classification, :class:`MultiModalPredictor.predict_proba(as_multiclass=False)`\n returns the estimated probability that each row belongs to the positive class.\n Will print a warning and return None if called when `predictor.problem_type != 'binary'`.\n\n Returns\n -------\n The positive class name in binary classification or None if the problem is not binary classification.\n \"\"\"\n return self._learner.positive_class\n\n # This func is required by the abstract trainer of TabularPredictor.\n def set_verbosity(self, verbosity: int):\n \"\"\"Set the verbosity level of the log.\n\n Parameters\n ----------\n verbosity\n The verbosity level.\n\n 0 --> only errors\n 1 --> only warnings and critical print statements\n 2 --> key print statements which should be shown by default\n 3 --> more-detailed printing\n 4 --> everything\n\n \"\"\"\n self._verbosity = verbosity\n set_logger_verbosity(verbosity)\n # TODO: align verbosity2loglevel with https://huggingface.co/docs/transformers/main_classes/logging#transformers.utils.logging.get_verbosity\n\n def set_num_gpus(self, num_gpus):\n \"\"\"\n Set the number of GPUs in config.\n \"\"\"\n self._learner.set_num_gpus(num_gpus)\n\n def get_num_gpus(self):\n \"\"\"\n Get the number of GPUs from config.\n \"\"\"\n self._learner.get_num_gpus()\n\n def fit(\n self,\n train_data: Union[pd.DataFrame, str],\n presets: Optional[str] = None,\n tuning_data: Optional[Union[pd.DataFrame, str]] = None,\n max_num_tuning_data: Optional[int] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n time_limit: Optional[int] = None,\n save_path: Optional[str] = None,\n hyperparameters: Optional[Union[str, Dict, List[str]]] = None,\n column_types: Optional[dict] = None,\n holdout_frac: Optional[float] = None,\n teacher_predictor: Union[str, MultiModalPredictor] = None,\n seed: Optional[int] = 0,\n standalone: Optional[bool] = True,\n hyperparameter_tune_kwargs: Optional[dict] = None,\n clean_ckpts: Optional[bool] = True,\n predictions: Optional[List[np.ndarray]] = None,\n labels: Optional[np.ndarray] = None,\n predictors: Optional[List[Union[str, MultiModalPredictor]]] = None,\n ):\n \"\"\"\n Fit models to predict a column of a data table (label) based on the other columns (features).\n\n Parameters\n ----------\n train_data\n A pd.DataFrame containing training data.\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n Each quality has its corresponding HPO presets: 'best_quality_hpo', 'high_quality_hpo', and 'medium_quality_hpo'.\n tuning_data\n A pd.DataFrame containing validation data, which should have the same columns as the train_data.\n If `tuning_data = None`, `fit()` will automatically hold out some random validation data from `train_data`.\n max_num_tuning_data\n The maximum number of tuning samples (used for object detection).\n id_mappings\n Id-to-content mappings (used for semantic matching). The contents can be text, image, etc.\n This is used when the pd.DataFrame contains the query/response identifiers instead of their contents.\n time_limit\n How long `fit()` should run for (wall clock time in seconds).\n If not specified, `fit()` will run until the model has completed training.\n Note that, if use_ensemble=True, the total running time would be time_limit * N,\n where N is the number of models in the ensemble.\n save_path\n Path to directory where models and artifacts should be saved.\n hyperparameters\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n hyperparameters = \"model.hf_text.checkpoint_name=google/electra-small-discriminator model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n hyperparameters = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\", \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n hyperparameters = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n column_types\n A dictionary that maps column names to their data types.\n For example: `column_types = {\"item_name\": \"text\", \"image\": \"image_path\",\n \"product_description\": \"text\", \"height\": \"numerical\"}`\n may be used for a table with columns: \"item_name\", \"brand\", \"product_description\", and \"height\".\n If None, column_types will be automatically inferred from the data.\n The current supported types are:\n - \"image_path\": each row in this column is one image path.\n - \"text\": each row in this column contains text (sentence, paragraph, etc.).\n - \"numerical\": each row in this column contains a number.\n - \"categorical\": each row in this column belongs to one of K categories.\n holdout_frac\n Fraction of train_data to holdout as tuning_data for optimizing hyperparameters or\n early stopping (ignored unless `tuning_data = None`).\n Default value (if None) is selected based on the number of rows in the training data\n and whether hyperparameter optimization is utilized.\n teacher_predictor\n The pre-trained teacher predictor or its saved path. If provided, `fit()` can distill its\n knowledge to a student predictor, i.e., the current predictor.\n seed\n The random seed to be used for training (default 0).\n standalone\n Whether to save the entire model for offline deployment.\n hyperparameter_tune_kwargs\n Hyperparameter tuning strategy and kwargs (for example, how many HPO trials to run).\n If None, then hyperparameter tuning will not be performed.\n num_trials: int\n How many HPO trials to run. Either `num_trials` or `time_limit` to `fit` needs to be specified.\n scheduler: Union[str, ray.tune.schedulers.TrialScheduler]\n If str is passed, AutoGluon will create the scheduler for you with some default parameters.\n If ray.tune.schedulers.TrialScheduler object is passed, you are responsible for initializing the object.\n scheduler_init_args: Optional[dict] = None\n If provided str to `scheduler`, you can optionally provide custom init_args to the scheduler\n searcher: Union[str, ray.tune.search.SearchAlgorithm, ray.tune.search.Searcher]\n If str is passed, AutoGluon will create the searcher for you with some default parameters.\n If ray.tune.schedulers.TrialScheduler object is passed, you are responsible for initializing the object.\n You don't need to worry about `metric` and `mode` of the searcher object. AutoGluon will figure it out by itself.\n scheduler_init_args: Optional[dict] = None\n If provided str to `searcher`, you can optionally provide custom init_args to the searcher\n You don't need to worry about `metric` and `mode`. AutoGluon will figure it out by itself.\n clean_ckpts\n Whether to clean the intermediate checkpoints after training.\n\n Returns\n -------\n An \"MultiModalPredictor\" object (itself).\n \"\"\"\n\n if teacher_predictor is None:\n teacher_learner = None\n elif isinstance(teacher_predictor, str):\n teacher_learner = teacher_predictor\n else:\n teacher_learner = teacher_predictor._learner\n\n if predictors is None:\n learners = None\n else:\n assert isinstance(predictors, list)\n learners = [ele if isinstance(ele, str) else ele._learner for ele in predictors]\n\n self._learner.fit(\n train_data=train_data,\n presets=presets,\n tuning_data=tuning_data,\n max_num_tuning_data=max_num_tuning_data,\n time_limit=time_limit,\n save_path=save_path,\n hyperparameters=hyperparameters,\n column_types=column_types,\n holdout_frac=holdout_frac,\n teacher_learner=teacher_learner,\n seed=seed,\n standalone=standalone,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n clean_ckpts=clean_ckpts,\n id_mappings=id_mappings,\n predictions=predictions,\n labels=labels,\n learners=learners,\n )\n\n return self\n\n def evaluate(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n query_data: Optional[list] = None,\n response_data: Optional[list] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n metrics: Optional[Union[str, List[str]]] = None,\n chunk_size: Optional[int] = 1024,\n similarity_type: Optional[str] = \"cosine\",\n cutoffs: Optional[List[int]] = [1, 5, 10],\n label: Optional[str] = None,\n return_pred: Optional[bool] = False,\n realtime: Optional[bool] = False,\n eval_tool: Optional[str] = None,\n predictions: Optional[List[np.ndarray]] = None,\n labels: Optional[np.ndarray] = None,\n ):\n \"\"\"\n Evaluate the model on a given dataset.\n\n Parameters\n ----------\n data\n A pd.DataFrame, containing the same columns as the training data.\n Or a str, that is a path of the annotation file for detection.\n query_data\n Query data used for ranking.\n response_data\n Response data used for ranking.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data/query_data/response_data contain the query/response identifiers instead of their contents.\n metrics\n A list of metric names to report.\n If None, we only return the score for the stored `_eval_metric_name`.\n chunk_size\n Scan the response data by chunk_size each time. Increasing the value increases the speed, but requires more memory.\n similarity_type\n Use what function (cosine/dot_prod) to score the similarity (default: cosine).\n cutoffs\n A list of cutoff values to evaluate ranking.\n label\n The label column name in data. Some tasks, e.g., image<-->text matching, have no label column in training data,\n but the label column may be still required in evaluation.\n return_pred\n Whether to return the prediction result of each row.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n eval_tool\n The eval_tool for object detection. Could be \"pycocotools\" or \"torchmetrics\".\n\n Returns\n -------\n A dictionary with the metric names and their corresponding scores.\n Optionally return a pd.DataFrame of prediction results.\n \"\"\"\n return self._learner.evaluate(\n data=data,\n metrics=metrics,\n return_pred=return_pred,\n realtime=realtime,\n eval_tool=eval_tool,\n query_data=query_data,\n response_data=response_data,\n id_mappings=id_mappings,\n chunk_size=chunk_size,\n similarity_type=similarity_type,\n cutoffs=cutoffs,\n label=label,\n predictions=predictions,\n labels=labels,\n )\n\n def predict(\n self,\n data: Union[pd.DataFrame, dict, list, str],\n candidate_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n as_pandas: Optional[bool] = None,\n realtime: Optional[bool] = False,\n save_results: Optional[bool] = None,\n **kwargs,\n ):\n \"\"\"\n Predict the label column values for new data.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n candidate_data\n The candidate data from which to search the query data's matches.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n save_results\n Whether to save the prediction results (only works for detection now)\n **kwargs\n Additional keyword arguments to pass to the underlying learner's predict method.\n For example, `as_coco` for object detection tasks.\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset.\n Format depends on the specific learner and provided arguments.\n \"\"\"\n return self._learner.predict(\n data=data,\n candidate_data=candidate_data,\n as_pandas=as_pandas,\n realtime=realtime,\n save_results=save_results,\n id_mappings=id_mappings,\n **kwargs,\n )\n\n def predict_proba(\n self,\n data: Union[pd.DataFrame, dict, list],\n candidate_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n as_pandas: Optional[bool] = None,\n as_multiclass: Optional[bool] = True,\n realtime: Optional[bool] = False,\n ):\n \"\"\"\n Predict class probabilities rather than class labels.\n Note that this is only for the classification tasks.\n Calling it for a regression task will throw an exception.\n\n Parameters\n ----------\n data\n The data to make predictions for. Should contain same column names as training data and\n follow same format (except for the `label` column).\n candidate_data\n The candidate data from which to search the query data's matches.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n as_pandas\n Whether to return the output as a pandas DataFrame(Series) (True) or numpy array (False).\n as_multiclass\n Whether to return the probability of all labels or\n just return the probability of the positive class for binary classification problems.\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n When as_multiclass is True, the output will always have shape (#samples, #classes).\n Otherwise, the output will have shape (#samples,)\n \"\"\"\n return self._learner.predict_proba(\n data=data,\n candidate_data=candidate_data,\n as_pandas=as_pandas,\n as_multiclass=as_multiclass,\n realtime=realtime,\n id_mappings=id_mappings,\n )\n\n def extract_embedding(\n self,\n data: Union[pd.DataFrame, dict, list],\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n return_masks: Optional[bool] = False,\n as_tensor: Optional[bool] = False,\n as_pandas: Optional[bool] = False,\n realtime: Optional[bool] = False,\n signature: Optional[str] = None,\n ):\n \"\"\"\n Extract features for each sample, i.e., one row in the provided pd.DataFrame `data`.\n\n Parameters\n ----------\n data\n The data to extract embeddings for. Should contain same column names as training dataset and\n follow same format (except for the `label` column).\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when data contain the query/response identifiers instead of their contents.\n return_masks\n If true, returns a mask dictionary, whose keys are the same as those in the features dictionary.\n If a sample has empty input in feature column `image_0`, the sample will has mask 0 under key `image_0`.\n as_tensor\n Whether to return a Pytorch tensor.\n as_pandas\n Whether to return the output as a pandas DataFrame (True) or numpy array (False).\n realtime\n Whether to do realtime inference, which is efficient for small data (default False).\n If provided None, we would infer it on based on the data modalities\n and sample number.\n signature\n When using matcher, it can be query or response.\n\n Returns\n -------\n Array of embeddings, corresponding to each row in the given data.\n It will have shape (#samples, D) where the embedding dimension D is determined\n by the neural network's architecture.\n \"\"\"\n return self._learner.extract_embedding(\n data=data,\n return_masks=return_masks,\n as_tensor=as_tensor,\n as_pandas=as_pandas,\n realtime=realtime,\n signature=signature,\n id_mappings=id_mappings,\n )\n\n def save(self, path: str, standalone: Optional[bool] = True):\n \"\"\"\n Save this predictor to file in directory specified by `path`.\n\n Parameters\n ----------\n path\n The directory to save this predictor.\n standalone\n Whether to save the downloaded model for offline deployment.\n When standalone = True, save the transformers.CLIPModel and transformers.AutoModel to os.path.join(path,model_name),\n and reset the associate model.model_name.checkpoint_name start with `local://` in config.yaml.\n When standalone = False, the saved artifact may require an online environment to process in load().\n \"\"\"\n self._learner.save(path=path, standalone=standalone)\n\n @classmethod\n def load(\n cls,\n path: str,\n resume: Optional[bool] = False,\n verbosity: Optional[int] = 3,\n ):\n \"\"\"\n Load a predictor object from a directory specified by `path`. The to-be-loaded predictor\n can be completely or partially trained by .fit(). If a previous training has completed,\n it will load the checkpoint `model.ckpt`. Otherwise, if a previous training accidentally\n collapses in the middle, it can load the `last.ckpt` checkpoint by setting `resume=True`.\n It also supports loading one specific checkpoint given its path.\n\n .. warning::\n\n :meth:`autogluon.multimodal.MultiModalPredictor.load` uses `pickle` module implicitly, which is known to\n be insecure. It is possible to construct malicious pickle data which will execute arbitrary code during\n unpickling. Never load data that could have come from an untrusted source, or that could have been tampered\n with. **Only load data you trust.**\n\n Parameters\n ----------\n path\n The directory to load the predictor object.\n resume\n Whether to resume training from `last.ckpt`. This is useful when a training was accidentally\n broken during the middle, and we want to resume the training from the last saved checkpoint.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements.\n You can set verbosity = 0 to suppress warnings.\n\n Returns\n -------\n The loaded predictor object.\n \"\"\"\n dir_path, ckpt_path = get_dir_ckpt_paths(path=path)\n\n assert os.path.isdir(dir_path), f\"'{dir_path}' must be an existing directory.\"\n predictor = cls(label=\"dummy_label\")\n\n with open(os.path.join(dir_path, \"assets.json\"), \"r\") as fp:\n assets = json.load(fp)\n learner_class = BaseLearner\n if assets[\"learner_class\"] == \"MatchingLearner\":\n learner_class = MatchingLearner\n elif assets[\"learner_class\"] == \"EnsembleLearner\":\n learner_class = EnsembleLearner\n elif assets[\"learner_class\"] == \"FewShotSVMLearner\":\n learner_class = FewShotSVMLearner\n elif assets[\"learner_class\"] == \"ObjectDetectionLearner\":\n learner_class = ObjectDetectionLearner\n elif assets[\"learner_class\"] == \"NERLearner\":\n learner_class = NERLearner\n elif assets[\"learner_class\"] == \"SemanticSegmentationLearner\":\n learner_class = SemanticSegmentationLearner\n\n predictor._learner = learner_class.load(path=path, resume=resume, verbosity=verbosity)\n return predictor\n\n def dump_model(self, save_path: Optional[str] = None):\n \"\"\"\n Save model weights and config to a local directory.\n Model weights are saved in the file `pytorch_model.bin` (for `timm_image` or `hf_text`)\n or '.pth' (for `mmdet_image`).\n Configs are saved in the file `config.json` (for `timm_image` or `hf_text`)\n or '.py' (for `mmdet_image`).\n\n Parameters\n ----------\n save_path : str\n Path to directory where models and configs should be saved.\n \"\"\"\n return self._learner.dump_model(save_path=save_path)\n\n def export_onnx(\n self,\n data: Union[dict, pd.DataFrame],\n path: Optional[str] = None,\n batch_size: Optional[int] = None,\n verbose: Optional[bool] = False,\n opset_version: Optional[int] = 16,\n truncate_long_and_double: Optional[bool] = False,\n ):\n \"\"\"\n Export this predictor's model to an ONNX file.\n\n When `path` argument is not provided, the method would not save the model into disk.\n Instead, it would export the onnx model into BytesIO and return its binary as bytes.\n\n Parameters\n ----------\n data\n Raw data used to trace and export the model.\n If this is None, will check if a processed batch is provided.\n path : str, default=None\n The export path of onnx model. If path is not provided, the method would export model to memory.\n batch_size\n The batch_size of export model's input.\n Normally the batch_size is a dynamic axis, so we could use a small value for faster export.\n verbose\n verbose flag in torch.onnx.export.\n opset_version\n opset_version flag in torch.onnx.export.\n truncate_long_and_double: bool, default False\n Truncate weights provided in int64 or double (float64) to int32 and float32\n\n Returns\n -------\n onnx_path : str or bytes\n A string that indicates location of the exported onnx model, if `path` argument is provided.\n Otherwise, would return the onnx model as bytes.\n \"\"\"\n\n # Make sure _model is initialized\n self._learner.on_predict_start()\n\n return self._learner.export_onnx(\n data=data,\n path=path,\n batch_size=batch_size,\n verbose=verbose,\n opset_version=opset_version,\n truncate_long_and_double=truncate_long_and_double,\n )\n\n def optimize_for_inference(\n self,\n providers: Optional[Union[dict, List[str]]] = None,\n ):\n \"\"\"\n Optimize the predictor's model for inference.\n\n Under the hood, the implementation would convert the PyTorch module into an ONNX module, so that\n we can leverage efficient execution providers in onnxruntime for faster inference.\n\n Parameters\n ----------\n providers : dict or str, default=None\n A list of execution providers for model prediction in onnxruntime.\n\n By default, the providers argument is None. The method would generate an ONNX module that\n would perform model inference with TensorrtExecutionProvider in onnxruntime, if tensorrt\n package is properly installed. Otherwise, the onnxruntime would fallback to use CUDA or CPU\n execution providers instead.\n\n Returns\n -------\n onnx_module : OnnxModule\n The onnx-based module that can be used to replace predictor._model for model inference.\n \"\"\"\n return self._learner.optimize_for_inference(providers=providers)\n\n def fit_summary(self, verbosity=0, show_plot=False):\n \"\"\"\n Output the training summary information from `fit()`.\n\n Parameters\n ----------\n verbosity : int, default = 2\n Verbosity levels range from 0 to 4 and control how much information is printed.\n verbosity = 0 for no output printing.\n TODO: Higher levels correspond to more detailed print statements\n show_plot : bool, default = False\n If True, shows the model summary plot in browser when verbosity > 1.\n\n Returns\n -------\n Dict containing various detailed information.\n We do not recommend directly printing this dict as it may be very large.\n \"\"\"\n return self._learner.fit_summary(verbosity=verbosity, show_plot=show_plot)\n\n def list_supported_models(self, pretrained=True):\n \"\"\"\n List supported models for each problem type.\n\n Parameters\n ----------\n pretrained : bool, default = True\n If True, only return the models with pretrained weights.\n If False, return all the models as long as there is model definition.\n\n Returns\n -------\n a list of model names\n \"\"\"\n return self._learner.list_supported_models(pretrained=pretrained)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/__init__.py",
"content": "from .cache import DDPPredictionWriter\nfrom .checkpoint import AutoMMModelCheckpoint, AutoMMModelCheckpointIO, average_checkpoints\nfrom .config import (\n apply_omegaconf_overrides,\n customize_model_names,\n filter_hyperparameters,\n filter_search_space,\n filter_timm_pretrained_cfg,\n get_config,\n get_default_config,\n get_local_pretrained_config_paths,\n get_pretrain_configs_dir,\n parse_dotlist_conf,\n save_pretrained_model_configs,\n split_hyperparameters,\n update_config_by_rules,\n update_ensemble_hyperparameters,\n update_hyperparameters,\n update_tabular_config_by_resources,\n)\nfrom .device import compute_num_gpus, get_available_devices, move_to_device\nfrom .distillation import DistillationMixin\nfrom .download import download, is_url\nfrom .env import is_interactive_env\nfrom .export import ExportMixin\nfrom .hpo import hyperparameter_tune\nfrom .inference import RealtimeMixin, compute_inference_batch_size, extract_from_output\nfrom .install import check_if_packages_installed\nfrom .load import CustomUnpickler, get_dir_ckpt_paths, get_load_ckpt_paths, protected_zip_extraction\nfrom .log import (\n LogFilter,\n apply_log_filter,\n get_gpu_message,\n on_fit_end_message,\n on_fit_per_run_start_message,\n on_fit_start_message,\n)\nfrom .matcher import compute_semantic_similarity, convert_data_for_ranking, create_siamese_model, semantic_search\nfrom .misc import (\n logits_to_prob,\n merge_bio_format,\n path_expander,\n path_to_base64str_expander,\n path_to_bytearray_expander,\n shopee_dataset,\n tensor_to_ndarray,\n)\nfrom .mmcv import CollateMMDet, CollateMMOcr\nfrom .object_detection import (\n COCODataset,\n bbox_ratio_xywh_to_index_xyxy,\n bbox_xyxy_to_xywh,\n cocoeval,\n convert_pred_to_xywh,\n convert_result_df,\n from_coco,\n from_coco_or_voc,\n from_dict,\n from_voc,\n get_detection_classes,\n object_detection_data_to_df,\n object_detection_df_to_coco,\n save_result_coco_format,\n save_result_voc_format,\n visualize_detection,\n)\nfrom .precision import get_precision_context, infer_precision\nfrom .presets import get_basic_config, get_ensemble_presets, get_presets, list_presets, matcher_presets\nfrom .problem_types import PROBLEM_TYPES_REG, infer_problem_type_by_eval_metric\nfrom .save import make_exp_dir, process_save_path, setup_save_path\nfrom .strategy import is_interactive_strategy, run_ddp_only_once\nfrom .visualizer import NERVisualizer, ObjectDetectionVisualizer, SemanticSegmentationVisualizer, visualize_ner\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/cache.py",
"content": "import logging\nimport os\nimport shutil\nimport time\nimport uuid\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Sequence\n\nimport lightning.pytorch as pl\nimport torch\nfrom lightning.pytorch.callbacks import BasePredictionWriter\n\nfrom ..constants import (\n AUG_LOGITS,\n BBOX,\n LOGIT_SCALE,\n MULTIMODAL_FEATURES,\n MULTIMODAL_FEATURES_POST_AUG,\n MULTIMODAL_FEATURES_PRE_AUG,\n ORI_LOGITS,\n VAE_MEAN,\n VAE_VAR,\n WEIGHT,\n)\n\nlogger = logging.getLogger(__name__)\n\n\nclass DDPPredictionWriter(BasePredictionWriter):\n def __init__(\n self, output_dir: Optional[str], write_interval: Optional[str], strategy: Optional[str], sleep_time=5\n ):\n \"\"\"\n Parameters\n ----------\n output_dir\n The directory to save predictions.\n write_interval\n When to write. Could be \"batch\" or \"epoch\".\n See Lightning's source code at\n https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.callbacks.BasePredictionWriter.html\n sleep_time\n If other process does not finish writing, sleep for a few seconds and recheck.\n \"\"\"\n super().__init__(write_interval)\n if output_dir is None: # TODO: give the predictor a default save_path before calling DDPPredictionWriter\n output_dir = \"temp_cache_for_ddp_prediction\"\n logging.warning(\n f\"Current predictor's save_path is None, using a default cache folder which may cause an error in prediction I/O. Try init the predictor with a save_path.\"\n )\n assert isinstance(output_dir, (str, Path)), (\n f\"Only str and pathlib.Path types are supported for path, but got {output_dir} of type {type(output_dir)}.\"\n )\n self.sleep_time = sleep_time\n output_dir = os.path.abspath(os.path.expanduser(output_dir))\n if \"spawn\" in strategy:\n self.output_dir = os.path.join(output_dir, f\"predictions_{uuid.uuid4().hex}\")\n else:\n self.output_dir = os.path.join(output_dir, \"ddp_prediction_cache\")\n try:\n os.makedirs(self.output_dir, exist_ok=False)\n except FileExistsError: # assume the string is unique\n logger.warning(\n f\"{self.output_dir} already exists. This could be caused by DDP subprocess. Just make sure the previous cache is removed in {self.output_dir}.\"\n )\n\n def get_predictions_cache_dir(self, global_rank: int):\n \"\"\"\n Parameters\n ----------\n global_rank\n Global rank of current process.\n \"\"\"\n return os.path.join(self.output_dir, f\"predictions_rank_{global_rank}.pt\")\n\n def get_batch_indices_cache_dir(self, global_rank: int):\n \"\"\"\n Parameters\n ----------\n global_rank\n Global rank of current process.\n \"\"\"\n return os.path.join(self.output_dir, f\"sample_indices_rank_{global_rank}.pt\")\n\n def write_on_epoch_end(\n self,\n trainer: pl.Trainer,\n pl_module: pl.LightningModule,\n predictions: Sequence[Any],\n batch_indices: Sequence[Any],\n ):\n \"\"\"\n Parameters\n ----------\n trainer\n Pytorch Lightning trainer.\n pl_module\n Pytorch Lightning module.\n predictions\n The predicted results.\n batch_indices\n The corresponding batch indices for prediction results.\n \"\"\"\n # this will create N (num processes) files in `cache_dir` each containing\n # the predictions of its respective rank\n torch.save(predictions, self.get_predictions_cache_dir(trainer.global_rank)) # nosec B614\n # here we save `batch_indices` to get the information about the data index\n # from prediction data\n torch.save(batch_indices, self.get_batch_indices_cache_dir(trainer.global_rank)) # nosec B614\n\n def read_single_gpu_results(self, global_rank: Optional[int]):\n \"\"\"\n Parameters\n ----------\n global_rank\n Global rank of current process.\n \"\"\"\n sample_indices_file = self.get_batch_indices_cache_dir(global_rank)\n predictions_file = self.get_predictions_cache_dir(global_rank)\n while (not os.path.exists(sample_indices_file)) or (not os.path.exists(predictions_file)):\n logger.info(f\"waiting for rank #{global_rank} to finish saving predictions...\")\n time.sleep(self.sleep_time)\n sample_indices = torch.load(sample_indices_file) # nosec B614\n predictions = torch.load(predictions_file) # nosec B614\n\n return sample_indices, predictions\n\n def flatten(self, x):\n \"\"\"\n Flatten nested lists into one list.\n\n Parameters\n ----------\n x\n A nested list to be flattened.\n \"\"\"\n if isinstance(x, list):\n return [a for i in x for a in self.flatten(i)]\n else:\n return [x]\n\n def collate(self, x: List[Dict]):\n \"\"\"\n Collate a list of dictionaries into one.\n Each value is a tensor concatenated from a list of batch tensors.\n\n Parameters\n ----------\n x\n A list of batch results. Each batch is one (nested) dictionary.\n \"\"\"\n if BBOX in x[0]: # for detection outputs just sort the batches\n return x\n\n results = dict()\n if len(x[0]) == 0: # dict is empty\n return dict()\n\n for k, v in x[0].items():\n if k in [\n WEIGHT,\n LOGIT_SCALE,\n MULTIMODAL_FEATURES,\n MULTIMODAL_FEATURES_PRE_AUG,\n MULTIMODAL_FEATURES_POST_AUG,\n ORI_LOGITS,\n AUG_LOGITS,\n VAE_MEAN,\n VAE_VAR,\n ]: # ignore the keys\n continue\n elif isinstance(v, dict):\n results[k] = self.collate([i[k] for i in x])\n elif isinstance(v, torch.Tensor):\n results[k] = torch.cat([i[k] for i in x])\n else:\n raise ValueError(\n f\"Unsupported data type {type(v)} is found in prediction results. \"\n f\"We only support dictionary with torch tensor values.\"\n )\n\n return results\n\n def sort(self, x: Dict, indices: List):\n \"\"\"\n Sort the values of each key according to the indices.\n This is to make sure that prediction results follow the order of input samples.\n\n Parameters\n ----------\n x\n A dictionary with all the predictions.\n indices\n A list of indices.\n \"\"\"\n if isinstance(x, list) and BBOX in x[0]: # for detection outputs just sort the batches\n return [xi for _, xi in sorted(zip(indices, x), key=lambda ele: ele[0])]\n\n results = dict()\n for k, v in x.items():\n if isinstance(v, dict):\n results[k] = self.sort(v, indices)\n else:\n assert len(indices) == len(v), (\n f\"Size mismatch, {k}: {v} of len {len(v)} and indices {indices} of length {len(indices)}\"\n )\n results[k] = [x for _, x in sorted(zip(indices, v), key=lambda ele: ele[0])]\n results[k] = torch.stack(results[k])\n\n return results\n\n def collect_all_gpu_results(self, num_gpus):\n \"\"\"\n Parameters\n ----------\n num_gpus\n Number of gpus used.\n \"\"\"\n sample_indices = []\n predictions = []\n for global_rank in range(num_gpus):\n sample_indices_per_rank, predictions_per_rank = self.read_single_gpu_results(global_rank)\n sample_indices += self.flatten(sample_indices_per_rank)\n predictions += self.flatten(predictions_per_rank)\n\n assert sorted(sample_indices) == list(range(len(sample_indices)))\n predictions = self.collate(predictions)\n sorted_predictions = self.sort(predictions, indices=sample_indices)\n shutil.rmtree(self.output_dir)\n\n return [sorted_predictions]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/checkpoint.py",
"content": "\"\"\"\nSome utilities are copied from\nhttps://github.com/Lightning-AI/lightning/blob/master/src/lightning/fabric/utilities/cloud_io.py\nto address warnings:\nLightningDeprecationWarning: lightning.pytorch.utilities.cloud_io.atomic_save has been\ndeprecated in v1.8.0 and will be removed in v1.10.0. This function is internal but you\ncan copy over its implementation.\n\"\"\"\n\nimport io\nimport logging\nimport os\nimport re\nimport shutil\nfrom pathlib import Path\nfrom typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union\n\nimport fsspec\nimport lightning.pytorch as pl\nimport torch\nfrom lightning.pytorch.strategies import DeepSpeedStrategy\nfrom lightning.pytorch.utilities.rank_zero import rank_zero_warn\n\nfrom .env import get_filesystem\n\n_DEVICE = Union[torch.device, str, int]\n_MAP_LOCATION_TYPE = Optional[Union[_DEVICE, Callable[[_DEVICE], _DEVICE], Dict[_DEVICE, _DEVICE]]]\n\nlogger = logging.getLogger(__name__)\n\n\ndef average_checkpoints(\n checkpoint_paths: List[str],\n):\n \"\"\"\n Average a list of checkpoints' state_dicts.\n Reference: https://github.com/rwightman/pytorch-image-models/blob/master/avg_checkpoints.py\n\n Parameters\n ----------\n checkpoint_paths\n A list of model checkpoint paths.\n\n Returns\n -------\n The averaged state_dict.\n \"\"\"\n if len(checkpoint_paths) > 1:\n avg_state_dict = {}\n avg_counts = {}\n for per_path in checkpoint_paths:\n if os.path.isdir(per_path + \"-dir\"): # deepspeed save checkpoints into a directory\n from lightning.pytorch.utilities.deepspeed import convert_zero_checkpoint_to_fp32_state_dict\n\n convert_zero_checkpoint_to_fp32_state_dict(per_path + \"-dir\", per_path)\n shutil.rmtree(per_path + \"-dir\")\n state_dict = torch.load(per_path, map_location=torch.device(\"cpu\"), weights_only=False)[\"state_dict\"] # nosec B614\n else:\n state_dict = torch.load(per_path, map_location=torch.device(\"cpu\"), weights_only=False)[\"state_dict\"] # nosec B614\n for k, v in state_dict.items():\n if k not in avg_state_dict:\n avg_state_dict[k] = v.clone().to(dtype=torch.float64)\n avg_counts[k] = 1\n else:\n avg_state_dict[k] += v.to(dtype=torch.float64)\n avg_counts[k] += 1\n del state_dict\n\n for k, v in avg_state_dict.items():\n v.div_(avg_counts[k])\n\n # convert to float32.\n float32_info = torch.finfo(torch.float32)\n for k in avg_state_dict:\n avg_state_dict[k].clamp_(float32_info.min, float32_info.max).to(dtype=torch.float32)\n else:\n avg_state_dict = torch.load( # nosec B614\n checkpoint_paths[0], map_location=torch.device(\"cpu\"), weights_only=False\n )[\"state_dict\"]\n\n return avg_state_dict\n\n\ndef pl_load(\n path_or_url: Union[IO, str, Path],\n map_location: _MAP_LOCATION_TYPE = None,\n) -> Any:\n \"\"\"Loads a checkpoint.\n\n Args:\n path_or_url: Path or URL of the checkpoint.\n map_location: a function, ``torch.device``, string or a dict specifying how to remap storage locations.\n \"\"\"\n if not isinstance(path_or_url, (str, Path)):\n # any sort of BytesIO or similar\n return torch.load(path_or_url, map_location=map_location) # nosec B614\n if str(path_or_url).startswith(\"http\"):\n return torch.hub.load_state_dict_from_url(\n str(path_or_url),\n map_location=map_location, # type: ignore[arg-type] # upstream annotation is not correct\n )\n fs = get_filesystem(path_or_url)\n with fs.open(path_or_url, \"rb\") as f:\n return torch.load(f, map_location=map_location) # nosec B614\n\n\ndef pl_save(checkpoint: Dict[str, Any], filepath: Union[str, Path]) -> None:\n \"\"\"Saves a checkpoint atomically, avoiding the creation of incomplete checkpoints.\n\n Args:\n checkpoint: The object to save.\n Built to be used with the ``dump_checkpoint`` method, but can deal with anything which ``torch.save``\n accepts.\n filepath: The path to which the checkpoint will be saved.\n This points to the file that the checkpoint will be stored in.\n \"\"\"\n bytesbuffer = io.BytesIO()\n torch.save(checkpoint, bytesbuffer) # nosec B614\n with fsspec.open(filepath, \"wb\") as f:\n f.write(bytesbuffer.getvalue())\n\n\nclass AutoMMModelCheckpointIO(pl.plugins.CheckpointIO):\n \"\"\"\n Class that customizes how checkpoints are saved. Saves either the entire model or only parameters that have been explicitly updated during training. The latter reduces memory footprint substantially when training very large models with parameter-efficient finetuning methods.\n Class is based on plugins.TorchCheckpointIO.\n\n \"\"\"\n\n def __init__(self, trainable_param_names, model_name_to_id):\n \"\"\"\n Parameters\n ----------\n trainable_param_names\n A list of regular expressions or exact names of layers to filter which parameters should be saved. If empty save entire model.\n model_name_to_id\n A dictionary mapping the layer names (keys) of the model to their ids (values).\n \"\"\"\n super().__init__()\n self.trainable_param_names = trainable_param_names\n self.model_name_to_id = model_name_to_id\n\n def save_checkpoint(self, checkpoint: Dict[str, Any], path, storage_options: Optional[Any] = None) -> None:\n \"\"\"\n Save model/training states as a checkpoint file through state-dump and file-write.\n\n Parameters\n ----------\n checkpoint\n dict containing model and trainer state\n path\n write-target path\n storage_options\n Optional parameters when saving the model/training states. Not currently considered.\n \"\"\"\n if storage_options is not None:\n raise TypeError(\n \"`Trainer.save_checkpoint(..., storage_options=...)` with `storage_options` arg\"\n f\" is not supported for `{self.__class__.__name__}`.\"\n )\n\n if \"state_dict\" in checkpoint:\n if self.trainable_param_names:\n updated_params = {}\n for name, param in checkpoint[\"state_dict\"].items():\n adjusted_name = name.replace(\"model.\", \"\", 1)\n if adjusted_name in self.model_name_to_id and self.model_name_to_id[adjusted_name] == 0:\n updated_params[name] = param\n if any(\n [re.match(trainable_param_name, name) for trainable_param_name in self.trainable_param_names]\n ):\n updated_params[name] = param\n else:\n updated_params = checkpoint[\"state_dict\"]\n\n checkpoint[\"state_dict\"] = updated_params\n\n fs = get_filesystem(path)\n fs.makedirs(os.path.dirname(path), exist_ok=True)\n try:\n # write the checkpoint dictionary on the file\n pl_save(checkpoint, path)\n except AttributeError as err:\n # todo (sean): is this try catch necessary still?\n # https://github.com/Lightning-AI/lightning/pull/431\n key = pl.LightningModule.CHECKPOINT_HYPER_PARAMS_KEY\n checkpoint.pop(key, None)\n rank_zero_warn(f\"Warning, `{key}` dropped from checkpoint. An attribute is not picklable: {err}\")\n pl_save(checkpoint, path)\n\n def load_checkpoint(self, path, map_location: Optional[Any] = None) -> Dict[str, Any]:\n \"\"\"\n Load checkpoint from a path when resuming or loading ckpt for test/validate/predict stages.\n\n Parameters\n ----------\n path\n Path to checkpoint\n map_location\n a function, torch.device, string or a dict specifying how to remap storage locations.\n \"\"\"\n\n fs = get_filesystem(path)\n if not fs.exists(path):\n raise FileNotFoundError(f\"Checkpoint at {path} not found. Aborting training.\")\n\n return pl_load(path, map_location=map_location)\n\n def remove_checkpoint(self, path) -> None:\n \"\"\"\n Remove checkpoint file from the filesystem.\n\n Parameters\n ----------\n path\n Path to checkpoint\n \"\"\"\n fs = get_filesystem(path)\n if fs.exists(path):\n fs.rm(path, recursive=True)\n logger.debug(f\"Removed checkpoint: {path}\")\n\n\nclass AutoMMModelCheckpoint(pl.callbacks.ModelCheckpoint):\n \"\"\"\n Class that inherits callbacks.ModelCheckpoint. The purpose is to resolve the potential issues in lightning.\n\n - Issue1:\n\n It solves the issue described in https://github.com/Lightning-AI/lightning/issues/5582.\n For ddp_spawn, the checkpoint_callback.best_k_models will be empty.\n Here, we resolve it by storing the best_models to \"SAVE_DIR/best_k_models.yaml\".\n\n \"\"\"\n\n def _save_checkpoint(self, trainer, filepath):\n # Deepspeed saves model and optimizer states in a shared state in a separate folder\n if isinstance(trainer.strategy, DeepSpeedStrategy):\n trainer.save_checkpoint(filepath + \"-dir\", self.save_weights_only)\n else:\n trainer.save_checkpoint(filepath, self.save_weights_only)\n\n # Required to avoid redundant evaluation and checkpointing\n self._last_global_step_saved = trainer.global_step\n\n def _update_best_and_save(\n self,\n current: torch.Tensor,\n trainer: \"pl.Trainer\",\n monitor_candidates: Dict[str, torch.Tensor],\n ) -> None:\n super(AutoMMModelCheckpoint, self)._update_best_and_save(\n current=current, trainer=trainer, monitor_candidates=monitor_candidates\n )\n self.to_yaml()\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/colormap.py",
"content": "# ------------------------------------------------------------------------\n# Copyright (c) Facebook, Inc. and its affiliates.\n# ------------------------------------------------------------------------\n\n\"\"\"\nAn awesome colormap for really neat visualizations.\nCopied from Detectron, and removed gray colors.\n\"\"\"\n\nimport random\n\nimport numpy as np\n\n__all__ = [\"colormap\", \"random_color\", \"random_colors\"]\n\n# fmt: off\n# RGB:\n_COLORS = np.array(\n [\n 0.000, 0.447, 0.741,\n 0.850, 0.325, 0.098,\n 0.929, 0.694, 0.125,\n 0.494, 0.184, 0.556,\n 0.466, 0.674, 0.188,\n 0.301, 0.745, 0.933,\n 0.635, 0.078, 0.184,\n 0.300, 0.300, 0.300,\n 0.600, 0.600, 0.600,\n 1.000, 0.000, 0.000,\n 1.000, 0.500, 0.000,\n 0.749, 0.749, 0.000,\n 0.000, 1.000, 0.000,\n 0.000, 0.000, 1.000,\n 0.667, 0.000, 1.000,\n 0.333, 0.333, 0.000,\n 0.333, 0.667, 0.000,\n 0.333, 1.000, 0.000,\n 0.667, 0.333, 0.000,\n 0.667, 0.667, 0.000,\n 0.667, 1.000, 0.000,\n 1.000, 0.333, 0.000,\n 1.000, 0.667, 0.000,\n 1.000, 1.000, 0.000,\n 0.000, 0.333, 0.500,\n 0.000, 0.667, 0.500,\n 0.000, 1.000, 0.500,\n 0.333, 0.000, 0.500,\n 0.333, 0.333, 0.500,\n 0.333, 0.667, 0.500,\n 0.333, 1.000, 0.500,\n 0.667, 0.000, 0.500,\n 0.667, 0.333, 0.500,\n 0.667, 0.667, 0.500,\n 0.667, 1.000, 0.500,\n 1.000, 0.000, 0.500,\n 1.000, 0.333, 0.500,\n 1.000, 0.667, 0.500,\n 1.000, 1.000, 0.500,\n 0.000, 0.333, 1.000,\n 0.000, 0.667, 1.000,\n 0.000, 1.000, 1.000,\n 0.333, 0.000, 1.000,\n 0.333, 0.333, 1.000,\n 0.333, 0.667, 1.000,\n 0.333, 1.000, 1.000,\n 0.667, 0.000, 1.000,\n 0.667, 0.333, 1.000,\n 0.667, 0.667, 1.000,\n 0.667, 1.000, 1.000,\n 1.000, 0.000, 1.000,\n 1.000, 0.333, 1.000,\n 1.000, 0.667, 1.000,\n 0.333, 0.000, 0.000,\n 0.500, 0.000, 0.000,\n 0.667, 0.000, 0.000,\n 0.833, 0.000, 0.000,\n 1.000, 0.000, 0.000,\n 0.000, 0.167, 0.000,\n 0.000, 0.333, 0.000,\n 0.000, 0.500, 0.000,\n 0.000, 0.667, 0.000,\n 0.000, 0.833, 0.000,\n 0.000, 1.000, 0.000,\n 0.000, 0.000, 0.167,\n 0.000, 0.000, 0.333,\n 0.000, 0.000, 0.500,\n 0.000, 0.000, 0.667,\n 0.000, 0.000, 0.833,\n 0.000, 0.000, 1.000,\n 0.000, 0.000, 0.000,\n 0.143, 0.143, 0.143,\n 0.857, 0.857, 0.857,\n 1.000, 1.000, 1.000\n ]\n).astype(np.float32).reshape(-1, 3)\n# fmt: on\n\n\ndef colormap(rgb=False, maximum=255):\n \"\"\"\n Parameters\n ----------\n rgb (bool): whether to return RGB colors or BGR colors.\n maximum (int): either 255 or 1\n\n Returns\n -------\n ndarray: a float32 array of Nx3 colors, in range [0, 255] or [0, 1]\n \"\"\"\n assert maximum in [255, 1], maximum\n c = _COLORS * maximum\n if not rgb:\n c = c[:, ::-1]\n return c\n\n\ndef random_color(rgb=False, maximum=255):\n \"\"\"\n Parameters\n ----------\n rgb (bool): whether to return RGB colors or BGR colors.\n maximum (int): either 255 or 1\n\n Returns\n -------\n ndarray: a vector of 3 numbers\n \"\"\"\n idx = np.random.randint(0, len(_COLORS))\n ret = _COLORS[idx] * maximum\n if not rgb:\n ret = ret[::-1]\n return ret\n\n\ndef random_colors(N, rgb=False, maximum=255):\n \"\"\"\n Parameters\n ----------\n N (int): number of unique colors needed\n rgb (bool): whether to return RGB colors or BGR colors.\n maximum (int): either 255 or 1\n\n Returns\n -------\n ndarray: a list of random_color\n \"\"\"\n indices = random.sample(range(len(_COLORS)), N)\n ret = [_COLORS[i] * maximum for i in indices]\n if not rgb:\n ret = [x[::-1] for x in ret]\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/config.py",
"content": "import copy\nimport logging\nimport os\nimport re\nimport warnings\nfrom typing import Callable, Dict, List, Optional, Tuple, Union\n\nfrom omegaconf import DictConfig, ListConfig, OmegaConf\nfrom torch import nn\n\nfrom ..constants import DATA, FT_TRANSFORMER, FUSION_TRANSFORMER, HF_MODELS, MODEL, REGRESSION, VALID_CONFIG_KEYS\nfrom .presets import get_basic_config, get_ensemble_presets, get_presets\n\nlogger = logging.getLogger(__name__)\n\n\ndef filter_search_space(hyperparameters: Dict, keys_to_filter: Union[str, List[str]]):\n \"\"\"\n Filter search space within hyperparameters without the given keys as prefixes.\n Hyperparameters that are not search space will not be filtered.\n\n Parameters\n ----------\n hyperparameters\n A dictionary containing search space and overrides to config.\n keys_to_filter\n Keys that needs to be filtered out\n\n Returns\n -------\n hyperparameters being filtered\n \"\"\"\n if isinstance(keys_to_filter, str):\n keys_to_filter = [keys_to_filter]\n\n assert any(key.startswith(valid_keys) for valid_keys in VALID_CONFIG_KEYS for key in keys_to_filter), (\n f\"Invalid keys: {keys_to_filter}. Valid options are {VALID_CONFIG_KEYS}\"\n )\n from ray.tune.search.sample import Domain\n\n from autogluon.common import space\n\n hyperparameters = copy.deepcopy(hyperparameters)\n for hyperparameter, value in hyperparameters.copy().items():\n if not isinstance(value, (space.Space, Domain)):\n continue\n for key in keys_to_filter:\n if hyperparameter.startswith(key):\n del hyperparameters[hyperparameter]\n return hyperparameters\n\n\ndef get_default_config(config: Optional[Union[Dict, DictConfig]] = None, extra: Optional[List[str]] = None):\n \"\"\"\n Get the default config.\n\n Parameters\n ----------\n config\n A dictionary including four keys: \"model\", \"data\", \"optim\", and \"env\".\n If any key is not given, we will fill in with the default value.\n extra\n A list of extra config keys.\n\n Returns\n -------\n The default config.\n \"\"\"\n if isinstance(config, DictConfig):\n return config\n\n if config is None:\n config = {}\n\n basic_config = get_basic_config(extra=extra)\n for k, default_value in basic_config.items():\n if k not in config:\n config[k] = default_value\n\n all_configs = []\n for k, v in config.items():\n if isinstance(v, dict):\n per_config = OmegaConf.create(v)\n elif isinstance(v, DictConfig):\n per_config = v\n elif isinstance(v, str):\n if v.lower().endswith((\".yaml\", \".yml\")):\n per_config = OmegaConf.load(os.path.expanduser(v))\n else:\n cur_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n config_path = os.path.join(cur_path, \"configs\", k, f\"{v}.yaml\")\n per_config = OmegaConf.load(config_path)\n else:\n raise ValueError(f\"Unknown configuration type: {type(v)}\")\n\n all_configs.append(per_config)\n\n config = OmegaConf.merge(*all_configs)\n\n return config\n\n\ndef get_config(\n problem_type: Optional[str] = None,\n presets: Optional[str] = None,\n config: Optional[Union[Dict, DictConfig]] = None,\n overrides: Optional[Union[str, List[str], Dict]] = None,\n extra: Optional[List[str]] = None,\n):\n \"\"\"\n Construct configurations for model, data, optim, and env.\n It supports to overrides some default configurations.\n\n Parameters\n ----------\n problem_type\n Problem type.\n presets\n Presets regarding model quality, e.g., best_quality, high_quality, and medium_quality.\n config\n A dictionary including four keys: \"model\", \"data\", \"optim\", and \"env\".\n If any key is not given, we will fill in with the default value.\n\n The value of each key can be a string, yaml path, or DictConfig object. For example:\n config = {\n \"model\": \"default\",\n \"data\": \"default\",\n \"optim\": \"default\",\n \"env\": \"default\",\n }\n or\n config = {\n \"model\": \"/path/to/model/config.yaml\",\n \"data\": \"/path/to/data/config.yaml\",\n \"optim\": \"/path/to/optim/config.yaml\",\n \"env\": \"/path/to/env/config.yaml\",\n }\n or\n config = {\n \"model\": OmegaConf.load(\"/path/to/model/config.yaml\"),\n \"data\": OmegaConf.load(\"/path/to/data/config.yaml\"),\n \"optim\": OmegaConf.load(\"/path/to/optim/config.yaml\"),\n \"env\": OmegaConf.load(\"/path/to/env/config.yaml\"),\n }\n overrides\n This is to override some default configurations.\n For example, changing the text and image backbones can be done by formatting:\n\n a string\n overrides = \"model.hf_text.checkpoint_name=google/electra-small-discriminator\n model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"\n\n or a list of strings\n overrides = [\"model.hf_text.checkpoint_name=google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name=swin_small_patch4_window7_224\"]\n\n or a dictionary\n overrides = {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n extra\n A list of extra config keys.\n\n Returns\n -------\n Configurations as a DictConfig object\n \"\"\"\n\n if not config and not presets:\n presets = \"default\"\n\n if not isinstance(config, DictConfig):\n if presets is None:\n preset_overrides = None\n else:\n preset_overrides, _ = get_presets(problem_type=problem_type, presets=presets)\n\n config = get_default_config(config, extra=extra)\n # apply the preset's overrides\n if preset_overrides:\n config = apply_omegaconf_overrides(config, overrides=preset_overrides, check_key_exist=True)\n\n verify_model_names(config.model)\n logger.debug(f\"overrides: {overrides}\")\n if overrides is not None:\n # avoid manipulating the user-provided overrides\n overrides = copy.deepcopy(overrides)\n # apply customized model names\n overrides = parse_dotlist_conf(overrides) # convert to a dict\n config.model, _ = customize_model_names(\n config=config.model,\n customized_names=overrides.get(\"model.names\", None),\n )\n # remove `model.names` from overrides since it's already applied.\n overrides.pop(\"model.names\", None)\n # apply the user-provided overrides\n config = apply_omegaconf_overrides(config, overrides=overrides, check_key_exist=True)\n verify_model_names(config.model)\n return config\n\n\ndef verify_model_names(config: DictConfig):\n \"\"\"\n Verify whether provided model names are valid.\n\n Parameters\n ----------\n config\n Config should have a attribute `names`, which contains a list of\n attribute names, e.g., [\"timm_image\", \"hf_text\"]. And each string in\n `config.names` should also be a attribute of `config`, e.g, `config.timm_image`.\n \"\"\"\n # must have attribute `names`\n assert hasattr(config, \"names\")\n # return if no names available\n if not config.names:\n return\n # assure no duplicate names\n assert len(config.names) == len(set(config.names))\n # verify that strings in `config.names` match the keys of `config`.\n keys = list(config.keys())\n keys.remove(\"names\")\n assert set(config.names).issubset(set(keys)), f\"`{config.names}` do not match config keys {keys}\"\n\n # verify that no name starts with another one\n names = sorted(config.names, key=lambda ele: len(ele), reverse=True)\n for i in range(len(names)):\n if names[i].startswith(tuple(names[i + 1 :])):\n raise ValueError(f\"name {names[i]} starts with one of another name: {names[i + 1 :]}\")\n\n\ndef get_name_prefix(\n name: str,\n prefixes: List[str],\n):\n \"\"\"\n Get a name's prefix from some available candidates.\n\n Parameters\n ----------\n name\n A name string\n prefixes\n Available prefixes.\n\n Returns\n -------\n Prefix of the name.\n \"\"\"\n search_results = [pre for pre in prefixes if name.lower().startswith(pre)]\n if len(search_results) == 0:\n return None\n elif len(search_results) >= 2:\n raise ValueError(\n f\"Model name `{name}` is mapped to multiple models, \"\n f\"which means some names in `{prefixes}` have duplicate prefixes.\"\n )\n else:\n return search_results[0]\n\n\ndef customize_model_names(\n config: DictConfig,\n customized_names: Union[str, List[str]],\n advanced_hyperparameters: Optional[Dict] = None,\n):\n \"\"\"\n Customize attribute names of `config` with the provided names.\n A valid customized name string should start with one available name\n string in `config`. Customizing the model names in advanced_hyperparameters\n is only used for matcher query and response models currently.\n\n Parameters\n ----------\n config\n Config should have a attribute `names`, which contains a list of\n attribute names, e.g., [\"timm_image\", \"hf_text\"]. And each string in\n `config.names` should also be a attribute of `config`, e.g, `config.timm_image`.\n customized_names\n The provided names to replace the existing ones in `config.names` as well as\n the corresponding attribute names. For example, if `customized_names` is\n [\"timm_image_123\", \"hf_text_abc\"], then `config.timm_image` and `config.hf_text`\n are changed to `config.timm_image_123` and `config.hf_text_abc`.\n advanced_hyperparameters\n The hyperparameters whose values are complex objects, which can't be stored in config.\n\n Returns\n -------\n A new config with its first-level attributes customized by the provided names.\n \"\"\"\n if not customized_names:\n return config, advanced_hyperparameters\n\n if isinstance(customized_names, str):\n customized_names = OmegaConf.from_dotlist([f\"names={customized_names}\"]).names\n\n if advanced_hyperparameters:\n new_advanced_hyperparameters = copy.deepcopy(advanced_hyperparameters)\n else:\n new_advanced_hyperparameters = dict()\n\n new_config = OmegaConf.create()\n new_config.names = []\n available_prefixes = list(config.keys())\n available_prefixes.remove(\"names\")\n for per_name in customized_names:\n per_prefix = get_name_prefix(\n name=per_name,\n prefixes=available_prefixes,\n )\n if per_prefix:\n per_config = getattr(config, per_prefix)\n setattr(new_config, per_name, copy.deepcopy(per_config))\n new_config.names.append(per_name)\n\n if advanced_hyperparameters:\n for k, v in advanced_hyperparameters.items():\n if k.startswith(f\"{MODEL}.{per_prefix}\"):\n new_k = k.replace(f\"{MODEL}.{per_prefix}\", f\"{MODEL}.{per_name}\")\n new_advanced_hyperparameters.pop(k)\n new_advanced_hyperparameters[new_k] = v\n else:\n logger.debug(f\"Removing {per_name}, which doesn't start with any of these prefixes: {available_prefixes}.\")\n\n if len(new_config.names) == 0:\n raise ValueError(\n f\"No customized name in `{customized_names}` starts with name prefixes in `{available_prefixes}`.\"\n )\n\n return new_config, new_advanced_hyperparameters\n\n\ndef save_pretrained_model_configs(\n model: nn.Module,\n config: DictConfig,\n path: str,\n) -> DictConfig:\n \"\"\"\n Save the pretrained model configs to local to make future loading not dependent on Internet access.\n By initializing models with local configs, Huggingface doesn't need to download pretrained weights from Internet.\n\n Parameters\n ----------\n model\n One model.\n config\n A DictConfig object. The model config should be accessible by \"config.model\".\n path\n The path to save pretrained model configs.\n \"\"\"\n # TODO? Fix hardcoded model names.\n requires_saving = any([model_name.lower().startswith(HF_MODELS) for model_name in config.model.names])\n if not requires_saving:\n return config\n\n if (\n len(config.model.names) == 1\n ): # TODO: Not sure this is a sufficient check. Hyperparameter \"model.names\" : [\"hf_text\", \"fusion_mlp\"] fails here.\n model = nn.ModuleList([model])\n else: # assumes the fusion model has a model attribute, a nn.ModuleList\n model = model.model\n for per_model in model:\n if per_model.prefix.lower().startswith(HF_MODELS):\n per_model.config.save_pretrained(os.path.join(path, per_model.prefix))\n model_config = getattr(config.model, per_model.prefix)\n model_config.checkpoint_name = os.path.join(\"local://\", per_model.prefix)\n\n return config\n\n\ndef get_local_pretrained_config_paths(config: DictConfig, path: str) -> DictConfig:\n \"\"\"\n Get the local config paths of hugginface pretrained models. With a local config,\n Hugginface can initialize a model without having to download its pretrained weights.\n\n Parameters\n ----------\n config\n A DictConfig object. The model config should be accessible by \"config.model\".\n path\n The saving path to the pretrained model configs.\n \"\"\"\n for model_name in config.model.names:\n if model_name.lower().startswith(HF_MODELS):\n model_config = getattr(config.model, model_name)\n if model_config.checkpoint_name.startswith(\"local://\"):\n model_config.checkpoint_name = os.path.join(path, model_config.checkpoint_name[len(\"local://\") :])\n assert os.path.exists(\n os.path.join(model_config.checkpoint_name, \"config.json\")\n ) # guarantee the existence of local configs\n\n return config\n\n\ndef parse_dotlist_conf(conf):\n \"\"\"\n Parse the config files that is potentially in the dotlist format to a dictionary.\n\n Parameters\n ----------\n conf\n Apply the conf stored as dotlist, e.g.,\n 'aaa=a, bbb=b' or ['aaa=a, ', 'bbb=b'] to {'aaa': 'a', 'bbb': b}\n\n Returns\n -------\n new_conf\n \"\"\"\n if isinstance(conf, str):\n conf = conf.split()\n need_parse = True\n elif isinstance(conf, (list, tuple)):\n need_parse = True\n elif isinstance(conf, dict):\n need_parse = False\n else:\n raise ValueError(f\"Unsupported format of conf={conf}\")\n if need_parse:\n new_conf = dict()\n curr_key = None\n curr_value = \"\"\n for ele in conf:\n if \"=\" in ele:\n key, v = ele.split(\"=\")\n if curr_key is not None:\n new_conf[curr_key] = curr_value\n curr_key = key\n curr_value = v\n else:\n if curr_key is None:\n raise ValueError(f\"Cannot parse the conf={conf}\")\n curr_value = curr_value + \" \" + ele\n if curr_key is not None:\n new_conf[curr_key] = curr_value\n return new_conf\n else:\n return conf\n\n\ndef apply_omegaconf_overrides(\n conf: DictConfig,\n overrides: Union[List, Tuple, str, Dict, DictConfig],\n check_key_exist=True,\n):\n \"\"\"\n Apply omegaconf overrides.\n\n Parameters\n ----------\n conf\n The base configuration.\n overrides\n The overrides can be a string or a list.\n check_key_exist\n Whether to check if all keys in the overrides must exist in the conf.\n\n Returns\n -------\n new_conf\n The updated configuration.\n \"\"\"\n overrides = parse_dotlist_conf(overrides)\n overrides = make_overrides_backward_compatible(overrides)\n\n def _check_exist_dotlist(C, key_in_dotlist):\n if not isinstance(key_in_dotlist, list):\n key_in_dotlist = key_in_dotlist.split(\".\")\n if key_in_dotlist[0] in C:\n if len(key_in_dotlist) > 1:\n return _check_exist_dotlist(C[key_in_dotlist[0]], key_in_dotlist[1:])\n else:\n return True\n else:\n return False\n\n if check_key_exist:\n for ele in overrides.items():\n if not _check_exist_dotlist(conf, ele[0]):\n raise KeyError(\n f'\"{ele[0]}\" is not found in the config. You may need to check the overrides. '\n f\"overrides={overrides}\"\n )\n override_conf = OmegaConf.from_dotlist([f\"{ele[0]}={ele[1]}\" for ele in overrides.items()])\n replace_none_str(override_conf)\n conf = OmegaConf.merge(conf, override_conf)\n return conf\n\n\ndef replace_none_str(config: Union[DictConfig, ListConfig, dict, list]):\n \"\"\"\n In-place replace \"None\" and \"none\" strings in the config with None.\n\n Parameters\n ----------\n config\n A config of type DictConfig, ListConfig, dict, or list.\n \"\"\"\n if isinstance(config, (dict, DictConfig)):\n for key, value in config.items():\n if isinstance(value, str) and value.lower() == \"none\":\n config[key] = None\n elif isinstance(value, (dict, list, DictConfig, ListConfig)):\n replace_none_str(value)\n elif isinstance(config, (list, ListConfig)):\n for i, value in enumerate(config):\n if isinstance(value, str) and value.lower() == \"none\":\n config[i] = None\n elif isinstance(value, (dict, list, DictConfig, ListConfig)):\n replace_none_str(value)\n\n\ndef update_config_by_rules(\n problem_type: str,\n config: DictConfig,\n):\n \"\"\"\n Modify configs based on the need of loss func.\n Now it support changing the preprocessing of numerical label into Minmaxscaler while using BCEloss.\n\n Parameters\n ----------\n problem_type\n The type of the problem of the project.\n config\n The config of the project. It is a Dictconfig object.\n\n Returns\n -------\n The modified config.\n \"\"\"\n loss_func = config.optim.loss_func\n if loss_func is not None:\n if problem_type == REGRESSION and \"bce\" in loss_func.lower():\n # To use BCELoss for regression problems, need to first scale the labels.\n config.data.label.numerical_preprocessing = \"minmaxscaler\"\n\n return config\n\n\ndef update_tabular_config_by_resources(\n config: DictConfig,\n num_numerical_columns: Optional[int] = 0,\n num_categorical_columns: Optional[int] = 0,\n resource: Optional[int] = 16,\n):\n \"\"\"\n Modify configs based on the dataset statistics.\n Use Additive attention with large column count and tune batch size accordingly.\n Parameters\n ----------\n config\n The config of the project. It is a Dictconfig object.\n num_numerical_columns\n The number of numerical columns.\n num_categorical_columns\n The number of categorical columns.\n resource\n The maximum resource (memory in GB) a single GPU has.\n Returns\n -------\n The modified config.\n \"\"\"\n columns_per_model = {\n FUSION_TRANSFORMER: num_categorical_columns + num_numerical_columns,\n FT_TRANSFORMER: num_categorical_columns + num_numerical_columns,\n }\n\n # Threshold is expected to be ~= batch_size * num_tokens, for additive attention.\n # The multiplier 2e4 is a heuristic found from AutoML Benchmark.\n # TODO: determine the threshold/batch_size on training data directly\n threshold = resource * 2e4\n per_gpu_batch_size = config.env.per_gpu_batch_size\n for model in columns_per_model:\n if model in config.model.names:\n model_ = getattr(config.model, model)\n if columns_per_model[model] > 300 and model_.additive_attention == \"auto\":\n model_.additive_attention = True\n model_.share_qv_weights = True if model_.share_qv_weights == \"auto\" else model_.share_qv_weights\n warnings.warn(\n f\"Dataset contains >300 features, using additive attention for efficiency\",\n UserWarning,\n )\n if columns_per_model[model] * per_gpu_batch_size > threshold:\n per_gpu_batch_size = int(threshold / columns_per_model[model])\n\n model_.additive_attention = False if model_.additive_attention == \"auto\" else model_.additive_attention\n model_.share_qv_weights = False if model_.share_qv_weights == \"auto\" else model_.share_qv_weights\n\n per_gpu_batch_size = max(per_gpu_batch_size, 1)\n if per_gpu_batch_size < config.env.per_gpu_batch_size:\n config.env.per_gpu_batch_size = per_gpu_batch_size\n warnings.warn(\n f\"Setting per_gpu_batch_size to {per_gpu_batch_size} to fit into GPU memory\",\n UserWarning,\n )\n\n return config\n\n\ndef get_pretrain_configs_dir(subfolder: Optional[str] = None):\n import autogluon.multimodal\n\n pretrain_config_dir = os.path.join(autogluon.multimodal.__path__[0], \"configs\", \"pretrain\")\n if subfolder:\n pretrain_config_dir = os.path.join(pretrain_config_dir, subfolder)\n return pretrain_config_dir\n\n\ndef filter_timm_pretrained_cfg(cfg, remove_source=False, remove_null=True):\n filtered_cfg = {}\n keep_null = {\"pool_size\", \"first_conv\", \"classifier\"} # always keep these keys, even if none\n for k, v in cfg.items():\n if remove_source and k in {\"url\", \"file\", \"hf_hub_id\", \"hf_hub_id\", \"hf_hub_filename\", \"source\"}:\n continue\n if remove_null and v is None and k not in keep_null:\n continue\n filtered_cfg[k] = v\n return filtered_cfg\n\n\ndef update_hyperparameters(\n problem_type,\n presets,\n provided_hyperparameters,\n provided_hyperparameter_tune_kwargs,\n):\n \"\"\"\n Update preset hyperparameters hyperparameter_tune_kwargs by the provided.\n Currently, this is mainly used for HPO presets, which define some searchable hyperparameters.\n We need to combine these searchable hyperparameters with ones provided by users.\n\n Parameters\n ----------\n problem_type\n Problem type.\n presets\n A preset string regarding modality quality or hpo.\n provided_hyperparameters\n The hyperparameters provided by users.\n provided_hyperparameter_tune_kwargs\n The hyperparameter_tune_kwargs provided by users.\n\n Returns\n -------\n The updated hyperparameters and hyperparameter_tune_kwargs.\n \"\"\"\n hyperparameters, hyperparameter_tune_kwargs = get_presets(problem_type=problem_type, presets=presets)\n\n if hyperparameter_tune_kwargs and provided_hyperparameter_tune_kwargs:\n hyperparameter_tune_kwargs.update(provided_hyperparameter_tune_kwargs)\n elif provided_hyperparameter_tune_kwargs:\n hyperparameter_tune_kwargs = provided_hyperparameter_tune_kwargs\n\n if hyperparameter_tune_kwargs:\n if provided_hyperparameters:\n hyperparameters.update(provided_hyperparameters)\n else: # use the provided hyperparameters if no hpo. The preset hyperparameters will be also used later in get_config.\n hyperparameters = provided_hyperparameters\n\n if hyperparameter_tune_kwargs:\n assert isinstance(hyperparameters, dict), (\n \"Please provide hyperparameters as a dictionary if you want to do HPO\"\n )\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\ndef filter_hyperparameters(\n hyperparameters: Dict,\n column_types: Dict,\n config: Optional[Union[Dict, DictConfig]] = None,\n fit_called: Optional[bool] = False,\n):\n \"\"\"\n Filter out the hyperparameters that have no effect for HPO.\n\n Parameters\n ----------\n hyperparameters\n The hyperparameters to override the default config.\n column_types\n Dataframe's column types.\n config\n A config provided by users or from the previous training.\n fit_called\n Whether fit() has been called.\n\n Returns\n -------\n The filtered hyperparameters.\n \"\"\"\n model_names_key = f\"{MODEL}.names\"\n keys_to_filter = []\n\n if model_names_key in hyperparameters:\n # If continuous training or config is provided, make sure models are in config.model.\n config = get_default_config(config)\n selected_model_names = []\n config_model_names = list(config.model.keys())\n config_model_names.remove(\"names\")\n for name in hyperparameters[model_names_key]:\n if name in config_model_names:\n selected_model_names.append(name)\n hyperparameters[model_names_key] = selected_model_names\n assert len(selected_model_names) > 0, (\n f\"hyperparameters['model.names'] {hyperparameters[model_names_key]} doesn't match any config model names {config_model_names}.\"\n )\n\n # Filter models that are not in hyperparameters[model_names_key]\n # Avoid key not in config error when applying the overrides later.\n model_keys = [k for k in hyperparameters.keys() if k.startswith(MODEL)]\n if model_keys and model_names_key in model_keys:\n model_keys.remove(model_names_key)\n for k in model_keys:\n if k.split(\".\")[1] not in hyperparameters[model_names_key] and k not in keys_to_filter:\n keys_to_filter.append(k)\n\n # Filter models whose data types are not detected.\n # Avoid sampling unused checkpoints, e.g., hf_text models for image classification, to run jobs,\n # which wastes resources and time.\n from ..data.utils import get_detected_data_types\n\n detected_data_types = get_detected_data_types(column_types)\n selected_model_names = []\n for model_name in hyperparameters[model_names_key]:\n model_config = config.model[model_name]\n if model_config.data_types:\n model_data_status = [d_type in detected_data_types for d_type in model_config.data_types]\n if not all(model_data_status):\n keys_to_filter.append(f\"{MODEL}.{model_name}\")\n else:\n selected_model_names.append(model_name)\n else: # keep the fusion model, which will be handled by select_model().\n selected_model_names.append(model_name)\n hyperparameters[model_names_key] = selected_model_names\n assert len(selected_model_names) > 0, (\n f\"Model {hyperparameters[model_names_key]} can't handle the data with column types {column_types}\"\n )\n\n # Filter keys for continuous training.\n # Model and data processors would be reused.\n if fit_called:\n warnings.warn(\n \"HPO while continuous training.\"\n \"Hyperparameters related to Model and Data will NOT take effect.\"\n \"We will filter them out from the search space.\"\n )\n keys_to_filter.extend([MODEL, DATA])\n\n for key in keys_to_filter:\n hyperparameters = {k: v for k, v in hyperparameters.items() if not k.startswith(key)}\n\n return hyperparameters\n\n\ndef split_hyperparameters(hyperparameters: Dict):\n \"\"\"\n Split out some advanced hyperparameters whose values are complex objects instead of strings or numbers.\n\n Parameters\n ----------\n hyperparameters\n The user provided hyperparameters.\n\n Returns\n -------\n Hyperparameters and advanced hyperparameters.\n \"\"\"\n if not isinstance(hyperparameters, dict): # only support complex objects in dict.\n return hyperparameters, dict()\n\n if not hyperparameters:\n return hyperparameters, dict()\n\n advanced_hyperparameters = dict()\n for k, v in hyperparameters.items():\n if re.search(\"^model.*train_transforms$\", k) or re.search(\"^model.*val_transforms$\", k):\n if all([isinstance(trans, str) for trans in hyperparameters[k]]):\n pass\n elif all([isinstance(trans, Callable) for trans in hyperparameters[k]]):\n advanced_hyperparameters[k] = copy.deepcopy(v)\n hyperparameters[k] = str(v) # get the objects' class strings\n else:\n raise ValueError(f\"transform_types {v} contain neither all strings nor all callable objects.\")\n\n return hyperparameters, advanced_hyperparameters\n\n\ndef update_ensemble_hyperparameters(\n presets,\n provided_hyperparameters,\n):\n presets_hyperparameters, _ = get_ensemble_presets(presets=presets)\n if provided_hyperparameters:\n learner_names = provided_hyperparameters.pop(\"learner_names\", None)\n if learner_names:\n assert isinstance(learner_names, list), (\n f\"learner_names should be a list, but got type {type(learner_names)}\"\n )\n presets_hyperparameters = {k: v for k, v in presets_hyperparameters.items() if k in learner_names}\n provided_hyperparameters = {k: v for k, v in provided_hyperparameters.items() if k in learner_names}\n\n hyperparameters = copy.deepcopy(provided_hyperparameters)\n for k, v in presets_hyperparameters.items():\n if k not in hyperparameters:\n hyperparameters[k] = v\n else:\n for kk, vv in presets_hyperparameters[k].items():\n if kk not in hyperparameters[k]: # don't use presets to overwrite user-provided\n hyperparameters[k][kk] = vv\n else:\n hyperparameters = presets_hyperparameters\n\n return hyperparameters\n\n\ndef make_overrides_backward_compatible(overrides: Dict):\n \"\"\"\n Some config keys were changed in PR https://github.com/autogluon/autogluon/pull/4737\n This function is to make the changes backward compatible.\n\n Parameters\n ----------\n overrides\n A dictionary containing the user-provided hyperparameters,\n which may contain old config keys.\n\n Returns\n -------\n Overrides with up-to-date config keys.\n \"\"\"\n key_pairs = {\n \"optim.learning_rate\": \"optim.lr\",\n \"optim.efficient_finetune\": \"optim.peft\",\n \"optim.loss_function\": \"optim.loss_func\",\n \"env.num_workers_evaluation\": \"env.num_workers_inference\",\n \"env.eval_batch_size_ratio\": \"env.inference_batch_size_ratio\",\n \"data.label.numerical_label_preprocessing\": \"data.label.numerical_preprocessing\",\n \"model.categorical_mlp.drop_rate\": \"model.categorical_mlp.dropout\",\n \"model.numerical_mlp.drop_rate\": \"model.numerical_mlp.dropout\",\n \"model.numerical_mlp.d_token\": \"model.numerical_mlp.token_dim\",\n \"model.timm_image.max_img_num_per_col\": \"model.timm_image.max_image_num_per_column\",\n \"model.clip.max_img_num_per_col\": \"model.clip.max_image_num_per_column\",\n \"model.clip_image.max_img_num_per_col\": \"model.clip_image.max_image_num_per_column\",\n \"model.fusion_mlp.weight\": \"model.fusion_mlp.aux_loss_weight\",\n \"model.fusion_mlp.drop_rate\": \"model.fusion_mlp.dropout\",\n \"model.fusion_transformer.n_blocks\": \"model.fusion_transformer.num_blocks\",\n \"model.fusion_transformer.attention_n_heads\": \"model.fusion_transformer.attention_num_heads\",\n \"model.fusion_transformer.ffn_d_hidden\": \"model.fusion_transformer.ffn_hidden_size\",\n \"model.ft_transformer.attention_n_heads\": \"model.ft_transformer.attention_num_heads\",\n }\n for k in list(overrides.keys()):\n provided_k = k\n if k.startswith(\"optimization.\"):\n k = \"optim.\" + k[len(\"optimization.\") :]\n logger.warning(\n f\"The provided hyperparameter name {provided_k} contains a deprecated key `optimization.`. \"\n f\"Please replace `optimization.` with `optim.` when customizing the optimization hyperparameters.\"\n )\n\n if k in key_pairs:\n overrides[key_pairs[k]] = overrides.pop(provided_k)\n logger.warning(\n f\"The hyperparameter name {provided_k} is depreciated. \"\n f\"We recommend using the new name {key_pairs[k]} instead.\"\n f\"The deprecated hyperparameter will raise an exception starting in AutoGluon 1.4.0\"\n )\n\n return overrides\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/device.py",
"content": "import logging\nimport math\nimport warnings\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport torch\nfrom lightning.pytorch.accelerators import find_usable_cuda_devices\nfrom torch import nn\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom .env import is_interactive_env\n\nlogger = logging.getLogger(__name__)\n\n\ndef compute_num_gpus(config_num_gpus: Union[int, float, List], accelerator: str):\n \"\"\"\n Compute the gpu number to initialize the lightning trainer.\n\n Parameters\n ----------\n config_num_gpus\n The gpu number provided by config.\n accelerator\n # \"cpu\", \"gpu\", or \"auto\".\n\n Returns\n -------\n A valid gpu number for the current environment and config.\n \"\"\"\n if isinstance(accelerator, str) and accelerator.lower() not in [\"gpu\", \"auto\"]:\n return 0\n\n config_num_gpus = (\n math.floor(config_num_gpus) if isinstance(config_num_gpus, (int, float)) else len(config_num_gpus)\n )\n detected_num_gpus = ResourceManager.get_gpu_count_torch()\n\n if config_num_gpus < 0: # In case config_num_gpus is -1, meaning using all gpus.\n num_gpus = detected_num_gpus\n else:\n num_gpus = min(config_num_gpus, detected_num_gpus)\n if detected_num_gpus < config_num_gpus:\n warnings.warn(\n f\"Using the detected GPU number {detected_num_gpus}, \"\n f\"smaller than the GPU number {config_num_gpus} in the config.\",\n UserWarning,\n )\n\n return num_gpus\n\n\ndef move_to_device(obj: Union[torch.Tensor, nn.Module, Dict, List, Tuple], device: torch.device):\n \"\"\"\n Move an object to the given device.\n\n Parameters\n ----------\n obj\n An object, which can be a tensor, a module, a dict, or a list.\n device\n A Pytorch device instance.\n\n Returns\n -------\n The object on the device.\n \"\"\"\n if not isinstance(device, torch.device):\n raise ValueError(f\"Invalid device: {device}. Ensure the device type is `torch.device`.\")\n\n if torch.is_tensor(obj) or isinstance(obj, nn.Module):\n return obj.to(device)\n elif isinstance(obj, dict):\n res = {}\n for k, v in obj.items():\n res[k] = move_to_device(v, device)\n return res\n elif isinstance(obj, list) or isinstance(obj, tuple):\n res = []\n for v in obj:\n res.append(move_to_device(v, device))\n return res\n elif isinstance(obj, (int, float, str)):\n return obj\n else:\n raise TypeError(\n f\"Invalid type {type(obj)} for move_to_device. \"\n f\"Make sure the object is one of these: a Pytorch tensor, a Pytorch module, \"\n f\"a dict or list of tensors or modules.\"\n )\n\n\ndef get_available_devices(num_gpus: int, auto_select_gpus: bool):\n \"\"\"\n Get the available devices.\n\n Parameters\n ----------\n num_gpus\n Number of GPUs.\n auto_select_gpus\n Whether to pick GPU indices that are \"accessible\". See here: https://github.com/Lightning-AI/lightning/blob/accd2b9e61063ba3c683764043030545ed87c71f/src/lightning/fabric/accelerators/cuda.py#L79\n\n Returns\n -------\n The available devices.\n \"\"\"\n if num_gpus > 0:\n if auto_select_gpus:\n if is_interactive_env():\n devices = list(range(num_gpus))\n else:\n devices = find_usable_cuda_devices(num_gpus)\n else:\n devices = num_gpus\n else:\n devices = \"auto\"\n\n return devices\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/distillation.py",
"content": "import logging\nfrom typing import Callable, Dict, List, Optional, Union\n\nfrom omegaconf import DictConfig, OmegaConf\nfrom torch import nn\n\nfrom ..constants import REGRESSION\nfrom ..data import turn_on_off_feature_column_info\nfrom ..models import modify_duplicate_model_names\nfrom ..optim.losses import RKDLoss\n\nlogger = logging.getLogger(__name__)\n\n\nclass DistillationMixin:\n def setup_distillation(\n self,\n model: nn.Module,\n loss_func: Callable,\n config: DictConfig,\n data_processors: Dict,\n ):\n \"\"\"\n Prepare for distillation. It verifies whether the student and teacher learners have consistent\n configurations. If teacher and student have duplicate model names, it modifies teacher's model names.\n\n Returns\n -------\n distillation_kwargs\n Distillation related keyword arguments.\n \"\"\"\n if self._teacher_learner is None:\n return dict()\n logger.debug(\"setting up distillation...\")\n if isinstance(self._teacher_learner, str):\n from ..learners.base import BaseLearner\n\n self._teacher_learner = BaseLearner.load(self._teacher_learner)\n\n # verify that student and teacher configs are consistent.\n assert self._problem_type == self._teacher_learner.problem_type\n assert self._label_column == self._teacher_learner._label_column\n assert self._output_shape == self._teacher_learner._output_shape\n\n # if teacher and student have duplicate model names, change teacher's model names\n # we don't change student's model names to avoid changing the names back when saving the model.\n self._teacher_learner = modify_duplicate_model_names(\n learner=self._teacher_learner,\n postfix=\"teacher\",\n blacklist=config.model.names,\n )\n\n critics, baseline_funcs = None, None\n if not config.distiller.soft_label_loss_type:\n # automatically infer loss func based on problem type if not specified\n if self._problem_type == REGRESSION:\n soft_label_loss_func = nn.MSELoss()\n else:\n assert self._output_shape > 1\n soft_label_loss_func = nn.CrossEntropyLoss()\n elif config.distiller.soft_label_loss_type == \"mse\":\n soft_label_loss_func = nn.MSELoss()\n elif config.distiller.soft_label_loss_type == \"cross_entropy\":\n soft_label_loss_func = nn.CrossEntropyLoss()\n else:\n raise ValueError(f\"Unknown soft_label_loss_type: {config.distiller.soft_label_loss_type}\")\n\n if not config.distiller.softmax_regression_loss_type:\n # automatically infer loss func based on problem type if not specified\n if self._problem_type == REGRESSION:\n softmax_regression_loss_func = nn.MSELoss()\n else:\n assert self._output_shape > 1\n softmax_regression_loss_func = nn.CrossEntropyLoss()\n elif config.distiller.softmax_regression_loss_type == \"mse\":\n softmax_regression_loss_func = nn.MSELoss()\n elif config.distiller.softmax_regression_loss_type == \"cross_entropy\":\n softmax_regression_loss_func = nn.CrossEntropyLoss()\n else:\n raise ValueError(f\"Unknown soft_label_loss_type: {config.distiller.softmax_regression_loss_type}\")\n\n output_feature_loss_type = config.distiller.output_feature_loss_type\n if output_feature_loss_type == \"cosine\":\n output_feature_loss_func = nn.CosineEmbeddingLoss()\n elif output_feature_loss_type == \"mse\":\n output_feature_loss_func = nn.MSELoss()\n else:\n raise ValueError(f\"Unknown output_feature_loss_type: {output_feature_loss_type}\")\n\n # Adapt student's output_feature feature to teacher's\n # Refer to FitNet: https://arxiv.org/abs/1412.6550\n teacher_model_dim = self._teacher_learner._model.out_features\n student_model_dim = model.out_features\n output_feature_adaptor = (\n nn.Linear(student_model_dim, teacher_model_dim)\n if teacher_model_dim != student_model_dim\n else nn.Identity()\n )\n\n rkd_distance_loss_weight = config.distiller.rkd_distance_loss_weight\n rkd_angle_loss_weight = config.distiller.rkd_angle_loss_weight\n rkd_loss_func = RKDLoss(rkd_distance_loss_weight, rkd_angle_loss_weight)\n output_feature_loss_weight = config.distiller.output_feature_loss_weight\n softmax_regression_weight = config.distiller.softmax_regression_weight\n\n # turn on returning column information in data processors\n turn_on_off_feature_column_info(\n data_processors=data_processors,\n flag=True,\n )\n turn_on_off_feature_column_info(\n data_processors=self._teacher_learner._data_processors,\n flag=True,\n )\n\n return dict(\n matches=config.distiller.matches,\n critics=critics,\n baseline_funcs=baseline_funcs,\n hard_label_weight=config.distiller.hard_label_weight,\n soft_label_weight=config.distiller.soft_label_weight,\n softmax_regression_weight=softmax_regression_weight,\n temperature=config.distiller.temperature,\n output_feature_loss_weight=output_feature_loss_weight,\n hard_label_loss_func=loss_func,\n soft_label_loss_func=soft_label_loss_func,\n softmax_regression_loss_func=softmax_regression_loss_func,\n output_feature_adaptor=output_feature_adaptor,\n output_feature_loss_func=output_feature_loss_func,\n rkd_loss_func=rkd_loss_func,\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/download.py",
"content": "import functools\nimport hashlib\nimport logging\nimport os\nimport re\nimport sys\nimport uuid\nimport warnings\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport boto3\nimport requests\nimport tqdm\n\nfrom ..constants import S3_PREFIX\n\nlogger = logging.getLogger(__name__)\n\n\n_URL_REGEX = re.compile(\n r\"^(?:http|ftp)s?://\" # http:// or https://\n r\"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+(?:[A-Z]{2,6}\\.?|[A-Z0-9-]{2,}\\.?)|\" # domain...\n r\"localhost|\" # localhost...\n r\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\" # ...or ip\n r\"(?::\\d+)?\" # optional port\n r\"(?:/?|[/?]\\S+)$\",\n re.IGNORECASE,\n)\n\n\ndef is_url(url_like: str):\n \"\"\"\n Check if a path is url or local.\n\n Parameters\n ----------\n url_like\n A provided path.\n\n Returns\n -------\n A boolean indicate if the path is an url.\n \"\"\"\n if not isinstance(url_like, str):\n return False\n return re.match(_URL_REGEX, url_like) is not None\n\n\ndef download(\n url: str,\n path: Optional[str] = None,\n overwrite: Optional[bool] = False,\n sha1_hash: Optional[str] = None,\n retries: Optional[int] = 5,\n verify_ssl: Optional[bool] = True,\n) -> str:\n \"\"\"\n Download a file from a given URL. Some util functions are also included in this function.\n https://github.com/sxjscience/automl_multimodal_benchmark/blob/main/multimodal_text_benchmark/src/auto_mm_bench/utils.py\n Parameters\n ----------\n url\n URL to download\n path\n Destination path to store downloaded file. By default stores to the\n current directory with same name as in url.\n overwrite\n Whether to overwrite destination file if already exists.\n sha1_hash\n Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified\n but doesn't match.\n retries\n The number of times to attempt the download in case of failure or non 200 return codes\n verify_ssl\n Verify SSL certificates.\n Returns\n -------\n fname\n The file path of the downloaded file.\n \"\"\"\n\n if not sys.platform.startswith(\"win32\"):\n # refer to https://github.com/untitaker/python-atomicwrites\n def replace_file(src, dst):\n \"\"\"Implement atomic os.replace with linux and OSX.\n Parameters\n ----------\n src : source file path\n dst : destination file path\n \"\"\"\n try:\n os.rename(src, dst)\n except OSError:\n try:\n os.remove(src)\n except OSError:\n pass\n finally:\n raise OSError(\n \"Moving downloaded temp file - {}, to {} failed. \\\n Please retry the download.\".format(src, dst)\n )\n\n else:\n import ctypes\n\n _MOVEFILE_REPLACE_EXISTING = 0x1\n # Setting this value guarantees that a move performed as a copy\n # and delete operation is flushed to disk before the function returns.\n # The flush occurs at the end of the copy operation.\n _MOVEFILE_WRITE_THROUGH = 0x8\n _windows_default_flags = _MOVEFILE_WRITE_THROUGH\n\n def _str_to_unicode(x):\n \"\"\"Handle text decoding. Internal use only\"\"\"\n if not isinstance(x, str):\n return x.decode(sys.getfilesystemencoding())\n return x\n\n def _handle_errors(rv, src):\n \"\"\"Handle WinError. Internal use only\"\"\"\n if not rv:\n msg = ctypes.FormatError(ctypes.GetLastError())\n # if the MoveFileExW fails(e.g. fail to acquire file lock), removes the tempfile\n try:\n os.remove(src)\n except OSError:\n pass\n finally:\n raise OSError(msg)\n\n def replace_file(src, dst):\n \"\"\"Implement atomic os.replace with windows.\n refer to https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-movefileexw\n The function fails when one of the process(copy, flush, delete) fails.\n Parameters\n ----------\n src : source file path\n dst : destination file path\n \"\"\"\n _handle_errors(\n ctypes.windll.kernel32.MoveFileExW(\n _str_to_unicode(src), _str_to_unicode(dst), _windows_default_flags | _MOVEFILE_REPLACE_EXISTING\n ),\n src,\n )\n\n def sha1sum(filename: str):\n \"\"\"\n Calculate the sha1sum of a file.\n Parameters\n ----------\n filename\n Name of the file\n Returns\n -------\n ret\n The sha1sum\n \"\"\"\n with open(filename, mode=\"rb\") as f:\n # Disable bandit check because we are using sha1sum for evaluating the checksums.\n # It is not used for hosting credentials.\n d = hashlib.new(\"sha1\", usedforsecurity=False) # nosec\n for buf in iter(functools.partial(f.read, 1024 * 100), b\"\"):\n d.update(buf)\n return d.hexdigest()\n\n is_s3 = url.startswith(S3_PREFIX)\n if is_s3:\n s3 = boto3.resource(\"s3\")\n if boto3.session.Session().get_credentials() is None:\n from botocore.handlers import disable_signing\n\n s3.meta.client.meta.events.register(\"choose-signer.s3.*\", disable_signing)\n components = url[len(S3_PREFIX) :].split(\"/\")\n if len(components) < 2:\n raise ValueError(\"Invalid S3 url. Received url={}\".format(url))\n s3_bucket_name = components[0]\n s3_key = \"/\".join(components[1:])\n if path is None:\n fname = url.split(\"/\")[-1]\n # Empty filenames are invalid\n assert fname, \"Can't construct file-name from this URL. Please set the `path` option manually.\"\n else:\n path = os.path.expanduser(path)\n if os.path.isdir(path):\n fname = os.path.join(path, url.split(\"/\")[-1])\n else:\n fname = path\n assert retries >= 0, \"Number of retries should be at least 0, currently it's {}\".format(retries)\n\n if not verify_ssl:\n warnings.warn(\n \"Unverified HTTPS request is being made (verify_ssl=False). \"\n \"Adding certificate verification is strongly advised.\"\n )\n\n if overwrite or not os.path.exists(fname) or (sha1_hash and not sha1sum(fname) == sha1_hash):\n dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname)))\n if not os.path.exists(dirname):\n os.makedirs(dirname, exist_ok=True)\n while retries + 1 > 0:\n # Disable pyling too broad Exception\n # pylint: disable=W0703\n try:\n print(\"Downloading {} from {}...\".format(fname, url))\n if is_s3:\n response = s3.meta.client.head_object(Bucket=s3_bucket_name, Key=s3_key)\n total_size = int(response.get(\"ContentLength\", 0))\n random_uuid = str(uuid.uuid4())\n tmp_path = \"{}.{}\".format(fname, random_uuid)\n if tqdm is not None:\n\n def hook(t_obj):\n def inner(bytes_amount):\n t_obj.update(bytes_amount)\n\n return inner\n\n with tqdm.tqdm(total=total_size, unit=\"iB\", unit_scale=True) as t:\n s3.meta.client.download_file(s3_bucket_name, s3_key, tmp_path, Callback=hook(t))\n else:\n s3.meta.client.download_file(s3_bucket_name, s3_key, tmp_path)\n else:\n r = requests.get(url, stream=True, verify=verify_ssl, timeout=(10, 10000))\n if r.status_code != 200:\n raise RuntimeError(\"Failed downloading url {}\".format(url))\n # create uuid for temporary files\n random_uuid = str(uuid.uuid4())\n total_size = int(r.headers.get(\"content-length\", 0))\n chunk_size = 1024\n if tqdm is not None:\n t = tqdm.tqdm(total=total_size, unit=\"iB\", unit_scale=True, leave=False)\n with open(\"{}.{}\".format(fname, random_uuid), \"wb\") as f:\n for chunk in r.iter_content(chunk_size=chunk_size):\n if chunk: # filter out keep-alive new chunks\n if tqdm is not None:\n t.update(len(chunk))\n f.write(chunk)\n if tqdm is not None:\n t.close()\n # if the target file exists(created by other processes)\n # and have the same hash with target file\n # delete the temporary file\n if not os.path.exists(fname) or (sha1_hash and not sha1sum(fname) == sha1_hash):\n # atomic operation in the same file system\n replace_file(\"{}.{}\".format(fname, random_uuid), fname)\n else:\n try:\n os.remove(\"{}.{}\".format(fname, random_uuid))\n except OSError:\n pass\n finally:\n warnings.warn(\"File {} exists in file system so the downloaded file is deleted\".format(fname))\n if sha1_hash and not sha1sum(fname) == sha1_hash:\n raise UserWarning(\n \"File {} is downloaded but the content hash does not match.\"\n \" The repo may be outdated or download may be incomplete. \"\n 'If the \"repo_url\" is overridden, consider switching to '\n \"the default repo.\".format(fname)\n )\n break\n except Exception as e:\n retries -= 1\n if retries <= 0:\n raise e\n\n print(\n \"download failed due to {}, retrying, {} attempt{} left\".format(\n repr(e), retries, \"s\" if retries > 1 else \"\"\n )\n )\n\n return fname\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/env.py",
"content": "import logging\nimport sys\nfrom pathlib import Path\nfrom typing import Any, Union\n\nfrom fsspec.core import url_to_fs\nfrom fsspec.implementations.local import AbstractFileSystem\n\nlogger = logging.getLogger(__name__)\n\n\ndef is_interactive_env():\n \"\"\"\n Return whether the current process is running under the interactive mode.\n Check also https://stackoverflow.com/a/64523765\n \"\"\"\n return hasattr(sys, \"ps1\")\n\n\ndef get_filesystem(path: Union[str, Path], **kwargs: Any) -> AbstractFileSystem:\n fs, _ = url_to_fs(str(path), **kwargs)\n return fs\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/export.py",
"content": "import io\nimport logging\nimport os\nimport warnings\nfrom collections import defaultdict\nfrom typing import Dict, List, Optional, Union\n\nimport pandas as pd\nimport torch\n\nfrom ..constants import CATEGORICAL, HF_TEXT, IMAGE_PATH, MMDET_IMAGE, NULL, NUMERICAL, TEXT, TIMM_IMAGE\nfrom ..models.fusion import AbstractMultimodalFusionModel\nfrom ..models.hf_text import HFAutoModelForTextPrediction\nfrom ..models.mmdet_image import MMDetAutoModelForObjectDetection\nfrom ..models.timm_image import TimmAutoModelForImagePrediction\nfrom .onnx import OnnxModule, onnx_get_dynamic_axes\nfrom .precision import infer_precision\n\nlogger = logging.getLogger(__name__)\n\n\nclass ExportMixin:\n def dump_model(self, save_path: Optional[str] = None):\n \"\"\"\n Save model weights and config to local directory.\n Model weights are saved in file `pytorch_model.bin` (timm, hf) or '.pth' (mmdet);\n Configs are saved in file `config.json` (timm, hf) or '.py' (mmdet).\n\n Parameters\n ----------\n save_path : str\n Path to directory where models and configs should be saved.\n \"\"\"\n\n if not save_path:\n save_path = self._save_path if self._save_path else \"./\"\n\n supported_models = {\n TIMM_IMAGE: TimmAutoModelForImagePrediction,\n HF_TEXT: HFAutoModelForTextPrediction,\n MMDET_IMAGE: MMDetAutoModelForObjectDetection,\n }\n\n models = defaultdict(list)\n # TODO: simplify the code\n if isinstance(self._model, AbstractMultimodalFusionModel) and isinstance(\n self._model.model, torch.nn.modules.container.ModuleList\n ):\n for per_model in self._model.model:\n for model_key, model_type in supported_models.items():\n if isinstance(per_model, model_type):\n models[model_key].append(per_model)\n else:\n for model_key, model_type in supported_models.items():\n if isinstance(self._model, model_type):\n models[model_key].append(self._model)\n\n if not models:\n raise NotImplementedError(\n f\"No models available for dump. Current supported models are: {supported_models.keys()}\"\n )\n\n for model_key in models:\n for per_model in models[model_key]:\n subdir = os.path.join(save_path, per_model.prefix)\n os.makedirs(subdir, exist_ok=True)\n per_model.save(save_path=subdir)\n\n return save_path\n\n def export_onnx(\n self,\n data: Union[dict, pd.DataFrame],\n path: Optional[str] = None,\n batch_size: Optional[int] = None,\n verbose: Optional[bool] = False,\n opset_version: Optional[int] = 16,\n truncate_long_and_double: Optional[bool] = False,\n ):\n \"\"\"\n Export this predictor's model to ONNX file.\n\n When `path` argument is not provided, the method would not save the model into disk.\n Instead, it would export the onnx model into BytesIO and return its binary as bytes.\n\n Parameters\n ----------\n data\n Raw data used to trace and export the model.\n If this is None, will check if a processed batch is provided.\n path : str, default=None\n The export path of onnx model. If path is not provided, the method would export model to memory.\n batch_size\n The batch_size of export model's input.\n Normally the batch_size is a dynamic axis, so we could use a small value for faster export.\n verbose\n verbose flag in torch.onnx.export.\n opset_version\n opset_version flag in torch.onnx.export.\n truncate_long_and_double: bool, default False\n Truncate weights provided in int64 or double (float64) to int32 and float32\n\n Returns\n -------\n onnx_path : str or bytes\n A string that indicates location of the exported onnx model, if `path` argument is provided.\n Otherwise, would return the onnx model as bytes.\n \"\"\"\n\n import torch.jit\n\n from ..models.fusion.fusion_mlp import MultimodalFusionMLP\n from ..models.hf_text import HFAutoModelForTextPrediction\n from ..models.timm_image import TimmAutoModelForImagePrediction\n\n supported_models = (TimmAutoModelForImagePrediction, HFAutoModelForTextPrediction, MultimodalFusionMLP)\n if not isinstance(self._model, supported_models):\n raise NotImplementedError(f\"export_onnx doesn't support model type {type(self._model)}\")\n warnings.warn(\"Currently, the functionality of exporting to ONNX is experimental.\")\n\n # Data preprocessing, loading, and filtering\n batch = self.get_processed_batch_for_deployment(\n data=data,\n onnx_tracing=True,\n batch_size=batch_size,\n truncate_long_and_double=truncate_long_and_double,\n )\n input_keys = self._model.input_keys\n input_vec = [batch[k] for k in input_keys]\n\n # Write to BytesIO if path argument is not provided\n if path is None:\n onnx_path = io.BytesIO()\n else:\n onnx_path = os.path.join(path, \"model.onnx\")\n dirname = os.path.dirname(os.path.abspath(onnx_path))\n if not os.path.exists(dirname):\n os.makedirs(dirname)\n\n # Infer dynamic dimensions\n dynamic_axes = onnx_get_dynamic_axes(input_keys)\n\n torch.onnx.export(\n self._model.eval(),\n args=tuple(input_vec),\n f=onnx_path,\n opset_version=opset_version,\n verbose=verbose,\n input_names=input_keys,\n dynamic_axes=dynamic_axes,\n dynamo=False,\n )\n\n if isinstance(onnx_path, io.BytesIO):\n onnx_path = onnx_path.getvalue()\n\n return onnx_path\n\n def optimize_for_inference(\n self,\n providers: Optional[Union[dict, List[str]]] = None,\n ):\n \"\"\"\n Optimize the predictor's model for inference.\n\n Under the hood, the implementation would convert the PyTorch module into an ONNX module, so that\n we can leverage efficient execution providers in onnxruntime for faster inference.\n\n Parameters\n ----------\n data\n Raw data used to trace and export the model.\n If this is None, will check if a processed batch is provided.\n providers : dict or str, default=None\n A list of execution providers for model prediction in onnxruntime.\n\n By default, the providers argument is None. The method would generate an ONNX module that\n would perform model inference with TensorrtExecutionProvider in onnxruntime, if tensorrt\n package is properly installed. Otherwise, the onnxruntime would fallback to use CUDA or CPU\n execution providers instead.\n\n Returns\n -------\n onnx_module : OnnxModule\n The onnx-based module that can be used to replace predictor._model for model inference.\n \"\"\"\n data_dict = {}\n for col_name, col_type in self._column_types.items():\n if col_type in [NUMERICAL, CATEGORICAL, NULL]:\n data_dict[col_name] = [0, 1]\n elif col_type == TEXT:\n data_dict[col_name] = [\"some text\", \"some other text\"]\n elif col_type in [IMAGE_PATH]:\n data_dict[col_name] = [\"/not-exist-dir/xxx.jpg\", \"/not-exist-dir/yyy.jpg\"]\n else:\n raise ValueError(f\"unsupported column type: {col_type}\")\n data = pd.DataFrame.from_dict(data_dict)\n\n onnx_module = None\n onnx_path = self.export_onnx(data=data, truncate_long_and_double=True)\n\n onnx_module = OnnxModule(onnx_path, providers)\n onnx_module.input_keys = self._model.input_keys\n onnx_module.prefix = self._model.prefix\n onnx_module.get_output_dict = self._model.get_output_dict\n\n # To use the TensorRT module for prediction, simply replace the _model in the predictor\n self._model = onnx_module\n\n # Evaluate and cache TensorRT engine files\n logger.info(\"Compiling ... (this may take a few minutes)\")\n _ = self.predict(data)\n logger.info(\"Finished compilation!\")\n\n return onnx_module\n\n def get_processed_batch_for_deployment(\n self,\n data: Union[pd.DataFrame, dict],\n onnx_tracing: bool = False,\n batch_size: int = None,\n to_numpy: bool = True,\n requires_label: bool = False,\n truncate_long_and_double: bool = False,\n ):\n \"\"\"\n Get the processed batch of raw data given.\n\n Parameters\n ----------\n data\n The raw data to process\n onnx_tracing\n If the output is used for onnx tracing.\n batch_size\n The batch_size of output batch.\n If onnx_tracing, it will only output one mini-batch, and all int tensor values will be converted to long.\n to_numpy\n Output numpy array if True. Only valid if not onnx_tracing.\n require_label\n Whether do we put label data into the output batch\n\n Returns\n -------\n Tensor or numpy array.\n The output processed batch could be used for export/evaluate deployed model.\n \"\"\"\n data = self.data_to_df(data=data)\n column_types = self.infer_column_types(\n column_types=self._column_types,\n data=data,\n is_train=False,\n )\n df_preprocessor = self.get_df_preprocessor_per_run(\n df_preprocessor=self._df_preprocessor,\n data=data,\n column_types=column_types,\n is_train=False,\n )\n if self._fit_called:\n df_preprocessor._column_types = self.update_image_column_types(data=data)\n data_processors = self.get_data_processors_per_run(\n data_processors=self._data_processors,\n requires_label=requires_label,\n is_train=False,\n )\n\n batch = self.process_batch(\n data=data,\n df_preprocessor=df_preprocessor,\n data_processors=data_processors,\n )\n\n input_keys = self._model.input_keys\n\n # Perform tracing on cpu\n device_type = \"cpu\"\n num_gpus = 0\n strategy = \"dp\" # default used in inference.\n device = torch.device(device_type)\n dtype = infer_precision(\n num_gpus=num_gpus, precision=self._config.env.precision, cpu_only_warning=False, as_torch=True\n )\n\n # Move model data to the specified device\n for key in input_keys:\n inp = batch[key]\n # support mixed precision on floating point inputs, and leave integer inputs (for language models) untouched.\n if inp.dtype.is_floating_point:\n batch[key] = inp.to(device, dtype=dtype)\n else:\n batch[key] = inp.to(device)\n self._model.to(device)\n\n # Truncate input data types for TensorRT (only support: bool, int32, half, float)\n if truncate_long_and_double:\n for k in batch:\n if batch[k].dtype == torch.int64:\n batch[k] = batch[k].to(torch.int32)\n\n # Data filtering\n ret = {}\n for k in batch:\n if input_keys and k not in input_keys:\n continue\n if onnx_tracing:\n ret[k] = batch[k].long() if isinstance(batch[k], torch.IntTensor) else batch[k]\n elif to_numpy:\n ret[k] = batch[k].cpu().detach().numpy().astype(int)\n else:\n ret[k] = batch[k]\n if not onnx_tracing:\n if batch_size:\n raise NotImplementedError(\"We should split the batch here.\") # TODO\n return ret\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/hpo.py",
"content": "import logging\nimport os\nimport shutil\n\nimport lightning.pytorch as pl\nimport yaml\n\nfrom autogluon.common.utils.context import set_torch_num_threads\n\nfrom ..constants import BEST_K_MODELS_FILE, RAY_TUNE_CHECKPOINT\nfrom ..models import create_fusion_model\nfrom .matcher import create_siamese_model\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_ray_tune_ckpt_callback():\n \"\"\"\n This is a workaround for the issue caused by the mixed use of old and new lightning's import style.\n https://github.com/optuna/optuna/issues/4689\n We can remove this function after ray adopts the new lightning import style.\n \"\"\"\n from ray.tune.integration.pytorch_lightning import TuneReportCheckpointCallback\n\n class _TuneReportCheckpointCallback(TuneReportCheckpointCallback, pl.Callback):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n return _TuneReportCheckpointCallback\n\n\ndef hpo_trial(sampled_hyperparameters, learner, checkpoint_dir=None, **_fit_args):\n \"\"\"\n Run one HPO trial.\n\n Parameters\n ----------\n sampled_hyperparameters\n The sampled hyperparameters for this trial.\n learner\n A learner object.\n checkpoint_dir\n The checkpoint directory.\n _fit_args\n The keyword arguments for learner.fit_per_run().\n \"\"\"\n from ray import tune\n\n context = tune.get_context()\n resources = context.get_trial_resources().required_resources\n num_cpus = int(resources.get(\"CPU\"))\n\n # The original hyperparameters is the search space, replace it with the hyperparameters sampled\n _fit_args[\"hyperparameters\"] = sampled_hyperparameters\n\n _fit_args[\"save_path\"] = context.get_trial_dir() # We want to save each trial to a separate directory\n logger.debug(f\"hpo trial save_path: {_fit_args['save_path']}\")\n if checkpoint_dir is not None:\n _fit_args[\"resume\"] = True\n _fit_args[\"ckpt_path\"] = os.path.join(checkpoint_dir, RAY_TUNE_CHECKPOINT)\n with set_torch_num_threads(num_cpus=num_cpus):\n learner.fit_per_run(**_fit_args)\n\n\ndef build_final_learner(\n learner,\n best_trial_path,\n save_path,\n last_ckpt_path,\n is_matching,\n standalone,\n clean_ckpts,\n):\n \"\"\"\n Build the final learner after HPO is finished.\n\n Parameters\n ----------\n learner\n A learner object.\n best_trial_path\n The best trial's saving path.\n save_path\n The saving path.\n last_ckpt_path\n The last checkpoint's path.\n is_matching\n Whether is matching.\n\n Returns\n -------\n The constructed learner.\n \"\"\"\n if is_matching:\n from ..learners import MatchingLearner\n\n # reload the learner metadata\n matcher = MatchingLearner._load_metadata(matcher=learner, path=best_trial_path)\n # construct the model\n matcher._query_model, matcher._response_model = create_siamese_model(\n query_config=matcher._query_config,\n response_config=matcher._response_config,\n pretrained=False,\n )\n # average checkpoint\n matcher.top_k_average(\n save_path=best_trial_path,\n last_ckpt_path=last_ckpt_path,\n top_k_average_method=matcher._config.optim.top_k_average_method,\n )\n matcher._save_path = save_path\n\n return matcher\n else:\n from ..learners import BaseLearner\n\n # reload the learner metadata\n learner = BaseLearner._load_metadata(learner=learner, path=best_trial_path)\n # construct the model\n model = create_fusion_model(\n config=learner._config,\n num_classes=learner._output_shape,\n classes=learner._classes if hasattr(learner, \"_classes\") else None,\n num_numerical_columns=len(learner._df_preprocessor.numerical_feature_names),\n num_categories=learner._df_preprocessor.categorical_num_categories,\n pretrained=False, # set \"pretrain=False\" to prevent downloading online models\n )\n learner._model = model\n # average checkpoint\n learner.top_k_average(\n save_path=best_trial_path,\n last_ckpt_path=last_ckpt_path,\n top_k_average_method=learner._config.optim.top_k_average_method,\n standalone=standalone,\n clean_ckpts=clean_ckpts,\n )\n\n learner._save_path = save_path\n\n return learner\n\n\ndef hyperparameter_tune(hyperparameter_tune_kwargs, resources, is_matching=False, **_fit_args):\n \"\"\"\n Tune hyperparameters of learner.\n\n Parameters\n ----------\n hyperparameter_tune_kwargs\n The hyperparameters for HPO, such as, searcher, scheduler, and num_trials.\n resources\n The resources for HPO.\n is_matching\n Whether is matching.\n _fit_args\n The keyword arguments for learner.fit_per_run().\n\n Returns\n -------\n The learner after tuning hyperparameters.\n \"\"\"\n from ray.air.config import CheckpointConfig\n\n from autogluon.core.hpo.ray_hpo import (\n AutommRayTuneAdapter,\n EmptySearchSpace,\n cleanup_checkpoints,\n cleanup_trials,\n run,\n )\n\n ray_tune_adapter = AutommRayTuneAdapter()\n search_space = _fit_args.get(\"hyperparameters\", dict())\n metric = \"val_\" + _fit_args.get(\"learner\")._validation_metric_name\n mode = _fit_args.get(\"learner\")._minmax_mode\n save_path = _fit_args.get(\"save_path\")\n time_budget_s = _fit_args.get(\"max_time\")\n num_to_keep = hyperparameter_tune_kwargs.pop(\"num_to_keep\", 3)\n if time_budget_s is not None:\n time_budget_s *= 0.95 # give some buffer time to ray\n try:\n run_config_kwargs = {\n \"checkpoint_config\": CheckpointConfig(\n num_to_keep=num_to_keep,\n checkpoint_score_attribute=metric,\n ),\n }\n analysis = run(\n trainable=hpo_trial,\n trainable_args=_fit_args,\n search_space=search_space,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n metric=metric,\n mode=mode,\n save_dir=save_path,\n ray_tune_adapter=ray_tune_adapter,\n total_resources=resources,\n minimum_gpu_per_trial=1.0 if resources[\"num_gpus\"] > 0 else 0.0,\n time_budget_s=time_budget_s,\n tune_config_kwargs={\"reuse_actors\": False}, # reuse_actors cause crashing in ray tune\n run_config_kwargs=run_config_kwargs,\n verbose=2,\n )\n except EmptySearchSpace:\n raise ValueError(\"Please provide a search space using `hyperparameters` in order to do hyperparameter tune\")\n except Exception as e:\n raise e\n else:\n # find the best trial\n best_trial = analysis.get_best_trial(\n metric=metric,\n mode=mode,\n )\n if best_trial is None:\n raise ValueError(\n \"MultiModalPredictor wasn't able to find the best trial.\"\n \"Either all trials failed or\"\n \"it's likely that the time is not enough to train a single epoch for trials.\"\n )\n # clean up other trials\n logger.info(\"Removing non-optimal trials and only keep the best one.\")\n cleanup_trials(save_path, best_trial.trial_id)\n best_trial_path = os.path.join(save_path, best_trial.trial_id)\n\n checkpoints_paths_and_scores = dict(\n (os.path.join(checkpoint.path, RAY_TUNE_CHECKPOINT), score)\n for checkpoint, score in analysis._get_trial_checkpoints_with_metric(best_trial, metric=metric)\n )\n # write checkpoint paths and scores to yaml file so that top_k_average could read it\n best_k_model_path = os.path.join(best_trial_path, BEST_K_MODELS_FILE)\n with open(best_k_model_path, \"w\") as yaml_file:\n yaml.dump(checkpoints_paths_and_scores, yaml_file, default_flow_style=False)\n\n with analysis.get_last_checkpoint(best_trial).as_directory() as last_ckpt_path:\n learner = build_final_learner(\n learner=_fit_args.get(\"learner\"),\n best_trial_path=best_trial_path,\n save_path=save_path,\n last_ckpt_path=last_ckpt_path,\n is_matching=is_matching,\n standalone=_fit_args.get(\"standalone\"),\n clean_ckpts=_fit_args.get(\"clean_ckpts\"),\n )\n\n cleanup_checkpoints(best_trial_path)\n # move trial learner one level up\n contents = os.listdir(best_trial_path)\n for content in contents:\n shutil.move(\n os.path.join(best_trial_path, content),\n os.path.join(save_path, content),\n )\n shutil.rmtree(best_trial_path)\n\n return learner\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/inference.py",
"content": "import logging\nfrom typing import Callable, Dict, List, Optional, Tuple, Union\n\nimport pandas as pd\nimport torch\nfrom scipy.special import softmax\nfrom torch import nn\n\nfrom ..constants import (\n BBOX,\n COLUMN_FEATURES,\n FEATURES,\n IMAGE,\n LOGITS,\n MASKS,\n NER_ANNOTATION,\n NER_RET,\n PROBABILITY,\n QUERY,\n RESPONSE,\n SCORE,\n SEMANTIC_MASK,\n TEXT,\n TOKEN_WORD_MAPPING,\n WORD_OFFSETS,\n)\nfrom ..data.preprocess_dataframe import MultiModalFeaturePreprocessor\nfrom ..data.utils import apply_data_processor, apply_df_preprocessor, get_collate_fn, get_per_sample_features\nfrom ..models.utils import run_model\nfrom .device import move_to_device\nfrom .matcher import compute_matching_probability\nfrom .misc import tensor_to_ndarray\nfrom .precision import get_precision_context\n\nlogger = logging.getLogger(__name__)\n\n\ndef compute_inference_batch_size(\n per_gpu_batch_size: int,\n inference_batch_size_ratio: Union[int, float],\n num_gpus: int,\n strategy: str,\n):\n \"\"\"\n Compute the batch size for inference.\n\n Parameters\n ----------\n per_gpu_batch_size\n Per gpu batch size from the config.\n inference_batch_size_ratio\n per_gpu_batch_size_for_inference = per_gpu_batch_size * inference_batch_size_ratio.\n num_gpus\n Number of GPUs.\n strategy\n A pytorch lightning strategy.\n\n Returns\n -------\n Batch size for inference.\n \"\"\"\n batch_size = per_gpu_batch_size * inference_batch_size_ratio\n\n if num_gpus > 1 and strategy == \"dp\":\n # If using 'dp', the per_gpu_batch_size would be split by all GPUs.\n # So, we need to use the GPU number as a multiplier to compute the batch size.\n batch_size = batch_size * num_gpus\n\n return batch_size\n\n\ndef extract_from_output(outputs: List[Dict], ret_type: str, as_ndarray: Optional[bool] = True):\n \"\"\"\n Extract desired information, e.g., logits or features, from a list of model outputs.\n Support returning a concatenated tensor/ndarray or a dictionary of tensors/ndarrays.\n\n Parameters\n ----------\n ret_type\n What kind of information to extract from model outputs.\n outputs\n A list of model outputs.\n as_ndarray\n Whether to convert Pytorch tensor to numpy array. (Default True)\n\n Returns\n -------\n The desired information from model outputs.\n \"\"\"\n if ret_type == LOGITS:\n logits = [ele[LOGITS] for ele in outputs]\n ret = torch.cat(logits).nan_to_num(nan=-1e4)\n elif ret_type == PROBABILITY:\n probability = [ele[PROBABILITY] for ele in outputs]\n ret = torch.cat(probability).nan_to_num(nan=0)\n elif ret_type == FEATURES:\n features = [ele[FEATURES] for ele in outputs]\n ret = torch.cat(features).nan_to_num(nan=0)\n elif ret_type == COLUMN_FEATURES:\n ret = {}\n column_features = [ele[COLUMN_FEATURES][FEATURES] for ele in outputs] # a list of dicts\n for feature_name in column_features[0].keys():\n ret[feature_name] = torch.cat([ele[feature_name] for ele in column_features])\n elif ret_type == MASKS:\n ret = {}\n feature_masks = [ele[COLUMN_FEATURES][MASKS] for ele in outputs] # a list of dicts\n for feature_name in feature_masks[0].keys():\n ret[feature_name] = torch.cat([ele[feature_name] for ele in feature_masks])\n elif ret_type == BBOX:\n if isinstance(outputs, list) and isinstance(outputs[0], list) and len(outputs) == 1:\n # TODO: the output should be a list of dict\n # find the cause of this (should be in cache.py)\n outputs = outputs[0]\n return [ele[BBOX] for ele in outputs]\n elif ret_type == TEXT:\n return [ele[TEXT] for ele in outputs] # single image\n elif ret_type == SCORE:\n return [ele[SCORE] for ele in outputs]\n elif ret_type == NER_RET:\n ner_pred = []\n as_ndarray = False\n for ele in outputs:\n logits_label = ele[NER_ANNOTATION].detach().cpu().numpy()\n logits = softmax(ele[LOGITS].detach().cpu().numpy(), axis=-1)\n token_word_mapping = ele[TOKEN_WORD_MAPPING].detach().cpu().numpy()\n word_offsets = ele[WORD_OFFSETS].detach().cpu().numpy()\n for token_preds, logit, mappings, offsets in zip(logits_label, logits, token_word_mapping, word_offsets):\n pred_one_sentence, word_offset, pred_proba = [], [], []\n counter = 0\n temp = set()\n for token_pred, mapping, lt in zip(token_preds, mappings, logit):\n if mapping != -1 and mapping not in temp:\n temp.add(mapping)\n word_offset.append(list(offsets[counter]))\n pred_one_sentence.append(token_pred)\n pred_proba.append(lt)\n counter += 1\n ner_pred.append((pred_one_sentence, word_offset, pred_proba))\n return ner_pred\n elif ret_type == SEMANTIC_MASK:\n masks = [ele[SEMANTIC_MASK] for ele in outputs]\n ret = torch.cat(masks)\n else:\n raise ValueError(f\"Unknown return type: {ret_type}\")\n\n if as_ndarray:\n if isinstance(ret, torch.Tensor):\n ret = tensor_to_ndarray(ret)\n elif isinstance(ret, dict):\n ret = {k: tensor_to_ndarray(v) for k, v in ret.items()}\n else:\n raise ValueError(f\"Unsupported ret type: {type(ret)}\")\n return ret\n\n\nclass RealtimeMixin:\n def predict_batch(\n self,\n batch: Dict,\n model: nn.Module,\n precision: Union[str, int],\n num_gpus: int,\n ):\n \"\"\"\n Perform inference for a batch.\n\n Parameters\n ----------\n batch\n The batch data.\n model\n A Pytorch model. This is to align with matcher which passes either query or response model.\n precision\n The desired precision used in inference.\n num_gpus\n Number of GPUs.\n\n Returns\n -------\n Model output.\n \"\"\"\n device_type = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n device = torch.device(device_type)\n batch_size = len(batch[next(iter(batch))])\n if 1 < num_gpus <= batch_size:\n model = nn.DataParallel(model)\n model.to(device).eval()\n batch = move_to_device(batch, device=device)\n precision_context = get_precision_context(precision=precision, device_type=device_type)\n with precision_context, torch.no_grad():\n output = run_model(model, batch)\n if hasattr(self, \"_model_postprocess_fn\") and self._model_postprocess_fn:\n output = self._model_postprocess_fn(output)\n\n if isinstance(model, nn.DataParallel):\n model = model.module\n else:\n model = model\n output = move_to_device(output, device=torch.device(\"cpu\"))\n return output[model.prefix]\n\n def predict_matcher_batch(\n self,\n batch: Dict,\n query_model: nn.Module,\n response_model: nn.Module,\n signature: str,\n match_label: int,\n precision: Union[str, int],\n num_gpus: int,\n ):\n \"\"\"\n Perform matcher inference for a batch.\n\n Parameters\n ----------\n batch\n The batch data.\n query_model\n Query model.\n response_model\n Response model.\n signature\n query, response, or None.\n match_label\n 0 or 1.\n precision\n The desired precision used in inference.\n num_gpus\n Number of GPUs.\n\n Returns\n -------\n Model output.\n \"\"\"\n if signature is None or signature == QUERY:\n output = self.predict_batch(\n batch=batch,\n model=query_model,\n precision=precision,\n num_gpus=num_gpus,\n )\n query_embeddings = output[FEATURES]\n\n if signature is None or signature == RESPONSE:\n output = self.predict_batch(\n batch=batch,\n model=response_model,\n precision=precision,\n num_gpus=num_gpus,\n )\n response_embeddings = output[FEATURES]\n\n if signature == QUERY:\n return {FEATURES: query_embeddings}\n elif signature == RESPONSE:\n return {FEATURES: response_embeddings}\n else:\n match_prob = compute_matching_probability(\n embeddings1=query_embeddings,\n embeddings2=response_embeddings,\n )\n if match_label == 0:\n probability = torch.stack([match_prob, 1 - match_prob]).t()\n else:\n probability = torch.stack([1 - match_prob, match_prob]).t()\n\n return {PROBABILITY: probability}\n\n def use_realtime(\n self, realtime: bool, data: pd.DataFrame, data_processors: Union[Dict, List[Dict]], batch_size: int\n ):\n \"\"\"\n Determine whether to use the realtime inference based on the sample number\n and the data modalities. Loading image data requires more time than text.\n Thus, we set a small threshold for image data. We may also consider the\n model size in future, but we need to ensure this function is efficient since\n using this function also costs additional inference time.\n\n Parameters\n ----------\n realtime\n The provided realtime flag.\n data\n A dataframe.\n data_processors\n A dict of data processors.\n batch_size\n The batch size from config.\n\n Returns\n -------\n Whether to use the realtime inference.\n \"\"\"\n if realtime is not None:\n return realtime\n\n realtime = False\n sample_num = len(data)\n if IMAGE in data_processors and len(data_processors[IMAGE]) > 0: # has image\n if sample_num <= min(10, batch_size):\n realtime = True\n elif TEXT in data_processors and len(data_processors[TEXT]) > 0: # has text but no image\n if sample_num <= min(100, batch_size):\n realtime = True\n else: # only has tabular data\n if sample_num <= min(200, batch_size):\n realtime = True\n\n return realtime # TODO: this should be disabled after multi GPU runs\n\n def process_batch(\n self,\n data: pd.DataFrame,\n df_preprocessor: Union[MultiModalFeaturePreprocessor, List[MultiModalFeaturePreprocessor]],\n data_processors: Union[Dict, List[Dict]],\n id_mappings: Union[Dict[str, Dict], Dict[str, pd.Series]] = None,\n ):\n \"\"\"\n process data to get a batch.\n\n Parameters\n ----------\n data\n A dataframe.\n df_preprocessor\n Dataframe preprocessors.\n data_processors\n Data processors.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n\n Returns\n -------\n A dict of tensors.\n \"\"\"\n if isinstance(df_preprocessor, MultiModalFeaturePreprocessor):\n df_preprocessor = [df_preprocessor]\n if isinstance(data_processors, dict):\n data_processors = [data_processors]\n\n modality_features = dict()\n modality_types = dict()\n sample_num = dict()\n\n for i, (per_preprocessor, per_processors_group) in enumerate(zip(df_preprocessor, data_processors)):\n modality_features[i], modality_types[i], sample_num[i] = apply_df_preprocessor(\n data=data,\n df_preprocessor=per_preprocessor,\n modalities=per_processors_group.keys(),\n )\n sample_num = list(sample_num.values())\n assert len(set(sample_num)) == 1\n sample_num = sample_num[0]\n\n processed_features = []\n for i in range(sample_num):\n per_sample_features = dict()\n for group_id, per_processors_group in enumerate(data_processors):\n per_sample_features_group = get_per_sample_features(\n modality_features=modality_features[group_id],\n modality_types=modality_types[group_id],\n idx=i,\n id_mappings=id_mappings,\n )\n per_sample_features_group = apply_data_processor(\n per_sample_features=per_sample_features_group,\n data_processors=per_processors_group,\n data_types=modality_types[group_id],\n is_training=False,\n )\n per_sample_features.update(per_sample_features_group)\n\n processed_features.append(per_sample_features)\n\n collate_fn = get_collate_fn(\n df_preprocessor=df_preprocessor, data_processors=data_processors, per_gpu_batch_size=sample_num\n )\n batch = collate_fn(processed_features)\n\n return batch\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/install.py",
"content": "import logging\nimport warnings\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nfrom ..constants import OBJECT_DETECTION, OCR\n\nlogger = logging.getLogger(__name__)\n\n\ndef check_if_packages_installed(problem_type: str = None, package_names: List[str] = None):\n \"\"\"\n Check if necessary packages are installed for some problem types.\n Raise an error if an package can't be imported.\n\n Parameters\n ----------\n problem_type\n Problem type\n \"\"\"\n if problem_type:\n problem_type = problem_type.lower()\n if any(p in problem_type for p in [OBJECT_DETECTION, OCR]):\n try:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n import mmcv\n except ImportError as e:\n raise ValueError(\n f\"Encountered error while importing mmcv: {e}. {_get_mmlab_installation_guide('mmcv')}\"\n )\n\n try:\n import mmdet\n except ImportError as e:\n raise ValueError(\n f\"Encountered error while importing mmdet: {e}. {_get_mmlab_installation_guide('mmdet')}\"\n )\n\n if OCR in problem_type:\n try:\n import mmocr\n except ImportError as e:\n raise ValueError(\n f'Encountered error while importing mmocr: {e}. Try to install mmocr: pip install \"mmocr<1.0\".'\n )\n if package_names:\n for package_name in package_names:\n if package_name == \"mmcv\":\n try:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n import mmcv\n from mmcv import ConfigDict\n from mmcv.runner import load_checkpoint\n from mmcv.transforms import Compose\n except ImportError as e:\n f\"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}\"\n elif package_name == \"mmdet\":\n try:\n import mmdet\n from mmdet.datasets.transforms import ImageToTensor\n from mmdet.registry import MODELS\n except ImportError as e:\n f\"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}\"\n elif package_name == \"mmengine\":\n try:\n import mmengine\n from mmengine.dataset import pseudo_collate as collate\n from mmengine.runner import load_checkpoint\n except ImportError as e:\n warnings.warn(e)\n raise ValueError(\n f\"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}\"\n )\n else:\n raise ValueError(f\"package_name {package_name} is not required.\")\n\n\ndef _get_mmlab_installation_guide(package_name):\n if package_name == \"mmdet\":\n err_msg = 'Please install MMDetection by: pip install \"mmdet==3.2.0\"'\n elif package_name == \"mmcv\":\n err_msg = 'Please install MMCV by: mim install \"mmcv==2.1.0\"'\n elif package_name == \"mmengine\":\n err_msg = \"Please install MMEngine by: mim install mmengine\"\n else:\n raise ValueError(\"Available package_name are: mmdet, mmcv, mmengine.\")\n\n err_msg += \" Pytorch version larger than 2.1 is not supported yet. To use Autogluon for object detection, please downgrade PyTorch version to <=2.1.\"\n\n return err_msg\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/label_studio.py",
"content": "import json\nimport os\nfrom enum import Enum\n\nimport pandas as pd\n\n\nclass TaskType(Enum):\n \"\"\"\n Currently supported task type\n \"\"\"\n\n IMAGE_CLASSIFICATION = \"image_classification\"\n OBJECT_DETECTION = \"object_detection\"\n CUSTOMIZE = \"customize\"\n TEXT_CLASSIFICATION = \"text_classification\"\n TEXT_SUMMARIZATION = \"text_summarization\"\n NAMED_ENTITY_RECOGNITION = \"named_entity_recognition\"\n\n\n# preset data and label columns of some given label-studio template (except customized)\ncolumns_template = {\n \"image_classification\": {\n \"csv\": {\"data_columns\": [\"image\"], \"label_columns\": [\"choice\"]},\n \"json\": {\"data_columns\": [\"image\"], \"label_columns\": [\"value.choices\"]},\n \"json_min\": {\"data_columns\": [\"image\"], \"label_columns\": [\"choice\"]},\n },\n \"object_detection\": {\n \"csv\": {\"data_columns\": [\"image\"], \"label_columns\": [\"label\"]},\n \"json\": {\n \"data_columns\": [\"image\"],\n \"label_columns\": [\n \"original_width\",\n \"original_height\",\n \"value.x\",\n \"value.y\",\n \"value.width\",\n \"value.height\",\n \"value.rotation\",\n \"value.rectanglelabels\",\n ],\n },\n \"json_min\": {\"data_columns\": [\"image\"], \"label_columns\": [\"label\"]},\n },\n \"named_entity_recognition\": {\n \"csv\": {\"data_columns\": [\"text\"], \"label_columns\": [\"label\"]},\n \"json\": {\n \"data_columns\": [\"text\"],\n \"label_columns\": [\"value.start\", \"value.end\", \"value.text\", \"value.labels\"],\n },\n \"json_min\": {\"data_columns\": [\"text\"], \"label_columns\": [\"label\"]},\n },\n}\n\n\ndef read_from_labelstudio_csv(path, data_columns, label_columns):\n \"\"\"\n read data from exported .csv files and return an Dataframe\n\n params:\n - path: str: the path of the exported file\n - data_columns: list[str]: the key/column names of the data\n - label_columns: list[str]: the key/column names of the label\n \"\"\"\n if not os.path.exists(path):\n raise OSError(\"annotation file path not exists.\")\n labelstudio_csv = pd.read_csv(path)\n df = pd.DataFrame()\n columns = labelstudio_csv.columns\n # extracting columns for given data and label column names\n for col in data_columns:\n if col in columns:\n df[col] = labelstudio_csv[col]\n else:\n print(\"skip '{}' for not in the export data columns\".format(col))\n for col in label_columns:\n if col in columns:\n df[col] = labelstudio_csv[col]\n else:\n print(\"skip '{}' for not in the export label columns\".format(col))\n return df\n\n\ndef read_from_labelstudio_json(path, data_columns, label_columns):\n \"\"\"\n read data from exported .json files and return an Dataframe\n (include the JSON and JSON-MIN files from Label-Studio)\n\n params:\n - path: str: the path of the exported file\n - data_columns: list[str]: the key/column names of the data\n - label_columns: list[str]: the key/column names of the label\n \"\"\"\n if not os.path.exists(path):\n raise OSError(\"annotation file path not exists.\")\n\n with open(path, mode=\"r\") as fp:\n label_studio_json = json.load(fp)\n if len(label_studio_json) == 0:\n raise ValueError(\"ERROR: empty export file\")\n\n if \"annotations\" in label_studio_json[0]:\n # the file is export through JSON\n # JSON file parsing is available to specify nested data\n annotation = pd.json_normalize(\n data=label_studio_json, record_path=[\"annotations\", \"result\"], meta=[\"data\"]\n )\n data = pd.DataFrame(annotation[\"data\"].values.tolist())\n\n # annotation: labeling content\n # data: data content/url/...\n df = pd.DataFrame()\n\n columns = data.columns\n for col in data_columns:\n if col in columns:\n df[col] = data[col]\n else:\n print(\"skip '{}' for not in the export data columns\".format(col))\n\n columns = annotation.columns\n for col in label_columns:\n if col in columns:\n df[col] = annotation[col]\n else:\n print(\"skip '{}' for not in the export label columns\".format(col))\n\n return df\n\n else:\n # the file is export through JSON-MIN\n\n df = pd.DataFrame()\n annotation_table = pd.json_normalize(data=label_studio_json)\n\n for col in data_columns:\n if col in annotation_table:\n df[col] = annotation_table[col]\n else:\n print(\"skip '{}' for not in the export data columns\".format(col))\n\n for col in label_columns:\n if col in annotation_table:\n df[col] = annotation_table[col]\n else:\n print(\"skip '{}' for not in the export label columns\".format(col))\n\n return df\n\n\ndef get_dataframes_by_path(path, data_columns, label_columns):\n \"\"\"\n get the data frames by path and given data, label column names,\n and return the Dataframe\n\n params:\n - path: str: the path of the exported file\n - data_columns: list[str]: the key/column names of the data\n - label_columns: list[str]: the key/column names of the label\n \"\"\"\n\n # get file extension through os\n _, file_extension = os.path.splitext(path)\n file_extension = file_extension[1:]\n\n if file_extension == \"csv\":\n return read_from_labelstudio_csv(path, data_columns, label_columns)\n elif file_extension == \"json\" or file_extension == \"json_min\":\n return read_from_labelstudio_json(path, data_columns, label_columns)\n else:\n raise OSError(\"current file extension {} is not supported.\".format(file_extension))\n\n\nclass LabelStudioReader:\n \"\"\"\n a tool that transfer label-studio export file to the dataframe for autogluon training. Docs available at\n https://github.com/autogluon/autogluon/tree/master/examples/automm/label_studio/label_studio_export_reader\n \"\"\"\n\n def __init__(self, host=None):\n self.default_host = \"http://localhost:8080\" if not host else host\n print(\n \"NOTE: the default label-studio host is {},if you want to get data from an running label-studio url, \"\n \"please set 'ls_host_on' to True\".format(self.default_host)\n )\n self.templates = columns_template\n\n def set_labelstudio_host(self, host):\n if host:\n self.default_host = host\n print(\"set label-studio default host to {}\".format(host))\n\n def get_columns_by_type(self, type, format):\n \"\"\"\n for tasks that labeled from a given Label-Studio Template with the names\n of the template not modified, the data_columns and label_columns can be\n achieved through the preset template.\n params:\n - type: Enum of the template type(e.g: image classification).See class TaskType.\n - format: str: file export format: (csv, json, json_min)\n \"\"\"\n if type.value != \"customize\":\n data_columns = self.templates[type.value][format][\"data_columns\"]\n label_columns = self.templates[type.value][format][\"label_columns\"]\n return data_columns, label_columns\n else:\n return [], []\n\n def process_data_str(self, s: str, ls_online):\n \"\"\"\n used for lambda expression of the data str. See the detailed documentation\n params:\n - s: str, the origin data str\n - ls_online: boolean: whether the label-studio host should be provided\n \"\"\"\n if ls_online:\n if s.startswith(\"/data/\"):\n # label-studio imported data.\n # data are accessible when label-studio host is on\n # data can be addressed through a temporal ls-host-based URL\n return self.default_host + s\n else:\n # url, text content,...\n # need no further process\n return s\n\n else:\n local_storage_prefix = \"/data/local-files/?d=\"\n upload_prefix = \"/data/upload\"\n if s.startswith(local_storage_prefix):\n # data imported from Label-Studio local storage\n # changed to the local path of the file\n return s[len(local_storage_prefix) :]\n elif s.startswith(upload_prefix):\n # the uploaded file can not be accessed\n print(\"Warning: cannot read {} with the label-studio host off.\".format(s))\n return s\n # TODO: add s3, gcd, redis support\n else:\n return s\n\n def from_image_classification(self, path, ls_host_on, data_columns=None, label_columns=None):\n \"\"\"\n data: image files\n process export file from label-studio image classification template,\n return the overall and label Dataframe for Autogluon input\n param:\n - path: str: the local path of exported file\n - ls_host_on: boolean: if the label-studio host is needed\n - data_columns: list[str](Optional) key or column names of data\n - label_columns: list[str](Optional) key or column names of label\n \"\"\"\n\n if not os.path.exists(path):\n raise OSError(\"annotation file path not exists.\")\n\n # get file extension with os, distinguish JSON and JSON-MIN\n _, file_extension = os.path.splitext(path)\n file_extension = file_extension[1:]\n if file_extension == \"json\":\n with open(path, mode=\"r\") as fp:\n json_content = json.load(fp)\n if \"annotations\" not in json_content[0]:\n file_extension = \"json_min\"\n\n # get the data and label columns through default image classification template\n default_data, default_label = self.get_columns_by_type(TaskType.IMAGE_CLASSIFICATION, format=file_extension)\n # use user preset if there's any\n data = data_columns if data_columns else default_data\n label = label_columns if label_columns else default_label\n\n df = get_dataframes_by_path(path, data, label)\n\n columns = df.columns\n for col in data:\n if col in columns:\n # process the data file content/url according to ls_host_on\n df[col] = df[col].apply(lambda s: self.process_data_str(s, ls_host_on))\n else:\n print(\"skip '{}' for not in the data column names.\".format(col))\n\n return df, df[label]\n\n def from_named_entity_recognition(self, path, data_columns=None, label_columns=None):\n \"\"\"\n data: text\n process export file from label-studio NER template,\n return the overall and label Dataframe for Autogluon input\n param:\n - path: str: the local path of exported file\n - data_columns: list[str](Optional) key or column names of data\n - label_columns: list[str](Optional) key or column names of label\n \"\"\"\n\n if not os.path.exists(path):\n raise OSError(\"annotation file path not exists.\")\n\n # get file extension with os,distinguish JSON and JSON-MIN\n _, file_extension = os.path.splitext(path)\n file_extension = file_extension[1:]\n if file_extension == \"json\":\n with open(path, mode=\"r\") as fp:\n json_content = json.load(fp)\n if \"annotations\" not in json_content[0]:\n file_extension = \"json_min\"\n\n # get the data and label columns through default NER template\n default_data, default_label = self.get_columns_by_type(\n TaskType.NAMED_ENTITY_RECOGNITION, format=file_extension\n )\n data = data_columns if data_columns else default_data\n label = label_columns if label_columns else default_label\n\n df = get_dataframes_by_path(path, data, label)\n return df, df[label]\n\n def from_customize(self, path, ls_host_on, data_columns, label_columns):\n \"\"\"\n process export file from customized template,\n return the overall and label Dataframe for Autogluon input\n param:\n - path: str: the local path of exported file\n - ls_host_on: boolean: if the label-studio host is needed\n - data_columns: list[str](Optional) key or column names of data\n - label_columns: list[str](Optional) key or column names of label\n \"\"\"\n\n if not os.path.exists(path):\n raise OSError(\"annotation file path not exists.\")\n\n _, file_extension = os.path.splitext(path)\n file_extension = file_extension[1:]\n if file_extension == \"json\":\n with open(path, mode=\"r\") as fp:\n json_content = json.load(fp)\n if \"annotations\" not in json_content[0]:\n file_extension = \"json_min\"\n\n data, label = data_columns, label_columns\n\n df = get_dataframes_by_path(path, data, label)\n\n columns = df.columns\n for col in data:\n if col in columns:\n df[col] = df[col].apply(lambda s: self.process_data_str(s, ls_host_on))\n else:\n print(\"skip '{}' for not in the data column names.\".format(col))\n\n return df, df[label]\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/load.py",
"content": "import logging\nimport os\nimport pickle\nimport zipfile\n\nfrom ..constants import LAST_CHECKPOINT, MODEL_CHECKPOINT\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_dir_ckpt_paths(path: str):\n \"\"\"\n Get the dir path and ckpt path from a path.\n\n Parameters\n ----------\n path\n A path which can be either a dir or ckpt path.\n\n Returns\n -------\n The dir and ckpt paths.\n \"\"\"\n path = os.path.abspath(os.path.expanduser(path))\n if os.path.isfile(path):\n dir_path = os.path.dirname(path)\n ckpt_path = path\n else:\n dir_path = path\n ckpt_path = None\n\n return dir_path, ckpt_path\n\n\ndef get_load_ckpt_paths(ckpt_path: str, dir_path: str, resume: bool):\n \"\"\"\n Get the load_path and ckpt_path. They can be the same or different.\n #TODO: merging load_path and ckpt_path.\n\n Parameters\n ----------\n ckpt_path\n The path of one checkpoint, which can be None.\n dir_path\n The dir path from where to load model.\n resume\n Whether to resume training.\n\n Returns\n -------\n load_path and ckpt_path\n \"\"\"\n if ckpt_path:\n load_path = ckpt_path\n logger.info(f\"Loading checkpoint: '{ckpt_path}'\")\n else:\n resume_ckpt_path = os.path.join(dir_path, LAST_CHECKPOINT)\n final_ckpt_path = os.path.join(dir_path, MODEL_CHECKPOINT)\n if resume: # resume training which crashed before\n if not os.path.isfile(resume_ckpt_path):\n if os.path.isfile(final_ckpt_path):\n raise ValueError(\n f\"Resuming checkpoint '{resume_ckpt_path}' doesn't exist, but \"\n f\"final checkpoint '{final_ckpt_path}' exists, which means training \"\n f\"is already completed.\"\n )\n else:\n raise ValueError(\n f\"Resuming checkpoint '{resume_ckpt_path}' and \"\n f\"final checkpoint '{final_ckpt_path}' both don't exist. \"\n f\"Consider starting training from scratch.\"\n )\n load_path = resume_ckpt_path\n logger.info(f\"Resume training from checkpoint: '{resume_ckpt_path}'\")\n ckpt_path = resume_ckpt_path\n else: # load a model checkpoint for prediction, evaluation, or continuing training on new data\n if not os.path.isfile(final_ckpt_path):\n if os.path.isfile(resume_ckpt_path):\n raise ValueError(\n f\"Final checkpoint '{final_ckpt_path}' doesn't exist, but \"\n f\"resuming checkpoint '{resume_ckpt_path}' exists, which means training \"\n f\"is not done yet. Consider resume training from '{resume_ckpt_path}'.\"\n )\n else:\n raise ValueError(\n f\"Resuming checkpoint '{resume_ckpt_path}' and \"\n f\"final checkpoint '{final_ckpt_path}' both don't exist. \"\n f\"Consider starting training from scratch.\"\n )\n load_path = final_ckpt_path\n logger.info(f\"Load pretrained checkpoint: {os.path.join(dir_path, MODEL_CHECKPOINT)}\")\n ckpt_path = None # must set None since we do not resume training\n\n return load_path, ckpt_path\n\n\nclass CustomUnpickler(pickle.Unpickler):\n \"\"\"\n This is to make pickle loading an object backward compatible.\n A df_preprocessor object saved with old name space `xxx.yyy` has errors\n when being loaded under the context of new name `aaa.bbb`.\n \"\"\"\n\n def find_class(self, module, name):\n renamed_module = module\n if module.startswith(\"autogluon.text.automm\"):\n renamed_module = module.replace(\"autogluon.text.automm\", \"autogluon.multimodal\")\n\n return super(CustomUnpickler, self).find_class(renamed_module, name)\n\n\ndef protected_zip_extraction(zipfile_path, sha1_hash, folder):\n \"\"\"\n Extract zip file to the folder.\n\n A signature file named \".SHA1HASH.sig\" will be created if the extraction has been finished.\n\n Returns\n -------\n folder\n The directory to extract the zipfile\n \"\"\"\n os.makedirs(folder, exist_ok=True)\n\n if sha1_hash:\n sha1_hash = sha1_hash[:6]\n signature = \".{}.sig\".format(sha1_hash)\n\n if os.path.exists(os.path.join(folder, signature)):\n # We have found the signature file. Thus, we will not extract again.\n return folder\n else:\n signature = None\n\n # Extract the file\n logging.info(\"Extract files...\")\n with zipfile.ZipFile(zipfile_path, \"r\") as zip_ref:\n zip_ref.extractall(folder)\n\n if signature:\n # Create the signature\n with open(os.path.join(folder, signature), \"w\"):\n pass\n\n return folder\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/log.py",
"content": "import logging\nfrom contextlib import contextmanager\nfrom typing import List, Optional, Union\n\nimport torch\n\nfrom autogluon.common.utils.system_info import get_ag_system_info\n\nfrom .strategy import is_interactive_strategy\n\nlogger = logging.getLogger(__name__)\n\n\nclass LogFilter(logging.Filter):\n \"\"\"\n Filter log messages with patterns.\n \"\"\"\n\n def __init__(self, blacklist: Union[str, List[str]]):\n \"\"\"\n Parameters\n ----------\n blacklist\n Patterns to be suppressed in logging.\n \"\"\"\n super().__init__()\n if isinstance(blacklist, str):\n blacklist = [blacklist]\n self._blacklist = blacklist\n\n def filter(self, record):\n \"\"\"\n Check whether to suppress a logging message.\n\n Parameters\n ----------\n record\n A logging message.\n\n Returns\n -------\n If True, no pattern exists in the message, hence printed out.\n If False, some pattern is in the message, hence filtered out.\n \"\"\"\n matches = [pattern not in record.msg for pattern in self._blacklist]\n return all(matches)\n\n\ndef add_log_filter(target_logger, log_filter):\n \"\"\"\n Add one log filter to the target logger.\n\n Parameters\n ----------\n target_logger\n Target logger\n log_filter\n Log filter\n \"\"\"\n for handler in target_logger.handlers:\n handler.addFilter(log_filter)\n\n\ndef remove_log_filter(target_logger, log_filter):\n \"\"\"\n Remove one log filter to the target logger.\n\n Parameters\n ----------\n target_logger\n Target logger\n log_filter\n Log filter\n \"\"\"\n for handler in target_logger.handlers:\n handler.removeFilter(log_filter)\n\n\n@contextmanager\ndef apply_log_filter(log_filter):\n \"\"\"\n User contextmanager to control the scope of applying one log filter.\n Currently, it is to filter some lightning's log messages.\n But we can easily extend it to cover more loggers.\n\n Parameters\n ----------\n log_filter\n Log filter.\n \"\"\"\n try:\n add_log_filter(logging.getLogger(), log_filter)\n add_log_filter(logging.getLogger(\"lightning\"), log_filter)\n add_log_filter(logging.getLogger(\"lightning.pytorch\"), log_filter)\n yield\n\n finally:\n remove_log_filter(logging.getLogger(), log_filter)\n remove_log_filter(logging.getLogger(\"lightning\"), log_filter)\n remove_log_filter(logging.getLogger(\"lightning.pytorch\"), log_filter)\n\n\ndef on_fit_start_message(path: Optional[str] = None):\n return get_ag_system_info(\n path=path,\n include_gpu_count=True,\n include_pytorch=True,\n include_cuda=True,\n )\n\n\ndef on_fit_per_run_start_message(save_path, validation_metric_name):\n return f\"\"\"\\\n\nAutoMM starts to create your model. â¨â¨â¨\n\nTo track the learning progress, you can open a terminal and launch Tensorboard:\n ```shell\n # Assume you have installed tensorboard\n tensorboard --logdir {save_path}\n ```\n\"\"\"\n\n\ndef on_fit_end_message(save_path):\n return f\"\"\"\\\nAutoMM has created your model. đđđ\n\nTo load the model, use the code below:\n ```python\n from autogluon.multimodal import MultiModalPredictor\n predictor = MultiModalPredictor.load(\"{save_path}\")\n ```\n\nIf you are not satisfied with the model, try to increase the training time, \nadjust the hyperparameters (https://auto.gluon.ai/stable/tutorials/multimodal/advanced_topics/customization.html),\nor post issues on GitHub (https://github.com/autogluon/autogluon/issues).\n\n\"\"\"\n\n\ndef get_gpu_message(detected_num_gpus: int, used_num_gpus: int, strategy: str):\n \"\"\"\n Get the GPU related info (GPU name, total memory, free memory, and CUDA version) for logging.\n\n Parameters\n ----------\n detected_num_gpus\n Number of detected GPUs.\n used_num_gpus\n Number of GPUs to be used.\n\n Returns\n -------\n A string with the GPU info.\n \"\"\"\n\n def _bytes_to_gigabytes(bytes):\n return round((bytes / 1024) / 1024 / 1024, 2)\n\n gpu_message = f\"GPU Count: {detected_num_gpus}\\nGPU Count to be Used: {used_num_gpus}\\n\"\n\n if is_interactive_strategy(strategy): # avoid pre-initializing cuda when using ddp_fork\n return gpu_message\n\n try:\n import nvidia_smi\n except:\n return gpu_message\n\n for i in range(detected_num_gpus):\n nvidia_smi.nvmlInit()\n handle = nvidia_smi.nvmlDeviceGetHandleByIndex(i)\n info = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)\n\n gpu_mem_used = _bytes_to_gigabytes(info.used)\n gpu_mem_total = _bytes_to_gigabytes(info.total)\n if torch.cuda.is_available():\n gpu_message += f\"GPU {i} Name: {torch.cuda.get_device_name(i)}\\n\"\n gpu_message += f\"GPU {i} Memory: {gpu_mem_used}GB/{gpu_mem_total}GB (Used/Total)\\n\"\n\n return gpu_message\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/matcher.py",
"content": "import copy\nimport functools\nimport heapq\nimport logging\nfrom typing import Dict, List, Optional, Union\n\nimport pandas as pd\nimport torch\nfrom omegaconf import DictConfig\nfrom torch import nn\nfrom torch.nn import functional as F\n\nfrom ..constants import FUSION, QUERY, RESPONSE\nfrom ..data import data_to_df\nfrom ..models import create_model\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_fusion_model_dict(\n model,\n single_models: Optional[Dict] = None,\n):\n \"\"\"\n Take apart a late-fusion model into a dict of single models and a fusion piece.\n\n Parameters\n ----------\n model\n A late-fusion model.\n single_models\n A dict of single models.\n\n Returns\n -------\n single_models\n A dict of single models.\n fusion_model\n The fusion part of a late-fusion model.\n \"\"\"\n if not single_models:\n single_models = {}\n fusion_model = None\n if model.prefix.startswith(FUSION): # fusion model\n fusion_model = model\n models = model.model\n model.model = None\n else:\n models = [model]\n\n for per_model in models:\n if per_model.prefix.endswith(QUERY):\n model_name = per_model.prefix[:-6] # cut off query\n elif per_model.prefix.endswith(RESPONSE):\n model_name = per_model.prefix[:-9]\n else:\n raise ValueError(f\"Model prefix {per_model.prefix} doesn't end with {QUERY} or {RESPONSE}.\")\n\n if model_name not in single_models:\n single_models[model_name] = per_model\n\n return single_models, fusion_model\n\n\ndef create_fusion_model_dict(\n config: DictConfig,\n single_models: Optional[Dict] = None,\n pretrained: Optional[bool] = True,\n):\n \"\"\"\n Create a dict of single models and fusion piece based on a late-fusion config.\n\n Parameters\n ----------\n config\n The model config.\n single_models\n A dict of single models used in the late-fusion.\n\n Returns\n -------\n single_models\n A dict of single models.\n fusion_model\n The fusion part of a late-fusion model.\n \"\"\"\n if not single_models:\n single_models = {}\n fusion_model = None\n for model_name in config.names:\n model_config = getattr(config, model_name)\n if not model_name.lower().startswith(FUSION):\n if model_name.endswith(QUERY):\n model_name = model_name[:-6] # cut off query\n elif model_name.endswith(RESPONSE):\n model_name = model_name[:-9]\n else:\n raise ValueError(f\"Model name {model_name} doesn't end with {QUERY} or {RESPONSE}.\")\n\n if model_name in single_models:\n continue\n model = create_model(\n model_name=model_name,\n model_config=model_config,\n pretrained=pretrained,\n is_matching=True, # clip needs to use this to init attributes for both image and text\n )\n if model_name.lower().startswith(FUSION):\n fusion_model = model\n else:\n single_models[model_name] = model\n\n return single_models, fusion_model\n\n\ndef make_siamese(\n query_config: DictConfig,\n response_config: DictConfig,\n single_models: Dict,\n query_fusion_model: Union[nn.Module, functools.partial],\n response_fusion_model: Union[nn.Module, functools.partial],\n share_fusion: bool,\n initialized: Optional[bool] = False,\n):\n \"\"\"\n Build a siamese network, in which the query and response share the same encoders for the same modalities.\n\n Parameters\n ----------\n query_config\n The query config.\n response_config\n The response config.\n single_models\n A dict of single models used in the late-fusion.\n query_fusion_model\n The fusion piece of the query model.\n response_fusion_model\n The fusion piece of the response model.\n share_fusion\n Whether the query and response share the fusion piece.\n initialized\n Whether the fusion piece is initialized.\n\n Returns\n -------\n The query and response models satisfying the siamese constraint.\n \"\"\"\n query_model_names = [n for n in query_config.model.names if not n.lower().startswith(FUSION)]\n query_fusion_model_name = [n for n in query_config.model.names if n.lower().startswith(FUSION)]\n assert len(query_fusion_model_name) <= 1\n if len(query_fusion_model_name) == 1:\n query_fusion_model_name = query_fusion_model_name[0]\n response_model_names = [n for n in response_config.model.names if not n.lower().startswith(FUSION)]\n response_fusion_model_name = [n for n in response_config.model.names if n.lower().startswith(FUSION)]\n assert len(response_fusion_model_name) <= 1\n if len(response_fusion_model_name) == 1:\n response_fusion_model_name = response_fusion_model_name[0]\n\n logger.debug(f\"single model names: {list(single_models.keys())}\")\n logger.debug(f\"query fusion model name: {query_fusion_model_name}\")\n logger.debug(f\"response fusion model name: {response_fusion_model_name}\")\n\n # use shallow copy to create query single models\n query_single_models = []\n for model_name in query_model_names:\n model = copy.copy(single_models[model_name[:-6]]) # cut off _query\n model.prefix = model_name\n query_single_models.append(model)\n\n # use shallow copy to create response single models\n response_single_models = []\n for model_name in response_model_names:\n model = copy.copy(single_models[model_name[:-9]]) # cut off _response\n model.prefix = model_name\n response_single_models.append(model)\n\n if len(query_single_models) == 1:\n query_model = query_single_models[0]\n else:\n if initialized:\n query_model = query_fusion_model\n query_model.model = nn.ModuleList(query_single_models)\n else:\n query_model = query_fusion_model(models=query_single_models)\n\n query_model.prefix = query_fusion_model_name\n\n if len(response_single_models) == 1:\n response_model = response_single_models[0]\n else:\n if share_fusion:\n response_model = copy.copy(query_model) # copy query_model rather than query_fusion_model\n response_model.model = nn.ModuleList(response_single_models)\n else:\n if initialized:\n response_model = response_fusion_model\n response_model.model = nn.ModuleList(response_single_models)\n else:\n response_model = response_fusion_model(models=response_single_models)\n\n response_model.prefix = response_fusion_model_name\n\n return query_model, response_model\n\n\ndef is_share_fusion(\n query_model_names: List[str],\n response_model_names: List[str],\n):\n \"\"\"\n Check whether the query and response models share the same fusion part.\n\n Parameters\n ----------\n query_model_names\n Names of single models in the query late-fusion model.\n response_model_names\n Names of single models in the response late-fusion model.\n\n Returns\n -------\n Whether to share the same fusion part.\n \"\"\"\n query_model_names = [n for n in query_model_names if not n.lower().startswith(FUSION)]\n response_model_names = [n for n in response_model_names if not n.lower().startswith(FUSION)]\n return sorted(query_model_names) == sorted(response_model_names)\n\n\ndef create_siamese_model(\n query_config: DictConfig,\n response_config: DictConfig,\n query_model: Optional[nn.Module] = None,\n response_model: Optional[nn.Module] = None,\n pretrained: Optional[bool] = True,\n):\n \"\"\"\n Create the query and response models and make them share the same encoders for the same modalities.\n\n Parameters\n ----------\n query_config\n The query config.\n response_config\n The response config.\n query_model\n The query model if already created.\n response_model\n The response model if already created.\n\n Returns\n -------\n The query and response models satisfying the siamese constraint.\n \"\"\"\n if query_model is None:\n single_models, query_fusion_model = create_fusion_model_dict(\n config=query_config.model,\n pretrained=pretrained,\n )\n else:\n single_models, query_fusion_model = get_fusion_model_dict(\n model=query_model,\n )\n\n if response_model is None:\n single_models, response_fusion_model = create_fusion_model_dict(\n config=response_config.model,\n single_models=single_models,\n pretrained=pretrained,\n )\n else:\n single_models, response_fusion_model = get_fusion_model_dict(\n model=response_model,\n single_models=single_models,\n )\n\n share_fusion = is_share_fusion(\n query_model_names=query_config.model.names,\n response_model_names=response_config.model.names,\n )\n query_model, response_model = make_siamese(\n query_config=query_config,\n response_config=response_config,\n single_models=single_models,\n query_fusion_model=query_fusion_model,\n response_fusion_model=response_fusion_model,\n share_fusion=share_fusion,\n initialized=False,\n )\n\n return query_model, response_model\n\n\ndef compute_semantic_similarity(a: torch.Tensor, b: torch.Tensor, similarity_type: Optional[str] = \"cosine\"):\n \"\"\"\n Compute the semantic similarity of each vector in `a` with each vector in `b`.\n\n Parameters\n ----------\n a\n A tensor with shape (n, dim).\n b\n A tensor with shape (m, dim).\n similarity_type\n Use what function (cosine/dot_prod) to score the similarity (default: cosine).\n\n Returns\n -------\n A similarity matrix with shape (n, m).\n \"\"\"\n if not isinstance(a, torch.Tensor):\n a = torch.as_tensor(a)\n\n if not isinstance(b, torch.Tensor):\n b = torch.as_tensor(b)\n\n if len(a.shape) == 1:\n a = a.unsqueeze(0)\n\n if len(b.shape) == 1:\n b = b.unsqueeze(0)\n\n if similarity_type == \"cosine\":\n a = torch.nn.functional.normalize(a, p=2, dim=1)\n b = torch.nn.functional.normalize(b, p=2, dim=1)\n elif similarity_type == \"dot_prod\":\n pass\n else:\n raise ValueError(\n f\"Invalid similarity type: {similarity_type}. The supported types are `cosine` and `dot_prod`.\"\n )\n\n return torch.mm(a, b.transpose(0, 1))\n\n\ndef semantic_search(\n matcher,\n query_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n response_data: Optional[Union[pd.DataFrame, dict, list]] = None,\n query_embeddings: Optional[torch.Tensor] = None,\n response_embeddings: Optional[torch.Tensor] = None,\n query_chunk_size: int = 128,\n response_chunk_size: int = 500000,\n top_k: int = 10,\n id_mappings: Optional[Union[Dict[str, Dict], Dict[str, pd.Series]]] = None,\n similarity_type: Optional[str] = \"cosine\",\n):\n \"\"\"\n Perform a cosine similarity search between query data and response data.\n\n Parameters\n ----------\n query_data\n The query data.\n response_data\n The response data.\n query_embeddings\n 2-D query embeddings.\n response_embeddings\n 2-D response embeddings.\n id_mappings\n Id-to-content mappings. The contents can be text, image, etc.\n This is used when the dataframe contains the query/response indexes instead of their contents.\n query_chunk_size\n Process queries by query_chunk_size each time.\n response_chunk_size\n Process response data by response_chunk_size each time.\n top_k\n Retrieve top k matching entries.\n similarity_type\n Use what function (cosine/dot_prod) to score the similarity (default: cosine).\n\n Returns\n -------\n Search results.\n \"\"\"\n assert query_data is None or query_embeddings is None, (\n \"Both query_data and query_embeddings are detected, but you can only use one of them.\"\n )\n assert query_data is not None or query_embeddings is not None, \"Both query_data and query_embeddings are None.\"\n assert response_data is None or response_embeddings is None, (\n \"Both response_data and response_embeddings are detected, but you can only use one of them.\"\n )\n assert response_data is not None or response_embeddings is not None, (\n \"Both response_data and response_embeddings are None.\"\n )\n\n if query_embeddings is None:\n query_header = matcher.query[0] if matcher.query is not None else QUERY\n query_data = data_to_df(query_data, header=query_header)\n if response_embeddings is None:\n response_header = matcher.response[0] if matcher.response else RESPONSE\n response_data = data_to_df(response_data, header=response_header)\n\n if query_embeddings is None:\n num_queries = len(query_data)\n else:\n num_queries = len(query_embeddings)\n\n if response_embeddings is None:\n num_responses = len(response_data)\n else:\n num_responses = len(response_embeddings)\n\n queries_result_list = [[] for _ in range(num_queries)]\n\n for query_start_idx in range(0, num_queries, query_chunk_size):\n if query_embeddings is None:\n batch_query_embeddings = matcher.extract_embedding(\n query_data[query_start_idx : query_start_idx + query_chunk_size],\n id_mappings=id_mappings,\n as_tensor=True,\n )\n else:\n batch_query_embeddings = query_embeddings[query_start_idx : query_start_idx + query_chunk_size]\n # Iterate over chunks of the corpus\n for response_start_idx in range(0, num_responses, response_chunk_size):\n if response_embeddings is None:\n batch_response_embeddings = matcher.extract_embedding(\n response_data[response_start_idx : response_start_idx + response_chunk_size],\n id_mappings=id_mappings,\n as_tensor=True,\n )\n else:\n batch_response_embeddings = response_embeddings[\n response_start_idx : response_start_idx + response_chunk_size\n ]\n # Compute cosine similarities\n scores = compute_semantic_similarity(\n a=batch_query_embeddings,\n b=batch_response_embeddings,\n similarity_type=similarity_type,\n )\n\n # Get top-k scores\n scores_top_k_values, scores_top_k_idx = torch.topk(\n scores,\n k=min(top_k, len(scores[0])),\n dim=1,\n largest=True,\n sorted=False,\n )\n scores_top_k_values = scores_top_k_values.cpu().tolist()\n scores_top_k_idx = scores_top_k_idx.cpu().tolist()\n\n for query_itr in range(len(scores)):\n for sub_response_id, score in zip(scores_top_k_idx[query_itr], scores_top_k_values[query_itr]):\n corpus_id = response_start_idx + sub_response_id\n query_id = query_start_idx + query_itr\n if len(queries_result_list[query_id]) < top_k:\n heapq.heappush(\n queries_result_list[query_id], (score, corpus_id)\n ) # heaqp tracks the quantity of the first element in the tuple\n else:\n heapq.heappushpop(queries_result_list[query_id], (score, corpus_id))\n\n # change the data format and sort\n for query_id in range(len(queries_result_list)):\n for doc_itr in range(len(queries_result_list[query_id])):\n score, corpus_id = queries_result_list[query_id][doc_itr]\n queries_result_list[query_id][doc_itr] = {\"response_id\": corpus_id, \"score\": score}\n queries_result_list[query_id] = sorted(queries_result_list[query_id], key=lambda x: x[\"score\"], reverse=True)\n\n return queries_result_list\n\n\ndef convert_data_for_ranking(\n data: pd.DataFrame, query_column: str, response_column: str, label_column: Optional[str] = None\n):\n \"\"\"\n Extract query and response data from a dataframe.\n If no label column exists, append one label column with all 1 labels,\n which assumes (a_i, p_i) are a positive pair and (a_i, p_j) for i!=j a negative pair.\n\n Parameters\n ----------\n data\n A dataframe with query, response, and label (optional) columns.\n query_column\n Name of the query column.\n response_column\n Name of the response column.\n label_column\n Name of the label column. If None, use `relevance` by default.\n\n Returns\n -------\n data_with_label\n A dataframe with query, response, and label columns.\n query_data\n The unique query data in the dataframe format.\n response_data\n The unique response data in the dataframe format.\n label_column\n Name of the label column.\n \"\"\"\n data_with_label = data.copy()\n if label_column is None:\n data_with_label[\"relevance\"] = [1] * len(data)\n label_column = \"relevance\"\n\n query_data = pd.DataFrame({query_column: data[query_column].unique().tolist()})\n response_data = pd.DataFrame({response_column: data[response_column].unique().tolist()})\n\n return data_with_label, query_data, response_data, label_column\n\n\ndef compute_matching_probability(\n logits: Optional[torch.Tensor] = None,\n embeddings1: Optional[torch.Tensor] = None,\n embeddings2: Optional[torch.Tensor] = None,\n reverse_prob: Optional[bool] = False,\n):\n \"\"\"\n Compute probabilities from logits or embedding pairs.\n\n Parameters\n ----------\n logits\n The output of a model's head layer.\n embeddings1\n Feature embeddings of one side in matching.\n embeddings2\n Feature embeddings 2 of the other side in matching.\n reverse_prob\n Whether to reverse the probability.\n\n Returns\n -------\n Probabilities.\n \"\"\"\n if logits is not None:\n prob = F.softmax(logits.float(), dim=1)[:, 1]\n else:\n cosine_similarity = F.cosine_similarity(embeddings1, embeddings2)\n prob = 0.5 * (cosine_similarity + 1)\n\n if reverse_prob:\n prob = 1 - prob\n\n return prob\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/misc.py",
"content": "import base64\nimport json\nimport logging\nimport os\nimport re\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom scipy.special import expit, softmax\n\nlogger = logging.getLogger(__name__)\n\n\ndef logits_to_prob(logits: np.ndarray):\n \"\"\"\n Convert logits to probabilities.\n\n Parameters\n ----------\n logits\n The logits output of a classification head.\n\n Returns\n -------\n Probabilities.\n \"\"\"\n if logits.ndim == 1:\n return expit(logits)\n elif logits.ndim == 2:\n return softmax(logits, axis=1)\n else:\n raise ValueError(f\"Unsupported logit dim: {logits.ndim}.\")\n\n\ndef tensor_to_ndarray(tensor: torch.Tensor):\n \"\"\"\n Convert Pytorch tensor to numpy array.\n\n Parameters\n ----------\n tensor\n A Pytorch tensor.\n\n Returns\n -------\n A ndarray.\n \"\"\"\n return tensor.detach().cpu().float().numpy()\n\n\ndef path_expander(path, base_folder):\n path_l = path.split(\";\")\n return \";\".join([os.path.abspath(os.path.join(base_folder, path)) for path in path_l])\n\n\ndef _read_byte(file):\n with open(file, \"rb\") as image:\n f = image.read()\n b = bytearray(f)\n return b\n\n\ndef path_to_bytearray_expander(path, base_folder):\n path_l = path.split(\";\")\n return [_read_byte(os.path.abspath(os.path.join(base_folder, path))) for path in path_l]\n\n\ndef _read_base64str(file):\n with open(file, \"rb\") as image:\n f = image.read()\n image_base64 = base64.b64encode(f)\n image_base64_str = image_base64.decode(\"utf-8\")\n return image_base64_str\n\n\ndef path_to_base64str_expander(path, base_folder):\n path_l = path.split(\";\")\n return [_read_base64str(os.path.abspath(os.path.join(base_folder, path))) for path in path_l]\n\n\ndef shopee_dataset(\n download_dir: str,\n is_bytearray=False,\n is_base64str=False,\n):\n \"\"\"\n Download Shopee dataset for demo.\n\n Parameters\n ----------\n download_dir\n Path to save the dataset locally.\n\n Returns\n -------\n train and test set of Shopee dataset in pandas DataFrame format.\n \"\"\"\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/vision_datasets/shopee.zip\"\n from autogluon.core.utils.loaders import load_zip\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n\n dataset_path = os.path.join(download_dir, \"shopee\")\n\n train_data = pd.read_csv(f\"{dataset_path}/train.csv\")\n test_data = pd.read_csv(f\"{dataset_path}/test.csv\")\n\n expander = (\n path_to_bytearray_expander if is_bytearray else (path_to_base64str_expander if is_base64str else path_expander)\n )\n train_data[\"image\"] = train_data[\"image\"].apply(lambda ele: expander(ele, base_folder=dataset_path))\n test_data[\"image\"] = test_data[\"image\"].apply(lambda ele: expander(ele, base_folder=dataset_path))\n return train_data, test_data\n\n\ndef merge_spans(sent, pred, for_visualizer=False):\n \"\"\"Merge subsequent predictions.\"\"\"\n if isinstance(pred, str):\n # For string values, we assume that it is json-encoded string of the sentences.\n try:\n pred = json.loads(pred)\n except Exception as exp:\n raise RuntimeError(\n f\"The received entity annotations is {pred}, \"\n f\"which can not be encoded with the json format. \"\n f\"Check your input again, or running `json.loads(pred)` to verify your data.\"\n )\n spans = {}\n last_start = -1\n last_end = -1\n last_label = \"\"\n for entity in pred:\n entity_group = entity[\"entity_group\"]\n start = entity[\"start\"]\n end = entity[\"end\"]\n if (\n last_start >= 0\n and not for_visualizer\n and (not re.match(\"B-\", entity_group, re.IGNORECASE))\n and (\n (re.match(\"I-\", entity_group, re.IGNORECASE) and last_label[2:] == entity_group[2:])\n or last_label == entity_group\n )\n and (sent[last_end:start].isspace() or (last_end == start))\n ):\n last_end = end\n else:\n last_start = start\n last_end = end\n last_label = entity_group\n\n if re.match(\"B-\", last_label, re.IGNORECASE) or re.match(\"I-\", last_label, re.IGNORECASE):\n spans.update({last_start: (last_end, last_label[2:])})\n else:\n spans.update({last_start: (last_end, last_label)})\n return spans\n\n\ndef merge_bio_format(data, preds):\n \"\"\"Merge predictions with BIO format during prediction.\"\"\"\n results = []\n for sent, pred in zip(data, preds):\n results.append(\n [\n {\"entity_group\": value[-1], \"start\": key, \"end\": value[0]}\n for key, value in merge_spans(sent, pred).items()\n ]\n )\n return results\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/mmcv.py",
"content": "import functools\nfrom typing import Callable\n\nimport torch\n\ntry:\n from mmengine.dataset import pseudo_collate as collate\nexcept ImportError as e:\n collate = None\n\n\ndef assert_tensor_type(func: Callable) -> Callable:\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n if not isinstance(args[0].data, torch.Tensor):\n raise AttributeError(\n f\"{args[0].__class__.__name__} has no attribute {func.__name__} for type {args[0].datatype}\"\n )\n return func(*args, **kwargs)\n\n return wrapper\n\n\nclass CollateMMDet:\n def __init__(self, samples_per_gpu):\n self.samples_per_gpu = samples_per_gpu\n\n def __call__(self, x):\n return collate(x)\n\n\nclass CollateMMOcr:\n def __init__(self, samples_per_gpu):\n self.samples_per_gpu = samples_per_gpu\n\n def __call__(self, x):\n return collate(x, samples_per_gpu=self.samples_per_gpu)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/object_detection.py",
"content": "import json\nimport logging\nimport os\nimport warnings\nfrom pathlib import Path\nfrom typing import Dict, Iterable, List, Optional, Union\n\nimport defusedxml.ElementTree as ET\nimport numpy as np\nimport pandas as pd\nimport PIL\nimport torch\nfrom torchmetrics.detection.mean_ap import MeanAveragePrecision\n\nfrom ..constants import (\n BBOX,\n LABEL,\n MAP,\n MAP_50,\n MAP_75,\n MAP_LARGE,\n MAP_MEDIUM,\n MAP_SMALL,\n MAR_1,\n MAR_10,\n MAR_100,\n MAR_LARGE,\n MAR_MEDIUM,\n MAR_SMALL,\n MEAN_AVERAGE_PRECISION,\n)\nfrom .download import download, is_url\n\nlogger = logging.getLogger(__name__)\n\n\ndef _get_image_info(image_path: str):\n \"\"\"\n Get the image width and height info\n Parameters\n ----------\n image_path\n str representing the path to the image\n Returns\n -------\n dict containing image info. None if cannot open image\n \"\"\"\n info_dict = {}\n try:\n with PIL.Image.open(image_path) as im:\n height, width = im.size\n info_dict[\"height\"] = height\n info_dict[\"width\"] = width\n return info_dict\n except Exception as err:\n warnings.warn(f\"Skip image {image_path} due to {err}\")\n return None\n\n\ndef get_df_unique_classes(data: pd.DataFrame):\n \"\"\"\n Get the unique classes and their category IDs from the dataframe for object detection.\n\n Parameters\n ----------\n data : pd.DataFrame\n DataFrame holding the data for object detection. Each row should contain a 'rois'\n column with detection boxes and class labels.\n\n Returns\n -------\n tuple\n A tuple containing (class_names, category_ids) where:\n - class_names: list of unique class name strings\n - category_ids: dict mapping class names to their numeric IDs\n \"\"\"\n unique_classes = {}\n\n # Iterate through all rows in the dataframe\n for idx in range(data.shape[0]):\n row = data.iloc[idx]\n rois = row[\"rois\"]\n\n # Process each ROI in the current row\n for roi in rois:\n # Unpack ROI values (assuming last element is class label)\n *_, class_label = roi\n\n # Add new classes to the dictionary with auto-incrementing IDs\n if class_label not in unique_classes:\n # Start IDs from 1, as 0 is often reserved for background\n unique_classes[class_label] = len(unique_classes) + 1\n\n # Create the output lists/dicts\n class_names = list(unique_classes.keys())\n category_ids = unique_classes\n\n return class_names, category_ids\n\n\ndef object_detection_df_to_coco(data: pd.DataFrame, save_path: Optional[str] = None):\n \"\"\"\n If the user already has dataframe format data and wants to convert to coco format .json files, this function\n completes the task\n Parameters\n ----------\n data\n pd.DataFrame format of object detection data\n save_path\n str path to save the output\n Returns\n -------\n Dict\n \"\"\"\n output_json_dict = {\"images\": [], \"type\": \"instances\", \"annotations\": [], \"categories\": []}\n bbox_count = 0\n unique_classes = {}\n for idx in range(data.shape[0]):\n row = data.iloc[idx]\n image_path = row[\"image\"]\n rois = row[\"rois\"]\n # label = row[\"label\"]\n image_id = idx\n\n image_info = _get_image_info(image_path)\n if image_info:\n image_entry = {\n \"file_name\": image_path,\n \"height\": image_info[\"height\"],\n \"width\": image_info[\"width\"],\n \"id\": image_id,\n }\n output_json_dict[\"images\"].append(image_entry)\n else:\n continue\n\n for roi in rois:\n xmin, ymin, xmax, ymax, class_label = roi\n x, y, w, h = bbox_xyxy_to_xywh([xmin, ymin, xmax, ymax])\n\n ann = {\n \"area\": w * h,\n \"iscrowd\": 0,\n \"bbox\": [x, y, w, h],\n \"category_id\": class_label,\n \"ignore\": 0,\n \"segmentation\": [], # This script is not for segmentation\n \"image_id\": image_id,\n \"id\": bbox_count,\n }\n bbox_count += 1\n\n output_json_dict[\"annotations\"].append(ann)\n\n if class_label not in unique_classes:\n unique_classes[class_label] = len(unique_classes)\n\n for class_name, id in unique_classes.items():\n output_json_dict[\"categories\"].append({\"supercategory\": \"none\", \"id\": id, \"name\": class_name})\n\n if save_path and save_path.endswith(\".json\"):\n with open(save_path, \"w\") as fp:\n json.dump(output_json_dict, fp)\n\n return output_json_dict\n\n\ndef object_detection_data_to_df(\n data: Union[pd.DataFrame, dict, list, str], coco_root: Optional[str] = None\n) -> pd.DataFrame:\n \"\"\"\n Construct a dataframe from a data dictionary, json file path (for COCO), folder path (for VOC),\n image path (for single image), list of image paths (for multiple images)\n Parameters\n ----------\n data (dict, str, list)\n\n Returns\n -------\n a pandas DataFrame with columns \"image\", \"rois\", and \"label\".\n \"\"\"\n if isinstance(data, dict):\n return from_dict(data)\n if isinstance(data, list):\n return from_list(data)\n if isinstance(data, str):\n if os.path.isdir(data) or data.endswith(\".json\"):\n return from_coco_or_voc(data, coco_root=coco_root)\n return from_str(data)\n if isinstance(data, pd.DataFrame):\n sanity_check_dataframe(data)\n return data\n\n raise TypeError(\n \"Expected data to be an instance of dict, list, str or pd.DataFrame, but got {} of type {}\".format(\n data, type(data)\n )\n )\n\n\ndef sanity_check_dataframe(data: pd.DataFrame):\n \"\"\"\n Checking if the dataframe contains valid headers and values\n Parameters\n ----------\n data\n dataframe holding the data for object detection\n Returns\n -------\n\n \"\"\"\n if \"image\" not in data:\n raise ValueError(f\"column 'image' not found in data column names: {data.columns.to_list()}\")\n if \"rois\" not in data and \"label\" not in data:\n raise ValueError(f\"Both column 'rois' and 'label' not found in data column names: {data.columns.to_list()}\")\n else:\n if \"rois\" not in data:\n warnings.warn(\n f\"column 'rois' not found in data column names: {data.columns.to_list()}. Copying from 'label' column...\"\n )\n data[\"rois\"] = data[\"label\"]\n if \"label\" not in data:\n warnings.warn(\n f\"column 'label' not found in data column names: {data.columns.to_list()}. Copying from 'rois' column...\"\n )\n data[\"label\"] = data[\"rois\"]\n assert data.shape[0] > 0, \"data frame is empty\"\n\n\ndef from_str(data: str) -> pd.DataFrame:\n \"\"\"\n Construct a dataframe a string representing a single image path\n Parameters\n ----------\n data\n string of the image path\n Returns\n -------\n a pandas DataFrame with columns \"image\", \"rois\", and \"label\".\n \"\"\"\n d = {\"image\": [], \"rois\": [], \"label\": []}\n d[\"image\"].append(data)\n # Dummy rois\n d[\"rois\"].append([[-1, -1, -1, -1, 0]])\n d[\"label\"].append([[-1, -1, -1, -1, 0]])\n df = pd.DataFrame(d)\n return df.sort_values(\"image\").reset_index(drop=True)\n\n\ndef from_list(data: List[str]) -> pd.DataFrame:\n \"\"\"\n Construct a dataframe from list of image paths\n Parameters\n ----------\n data\n List containing the image paths\n Returns\n -------\n a pandas DataFrame with columns \"image\", \"rois\", and \"label\".\n \"\"\"\n d = {\"image\": [], \"rois\": [], \"label\": []}\n for image_name in data:\n d[\"image\"].append(image_name)\n # Dummy rois\n d[\"rois\"].append([[-1, -1, -1, -1, 0]])\n d[\"label\"].append([[-1, -1, -1, -1, 0]])\n df = pd.DataFrame(d)\n return df.sort_values(\"image\").reset_index(drop=True)\n\n\ndef from_dict(data: dict) -> pd.DataFrame:\n \"\"\"\n Construct a dataframe (dummy) from a data dictionary, with the form {\"image\": [\"img1.jpg\", \"img2.jpg\", ...]}\n Parameters\n ----------\n data\n Dict containing the image paths\n Returns\n -------\n a pandas DataFrame with columns \"image\", \"rois\", and \"label\".\n \"\"\"\n # TODO: Remove this function after refactoring\n d = {\"image\": [], \"rois\": [], \"label\": []}\n\n for image in data[\"image\"]:\n d[\"image\"].append(image)\n # Dummy rois\n d[\"rois\"].append([[-1, -1, -1, -1, 0]])\n d[\"label\"].append([[-1, -1, -1, -1, 0]])\n df = pd.DataFrame(d)\n return df.sort_values(\"image\").reset_index(drop=True)\n\n\ndef from_voc(\n root: str,\n splits: Optional[Union[str, tuple]] = None,\n exts: Optional[Union[str, tuple]] = (\".jpg\", \".jpeg\", \".png\"),\n):\n \"\"\"\n Construct dataframe from pascal VOC format. Modified from gluon cv.\n Normally you will see a structure like:\n âââ VOC2007\n â âââ Annotations\n â âââ ImageSets\n | | âââ Main\n | | | âââ train.txt\n | | | âââ test.txt\n â âââ JPEGImages\n\n Parameters\n ----------\n root\n The root directory for VOC, e.g., the `VOC2007`. If an url is provided, it will be downloaded and extracted.\n splits\n If given, will search for this name in `ImageSets/Main/`, e.g., ('train', 'test')\n exts\n The supported image formats.\n\n Returns\n -------\n A dataframe with columns \"image\", \"rois\", and \"image_attr\".\n \"\"\"\n if is_url(root):\n root = download(root)\n rpath = Path(root).expanduser()\n img_list = []\n\n class_names, _ = get_detection_classes(root)\n\n NAME_TO_IDX = dict(zip(class_names, range(len(class_names))))\n name_to_index = lambda name: NAME_TO_IDX[name]\n\n if splits:\n logger.debug(\"Use splits: %s for root: %s\", str(splits), root)\n if isinstance(splits, str):\n splits = [splits]\n for split in splits:\n split_file = rpath / \"ImageSets\" / \"Main\" / split\n if not split_file.resolve().exists():\n split_file = rpath / \"ImageSets\" / \"Main\" / (split + \".txt\")\n if not split_file.resolve().exists():\n raise FileNotFoundError(split_file)\n with split_file.open(mode=\"r\") as fi:\n img_list += [line.split()[0].strip() for line in fi.readlines()]\n else:\n logger.debug(\n \"No split provided, use full image list in %s, with extension %s\", str(rpath / \"JPEGImages\"), str(exts)\n )\n if not isinstance(exts, (list, tuple)):\n exts = [exts]\n for ext in exts:\n img_list.extend([rp.stem for rp in rpath.glob(\"JPEGImages/*\" + ext)])\n d = {\"image\": [], \"rois\": []}\n logger.info(f\"Number of Images: {len(img_list)}\")\n for stem in img_list:\n basename = stem + \".xml\"\n anno_file = (rpath / \"Annotations\" / basename).resolve()\n tree = ET.parse(anno_file)\n xml_root = tree.getroot()\n size = xml_root.find(\"size\")\n im_path = xml_root.find(\"filename\").text\n if \".\" not in im_path:\n im_path += \".jpg\"\n width = float(size.find(\"width\").text)\n height = float(size.find(\"height\").text)\n rois = []\n for obj in xml_root.iter(\"object\"):\n class_label = name_to_index(obj.find(\"name\").text.strip().lower())\n xml_box = obj.find(\"bndbox\")\n xmin = max(0, float(xml_box.find(\"xmin\").text) - 1)\n ymin = max(0, float(xml_box.find(\"ymin\").text) - 1)\n xmax = min(width, float(xml_box.find(\"xmax\").text) - 1)\n ymax = min(height, float(xml_box.find(\"ymax\").text) - 1)\n if xmin >= xmax or ymin >= ymax:\n logger.warning(\"Invalid bbox: {%s} for {%s}\", str(xml_box), anno_file.name)\n else:\n rois.append(\n [\n xmin,\n ymin,\n xmax,\n ymax,\n class_label,\n ]\n )\n if rois:\n d[\"image\"].append(str(rpath / \"JPEGImages\" / im_path))\n d[\"rois\"].append(rois)\n df = pd.DataFrame(d)\n df[\"label\"] = df.loc[:, \"rois\"].copy() # TODO: remove duplicate column\n\n return df.sort_values(\"image\").reset_index(drop=True)\n\n\ndef import_try_install(package: str, extern_url: Optional[str] = None):\n \"\"\"\n Try import the specified package. Modified from gluon cv.\n If the package not installed, try use pip to install and import if success.\n\n Parameters\n ----------\n package\n The name of the package trying to import.\n extern_url\n The external url if package is not hosted on PyPI.\n For example, you can install a package using:\n \"pip install git+http://github.com/user/repo/tarball/master/egginfo=xxx\".\n In this case, you can pass the url to the extern_url.\n\n Returns\n -------\n \n The imported python module.\n \"\"\"\n import tempfile\n\n import portalocker\n\n lockfile = os.path.join(tempfile.gettempdir(), package + \"_install.lck\")\n with portalocker.Lock(lockfile):\n try:\n return __import__(package)\n except ImportError:\n try:\n from pip import main as pipmain\n except ImportError:\n from types import ModuleType\n\n from pip._internal import main as pipmain\n\n # fix for pip 19.3\n if isinstance(pipmain, ModuleType):\n from pip._internal.main import main as pipmain\n\n # trying to install package\n url = package if extern_url is None else extern_url\n pipmain([\"install\", \"--user\", url]) # will raise SystemExit Error if fails\n\n # trying to load again\n try:\n return __import__(package)\n except ImportError:\n import site\n import sys\n\n user_site = site.getusersitepackages()\n if user_site not in sys.path:\n sys.path.append(user_site)\n return __import__(package)\n return __import__(package)\n\n\ndef try_import_pycocotools():\n \"\"\"\n Tricks to optionally install and import pycocotools. Modified from gluon cv.\n \"\"\"\n # first we can try import pycocotools\n try:\n import pycocotools as _\n except ImportError:\n # we need to install pycootools, which is a bit tricky\n # pycocotools sdist requires Cython, numpy(already met)\n import_try_install(\"cython\")\n # pypi pycocotools is not compatible with windows\n win_url = \"git+https://github.com/zhreshold/cocoapi.git#subdirectory=PythonAPI\"\n try:\n if os.name == \"nt\":\n import_try_install(\"pycocotools\", win_url)\n else:\n import_try_install(\"pycocotools\")\n except ImportError:\n faq = \"cocoapi FAQ\"\n raise ImportError(\"Cannot import or install pycocotools, please refer to %s.\" % faq)\n\n\ndef bbox_xywh_to_xyxy(xywh: Optional[Union[list, tuple, np.ndarray]]):\n \"\"\"\n Convert bounding boxes from format (xmin, ymin, w, h) to (xmin, ymin, xmax, ymax). Modified from gluon cv.\n\n Parameters\n ----------\n xywh\n The bbox in format (x, y, w, h).\n If numpy.ndarray is provided, we expect multiple bounding boxes with\n shape `(N, 4)`.\n\n Returns\n -------\n A tuple or numpy.ndarray.\n The converted bboxes in format (xmin, ymin, xmax, ymax).\n If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n \"\"\"\n if isinstance(xywh, (tuple, list)):\n if not len(xywh) == 4:\n raise IndexError(\"Bounding boxes must have 4 elements, given {}\".format(len(xywh)))\n w, h = np.maximum(xywh[2] - 1, 0), np.maximum(xywh[3] - 1, 0)\n return xywh[0], xywh[1], xywh[0] + w, xywh[1] + h\n elif isinstance(xywh, np.ndarray):\n if not xywh.size % 4 == 0:\n raise IndexError(\"Bounding boxes must have n * 4 elements, given {}\".format(xywh.shape))\n xyxy = np.hstack((xywh[:, :2], xywh[:, :2] + np.maximum(0, xywh[:, 2:4] - 1)))\n return xyxy\n else:\n raise TypeError(\"Expect input xywh a list, tuple or numpy.ndarray, given {}\".format(type(xywh)))\n\n\ndef bbox_ratio_xywh_to_index_xyxy(\n xywh: Optional[Union[list, tuple, np.ndarray]], image_wh: Optional[Union[list, tuple, np.ndarray]]\n):\n \"\"\"\n Convert bounding boxes from format (x_center_ratio, y_center_ratio, w_ratio, h_ratio) to (xmin, ymin, xmax, ymax) in pixel index.\n\n Parameters\n ----------\n xywh\n The bbox in format (x_center_ratio, y_center_ratio, w_ratio, h_ratio).\n If numpy.ndarray is provided, we expect multiple bounding boxes with\n shape `(N, 4)`.\n\n Returns\n -------\n A tuple or numpy.ndarray.\n The converted bboxes in format (xmin, ymin, xmax, ymax).\n If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n \"\"\"\n if isinstance(xywh, (tuple, list)):\n assert isinstance(image_wh, (tuple, list)), (\n f\"image_wh (type: {type(image_wh)} should have the same type with xywh (type: {type(xywh)})\"\n )\n\n if not len(xywh) == 4:\n raise IndexError(\"Bounding boxes must have 4 elements, given {}\".format(len(xywh)))\n if not len(image_wh) == 2:\n raise IndexError(\"Image Width and Height must have 2 elements, given {}\".format(len(image_wh)))\n\n x, y = xywh[:2]\n w, h = np.maximum(xywh[2] - 1, 0), np.maximum(xywh[3] - 1, 0)\n W, H = np.maximum(image_wh[0] - 1, 0), np.maximum(image_wh[1] - 1, 0)\n\n # ratio to index\n x *= W\n y *= H\n W *= W\n H *= H\n\n # mid to upper left corner\n x -= W / 2\n y -= H / 2\n\n return x, y, x + w, y + h # xywh to xyxy\n elif isinstance(xywh, np.ndarray):\n if isinstance(image_wh, np.ndarray):\n if not xywh.size % 4 == 0:\n raise IndexError(\"Bounding boxes must have n * 4 elements, given {}\".format(xywh.shape))\n if not image_wh.size % 2 == 0:\n raise IndexError(\"Image Width and Height must have n * 2 elements, given {}\".format(image_wh.shape))\n\n xywh = xywh * np.concat([image_wh, image_wh], axis=1) # ratio to index\n\n # mid to upper left corner\n xywh[:, :2] -= xywh[:, 2:] / 2\n\n # xywh to xyxy\n xyxy = np.hstack((xywh[:, :2], xywh[:, :2] + np.maximum(0, xywh[:, 2:] - 1)))\n return xyxy\n elif isinstance(image_wh, (tuple, list)):\n if not xywh.size % 4 == 0:\n raise IndexError(\"Bounding boxes must have n * 4 elements, given {}\".format(xywh.shape))\n\n W, H = np.maximum(image_wh[0] - 1, 0), np.maximum(image_wh[1] - 1, 0)\n\n xywh = xywh * np.array([[W, H, W, H]]) # ratio to index\n\n # mid to upper left corner\n xywh[:, :2] -= xywh[:, 2:] / 2\n\n # xywh to xyxy\n xyxy = np.hstack((xywh[:, :2], xywh[:, :2] + np.maximum(0, xywh[:, 2:] - 1)))\n return xyxy\n else:\n raise TypeError(\"Expect input xywh a list, tuple or numpy.ndarray, given {}\".format(type(xywh)))\n\n\ndef bbox_xyxy_to_xywh(xyxy: Optional[Union[list, tuple, np.ndarray]]):\n \"\"\"\n Convert bounding boxes from format (xmin, ymin, xmax, ymax) to (x, y, w, h). Modified from gluon cv.\n\n Parameters\n ----------\n xyxy\n The bbox in format (xmin, ymin, xmax, ymax).\n If numpy.ndarray is provided, we expect multiple bounding boxes with\n shape `(N, 4)`.\n\n Returns\n -------\n A tuple or numpy.ndarray.\n The converted bboxes in format (x, y, w, h).\n If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n \"\"\"\n if isinstance(xyxy, (tuple, list)):\n if not len(xyxy) == 4:\n raise IndexError(\"Bounding boxes must have 4 elements, given {}\".format(len(xyxy)))\n x1, y1 = xyxy[0], xyxy[1]\n w, h = xyxy[2] - x1, xyxy[3] - y1\n return x1, y1, w, h\n elif isinstance(xyxy, np.ndarray):\n if not xyxy.size % 4 == 0:\n raise IndexError(\"Bounding boxes must have n * 4 elements, given {}\".format(xyxy.shape))\n return np.hstack((xyxy[:, :2], xyxy[:, 2:4] - xyxy[:, :2] + 1))\n else:\n raise TypeError(\"Expect input xywh a list, tuple or numpy.ndarray, given {}\".format(type(xyxy)))\n\n\ndef bbox_clip_xyxy(\n xyxy: Optional[Union[list, tuple, np.ndarray]],\n width: Optional[Union[int, float]],\n height: Optional[Union[int, float]],\n):\n \"\"\"\n Clip bounding box with format (xmin, ymin, xmax, ymax) to specified boundary. Modified from gluon cv.\n All bounding boxes will be clipped to the new region `(0, 0, width, height)`.\n\n Parameters\n ----------\n xyxy\n The bbox in format (xmin, ymin, xmax, ymax).\n If numpy.ndarray is provided, we expect multiple bounding boxes with\n shape `(N, 4)`.\n width\n Boundary width.\n height\n Boundary height.\n\n Returns\n -------\n A tuple or numpy.ndarray.\n The clipped bboxes in format (xmin, ymin, xmax, ymax).\n If input is numpy.ndarray, return is numpy.ndarray correspondingly.\n \"\"\"\n if isinstance(xyxy, (tuple, list)):\n if not len(xyxy) == 4:\n raise IndexError(\"Bounding boxes must have 4 elements, given {}\".format(len(xyxy)))\n x1 = np.minimum(width - 1, np.maximum(0, xyxy[0]))\n y1 = np.minimum(height - 1, np.maximum(0, xyxy[1]))\n x2 = np.minimum(width - 1, np.maximum(0, xyxy[2]))\n y2 = np.minimum(height - 1, np.maximum(0, xyxy[3]))\n return x1, y1, x2, y2\n elif isinstance(xyxy, np.ndarray):\n if not xyxy.size % 4 == 0:\n raise IndexError(\"Bounding boxes must have n * 4 elements, given {}\".format(xyxy.shape))\n x1 = np.minimum(width - 1, np.maximum(0, xyxy[:, 0]))\n y1 = np.minimum(height - 1, np.maximum(0, xyxy[:, 1]))\n x2 = np.minimum(width - 1, np.maximum(0, xyxy[:, 2]))\n y2 = np.minimum(height - 1, np.maximum(0, xyxy[:, 3]))\n return np.hstack((x1, y1, x2, y2))\n else:\n raise TypeError(\"Expect input xywh a list, tuple or numpy.ndarray, given {}\".format(type(xyxy)))\n\n\ndef _check_load_coco_bbox(\n coco,\n entry: dict,\n min_object_area: Optional[Union[int, float]] = 0,\n use_crowd: Optional[bool] = False,\n):\n \"\"\"\n Check and load ground-truth labels. Modified from gluon cv.\n\n Parameters\n ----------\n coco\n The COCO data class.\n entry\n The image annotation entry.\n min_object_area\n Minimum object area to consider.\n use_crowd\n Use crowd or not.\n\n Returns\n -------\n Valid objects to consider.\n \"\"\"\n entry_id = entry[\"id\"]\n # fix pycocotools _isArrayLike which don't work for str in python3\n entry_id = [entry_id] if not isinstance(entry_id, (list, tuple)) else entry_id\n ann_ids = coco.getAnnIds(imgIds=entry_id, iscrowd=None)\n objs = coco.loadAnns(ann_ids)\n # check valid bboxes\n rois = []\n is_crowds = []\n width = entry[\"width\"]\n height = entry[\"height\"]\n for obj in objs:\n if obj[\"area\"] < min_object_area:\n continue\n if obj.get(\"ignore\", 0) == 1:\n continue\n is_crowd = obj.get(\"iscrowd\", 0)\n if not use_crowd and is_crowd:\n continue\n # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound\n xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj[\"bbox\"]), width, height)\n # require non-zero box area\n if obj[\"area\"] > 0 and xmax > xmin and ymax > ymin:\n cat_ids = coco.getCatIds()\n id_to_idx = dict(zip(cat_ids, range(len(cat_ids))))\n class_label = id_to_idx[coco.loadCats(obj[\"category_id\"])[0][\"id\"]]\n rois.append(\n [\n float(xmin),\n float(ymin),\n float(xmax),\n float(ymax),\n class_label,\n ]\n )\n is_crowds.append(is_crowd)\n return rois, is_crowds\n\n\ndef from_coco(\n anno_file: Optional[str],\n coco_root: Optional[str] = None,\n min_object_area: Optional[Union[int, float]] = 0,\n use_crowd: Optional[bool] = False,\n):\n \"\"\"\n Load dataset from coco format annotations. Modified from gluon cv.\n The structure of a default coco 2017 dataset looks like:\n .\n âââ annotations\n | |ââ instances_val2017.json\n âââ train2017\n âââ val2017\n\n\n Parameters\n ----------\n anno_file\n The path to the annotation file.\n coco_root\n Root of the COCO folder. The default relative root folder (if set to `None`) is `anno_file/../`.\n min_object_area\n Minimum object area to consider.\n use_crowd\n Use crowd or not.\n\n Returns\n -------\n A dataframe with columns \"image\", \"rois\", and \"image_attr\".\n \"\"\"\n # construct coco object from COCO format\n try_import_pycocotools()\n from pycocotools.coco import COCO\n\n if isinstance(anno_file, Path):\n anno_file = str(anno_file.expanduser().resolve())\n elif isinstance(anno_file, str):\n anno_file = os.path.expanduser(anno_file)\n coco = COCO(anno_file)\n\n # get data root\n if isinstance(coco_root, Path):\n coco_root = str(coco_root.expanduser().resolve())\n elif isinstance(coco_root, str):\n coco_root = os.path.abspath(os.path.expanduser(coco_root))\n elif coco_root is None:\n # try to use the default coco structure\n coco_root = os.path.join(os.path.dirname(anno_file), \"..\")\n logger.info(\n f\"Using default root folder: {coco_root}. \"\n \"Specify `model.mmdet_image.coco_root=...` in hyperparameters if you think it is wrong.\"\n )\n else:\n raise ValueError(\"Unable to parse coco_root: {}\".format(coco_root))\n\n # support prediction using data with no annotations\n # note that data with annotation can be used for prediction without any changes\n try:\n num_annotations = len(coco.getAnnIds())\n except KeyError: # KeyError: 'annotations', there is no annotation entry\n num_annotations = 0\n\n # load entries\n d = {\"image\": [], \"rois\": []}\n image_ids = sorted(coco.getImgIds())\n for entry in coco.loadImgs(image_ids):\n if \"coco_url\" in entry:\n dirname, filename = entry[\"coco_url\"].split(\"/\")[-2:]\n abs_path = os.path.join(coco_root, dirname, filename)\n else:\n abs_path = os.path.join(coco_root, entry[\"file_name\"])\n if not os.path.exists(abs_path):\n logger.warning(f\"File skipped since not exists: {abs_path}.\")\n continue\n rois, _ = _check_load_coco_bbox(\n coco,\n entry,\n min_object_area=min_object_area,\n use_crowd=use_crowd,\n )\n if not rois:\n # discard the rows without valid annotation ONLY when data has annotation\n # add default placeholder to data without annotation for prediction\n if not num_annotations:\n d[\"image\"].append(abs_path)\n d[\"rois\"].append([[-1, -1, -1, -1, 0]]) # TODO: maybe remove this placeholder\n continue\n d[\"image\"].append(abs_path)\n d[\"rois\"].append(rois)\n df = pd.DataFrame(d)\n df[\"label\"] = df.loc[:, \"rois\"].copy()\n return df.sort_values(\"image\").reset_index(drop=True)\n\n\ndef get_image_filename(path: str):\n \"\"\"\n Get the filename (without extension) from its path.\n\n Parameters\n ----------\n path\n The path of image.\n\n Returns\n -------\n The file name of image.\n \"\"\"\n return Path(path.replace(\"\\\\\", \"/\")).stem\n\n\nclass COCODataset:\n # The class that load/save COCO data format.\n # TODO: refactor data loading into here\n def __init__(self, anno_file: str, category_ids: Optional[List] = None):\n \"\"\"\n Parameters\n ----------\n anno_file\n The path to COCO format json annotation file.\n \"\"\"\n self.anno_file = anno_file\n\n with open(anno_file, \"r\") as f:\n d = json.load(f)\n image_list = d[\"images\"]\n img_filename_list = []\n img_id_list = []\n for img in image_list:\n img_filename_list.append(get_image_filename(img[\"file_name\"]))\n img_id_list.append(int(img[\"id\"]))\n self.image_filename_to_id = dict(zip(img_filename_list, img_id_list))\n\n if category_ids is None:\n if \"categories\" in d:\n self.category_ids = [cat[\"id\"] for cat in d[\"categories\"]]\n else:\n self.category_ids = range(9999) # TODO: remove hardcoding here\n else:\n self.category_ids = category_ids\n\n def get_image_id_from_path(self, image_path: str):\n \"\"\"\n Get image id from its path.\n\n Parameters\n ----------\n image_path\n Image path.\n\n Returns\n -------\n Image ID.\n \"\"\"\n return self.image_filename_to_id[get_image_filename(image_path)]\n\n def save_result(self, ret: List, data: pd.DataFrame, save_path: str):\n \"\"\"\n Save COCO format result to given save path.\n\n Parameters\n ----------\n ret\n The returned prediction result.\n data\n The input data.\n save_path\n The save path given to store COCO format output.\n \"\"\"\n coco_format_result = []\n\n for i, row in data.reset_index(drop=True).iterrows():\n image_id = self.get_image_id_from_path(row[\"image\"])\n\n pred_result = ret[i]\n N_pred = len(pred_result[\"bboxes\"])\n for bbox_idx in range(N_pred):\n coco_format_result.append(\n {\n \"image_id\": image_id,\n \"category_id\": self.category_ids[int(pred_result[\"labels\"][bbox_idx].item())],\n \"bbox\": bbox_xyxy_to_xywh(pred_result[\"bboxes\"][bbox_idx].tolist()),\n \"score\": pred_result[\"scores\"][bbox_idx].item(),\n }\n )\n\n with open(save_path, \"w\") as f:\n print(f\"saving file at {save_path}\")\n json.dump(coco_format_result, f)\n\n\ndef cocoeval_torchmetrics(outputs: List):\n \"\"\"\n Evaluate predictor's output using torchmetrics' mAP implementation: https://github.com/Lightning-AI/metrics\n\n Parameters\n ----------\n outputs\n The predictor's output. It is a list with length equals number of images.\n\n Returns\n -------\n The mAP result.\n \"\"\"\n\n map_metric = MeanAveragePrecision(box_format=\"xyxy\", iou_type=\"bbox\", class_metrics=False)\n\n preds = []\n target = []\n for per_img_outputs in outputs: # TODO: refactor here\n preds.append(\n dict(\n boxes=per_img_outputs[BBOX][\"bboxes\"].to(\"cpu\"),\n scores=per_img_outputs[BBOX][\"scores\"].to(\"cpu\"),\n labels=per_img_outputs[BBOX][\"labels\"].to(\"cpu\"),\n )\n )\n\n target.append(\n dict(\n boxes=per_img_outputs[LABEL][\"bboxes\"].to(\"cpu\"),\n labels=per_img_outputs[LABEL][\"labels\"].to(\"cpu\"),\n )\n )\n\n map_metric.update(preds, target)\n\n return map_metric.compute()\n\n\ndef cocoeval_pycocotools(\n outputs: List,\n data: pd.DataFrame,\n anno_file: str,\n cache_path: str,\n):\n \"\"\"\n Evaluate predictor's output using pycocotool's mAP implementation: https://github.com/cocodataset/cocoapi\n Pycocotool's implementation takes COCO format prediction result file as input.\n So here requires a cache_path to store the prediction result file.\n\n Parameters\n ----------\n outputs\n The predictor's output. It is a list with length equals number of images.\n data\n The input data.\n anno_file\n The path to COCO format json annotation file.\n cache_path\n The cache path to store prediction result in COCO format.\n\n Returns\n -------\n The mAP result.\n \"\"\"\n try_import_pycocotools()\n from pycocotools.coco import COCO\n from pycocotools.cocoeval import COCOeval\n\n from ..constants import BBOX\n from . import extract_from_output\n\n coco_dataset = COCODataset(anno_file)\n\n ret = extract_from_output(ret_type=BBOX, outputs=outputs)\n\n coco_dataset.save_result(ret, data, cache_path)\n\n cocoGt = COCO(anno_file)\n # https://github.com/ppwwyyxx/cocoapi/commit/617836ce3551927ec94e2024b18d6c899226a742#diff-51af02f519555c402db0216fd229e4fcb51fe55c25f446e7af8890d73269c7bdR313-R314\n if \"info\" not in cocoGt.dataset:\n cocoGt.dataset[\"info\"] = \"\"\n cocoDt = cocoGt.loadRes(cache_path)\n annType = \"bbox\"\n\n cocoEval = COCOeval(cocoGt, cocoDt, annType)\n cocoEval.evaluate()\n cocoEval.accumulate()\n cocoEval.summarize()\n\n return cocoEval.stats\n\n\ndef parse_detection_result(\n result: Optional[Union[Dict, np.ndarray]],\n):\n if isinstance(result, np.ndarray):\n return {\n MAP: result[0],\n MEAN_AVERAGE_PRECISION: result[0],\n MAP_50: result[1],\n MAP_75: result[2],\n MAP_SMALL: result[3],\n MAP_MEDIUM: result[4],\n MAP_LARGE: result[5],\n MAR_1: result[6],\n MAR_10: result[7],\n MAR_100: result[8],\n MAR_SMALL: result[9],\n MAR_MEDIUM: result[10],\n MAR_LARGE: result[11],\n }\n else:\n result[MEAN_AVERAGE_PRECISION] = result[MAP]\n return result\n\n\ndef cocoeval(\n outputs: List,\n data: pd.DataFrame,\n anno_file: str,\n cache_path: str,\n metrics: Optional[Union[str, List]],\n tool=\"pycocotools\",\n):\n \"\"\"\n Evaluate predictor's output using mAP metrics per COCO's standard.\n\n Parameters\n ----------\n outputs\n The predictor's output. It is a list with length equals number of images.\n data\n The input data.\n anno_file\n The path to COCO format json annotation file.\n cache_path\n The cache path to store prediction result in COCO format.\n metrics\n The name of metrics to be reported.\n tool\n Use the mAP implementation of \"pycocotools\" or \"torchmetrics\".\n\n Returns\n -------\n The mAP result.\n \"\"\"\n if (not tool) or tool == \"pycocotools\":\n result = cocoeval_pycocotools(outputs, data, anno_file, cache_path)\n elif tool == \"torchmetrics\":\n result = cocoeval_torchmetrics(outputs)\n else:\n raise ValueError(f\"Unsupported eval_tool: {tool}\")\n\n result = parse_detection_result(result)\n\n if metrics:\n if isinstance(metrics, str) and metrics.lower() in result:\n return {metrics.lower(): result[metrics.lower()]}\n elif isinstance(metrics, list):\n return {metric.lower(): result[metric.lower()] for metric in metrics}\n\n return result\n\n\ndef dump_voc_classes(voc_annotation_path: str, voc_class_names_output_path: str = None) -> [str]:\n \"\"\"\n Reads annotations for a dataset in VOC format.\n Then\n dumps the unique class names into a labels.txt file.\n Parameters\n ----------\n voc_annotation_path\n root_path for annotations in VOC format\n voc_class_names_output_path\n output path for the labels.txt\n Returns\n -------\n list of strings, [class_name0, class_name1, ...]\n \"\"\"\n files = os.listdir(voc_annotation_path)\n class_names = set()\n for f in files:\n if f.endswith(\".xml\"):\n xml_path = os.path.join(voc_annotation_path, f)\n tree = ET.parse(xml_path)\n root = tree.getroot()\n for boxes in root.iter(\"object\"):\n class_names.add(boxes.find(\"name\").text)\n\n sorted_class_names = sorted(list(class_names))\n if voc_class_names_output_path:\n with open(voc_class_names_output_path, \"w\") as f:\n f.writelines(\"\\n\".join(sorted_class_names))\n\n return sorted_class_names\n\n\ndef dump_voc_xml_files(voc_annotation_path: str, voc_annotation_xml_output_path: str = None) -> [str]:\n \"\"\"\n Reads annotations for a dataset in VOC format.\n Then\n 1. dumps the unique class names into labels.txt file.\n 2. dumps the xml annotation file names into pathlist.txt file.\n Parameters\n ----------\n voc_annotation_path\n root_path for annotations in VOC format\n voc_annotation_xml_output_path\n output path for the pathlist.txt\n Returns\n -------\n list of strings, [xml_file0, xml_file1, ...]\n \"\"\"\n files = os.listdir(voc_annotation_path)\n annotation_path_base_name = os.path.basename(voc_annotation_path)\n xml_file_names = []\n for f in files:\n if f.endswith(\".xml\"):\n xml_file_names.append(os.path.join(annotation_path_base_name, f))\n\n if voc_annotation_xml_output_path:\n with open(voc_annotation_xml_output_path, \"w\") as f:\n f.writelines(\"\\n\".join(xml_file_names))\n\n return xml_file_names\n\n\ndef process_voc_annotations(\n voc_annotation_path: str, voc_class_names_output_path: str, voc_annotation_xml_output_path: str\n) -> None:\n \"\"\"\n Reads annotations for a dataset in VOC format.\n Then\n 1. dumps the unique class names into labels.txt file.\n 2. dumps the xml annotation file names into pathlist.txt file.\n Parameters\n ----------\n voc_annotation_path\n root_path for annotations in VOC format\n voc_class_names_output_path\n output path for the labels.txt\n voc_annotation_xml_output_path\n output path for the pathlist.txt\n Returns\n -------\n None\n \"\"\"\n files = os.listdir(voc_annotation_path)\n annotation_path_base_name = os.path.basename(voc_annotation_path)\n class_names = set()\n xml_file_names = []\n for f in files:\n xml_path = os.path.join(voc_annotation_path, f)\n tree = ET.parse(xml_path)\n root = tree.getroot()\n for boxes in root.iter(\"object\"):\n class_names.add(boxes.find(\"name\").text)\n\n xml_file_names.append(os.path.join(annotation_path_base_name, f))\n\n sorted_class_names = sorted(list(class_names))\n with open(voc_class_names_output_path, \"w\") as f:\n f.writelines(\"\\n\".join(sorted_class_names))\n\n with open(voc_annotation_xml_output_path, \"w\") as f:\n f.writelines(\"\\n\".join(xml_file_names))\n\n\ndef from_coco_or_voc(file_path: str, splits: Optional[Union[str]] = None, coco_root: Optional[str] = None):\n \"\"\"\n Convert the data from coco or voc format to pandas Dataframe.\n\n Parameters\n ----------\n file_path\n The path to data.\n If it is a file, it should be the COCO format json annotation file.\n If it is a directory, it should be the root folder of VOC format data.\n splits\n The splits to use for VOC format data.\n\n Returns\n -------\n The data in our pandas Dataframe format.\n \"\"\"\n if os.path.isdir(file_path):\n # VOC use dir as input\n return from_voc(root=file_path, splits=splits)\n else:\n return from_coco(file_path, coco_root=coco_root)\n\n\ndef get_coco_format_classes(sample_data_path: str):\n \"\"\"\n Get class names and category IDs for COCO format data.\n\n Parameters\n ----------\n sample_data_path : str\n The path to COCO format json annotation file. Could be any split, e.g. train/val/test/....\n\n Returns\n -------\n tuple\n A tuple containing (class_names, category_ids) where:\n - class_names: list of class name strings\n - category_ids: dict mapping class names to their COCO category IDs\n \"\"\"\n try:\n with open(sample_data_path, \"r\") as f:\n annotation = json.load(f)\n except:\n raise ValueError(f\"Failed to load json from provided json file: {sample_data_path}.\")\n\n # Extract both names and IDs from categories\n class_names = [cat[\"name\"] for cat in annotation[\"categories\"]]\n\n # Create mapping of names to their original COCO category IDs\n category_ids = [cat[\"id\"] for cat in annotation[\"categories\"]]\n\n return class_names, category_ids\n\n\ndef get_voc_format_classes(root: str):\n \"\"\"\n Get class names and category IDs for VOC format data.\n\n Parameters\n ----------\n root : str\n The path to the root directory of VOC data.\n\n Returns\n -------\n tuple\n A tuple containing (class_names, category_ids) where:\n - class_names: list of class name strings\n - category_ids: dict mapping class names to their numeric IDs\n \"\"\"\n if is_url(root):\n root = download(root)\n\n rpath = Path(root).expanduser()\n labels_file = os.path.join(rpath, \"labels.txt\")\n\n if os.path.exists(labels_file):\n with open(labels_file) as f:\n class_names = [line.rstrip().lower() for line in f]\n print(f\"using class_names in labels.txt: {class_names}\")\n else:\n # read the class names and save results\n logger.warning(\n \"labels.txt does not exist, using default VOC names. \"\n \"Creating labels.txt by scanning the directory: {}\".format(os.path.join(root, \"Annotations\"))\n )\n class_names = dump_voc_classes(\n voc_annotation_path=os.path.join(root, \"Annotations\"), voc_class_names_output_path=labels_file\n )\n\n # There are no category IDs in VOC format\n # Create category IDs dictionary starting from 1\n # 0 is typically reserved for background in many frameworks\n category_ids = [idx + 1 for idx, name in enumerate(class_names)]\n\n return class_names, category_ids\n\n\ndef get_detection_classes(sample_data_path):\n \"\"\"\n Get class names and category IDs from detection dataset in various formats.\n\n Parameters\n ----------\n sample_data_path : Union[str, pd.DataFrame]\n The input can be one of:\n - str (directory): Path to root directory of VOC format data\n - str (file): Path to COCO format JSON annotation file\n - pd.DataFrame: DataFrame containing detection data with 'rois' column\n\n Returns\n -------\n tuple\n A tuple containing (class_names, category_ids) where:\n - class_names: list of class name strings\n - category_ids: dict mapping class names to their numeric IDs\n\n For VOC: IDs start from 1\n For COCO: Original category IDs from annotation file\n For DataFrame: Sequential IDs starting from 1\n \"\"\"\n # Handle string paths for VOC and COCO formats\n if isinstance(sample_data_path, str):\n if os.path.isdir(sample_data_path):\n # Directory path indicates VOC format\n return get_voc_format_classes(sample_data_path)\n else:\n # File path indicates COCO format JSON\n return get_coco_format_classes(sample_data_path)\n # Handle DataFrame format\n elif isinstance(sample_data_path, pd.DataFrame):\n return get_df_unique_classes(sample_data_path)\n\n\ndef visualize_detection(\n pred: pd.DataFrame,\n detection_classes: List[str],\n conf_threshold: float,\n visualization_result_dir: str,\n):\n \"\"\"\n Visualize detection results for one image, and save to visualization_result_dir\n\n Parameters\n ----------\n pred\n Detection results as in pd.DataFrame format\n detection_classes\n All classes for detection\n conf_threshold\n Bounding box confidence threshold to filter unwanted detections\n visualization_result_dir\n Directory to save the visualization results\n Returns\n -------\n an List of np.ndarray of visualized images\n \"\"\"\n try:\n import cv2\n except:\n raise ImportError(\"No module named: cv2. Please install cv2 by 'pip install opencv-python'\")\n\n if not os.path.exists(visualization_result_dir):\n os.makedirs(visualization_result_dir, exist_ok=True)\n\n classname2idx = {classname: i for i, classname in enumerate(detection_classes)}\n idx2classname = {i: classname for i, classname in enumerate(detection_classes)}\n\n visualized_images = []\n for i in range(len(pred)):\n image_path = pred.iloc[i][\"image\"]\n image_pred = pred.iloc[i][\"bboxes\"]\n im = cv2.imread(image_path)\n tlwhs = []\n obj_ids = []\n conf_scores = []\n for data in image_pred:\n if data[\"score\"] > conf_threshold:\n obj_ids.append(classname2idx[data[\"class\"]])\n tlwhs.append(bbox_xyxy_to_xywh(data[\"bbox\"]))\n conf_scores.append(data[\"score\"])\n visualized_im = plot_detections(im, tlwhs, obj_ids, idx2classname, conf_threshold, scores=conf_scores)\n visualized_images.append(visualized_im)\n imgname = os.path.basename(image_path)\n cv2.imwrite(os.path.join(visualization_result_dir, imgname), visualized_im)\n logger.info(\"Saved visualizations to {}\".format(visualization_result_dir))\n return visualized_images\n\n\ndef plot_detections(\n image,\n tlwhs,\n obj_ids,\n idx2classname,\n conf_threshold,\n scores=None,\n text_scale=0.75,\n text_thickness=1,\n line_thickness=2,\n alpha=0.5,\n):\n \"\"\"\n Plot the detections on to the corresponding image\n\n Parameters\n ----------\n image\n np.ndarray: np array containing the image data\n tlwhs\n list: list containing the bounding boxes in (x1, y1, x2, y2) format\n obj_ids\n list: list containing the class indices of the bounding boxes, length should match tlwhs\n idx2classname\n dict: maps obj_ids to class name (str)\n conf_threshold\n float: confidence threshold to filter bounding boxes\n scores\n list: confidence scores of the bounding boxes, length should match tlwhs\n text_scale\n float: font size of the text display\n text_thickness\n int: font weight of the text display\n line_thickness\n int: line width of the bounding box display\n alpha\n float: opacity of the text display background color\n\n Returns\n -------\n an np.ndarray of visualized image\n \"\"\"\n # TODO: Convert to use mmdet package\n try:\n import cv2\n except:\n raise ImportError(\"No module named: cv2. Please install cv2 by 'pip install opencv-python'\")\n im = np.ascontiguousarray(np.copy(image))\n im_h, im_w = im.shape[:2]\n\n font = cv2.FONT_HERSHEY_DUPLEX\n text_scale = text_scale if im_w > 500 else text_scale * 0.8\n\n title = \"num_det: %d conf: %.2f\" % (len(tlwhs), conf_threshold)\n im = add_text_with_bg_color(\n im=im,\n text=title,\n tl=(0, 0),\n bg_color=(0, 0, 0),\n alpha=alpha,\n font=font,\n text_scale=text_scale,\n text_thickness=text_thickness,\n )\n\n for i, tlwh in enumerate(tlwhs):\n x1, y1, w, h = tlwh\n intbox = tuple(map(int, (x1, y1, x1 + w, y1 + h)))\n obj_id = int(obj_ids[i])\n id_text = idx2classname[obj_ids[i]]\n if scores is not None:\n id_text = id_text + \",{:.3f}\".format(float(scores[i]))\n color = get_color(abs(obj_id))\n im = add_bbox_with_alpha(\n im=im, tl=intbox[0:2], br=intbox[2:4], line_color=color, alpha=alpha, line_thickness=line_thickness\n )\n im = add_text_with_bg_color(\n im=im,\n text=id_text,\n tl=(intbox[0], intbox[1]),\n bg_color=color,\n alpha=0.75,\n font=font,\n text_scale=text_scale,\n text_thickness=text_thickness,\n )\n return im\n\n\ndef add_bbox_with_alpha(im: np.ndarray, tl: tuple, br: tuple, line_color: tuple, alpha: float, line_thickness: int):\n \"\"\"\n draw one box borders with transparency (alpha)\n\n Parameters\n ----------\n im\n np.ndarray: the image to draw bbox on\n tl\n tuple: bottom right corner of the bounding box: tl = (x1, y1)\n br\n tuple: bottom right corner of the bounding box: br = (x1, y1)\n line_color\n tuple: the color of the box borders, e.g. (0, 0, 0)\n alpha\n float: the opacity of the bbox borders\n line_thickness:\n int: thickness of the border\n Returns\n -------\n an np.ndarray of image with added bbox\n \"\"\"\n try:\n import cv2\n except:\n raise ImportError(\"No module named: cv2. Please install cv2 by 'pip install opencv-python'\")\n overlay = im.copy()\n cv2.rectangle(overlay, tl, br, line_color, thickness=line_thickness)\n im = cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0)\n return im\n\n\ndef add_text_with_bg_color(\n im: np.ndarray,\n text: str,\n tl: tuple,\n bg_color: tuple,\n alpha: float,\n font,\n text_scale: float,\n text_thickness: int,\n text_vert_padding: int = None,\n):\n \"\"\"\n Add text to im with background color\n\n Parameters\n ----------\n im\n np.ndarray: the image to add text on\n text\n string: the text content\n tl\n tuple: top left corner of the text region, tl = (x1, y1)\n bg_color\n tuple: the color of the background, e.g. (0, 0, 0)\n alpha\n float: the opacity of the background\n font\n the font of the text, e.g. cv2.FONT_HERSHEY_DUPLEX\n text_scale\n float: the scale (font size) of the text, e.g. 0.75\n text_thickness\n int: the font weight of the text, e.g. 1\n text_vert_padding\n int: vertical padding of the text on each side\n Returns\n -------\n an np.ndarray of image with added text\n \"\"\"\n try:\n import cv2\n except:\n raise ImportError(\"No module named: cv2. Please install cv2 by 'pip install opencv-python'\")\n\n x1, y1 = tl\n\n overlay = im.copy()\n text_size, _ = cv2.getTextSize(text, font, float(text_scale), text_thickness)\n text_w, text_h = text_size\n\n text_vert_padding = text_vert_padding if text_vert_padding else int(text_h * 0.1)\n\n y1 = max(y1 - text_h - text_vert_padding * 2, 0)\n\n cv2.rectangle(overlay, (x1, y1), (x1 + text_w, y1 + text_h + text_vert_padding * 2), bg_color, -1)\n im = cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0)\n cv2.putText(\n im, text, (x1, y1 + text_h + text_vert_padding), font, text_scale, (255, 255, 255), thickness=text_thickness\n )\n return im\n\n\ndef get_color(idx):\n idx = idx * 3\n color = ((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255)\n return color\n\n\ndef convert_result_df(\n pred: Iterable, data: Union[pd.DataFrame, Dict], detection_classes: List[str], result_path: Optional[str] = None\n):\n \"\"\"\n Saving detection results in pd.DataFrame format (per image)\n\n Parameters\n ----------\n pred\n List containing detection results for one image\n data\n pandas data frame or dict containing the image information to be tested\n result_path\n path to save result\n detection_classes\n all available classes for this detection\n Returns\n -------\n The detection results as pandas DataFrame\n \"\"\"\n if isinstance(data, dict):\n image_names = data[\"image\"]\n else:\n image_names = data[\"image\"].to_list()\n results = []\n idx2classname = {i: classname for (i, classname) in enumerate(detection_classes)}\n\n for image_pred, image_name in zip(pred, image_names):\n box_info = []\n N_preds = len(image_pred[\"bboxes\"])\n for i in range(N_preds):\n box_info.append(\n {\n \"class\": idx2classname[image_pred[\"labels\"][i].item()],\n \"class_id\": image_pred[\"labels\"][i].item(),\n \"bbox\": image_pred[\"bboxes\"][i].tolist(),\n \"score\": image_pred[\"scores\"][i].item(),\n }\n )\n results.append([image_name, box_info])\n result_df = pd.DataFrame(results, columns=[\"image\", \"bboxes\"])\n if result_path:\n result_df.to_csv(result_path, index=False)\n logger.info(\"Saved detection results to {}\".format(result_path))\n return result_df\n\n\ndef save_result_coco_format(data_path, pred, category_ids, result_path, coco_root: Optional[str] = None):\n coco_dataset = COCODataset(data_path, category_ids=category_ids)\n result_name, _ = os.path.splitext(result_path)\n result_path = result_name + \".json\"\n coco_dataset.save_result(pred, from_coco_or_voc(data_path, \"test\", coco_root=coco_root), save_path=result_path)\n logger.info(f\"Saved detection result to {result_path}\")\n\n\ndef save_result_voc_format(pred, result_path):\n result_name, _ = os.path.splitext(result_path)\n result_path = result_name + \".npy\"\n np.save(result_path, pred)\n logger.info(f\"Saved detection result to {result_path}\")\n\n\ndef convert_pred_to_xywh(pred: Optional[List]) -> Optional[List]:\n \"\"\"\n Convert prediction bounding boxes from XYXY to XYWH format.\n\n Args:\n pred: List of predictions, where each prediction contains 'bboxes' that can be\n either a torch.Tensor or numpy.ndarray\n\n Returns:\n Modified list of predictions with bboxes in XYWH format\n \"\"\"\n if not pred:\n return pred\n\n for i, pred_per_image in enumerate(pred):\n bboxes = pred_per_image[\"bboxes\"]\n\n # Handle numpy array case\n if isinstance(bboxes, np.ndarray):\n pred[i][\"bboxes\"] = bbox_xyxy_to_xywh(bboxes)\n # Handle torch.Tensor case\n elif torch.is_tensor(bboxes):\n pred[i][\"bboxes\"] = bbox_xyxy_to_xywh(bboxes.detach().numpy())\n else:\n raise TypeError(f\"Unsupported bbox type: {type(bboxes)}. Expected numpy.ndarray or torch.Tensor\")\n\n return pred\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/onnx.py",
"content": "import logging\nimport os\nfrom typing import Dict, List, Optional, Tuple, Union\n\nlogger = logging.getLogger(__name__)\n\n# TODO: Try a better workaround to lazy import tensorrt package.\ntensorrt_imported = False\nif not tensorrt_imported:\n try:\n import tensorrt # Unused but required by TensorrtExecutionProvider\n\n tensorrt_imported = True\n except:\n # We silently omit the import failure here to avoid overwhelming warning messages in case of multi-gpu.\n tensorrt_imported = False\n\n\ndef onnx_get_dynamic_axes(input_keys: List[str]):\n dynamic_axes = {}\n for k in input_keys:\n if \"token_ids\" in k or \"segment_ids\" in k:\n dynamic_axes[k] = {0: \"batch_size\", 1: \"seq_length\"}\n elif \"valid_length\" in k or k.startswith(\"numerical\") or k.startswith(\"timm_image\"):\n dynamic_axes[k] = {0: \"batch_size\"}\n\n return dynamic_axes\n\n\ndef get_provider_name(provider_config: Union[str, tuple]) -> str:\n if isinstance(provider_config, tuple):\n provider_name = provider_config[0]\n else:\n assert isinstance(provider_config, str), \"input provider config is expected to be either str or tuple\"\n provider_name = provider_config\n return provider_name\n\n\nclass OnnxModule(object):\n \"\"\"\n OnnxModule is as a replacement of torch.nn.Module for running forward pass with onnxruntime.\n\n The module can be generated with MultiModalPredictor.export_tensorrt(),\n so that we can predict with TensorRT by simply replacing predictor._model with OnnxModule.\n \"\"\"\n\n def __init__(self, onnx_path: Union[str, bytes], providers: Optional[Union[dict, List[str]]] = None):\n \"\"\"\n Parameters\n ----------\n onnx_path : str or bytes\n The file path (or bytes) of the onnx model that need to be executed in onnxruntime.\n providers : dict or str, default=None\n A list of execution providers for model prediction in onnxruntime.\n \"\"\"\n import onnx\n import onnxruntime as ort\n\n if isinstance(onnx_path, bytes):\n onnx_model = onnx.load_model_from_string(onnx_path)\n else:\n if not os.path.exists(onnx_path):\n raise FileNotFoundError(f\"failed to located onnx file at {onnx_path}\")\n\n logger.info(\"Loading ONNX file from path {}...\".format(onnx_path))\n onnx_model = onnx.load(onnx_path)\n\n if providers is None:\n if isinstance(onnx_path, str):\n dirname = os.path.dirname(os.path.abspath(onnx_path))\n cache_path = os.path.join(dirname, \"model_trt\")\n else:\n cache_path = None\n providers = [\n (\n \"TensorrtExecutionProvider\",\n {\n \"device_id\": 0,\n \"trt_max_workspace_size\": 2147483648,\n \"trt_fp16_enable\": True,\n \"trt_engine_cache_path\": cache_path,\n \"trt_engine_cache_enable\": True,\n },\n ),\n (\n \"CUDAExecutionProvider\",\n {\n \"device_id\": 0,\n \"arena_extend_strategy\": \"kNextPowerOfTwo\",\n \"gpu_mem_limit\": 2 * 1024 * 1024 * 1024,\n \"cudnn_conv_algo_search\": \"EXHAUSTIVE\",\n \"do_copy_in_default_stream\": True,\n },\n ),\n (\"CPUExecutionProvider\", {}),\n ]\n\n if len(providers) == 1 and get_provider_name(providers[0]) == \"TensorrtExecutionProvider\":\n if not tensorrt_imported:\n raise ImportError(\n \"tensorrt package is not installed. The package can be install via `pip install tensorrt`.\"\n )\n\n self.sess = ort.InferenceSession(onnx_model.SerializeToString(), providers=providers)\n\n if get_provider_name(providers[0]) == \"TensorrtExecutionProvider\" and tensorrt_imported:\n assert \"TensorrtExecutionProvider\" in self.sess.get_providers(), (\n f\"unexpected TensorRT compilation failure: TensorrtExecutionProvider not in providers ({self.sess.get_providers()}). \"\n \"Make sure onnxruntime package gets lazy imported everywhere.\"\n )\n\n inputs = self.sess.get_inputs()\n outputs = self.sess.get_outputs()\n self.input_names = [i.name for i in inputs]\n self.output_names = [i.name for i in outputs]\n\n def __call__(self, *args):\n \"\"\"\n Make the module callable like torch.nn.Module, while running forward pass with onnxruntime.\n\n Parameters\n ----------\n args : list of torch.Tensor\n A list of torch.Tensor that are inputs of the model.\n\n Returns\n -------\n onnx_outputs : list of torch.Tensor\n A list of torch.Tensor that are outputs of the model.\n \"\"\"\n import torch\n\n input_dict = {k: args[i].cpu().numpy() for i, k in enumerate(self.input_names)}\n onnx_outputs = self.sess.run(self.output_names, input_dict)\n onnx_outputs = onnx_outputs[:3]\n onnx_outputs = [torch.from_numpy(out) for out in onnx_outputs]\n return onnx_outputs\n\n def to(self, *args):\n \"\"\"A dummy function that act as torch.nn.Module.to() function\"\"\"\n\n class DummyModel:\n def eval():\n pass\n\n return DummyModel\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/path.py",
"content": "import logging\nimport os\nfrom datetime import datetime\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport pytz\n\nfrom ..constants import LAST_CHECKPOINT, MODEL_CHECKPOINT\n\nlogger = logging.getLogger(__name__)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/precision.py",
"content": "import contextlib\nimport logging\nimport warnings\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport torch\n\nlogger = logging.getLogger(__name__)\n\n\ndef convert_to_torch_precision(precision: Union[int, str]):\n \"\"\"\n Convert a precision integer or string to the corresponding torch precision.\n\n Parameters\n ----------\n precision\n a precision integer or string from the config.\n\n Returns\n -------\n A torch precision object.\n \"\"\"\n precision_mapping = {\n 16: torch.half,\n \"16\": torch.half,\n \"16-mixed\": torch.half,\n \"16-true\": torch.half,\n \"bf16\": torch.bfloat16,\n \"bf16-mixed\": torch.bfloat16,\n \"bf16-true\": torch.bfloat16,\n 32: torch.float32,\n \"32\": torch.float32,\n \"32-true\": torch.float32,\n 64: torch.float64,\n \"64\": torch.float64,\n \"64-true\": torch.float64,\n }\n\n if precision in precision_mapping:\n precision = precision_mapping[precision]\n else:\n raise ValueError(f\"Unknown precision: {precision}\")\n\n return precision\n\n\ndef infer_precision(\n num_gpus: int, precision: Union[int, str], as_torch: Optional[bool] = False, cpu_only_warning: bool = True\n):\n \"\"\"\n Infer the proper precision based on the environment setup and the provided precision.\n\n Parameters\n ----------\n num_gpus\n GPU number.\n precision\n The precision provided in config.\n as_torch\n Whether to convert the precision to the Pytorch format.\n cpu_only_warning\n Whether to turn on warning if the instance has only CPU.\n\n Returns\n -------\n The inferred precision.\n \"\"\"\n if num_gpus == 0: # CPU only prediction\n if cpu_only_warning:\n warnings.warn(\n \"Only CPU is detected in the instance. \"\n \"This may result in slow speed for MultiModalPredictor. \"\n \"Consider using an instance with GPU support.\",\n UserWarning,\n )\n precision = 32 # Force to use fp32 for training since 16-mixed is not available in CPU\n else:\n if isinstance(precision, str) and \"bf16\" in precision and not torch.cuda.is_bf16_supported():\n warnings.warn(\n f\"{precision} is not supported by the GPU device / cuda version. \"\n \"Consider using GPU devices with versions after Amphere or upgrading cuda to be >=11.0. \"\n f\"MultiModalPredictor is switching precision from {precision} to 32.\",\n UserWarning,\n )\n precision = 32\n\n if as_torch:\n precision = convert_to_torch_precision(precision=precision)\n\n return precision\n\n\n@contextlib.contextmanager\ndef double_precision_context():\n \"\"\"\n Double precision context manager.\n \"\"\"\n default_dtype = torch.get_default_dtype()\n torch.set_default_dtype(torch.float64)\n yield\n torch.set_default_dtype(default_dtype)\n\n\ndef get_precision_context(precision: Union[int, str], device_type: Optional[str] = None):\n \"\"\"\n Choose the proper context manager based on the precision.\n\n Parameters\n ----------\n precision\n The precision.\n device_type\n gpu or cpu.\n\n Returns\n -------\n A precision context manager.\n \"\"\"\n precision = convert_to_torch_precision(precision=precision)\n\n if precision in [torch.half, torch.float16, torch.bfloat16]:\n return torch.autocast(device_type=device_type, dtype=precision)\n if precision == torch.float32:\n assert torch.get_default_dtype() == torch.float32\n return contextlib.nullcontext()\n elif precision == torch.float64:\n return double_precision_context()\n else:\n raise ValueError(f\"Unknown precision: {precision}\")\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/presets.py",
"content": "from typing import List, Optional\n\nfrom autogluon.common.utils.try_import import try_import_ray\n\nfrom ..constants import (\n BEST_QUALITY,\n BINARY,\n DATA,\n DEFAULT,\n ENV,\n HIGH_QUALITY,\n MEDIUM_QUALITY,\n MODEL,\n MULTICLASS,\n OPTIM,\n REGRESSION,\n)\nfrom .registry import Registry\n\nautomm_presets = Registry(\"automm_presets\")\nmatcher_presets = Registry(\"matcher_presets\")\n\n\ndef get_default_hpo_setup():\n try_import_ray()\n from ray import tune\n\n default_hyperparameter_tune_kwargs = {\n \"searcher\": \"bayes\",\n \"scheduler\": \"ASHA\",\n \"num_trials\": 512,\n }\n\n default_tunable_hyperparameters = {\n \"optim.lr\": tune.loguniform(1e-5, 1e-2),\n \"optim.optim_type\": tune.choice([\"adamw\", \"sgd\"]),\n \"optim.max_epochs\": tune.choice(list(range(5, 31))),\n \"env.batch_size\": tune.choice([16, 32, 64, 128, 256]),\n }\n\n return default_tunable_hyperparameters, default_hyperparameter_tune_kwargs\n\n\ndef parse_presets_str(presets: str):\n use_hpo = False\n if presets.endswith(\"_hpo\"):\n presets = presets[:-4]\n use_hpo = True\n\n return presets, use_hpo\n\n\n@automm_presets.register()\ndef default(presets: str = DEFAULT):\n \"\"\"\n Register the presets for problem types: binary, multiclass, classification, and regression.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\n \"ft_transformer\",\n \"timm_image\",\n \"hf_text\",\n \"document_transformer\",\n \"fusion_mlp\",\n ],\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets in [HIGH_QUALITY, DEFAULT]:\n if use_hpo:\n from ray import tune\n\n hyperparameters.update(\n {\n \"env.per_gpu_batch_size\": 2, # Cover some corner cases of HPO on multimodal data.\n \"model.hf_text.checkpoint_name\": tune.choice(\n [\n \"google/electra-base-discriminator\",\n \"google/flan-t5-base\",\n \"microsoft/deberta-v3-small\",\n \"roberta-base\",\n \"albert-xlarge-v2\",\n ]\n ),\n \"model.timm_image.checkpoint_name\": tune.choice(\n [\n \"swin_base_patch4_window7_224\",\n \"convnext_base_in22ft1k\",\n \"vit_base_patch16_clip_224.laion2b_ft_in12k_in1k\",\n \"caformer_b36.sail_in22k_ft_in1k\",\n ]\n ),\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv3-base\",\n }\n )\n else:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"google/electra-base-discriminator\",\n \"model.timm_image.checkpoint_name\": \"caformer_b36.sail_in22k_ft_in1k\",\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv3-base\",\n }\n )\n elif presets == MEDIUM_QUALITY:\n if use_hpo:\n from ray import tune\n\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": tune.choice(\n [\n \"google/electra-small-discriminator\",\n \"google/flan-t5-small\",\n \"microsoft/deberta-v3-xsmall\",\n \"albert-base-v2\",\n \"microsoft/MiniLM-L12-H384-uncased\",\n ]\n ),\n \"model.timm_image.checkpoint_name\": tune.choice(\n [\"mobilenetv3_large_100\", \"gluon_resnet18_v1b\", \"maxvit_rmlp_pico_rw_256.sw_in1k\"]\n ),\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv2-base-uncased\",\n }\n )\n else:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"mobilenetv3_large_100\",\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv2-base-uncased\",\n \"optim.lr\": 4e-4,\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update({\"env.per_gpu_batch_size\": 1})\n if use_hpo:\n from ray import tune\n\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": tune.choice(\n [\n \"microsoft/deberta-v3-base\",\n \"google/flan-t5-large\",\n \"google/electra-large-discriminator\",\n \"roberta-large\",\n ]\n ),\n \"model.timm_image.checkpoint_name\": tune.choice(\n [\n \"swin_large_patch4_window7_224\",\n \"eva_large_patch14_336.in22k_ft_in22k_in1k\",\n \"vit_large_patch14_clip_336.openai_ft_in12k_in1k\",\n ]\n ),\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv3-large\",\n }\n )\n else:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"microsoft/deberta-v3-base\",\n \"model.timm_image.checkpoint_name\": \"swin_large_patch4_window7_224\",\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlmv3-large\",\n }\n )\n elif presets == \"multilingual\":\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"microsoft/mdeberta-v3-base\",\n \"optim.top_k\": 1,\n \"env.precision\": \"32\",\n \"env.per_gpu_batch_size\": 4,\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef few_shot_classification(presets: str = DEFAULT):\n \"\"\"\n Register the presets for few_shot_classification.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters, hyperparameter_tune_kwargs = default(presets=presets)\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-mpnet-base-v2\",\n \"model.hf_text.pooling_mode\": \"mean\",\n \"model.names\": [\"hf_text\", \"clip\"],\n \"model.clip.checkpoint_name\": \"openai/clip-vit-large-patch14-336\",\n \"model.clip.image_size\": 336,\n \"env.inference_batch_size_ratio\": 1,\n }\n )\n hyperparameter_tune_kwargs = {}\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef zero_shot_image_classification(presets: str = DEFAULT):\n \"\"\"\n Register the presets for zero_shot_image_classification.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"clip\"],\n \"model.clip.max_text_len\": 0,\n }\n hyperparameter_tune_kwargs = {}\n\n if presets in [DEFAULT, BEST_QUALITY]:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-large-patch14-336\",\n \"model.clip.image_size\": 336,\n \"env.inference_batch_size_ratio\": 1,\n }\n )\n elif presets == HIGH_QUALITY:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-large-patch14\",\n \"env.inference_batch_size_ratio\": 1,\n }\n )\n elif presets == MEDIUM_QUALITY:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef object_detection(presets: str = DEFAULT):\n \"\"\"\n Register the presets for object_detection.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"mmdet_image\"],\n \"model.mmdet_image.frozen_layers\": [],\n \"optim.patience\": 20,\n \"optim.val_check_interval\": 1.0,\n \"optim.check_val_every_n_epoch\": 1,\n \"env.batch_size\": 32,\n \"env.per_gpu_batch_size\": 1,\n \"env.num_workers\": 2,\n \"optim.lr\": 1e-5,\n \"optim.weight_decay\": 1e-4,\n \"optim.lr_mult\": 10,\n \"optim.lr_choice\": \"two_stages\",\n \"optim.lr_schedule\": \"multi_step\",\n \"optim.gradient_clip_val\": 0.1,\n \"optim.max_epochs\": 60,\n \"optim.warmup_steps\": 0.0,\n \"optim.top_k\": 1,\n \"optim.top_k_average_method\": \"best\",\n \"env.inference_batch_size_ratio\": 1,\n \"env.strategy\": \"ddp\",\n \"env.auto_select_gpus\": True, # Turn on for detection to return devices in a list, TODO: fix the extra GPU usage bug\n \"env.num_gpus\": -1,\n \"optim.lr_decay\": 0.9,\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets == MEDIUM_QUALITY:\n hyperparameters.update(\n {\n \"model.mmdet_image.checkpoint_name\": \"yolox_l\",\n \"env.per_gpu_batch_size\": 2, # Works on 8G GPU\n \"optim.lr\": 5e-5,\n \"optim.patience\": 5,\n \"optim.max_epochs\": 50,\n \"optim.val_check_interval\": 1.0,\n \"optim.check_val_every_n_epoch\": 3,\n \"optim.lr_mult\": 100,\n \"optim.weight_decay\": 1e-3,\n \"optim.lr_schedule\": \"cosine_decay\",\n \"optim.gradient_clip_val\": 1,\n }\n )\n elif presets in [DEFAULT, HIGH_QUALITY]:\n hyperparameters.update(\n {\n \"model.mmdet_image.checkpoint_name\": \"dino-4scale_r50_8xb2-12e_coco\",\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update(\n {\n \"model.mmdet_image.checkpoint_name\": \"dino-5scale_swin-l_8xb2-36e_coco\",\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef semantic_segmentation(presets: str = DEFAULT):\n \"\"\"\n Register the presets for semantic_segmentation.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"sam\"],\n \"model.sam.checkpoint_name\": \"facebook/sam-vit-huge\",\n \"env.batch_size\": 4,\n \"env.per_gpu_batch_size\": 1,\n \"env.inference_batch_size_ratio\": 1,\n \"env.strategy\": \"ddp_find_unused_parameters_true\",\n \"env.auto_select_gpus\": False,\n \"env.num_gpus\": -1,\n \"env.num_workers\": 4,\n \"env.precision\": \"16-mixed\",\n \"optim.lr\": 1e-4,\n \"optim.loss_func\": \"structure_loss\",\n \"optim.lr_decay\": 0,\n \"optim.lr_mult\": 1,\n \"optim.lr_choice\": \"single_stage\",\n \"optim.lr_schedule\": \"polynomial_decay\",\n \"optim.max_epochs\": 30,\n \"optim.top_k\": 3,\n \"optim.top_k_average_method\": \"best\",\n \"optim.warmup_steps\": 0.0,\n \"optim.weight_decay\": 0.0001,\n \"optim.patience\": 10,\n \"optim.val_check_interval\": 1.0,\n \"optim.check_val_every_n_epoch\": 1,\n \"optim.peft\": \"lora\",\n \"optim.lora.module_filter\": [\".*vision_encoder.*attn\"],\n \"optim.lora.filter\": [\"q\", \"v\"],\n \"optim.extra_trainable_params\": [\".*mask_decoder\"],\n \"optim.lora.r\": 3,\n \"optim.lora.alpha\": 32,\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef ocr_text_detection(presets: str = DEFAULT):\n \"\"\"\n Register the presets for ocr_text_detection.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"mmocr_text_detection\"],\n \"model.mmocr_text_detection.checkpoint_name\": \"TextSnake\",\n \"env.inference_batch_size_ratio\": 1,\n \"env.num_gpus\": 1,\n \"env.precision\": 32,\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef ocr_text_recognition(presets: str = DEFAULT):\n \"\"\"\n Register the presets for ocr_text_recognition.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"mmocr_text_recognition\"],\n \"model.mmocr_text_recognition.checkpoint_name\": \"ABINet\",\n \"env.inference_batch_size_ratio\": 1,\n \"env.num_gpus\": 1,\n \"env.precision\": 32,\n }\n hyperparameter_tune_kwargs = {}\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef feature_extraction(presets: str = DEFAULT): # TODO: rename the problem type as text_feature_extraction?\n \"\"\"\n Register the presets for feature_extraction.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/msmarco-MiniLM-L-12-v3\",\n \"model.hf_text.pooling_mode\": \"mean\",\n \"env.inference_batch_size_ratio\": 1,\n }\n hyperparameter_tune_kwargs = {}\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\n@matcher_presets.register()\ndef image_similarity(presets: str = DEFAULT):\n \"\"\"\n Register the presets for image_similarity.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"timm_image\"],\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets in [DEFAULT, HIGH_QUALITY]:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": \"caformer_b36.sail_in22k_ft_in1k\",\n }\n )\n elif presets == MEDIUM_QUALITY:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": \"swin_large_patch4_window7_224\",\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\n@matcher_presets.register()\ndef text_similarity(presets: str = DEFAULT):\n \"\"\"\n Register the presets for text_similarity.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.pooling_mode\": \"mean\",\n \"data.categorical.convert_to_text\": True,\n \"data.numerical.convert_to_text\": True,\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets in [DEFAULT, HIGH_QUALITY]:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L12-v2\",\n }\n )\n elif presets == MEDIUM_QUALITY:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-mpnet-base-v2\",\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\n@matcher_presets.register()\ndef image_text_similarity(presets: str = DEFAULT):\n \"\"\"\n Register the presets for image_text_similarity.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\"clip\"],\n \"matcher.loss.type\": \"multi_negatives_softmax_loss\",\n \"optim.lr\": 1e-5,\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets in [DEFAULT, MEDIUM_QUALITY]:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"env.per_gpu_batch_size\": 128,\n }\n )\n elif presets == HIGH_QUALITY:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-large-patch14\",\n \"env.per_gpu_batch_size\": 16,\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update(\n {\n \"model.clip.checkpoint_name\": \"openai/clip-vit-large-patch14-336\",\n \"model.clip.image_size\": 336,\n \"env.per_gpu_batch_size\": 8,\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef ner(presets: str = DEFAULT):\n \"\"\"\n Register the presets for ner.\n\n Parameters\n ----------\n presets\n The preset name.\n\n Returns\n -------\n hyperparameters\n The hyperparameters for a given preset.\n hyperparameter_tune_kwargs\n The hyperparameter tuning kwargs.\n \"\"\"\n hyperparameters = {\n \"model.names\": [\n \"ft_transformer\",\n \"timm_image\",\n \"ner_text\",\n \"fusion_ner\",\n ],\n }\n hyperparameter_tune_kwargs = {}\n\n presets, use_hpo = parse_presets_str(presets)\n if use_hpo:\n default_tunable_hyperparameters, default_hyperparameter_tune_kwargs = get_default_hpo_setup()\n hyperparameters.update(default_tunable_hyperparameters)\n hyperparameter_tune_kwargs.update(default_hyperparameter_tune_kwargs)\n\n if presets in [DEFAULT, HIGH_QUALITY]:\n hyperparameters.update(\n {\n \"model.ner_text.checkpoint_name\": \"microsoft/deberta-v3-base\",\n }\n )\n elif presets == MEDIUM_QUALITY:\n hyperparameters.update(\n {\n \"model.ner_text.checkpoint_name\": \"google/electra-small-discriminator\",\n }\n )\n elif presets == BEST_QUALITY:\n hyperparameters.update(\n {\n \"model.ner_text.checkpoint_name\": \"microsoft/deberta-v3-large\",\n \"env.per_gpu_batch_size\": 4,\n }\n )\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\n@automm_presets.register()\ndef ensemble(presets: str = DEFAULT):\n hyperparameters = {\n \"lf_mlp\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"lf_transformer\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_transformer\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"lf_clip\": {\n \"model.names\": [\"ft_transformer\", \"clip_image\", \"clip_text\", \"fusion_mlp\"],\n \"model.clip_image.data_types\": [\"image\"],\n \"model.clip_text.data_types\": [\"text\"],\n \"model.clip_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.clip_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.clip_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"early_fusion\": {\n \"model.names\": [\"meta_transformer\"],\n \"model.meta_transformer.checkpoint_path\": \"null\",\n \"model.meta_transformer.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.meta_transformer.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.meta_transformer.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"convert_categorical_to_text\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": True,\n \"data.categorical.convert_to_text_template\": \"latex\",\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"convert_numeric_to_text\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": True,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"cross_modal_align_pos_only\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"positive_only\",\n \"optim.cross_modal_align_weight\": 1,\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"input_aug\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\", \"trivial_augment\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0.1,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"feature_aug_lemda\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": True,\n \"optim.automatic_optimization\": False,\n },\n \"modality_dropout\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0.2,\n \"model.timm_image.use_learnable_image\": False,\n \"optim.lemda.turn_on\": False,\n },\n \"learnable_image\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0,\n \"model.timm_image.use_learnable_image\": True,\n \"optim.lemda.turn_on\": False,\n },\n \"modality_dropout_and_learnable_image\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"model.hf_text.text_trivial_aug_maxscale\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"optim.cross_modal_align\": \"null\",\n \"data.modality_dropout\": 0.2,\n \"model.timm_image.use_learnable_image\": True,\n \"optim.lemda.turn_on\": False,\n },\n }\n\n if presets in [DEFAULT, HIGH_QUALITY]:\n for v in hyperparameters.values():\n if \"timm_image\" in v[\"model.names\"]:\n v[\"model.timm_image.checkpoint_name\"] = \"caformer_b36.sail_in22k_ft_in1k\"\n if \"hf_text\" in v[\"model.names\"]:\n v[\"model.hf_text.checkpoint_name\"] = \"google/electra-base-discriminator\"\n if \"meta_transformer\" in v[\"model.names\"]:\n v[\"model.meta_transformer.model_version\"] = \"base\"\n if \"clip_image\" in v[\"model.names\"]:\n v[\"model.clip_image.checkpoint_name\"] = \"openai/clip-vit-base-patch32\"\n if \"clip_text\" in v[\"model.names\"]:\n v[\"model.clip_text.checkpoint_name\"] = \"openai/clip-vit-base-patch32\"\n\n elif presets == MEDIUM_QUALITY:\n for v in hyperparameters.values():\n if \"timm_image\" in v[\"model.names\"]:\n v[\"model.timm_image.checkpoint_name\"] = \"mobilenetv3_large_100\"\n if \"hf_text\" in v[\"model.names\"]:\n v[\"model.hf_text.checkpoint_name\"] = \"google/electra-small-discriminator\"\n if \"meta_transformer\" in v[\"model.names\"]:\n v[\"model.meta_transformer.model_version\"] = \"base\"\n if \"clip_image\" in v[\"model.names\"]:\n v[\"model.clip_image.checkpoint_name\"] = \"openai/clip-vit-base-patch32\"\n if \"clip_text\" in v[\"model.names\"]:\n v[\"model.clip_text.checkpoint_name\"] = \"openai/clip-vit-base-patch32\"\n elif presets == BEST_QUALITY:\n for v in hyperparameters.values():\n if \"timm_image\" in v[\"model.names\"]:\n v[\"model.timm_image.checkpoint_name\"] = \"swin_large_patch4_window7_224\"\n if \"hf_text\" in v[\"model.names\"]:\n v[\"model.hf_text.checkpoint_name\"] = \"microsoft/deberta-v3-base\"\n if \"meta_transformer\" in v[\"model.names\"]:\n v[\"model.meta_transformer.model_version\"] = \"large\"\n if \"clip_image\" in v[\"model.names\"]:\n v[\"model.clip_image.checkpoint_name\"] = \"openai/clip-vit-large-patch14\"\n if \"clip_text\" in v[\"model.names\"]:\n v[\"model.clip_text.checkpoint_name\"] = \"openai/clip-vit-large-patch14\"\n else:\n raise ValueError(f\"Unknown preset type: {presets}\")\n\n return hyperparameters, None\n\n\ndef list_presets(verbose: bool = False):\n \"\"\"\n List all available presets.\n\n Returns\n -------\n A list of presets.\n \"\"\"\n preset_keys = automm_presets.list_keys()\n if not verbose:\n return preset_keys\n\n preset_details = {}\n for k in preset_keys:\n preset_details[k] = automm_presets.create(k)\n\n return preset_details\n\n\ndef get_basic_config(extra: Optional[List[str]] = None):\n \"\"\"\n Get the basic config of AutoMM.\n\n Parameters\n ----------\n extra\n A list of extra config keys.\n\n Returns\n -------\n A dict config with keys: MODEL, DATA, OPTIM, ENV, and their default values.\n \"\"\"\n config = {\n MODEL: DEFAULT,\n DATA: DEFAULT,\n OPTIM: DEFAULT,\n ENV: DEFAULT,\n }\n if extra:\n for k in extra:\n config[k] = DEFAULT\n\n return config\n\n\ndef get_presets(problem_type: str, presets: str):\n \"\"\"\n Get the default hyperparameters and hyperparameter_tune_kwargs given problem type and presets.\n\n Parameters\n ----------\n problem_type\n Problem type.\n presets\n Name of a preset.\n\n Returns\n -------\n hyperparameters\n The hyperparameter overrides of this preset.\n hyperparameter_tune_kwargs\n Hyperparameter tuning strategy and kwargs (for example, how many HPO trials to run).\n \"\"\"\n if not presets:\n presets = DEFAULT\n if presets == \"hpo\":\n presets = f\"{DEFAULT}_{presets}\"\n presets = presets.lower()\n if problem_type in [\n BINARY,\n MULTICLASS,\n REGRESSION,\n None,\n ]:\n problem_type = DEFAULT\n\n if problem_type in automm_presets.list_keys():\n hyperparameters, hyperparameter_tune_kwargs = automm_presets.create(problem_type, presets)\n else:\n raise ValueError(\n f\"Problem type '{problem_type}' doesn't have any presets yet. \"\n f\"Consider one of these: {automm_presets.list_keys()}\"\n )\n\n return hyperparameters, hyperparameter_tune_kwargs\n\n\ndef get_ensemble_presets(presets):\n if not presets:\n presets = DEFAULT\n return automm_presets.create(\"ensemble\", presets)\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/problem_types.py",
"content": "\"\"\"Problem types supported in MultiModalPredictor\"\"\"\n\nimport logging\nfrom dataclasses import dataclass, field\nfrom typing import List, Optional, Set\n\nfrom ..constants import (\n ACCURACY,\n BINARY,\n CATEGORICAL,\n CLASSIFICATION,\n EVALUATION_METRICS,\n FEATURE_EXTRACTION,\n FEW_SHOT_CLASSIFICATION,\n IMAGE,\n IMAGE_BASE64_STR,\n IMAGE_BYTEARRAY,\n IMAGE_SIMILARITY,\n IMAGE_TEXT_SIMILARITY,\n IOU,\n MAP,\n MULTICLASS,\n NAMED_ENTITY_RECOGNITION,\n NER,\n NER_ANNOTATION,\n NER_TOKEN_F1,\n NUMERICAL,\n OBJECT_DETECTION,\n REGRESSION,\n RMSE,\n ROC_AUC,\n ROIS,\n SEMANTIC_SEGMENTATION,\n TEXT,\n TEXT_NER,\n TEXT_SIMILARITY,\n VALIDATION_METRICS,\n ZERO_SHOT_IMAGE_CLASSIFICATION,\n)\nfrom .registry import Registry\n\nlogger = logging.getLogger(__name__)\n\nPROBLEM_TYPES_REG = Registry(\"problem_type_properties\")\n\n\n@dataclass\nclass ProblemTypeProperty:\n \"\"\"\n Property of the problem. Stores the name of the problem\n and its related properties. Some properties are used in the label / feature inference logic.\n \"\"\"\n\n name: str # Name of the problem\n support_fit: bool = True # Whether the problem type support `.fit()`\n support_zero_shot: bool = False # Support `.predict()` and `.evaluate()` without calling `.fit()`\n is_matching: bool = False # Whether the problem belongs to the matching category\n is_classification: bool = False # Whether the problem is a classification problem\n experimental: bool = False # Indicate whether the problem is experimental\n\n # The collection of modality types the problem supports.\n # Multiple column types may be parsed into the same modality. For example\n # IMAGE, IMAGE_PATH, IMAGE_BYTEARRAY, IMAGE_BASE64_STR --> IMAGE\n # It will be used to analyze the dataframe and detect the columns.\n supported_modality_type: Set[str] = field(default_factory=set)\n\n # The collection of label column types the problem supports.\n supported_label_type: Optional[Set[str]] = None\n\n # The modalities that have to appear in the table.\n force_exist_modality: Optional[Set[str]] = None\n\n # The fallback label type of the problem\n _fallback_label_type: Optional[str] = None\n\n # If evaluations are supported\n _support_eval: Optional[bool] = False\n\n # Overwrite the default setting (first in supported_evaluation_metrics)\n _fallback_evaluation_metric: Optional[str] = None\n\n # The validation metric fallback\n # It may be different from the evaluation metric\n _fallback_validation_metric: Optional[str] = None\n\n @property\n def fallback_label_type(self):\n if self._fallback_label_type is None and len(self.supported_label_type) == 1:\n return next(iter(self.supported_label_type))\n else:\n return self._fallback_label_type\n\n @property\n def supported_evaluation_metrics(self):\n if self._support_eval:\n return EVALUATION_METRICS[self.name]\n else:\n return []\n\n @property\n def fallback_evaluation_metric(self):\n if self._fallback_evaluation_metric:\n return self._fallback_evaluation_metric\n elif self._support_eval:\n return self.supported_evaluation_metrics[0]\n else:\n return None\n\n @property\n def supported_validation_metrics(self):\n if self._support_eval:\n return VALIDATION_METRICS[self.name]\n else:\n return []\n\n @property\n def fallback_validation_metric(self):\n if self._fallback_validation_metric:\n assert self._fallback_validation_metric in self.supported_validation_metrics\n return self._fallback_validation_metric\n else:\n return None\n\n\n# Classification: Arbitrary combination of image, text, tabular data --> categorical value\nPROBLEM_TYPES_REG.register(\n CLASSIFICATION,\n ProblemTypeProperty(\n name=CLASSIFICATION,\n supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},\n supported_label_type={CATEGORICAL},\n is_classification=True,\n ),\n)\nPROBLEM_TYPES_REG.register(\n BINARY,\n ProblemTypeProperty(\n name=BINARY,\n supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},\n supported_label_type={CATEGORICAL},\n is_classification=True,\n _support_eval=True,\n _fallback_evaluation_metric=ROC_AUC,\n _fallback_validation_metric=ROC_AUC,\n ),\n)\nPROBLEM_TYPES_REG.register(\n MULTICLASS,\n ProblemTypeProperty(\n name=MULTICLASS,\n supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},\n supported_label_type={CATEGORICAL},\n is_classification=True,\n _support_eval=True,\n _fallback_evaluation_metric=ACCURACY,\n _fallback_validation_metric=ACCURACY,\n ),\n)\n\n# Regression: Arbitrary combination of image, text, tabular data --> numeric value\nPROBLEM_TYPES_REG.register(\n REGRESSION,\n ProblemTypeProperty(\n name=REGRESSION,\n supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},\n supported_label_type={NUMERICAL},\n _support_eval=True,\n _fallback_evaluation_metric=RMSE,\n _fallback_validation_metric=RMSE,\n ),\n)\n\n# Object detection: image --> bounding boxes\nPROBLEM_TYPES_REG.register(\n OBJECT_DETECTION,\n ProblemTypeProperty(\n name=OBJECT_DETECTION,\n support_zero_shot=True,\n supported_modality_type={IMAGE},\n supported_label_type={ROIS},\n force_exist_modality={IMAGE},\n _support_eval=True,\n _fallback_validation_metric=MAP,\n ),\n)\n\n# Real-World Semantic Segmentation: image --> image\nPROBLEM_TYPES_REG.register(\n SEMANTIC_SEGMENTATION,\n ProblemTypeProperty(\n name=SEMANTIC_SEGMENTATION,\n support_zero_shot=True,\n support_fit=True,\n supported_modality_type={IMAGE},\n supported_label_type={IMAGE},\n force_exist_modality={IMAGE},\n _support_eval=True,\n _fallback_evaluation_metric=IOU,\n _fallback_validation_metric=IOU,\n ),\n)\n\n# Matching: text <--> text, image <--> image, text <--> image\nPROBLEM_TYPES_REG.register(\n TEXT_SIMILARITY,\n ProblemTypeProperty(\n name=TEXT_SIMILARITY,\n support_zero_shot=True,\n is_matching=True,\n supported_modality_type={TEXT},\n supported_label_type={CATEGORICAL, NUMERICAL},\n force_exist_modality={TEXT},\n ),\n)\nPROBLEM_TYPES_REG.register(\n IMAGE_SIMILARITY,\n ProblemTypeProperty(\n name=IMAGE_SIMILARITY,\n support_zero_shot=True,\n is_matching=True,\n supported_modality_type={IMAGE},\n supported_label_type={CATEGORICAL, NUMERICAL},\n force_exist_modality={IMAGE},\n ),\n)\nPROBLEM_TYPES_REG.register(\n IMAGE_TEXT_SIMILARITY,\n ProblemTypeProperty(\n name=IMAGE_TEXT_SIMILARITY,\n support_zero_shot=True,\n is_matching=True,\n supported_modality_type={IMAGE, TEXT},\n supported_label_type={CATEGORICAL, NUMERICAL},\n force_exist_modality={IMAGE, TEXT},\n ),\n)\n\n# Entity Extraction: text (tied to the entity), [other text], [image], [tabular] --> entity\n_ner_property = ProblemTypeProperty(\n name=NER,\n supported_modality_type={IMAGE, TEXT, CATEGORICAL, NUMERICAL, TEXT_NER},\n supported_label_type={NER_ANNOTATION},\n force_exist_modality={TEXT_NER},\n _support_eval=True,\n _fallback_validation_metric=NER_TOKEN_F1,\n)\nPROBLEM_TYPES_REG.register(NER, _ner_property)\nPROBLEM_TYPES_REG.register(NAMED_ENTITY_RECOGNITION, _ner_property)\n\n# Feature Extraction: text --> feature, image --> features\nPROBLEM_TYPES_REG.register(\n FEATURE_EXTRACTION,\n ProblemTypeProperty(\n name=FEATURE_EXTRACTION, support_fit=False, support_zero_shot=True, supported_modality_type={IMAGE, TEXT}\n ),\n)\n\n# Zero-shot Image classification\nPROBLEM_TYPES_REG.register(\n ZERO_SHOT_IMAGE_CLASSIFICATION,\n ProblemTypeProperty(\n name=ZERO_SHOT_IMAGE_CLASSIFICATION,\n support_fit=False,\n support_zero_shot=True,\n supported_modality_type={IMAGE},\n force_exist_modality={IMAGE},\n ),\n)\n\nPROBLEM_TYPES_REG.register(\n FEW_SHOT_CLASSIFICATION,\n ProblemTypeProperty(\n name=FEW_SHOT_CLASSIFICATION,\n support_fit=True,\n support_zero_shot=False,\n supported_modality_type={IMAGE, TEXT},\n supported_label_type={CATEGORICAL},\n _support_eval=True,\n _fallback_evaluation_metric=ACCURACY,\n _fallback_validation_metric=ACCURACY,\n ),\n)\n\n\ndef infer_problem_type_by_eval_metric(eval_metric_name: str, problem_type: str):\n if eval_metric_name is not None and eval_metric_name.lower() in [\n \"rmse\",\n \"r2\",\n \"pearsonr\",\n \"spearmanr\",\n ]:\n if problem_type is None:\n logger.debug(\n f\"Infer problem type to be a regression problem \"\n f\"since the evaluation metric is set as {eval_metric_name}.\"\n )\n problem_type = REGRESSION\n else:\n problem_prop = PROBLEM_TYPES_REG.get(problem_type)\n if NUMERICAL not in problem_prop.supported_label_type:\n raise ValueError(\n f\"The provided evaluation metric will require the problem \"\n f\"to support label type = {NUMERICAL}. However, \"\n f\"the provided problem type = {problem_type} only \"\n f\"supports label type = {problem_prop.supported_label_type}.\"\n )\n\n return problem_type\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/registry.py",
"content": "import json\nfrom collections import OrderedDict\nfrom json import JSONDecodeError\nfrom typing import List\nfrom typing import OrderedDict as t_OrderedDict\n\n\nclass Registry:\n \"\"\"\n Create the registry that will map name to object.\n This facilitates the users to create custom registry.\n \"\"\"\n\n def __init__(self, name: str) -> None:\n \"\"\"\n Parameters\n ----------\n name\n Registry name\n \"\"\"\n self._name: str = name\n self._obj_map: t_OrderedDict[str, object] = OrderedDict()\n\n def __contains__(self, item):\n return item in self._obj_map\n\n def _do_register(self, name: str, obj: object) -> None:\n assert name not in self._obj_map, \"An object named '{}' was already registered in '{}' registry!\".format(\n name, self._name\n )\n self._obj_map[name] = obj\n\n def register(self, *args):\n \"\"\"\n Register the given object under either the nickname or `obj.__name__`. It can be used as\n either a decorator or not. See docstring of this class for usage.\n \"\"\"\n if len(args) == 2:\n # Register an object with nick name by function call\n nickname, obj = args\n self._do_register(nickname, obj)\n elif len(args) == 1:\n if isinstance(args[0], str):\n # Register an object with nick name by decorator\n nickname = args[0]\n\n def deco(func_or_class: object) -> object:\n self._do_register(nickname, func_or_class)\n return func_or_class\n\n return deco\n else:\n # Register an object by function call\n self._do_register(args[0].__name__, args[0])\n elif len(args) == 0:\n # Register an object by decorator\n def deco(func_or_class: object) -> object:\n self._do_register(func_or_class.__name__, func_or_class)\n return func_or_class\n\n return deco\n else:\n raise ValueError(\"Do not support the usage!\")\n\n def get(self, name: str) -> object:\n ret = self._obj_map.get(name)\n if ret is None:\n raise KeyError(\"No object named '{}' found in '{}' registry!\".format(name, self._name))\n return ret\n\n def list_keys(self) -> List:\n return list(self._obj_map.keys())\n\n def __repr__(self) -> str:\n s = \"{name}(keys={keys})\".format(name=self._name, keys=self.list_keys())\n return s\n\n def create(self, name: str, *args, **kwargs) -> object:\n \"\"\"\n Create the class object with the given args and kwargs\n Parameters\n ----------\n name\n The name in the registry\n args\n kwargs\n Returns\n -------\n ret\n The created object\n \"\"\"\n return self.get(name)(*args, **kwargs)\n\n def create_with_json(self, name: str, json_str: str):\n \"\"\"\n Parameters\n ----------\n name\n json_str\n Returns\n -------\n \"\"\"\n try:\n args = json.loads(json_str)\n except JSONDecodeError:\n raise ValueError('Unable to decode the json string: json_str=\"{}\"'.format(json_str))\n if isinstance(args, (list, tuple)):\n return self.create(name, *args)\n elif isinstance(args, dict):\n return self.create(name, **args)\n else:\n raise NotImplementedError(\n 'The format of json string is not supported! We only support list/dict. json_str=\"{}\".'.format(\n json_str\n )\n )\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/save.py",
"content": "import logging\nimport os\nfrom datetime import datetime\nfrom typing import Dict, List, Optional, Tuple, Union\n\nimport pytz\n\nfrom autogluon.common.utils.utils import setup_outputdir\n\nfrom ..constants import LAST_CHECKPOINT\n\nlogger = logging.getLogger(__name__)\n\n\ndef process_save_path(path, resume: Optional[bool] = False, raise_if_exist: Optional[bool] = True):\n \"\"\"\n Convert the provided path to an absolute path and check whether it is valid.\n If a path exists, either raise error or return None.\n A None path can be identified by the `setup_outputdir` to generate a random path.\n\n Parameters\n ----------\n path\n A provided path.\n resume\n Whether this is a path to resume training.\n raise_if_exist\n Whether to raise error if the path exists.\n\n Returns\n -------\n A complete and verified path or None.\n \"\"\"\n path = os.path.abspath(os.path.expanduser(path))\n if resume:\n assert os.path.isfile(os.path.join(path, LAST_CHECKPOINT)), (\n f\"Trying to resume training from '{path}'. \"\n f\"However, it does not contain the last checkpoint file: '{LAST_CHECKPOINT}'. \"\n \"Are you using a correct path?\"\n )\n elif os.path.isdir(path) and len(os.listdir(path)) > 0:\n if raise_if_exist:\n raise ValueError(\n f\"Path {path} already exists.Specify a new path to avoid accidentally overwriting a saved predictor.\"\n )\n else:\n logger.warning(\n \"A new predictor save path is created. \"\n \"This is to prevent you to overwrite previous predictor saved here. \"\n \"You could check current save path at predictor._save_path. \"\n \"If you still want to use this path, set resume=True\"\n )\n path = None\n\n return path\n\n\ndef setup_save_path(\n resume: Optional[bool] = None,\n old_save_path: Optional[str] = None,\n proposed_save_path: Optional[str] = None,\n warn_if_exist: Optional[bool] = True,\n raise_if_exist: Optional[bool] = False,\n fit_called: Optional[bool] = None,\n):\n # TODO: remove redundant folders in DDP mode\n rank = int(os.environ.get(\"LOCAL_RANK\", 0))\n save_path = None\n if resume:\n save_path = process_save_path(path=old_save_path, resume=True)\n elif proposed_save_path is not None: # TODO: distinguish DDP and existed predictor\n save_path = process_save_path(path=proposed_save_path, raise_if_exist=(raise_if_exist and rank == 0))\n elif old_save_path is not None:\n if fit_called:\n save_path = process_save_path(path=old_save_path, raise_if_exist=False)\n else:\n save_path = process_save_path(path=old_save_path, raise_if_exist=(raise_if_exist and rank == 0))\n\n if not resume:\n save_path = setup_outputdir(\n path=save_path,\n warn_if_exist=warn_if_exist,\n )\n os.makedirs(save_path, exist_ok=True) # setup_outputdir doesn't create dir if warn_if_exist==False\n\n save_path = os.path.abspath(os.path.expanduser(save_path))\n logger.debug(f\"save path: {save_path}\")\n\n return save_path\n\n\ndef make_exp_dir(\n root_path: str,\n job_name: Optional[str] = None,\n create: Optional[bool] = True,\n):\n \"\"\"\n Creates the exp dir of format e.g.,: root_path/2022_01_01/job_name_12_00_00/\n This function is to better organize the training runs. It is recommended to call this\n function and pass the returned \"exp_dir\" to \"MultiModalPredictor.fit(save_path=exp_dir)\".\n\n Parameters\n ----------\n root_path\n The basic path where to create saving directories for training runs.\n job_name\n The job names to name training runs.\n create\n Whether to make the directory.\n\n Returns\n -------\n The formatted directory path.\n \"\"\"\n tz = pytz.timezone(\"US/Pacific\")\n ct = datetime.now(tz=tz)\n date_stamp = ct.strftime(\"%Y_%m_%d\")\n time_stamp = ct.strftime(\"%H_%M_%S\")\n\n # Group logs by day first\n exp_dir = os.path.join(root_path, date_stamp)\n\n # Then, group by run_name and hour + min + sec to avoid duplicates\n if job_name:\n exp_dir = os.path.join(exp_dir, \"_\".join([job_name, time_stamp]))\n else:\n exp_dir = os.path.join(exp_dir, time_stamp)\n\n if create:\n os.makedirs(exp_dir, mode=0o777, exist_ok=False)\n\n return exp_dir\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/strategy.py",
"content": "import logging\n\nfrom ..constants import DDP_STRATEGIES\n\nlogger = logging.getLogger(__name__)\n\n\ndef is_interactive_strategy(strategy: str):\n if isinstance(strategy, str) and strategy:\n return strategy.startswith((\"ddp_fork\", \"ddp_notebook\"))\n else:\n return False\n\n\ndef run_ddp_only_once(num_gpus: int, strategy: str):\n if strategy in DDP_STRATEGIES:\n global FIRST_DDP_RUN # Use the global variable to make sure it is tracked per process\n if \"FIRST_DDP_RUN\" in globals() and not FIRST_DDP_RUN:\n # not the first time running DDP, set number of devices to 1 (use single GPU)\n return min(1, num_gpus), \"auto\"\n else:\n if num_gpus > 1:\n FIRST_DDP_RUN = False # run DDP for the first time, disable the following runs\n return num_gpus, strategy\n"
},
{
"path": "multimodal/src/autogluon/multimodal/utils/visualizer.py",
"content": "# Copyright (c) Facebook, Inc. and its affiliates.\n# Disclaimer: Special thanks to the Detectron2 developers\n# https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/visualizer.py!\n# We use part of its provided, open-source functionalities.\nimport collections\nimport colorsys\nimport logging\nimport re\nfrom enum import Enum, unique\nfrom typing import List\n\nimport matplotlib as mpl\nimport matplotlib.colors as mplc\nimport matplotlib.figure as mplfigure\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom matplotlib.backends.backend_agg import FigureCanvasAgg\nfrom PIL import Image\n\nfrom .colormap import random_color\nfrom .misc import merge_spans\n\nlogger = logging.getLogger(__name__)\n\n\n_SMALL_OBJECT_AREA_THRESH = 1000\n_LARGE_MASK_AREA_THRESH = 120000\n_OFF_WHITE = (1.0, 1.0, 240.0 / 255)\n_BLACK = (0, 0, 0)\n_RED = (1.0, 0, 0)\n\n_KEYPOINT_THRESHOLD = 0.05\n\n\n@unique\nclass ColorMode(Enum):\n \"\"\"\n Enum of different color modes to use for instance visualizations.\n \"\"\"\n\n IMAGE = 0\n \"\"\"\n Picks a random color for every instance and overlay segmentations with low opacity.\n \"\"\"\n SEGMENTATION = 1\n \"\"\"\n Let instances of the same category have similar colors\n (from metadata.thing_colors), and overlay them with\n high opacity. This provides more attention on the quality of segmentation.\n \"\"\"\n IMAGE_BW = 2\n \"\"\"\n Same as IMAGE, but convert all areas without masks to gray-scale.\n Only available for drawing per-instance mask predictions.\n \"\"\"\n\n\ndef _create_text_labels(classes: List[str], scores: List[float]):\n \"\"\"\n Create the label tags for visualization\n Parameters\n ----------\n classes (list[str]): class names for all the detected instances\n scores (list[float]); detection confidence scores for all the detected instances\n\n Returns\n -------\n labels (list[str]): label tags for visualization\n \"\"\"\n labels = None\n if classes is not None:\n labels = classes\n\n if scores is not None:\n if labels is None:\n labels = [\"{:.0f}%\".format(s * 100) for s in scores]\n else:\n labels = [\"{} {:.0f}%\".format(l, s * 100) for l, s in zip(labels, scores)]\n return labels\n\n\nclass VisImage:\n def __init__(self, img, scale=1.0):\n \"\"\"\n Parameters\n ----------\n img (ndarray): an RGB image of shape (H, W, 3) in range [0, 255].\n scale (float): scale the input image\n \"\"\"\n self.img = img\n self.scale = scale\n self.width, self.height = img.shape[1], img.shape[0]\n self._setup_figure(img)\n\n def _setup_figure(self, img):\n \"\"\"\n Parameters\n ----------\n Same as in :meth:`__init__()`.\n\n Returns\n -------\n fig (matplotlib.pyplot.figure): top level container for all the image plot elements.\n ax (matplotlib.pyplot.Axes): contains figure elements and sets the coordinate system.\n \"\"\"\n fig = mplfigure.Figure(frameon=False)\n self.dpi = fig.get_dpi()\n # add a small 1e-2 to avoid precision lost due to matplotlib's truncation\n # (https://github.com/matplotlib/matplotlib/issues/15363)\n fig.set_size_inches(\n (self.width * self.scale + 1e-2) / self.dpi,\n (self.height * self.scale + 1e-2) / self.dpi,\n )\n self.canvas = FigureCanvasAgg(fig)\n # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig)\n ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])\n ax.axis(\"off\")\n self.fig = fig\n self.ax = ax\n self.reset_image(img)\n\n def reset_image(self, img):\n \"\"\"\n Parameters\n ----------\n img: same as in __init__\n \"\"\"\n img = img.astype(\"uint8\")\n self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation=\"nearest\")\n\n def save(self, filepath):\n \"\"\"\n Parameters\n ----------\n filepath (str): a string that contains the absolute path, including the file name, where\n the visualized image will be saved.\n \"\"\"\n self.fig.savefig(filepath)\n\n def get_image(self):\n \"\"\"\n Returns\n -------\n ndarray:\n the visualized image of shape (H, W, 3) (RGB) in uint8 type.\n The shape is scaled w.r.t the input image using the given `scale` argument.\n \"\"\"\n canvas = self.canvas\n s, (width, height) = canvas.print_to_buffer()\n # buf = io.BytesIO() # works for cairo backend\n # canvas.print_rgba(buf)\n # width, height = self.width, self.height\n # s = buf.getvalue()\n\n buffer = np.frombuffer(s, dtype=\"uint8\")\n\n img_rgba = buffer.reshape(height, width, 4)\n rgb, alpha = np.split(img_rgba, [3], axis=2)\n return rgb.astype(\"uint8\")\n\n\nclass ObjectDetectionVisualizer:\n \"\"\"\n Visualizer that draws data about detection on images.\n\n It contains methods like `draw_{text,box}`\n that draw primitive objects to images, as well as high-level wrappers like\n `draw_{instance_predictions}` that draw composite data in some pre-defined style.\n\n Note that the exact visualization style for the high-level wrappers are subject to change.\n Style such as color, opacity, label contents, visibility of labels, or even the visibility\n of objects themselves (e.g. when the object is too small) may change according\n to different heuristics, as long as the results still look visually reasonable.\n\n To obtain a consistent style, you can implement custom drawing functions with the\n abovementioned primitive methods instead. This class does not intend to satisfy\n everyone's preference on drawing styles.\n\n This visualizer focuses on high rendering quality rather than performance. It is not\n designed to be used for real-time applications.\n \"\"\"\n\n def __init__(self, img_path, scale=1.0, instance_mode=ColorMode.IMAGE):\n \"\"\"\n Parameters\n ----------\n img_rgb: a numpy array of shape (H, W, C), where H and W correspond to\n the height and width of the image respectively. C is the number of\n color channels. The image is required to be in RGB format since that\n is a requirement of the Matplotlib library. The image is also expected\n to be in the range [0, 255].\n metadata (Metadata): dataset metadata (e.g. class names and colors)\n instance_mode (ColorMode): defines one of the pre-defined style for drawing\n instances on an image.\n \"\"\"\n try:\n import cv2\n except:\n raise ImportError(\"No module named: cv2. Please install cv2 by 'pip install opencv-python'\")\n\n img_rgb = cv2.imread(img_path)\n img_rgb = img_rgb[:, :, ::-1]\n self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)\n self.output = VisImage(self.img, scale=scale)\n\n # too small texts are useless, therefore clamp to 9\n self._default_font_size = max(np.sqrt(self.output.height * self.output.width) // 90, 10 // scale)\n self._instance_mode = instance_mode\n\n @staticmethod\n def process_predictions(predictions: pd.DataFrame, conf_threshold: float = 0.4):\n \"\"\"\n Process the classes, box coordinates and confidence scores of the predictions in the image\n\n Parameters\n ----------\n predictions (pd.DataFrame): the output of object detection with 2 attributes:\n \"image\": containing paths to the source image\n \"bboxes\": containing detection results for the images with the following format\n {\"class\": , \"bbox\": [x1, y1, x2, y2], \"score\": }\n conf_threshold (float): detection confidence threshold to display instances\n\n Returns\n -------\n boxes: XYXY format of bounding boxes shape = (N, 4)\n scores: detection confidence scores, shape = (N, )\n classes: detection classes, shape = (N, )\n \"\"\"\n boxes, scores, classes = [], [], []\n instances = predictions[\"bboxes\"]\n for instance in instances:\n s = instance[\"score\"]\n if s >= conf_threshold:\n box = instance[\"bbox\"]\n c = instance[\"class\"]\n boxes.append(box)\n scores.append(s)\n classes.append(c)\n boxes = np.array(boxes)\n scores = np.array(scores)\n classes = np.array(classes)\n assert len(boxes) == len(scores) == len(classes), (\n \"Expected boxes, scores and classes to have the same length, but got len(boxes): {}, len(scores) = {}, len(classes) = {}\".format(\n len(boxes), len(scores), len(classes)\n )\n )\n if len(boxes) == 0:\n return None, None, None\n return boxes, scores, classes\n\n def draw_instance_predictions(self, predictions: pd.DataFrame, conf_threshold: float = 0.4):\n \"\"\"\n Draw instance-level prediction results on an image.\n\n Parameters\n ----------\n predictions (pd.DataFrame): the output of object detection for that image, with 2 attributes:\n \"image\": containing paths to the source image\n \"bboxes\": containing detection results for the images with the following format\n {\"class\": , \"bbox\": [x1, y1, x2, y2], \"score\": }\n conf_threshold (float): detection confidence threshold to display instances\n\n Returns\n -------\n output (VisImage): image object with visualizations.\n \"\"\"\n boxes, scores, classes = self.process_predictions(predictions, conf_threshold=conf_threshold)\n labels = _create_text_labels(classes, scores)\n colors = None\n\n if self._instance_mode == ColorMode.IMAGE_BW:\n self.output.reset_image(\n self._create_grayscale_image(\n (predictions.pred_masks.any(dim=0) > 0).numpy() if predictions.has(\"pred_masks\") else None\n )\n )\n\n self.overlay_instances(\n boxes=boxes,\n labels=labels,\n assigned_colors=colors,\n )\n return self.output\n\n def overlay_instances(\n self,\n *,\n boxes=None,\n labels=None,\n assigned_colors=None,\n ):\n \"\"\"\n Draw the visualizations\n Parameters\n ----------\n boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`,\n or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image,\n or a :class:`RotatedBoxes`,\n or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format\n for the N objects in a single image,\n labels (list[str]): the text to be displayed for each instance.\n assigned_colors (list[matplotlib.colors]): a list of colors, where each color\n corresponds to each mask or box in the image. Refer to 'matplotlib.colors'\n for full list of formats that the colors are accepted in.\n Returns\n -------\n output (VisImage): image object with visualizations.\n \"\"\"\n num_instances = 0\n if boxes is not None:\n num_instances = len(boxes)\n if labels is not None:\n assert len(labels) == num_instances\n if assigned_colors is None:\n assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)]\n if num_instances == 0:\n return self.output\n\n # Display in largest to smallest order to reduce occlusion.\n areas = None\n if boxes is not None:\n areas = np.prod(boxes[:, 2:] - boxes[:, :2], axis=1)\n\n if areas is not None:\n sorted_idxs = np.argsort(-areas).tolist()\n # Re-order overlapped instances in descending order.\n boxes = boxes[sorted_idxs] if boxes is not None else None\n labels = [labels[k] for k in sorted_idxs] if labels is not None else None\n assigned_colors = [assigned_colors[idx] for idx in sorted_idxs]\n\n for i in range(num_instances):\n color = assigned_colors[i]\n if boxes is not None:\n self.draw_box(boxes[i], edge_color=color)\n\n if labels is not None:\n # first get a box\n if boxes is not None:\n x0, y0, x1, y1 = boxes[i]\n text_pos = (x0, y0) # if drawing boxes, put text on the box corner.\n horiz_align = \"left\"\n else:\n continue # drawing the box confidence for keypoints isn't very useful.\n # for small objects, draw text at the side to avoid occlusion\n instance_area = (y1 - y0) * (x1 - x0)\n if instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale or y1 - y0 < 40 * self.output.scale:\n if y1 >= self.output.height - 5:\n text_pos = (x1, y0)\n else:\n text_pos = (x0, y1)\n\n height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width)\n lighter_color = self._change_color_brightness(color, brightness_factor=0.7)\n font_size = np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size\n self.draw_text(\n labels[i],\n text_pos,\n color=lighter_color,\n horizontal_alignment=horiz_align,\n font_size=font_size,\n )\n\n return self.output\n\n \"\"\"\n Primitive drawing functions:\n \"\"\"\n\n def draw_text(\n self,\n text,\n position,\n *,\n font_size=None,\n color=\"g\",\n horizontal_alignment=\"center\",\n rotation=0,\n ):\n \"\"\"\n Parameters\n ----------\n text (str): class label\n position (tuple): a tuple of the x and y coordinates to place text on image.\n font_size (int, optional): font of the text. If not provided, a font size\n proportional to the image width is calculated and used.\n color: color of the text. Refer to `matplotlib.colors` for full list\n of formats that are accepted.\n horizontal_alignment (str): see `matplotlib.text.Text`\n rotation: rotation angle in degrees CCW\n\n Returns\n -------\n output (VisImage): image object with text drawn.\n \"\"\"\n if not font_size:\n font_size = self._default_font_size\n\n # since the text background is dark, we don't want the text to be dark\n color = np.maximum(list(mplc.to_rgb(color)), 0.2)\n color[np.argmax(color)] = max(0.8, np.max(color))\n\n x, y = position\n self.output.ax.text(\n x,\n y,\n text,\n size=font_size * self.output.scale,\n family=\"sans-serif\",\n bbox={\"facecolor\": \"black\", \"alpha\": 0.8, \"pad\": 0.7, \"edgecolor\": \"none\"},\n verticalalignment=\"top\",\n horizontalalignment=horizontal_alignment,\n color=color,\n zorder=10,\n rotation=rotation,\n )\n return self.output\n\n def draw_box(self, box_coord, alpha=0.5, edge_color=\"g\", line_style=\"-\"):\n \"\"\"\n Parameters\n ----------\n box_coord (tuple): a tuple containing x0, y0, x1, y1 coordinates, where x0 and y0\n are the coordinates of the image's top left corner. x1 and y1 are the\n coordinates of the image's bottom right corner.\n alpha (float): blending efficient. Smaller values lead to more transparent masks.\n edge_color: color of the outline of the box. Refer to `matplotlib.colors`\n for full list of formats that are accepted.\n line_style (string): the string to use to create the outline of the boxes.\n\n Returns\n -------\n output (VisImage): image object with box drawn.\n \"\"\"\n x0, y0, x1, y1 = box_coord\n width = x1 - x0\n height = y1 - y0\n\n linewidth = max(self._default_font_size / 4, 1)\n\n self.output.ax.add_patch(\n mpl.patches.Rectangle(\n (x0, y0),\n width,\n height,\n fill=False,\n edgecolor=edge_color,\n linewidth=linewidth * self.output.scale,\n alpha=alpha,\n linestyle=line_style,\n )\n )\n return self.output\n\n \"\"\"\n Internal methods:\n \"\"\"\n\n def _create_grayscale_image(self, mask=None):\n \"\"\"\n Create a grayscale version of the original image.\n The colors in masked area, if given, will be kept.\n \"\"\"\n img_bw = self.img.astype(\"f4\").mean(axis=2)\n img_bw = np.stack([img_bw] * 3, axis=2)\n if mask is not None:\n img_bw[mask] = self.img[mask]\n return img_bw\n\n def _change_color_brightness(self, color, brightness_factor):\n \"\"\"\n Depending on the brightness_factor, gives a lighter or darker color i.e. a color with\n less or more saturation than the original color.\n\n Parameters\n ----------\n color: color of the polygon. Refer to `matplotlib.colors` for a full list of\n formats that are accepted.\n brightness_factor (float): a value in [-1.0, 1.0] range. A lightness factor of\n 0 will correspond to no change, a factor in [-1.0, 0) range will result in\n a darker color and a factor in (0, 1.0] range will result in a lighter color.\n\n Returns\n -------\n modified_color (tuple[double]): a tuple containing the RGB values of the\n modified color. Each value in the tuple is in the [0.0, 1.0] range.\n \"\"\"\n assert brightness_factor >= -1.0 and brightness_factor <= 1.0\n color = mplc.to_rgb(color)\n polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color))\n modified_lightness = polygon_color[1] + (brightness_factor * polygon_color[1])\n modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness\n modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness\n modified_color = colorsys.hls_to_rgb(polygon_color[0], modified_lightness, polygon_color[2])\n return modified_color\n\n\nclass SemanticSegmentationVisualizer:\n \"\"\"\n Visualize images and predicted semantic segmentation masks.\n \"\"\"\n\n def plot_image(self, img_path: str):\n \"\"\"\n Parameters\n ----------\n img_path\n File path of the image.\n \"\"\"\n image = Image.open(img_path)\n plt.imshow(image)\n\n def plot_mask(self, pred: np.array, output_path: str = None):\n \"\"\"\n Parameters\n ----------\n pred\n np.array of the mask prediction\n output_path\n The path to save the mask image.\n \"\"\"\n\n def show_mask(mask, ax):\n color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n h, w = mask.shape[-2:]\n mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n ax.imshow(mask_image)\n\n class_ids = np.unique(pred)\n for class_id in class_ids:\n if class_id == 0: # background\n continue\n show_mask(pred == class_id, plt.gca())\n\n if output_path:\n plt.savefig(output_path)\n plt.show()\n\n\nclass NERVisualizer:\n \"\"\"An NER visualizer that renders NER prediction as a string of HTML\n inline to any Python class Jupyter notebooks.\n \"\"\"\n\n def __init__(self, pred, sent, seed):\n self.pred = pred\n self.sent = sent\n self.colors = {}\n self.spans = merge_spans(sent, pred, for_visualizer=True)\n self.spans = collections.OrderedDict(sorted(self.spans.items()))\n self.rng = np.random.RandomState(seed)\n\n @staticmethod\n def escape_html(text: str) -> str:\n \"\"\"Replace <, >, &, \" with their HTML encoded representation. Intended to\n prevent HTML errors in rendered displaCy markup.\n text (str): The original text.\n RETURNS (str): Equivalent text to be safely used within HTML.\n \"\"\"\n text = text.replace(\"&\", \"&\")\n text = text.replace(\"<\", \"<\")\n text = text.replace(\">\", \">\")\n text = text.replace('\"', \""\")\n return text\n\n def html_template(self, text, label, color):\n \"\"\"\n Generate an HTML template for the given text and its label.\n\n Parameters\n ----------\n text\n The text to be highlighted.\n label\n The predicted label for the given text.\n color\n The background color of the mark tag.\n \"\"\"\n text = '{} \\\n {} \\\n '.format(color, self.escape_html(text), self.escape_html(label))\n return text\n\n def _repr_html_(self):\n entities = []\n new_sent = \"\"\n last = 0\n for key, value in self.spans.items():\n entity_group = value[-1]\n if re.match(\"B-\", entity_group, re.IGNORECASE) or re.match(\"I-\", entity_group, re.IGNORECASE):\n entity_group = entity_group[2:]\n if entity_group not in self.colors:\n self.colors.update({entity_group: \"#%06X\" % self.rng.randint(0, 0xFFFFFF)})\n start = key\n new_sent += self.sent[last:start]\n last = end = value[0]\n entity_text = self.html_template(self.sent[start:end], entity_group, color=self.colors[entity_group])\n new_sent += entity_text\n new_sent += self.sent[last:]\n\n return new_sent\n\n\ndef visualize_ner(sentence, prediction, seed=0):\n \"\"\"\n Visualize the prediction of NER.\n\n Parameters\n ----------\n sentence\n The input sentence.\n prediction\n The NER prediction for the sentence.\n seed\n The seed for colorpicker.\n\n Returns\n -------\n An NER html visualizer.\n \"\"\"\n visualizer = NERVisualizer(prediction, sentence, seed)\n return visualizer\n"
},
{
"path": "multimodal/tests/README.md",
"content": "# Welcome to Contributing to AutoGluon Multimodal!\n\nTo guarantee code quality and correctness, we do two kinds of testing:\n\n- code style check\n- unit testing\n\n## Code Style Check\n\n### ruff\n\n[ruff](https://github.com/astral-sh/ruff) is a PEP 8 compliant opinionated formatter with its own style. We use it to organize our code style. This is important to guarantee code quality as we have many developers work on the same codebase. The continuous integration (CI) would fail if your code doesn't meet `ruff`'s style.\n\nBefore submitting a pull request, you can run `ruff` locally to format your code. First, install it:\n\n```\npip install ruff\n```\n\nThen run it:\n\n```\nruff format source_file_or_directory --line-length 119\n```\n\nWe use line length 119 instead of the default. You can refer to the CI's [ruff configurations](https://github.com/autogluon/autogluon/blob/master/pyproject.toml).\n\nNote that if using `ruff` as a plugin in your IDE, the plugin may not use the configuration file `pyproject.toml` in our project. So, you need to configure the IDE plugin separately by following [here](https://docs.astral.sh/ruff/configuration/).\n\n\n## Unit Testing\n\n[Unit testing](https://en.wikipedia.org/wiki/Unit_testing) is necessary to automatically examine that our system's components meet their design and behave as intended.\nEvery time we add new features/functions, we need to add new unit tests for them. You can browse the files inside `unittests/` and determine where to add the new test cases properly. To run\nall the unit tests:\n\n```\npytest unittests/\n```\n\nYou can also run unit tests in one file, e.g., `unittests/test_utils.py`:\n\n```\npytest unittests/test_utils.py\n```\n\nFurthermore, testing one function is also easy:\n\n```\npytest unittests/test_utils.py -k test_inferring_pos_label\n```\n"
},
{
"path": "multimodal/tests/__init__.py",
"content": ""
},
{
"path": "multimodal/tests/conftest.py",
"content": "import pytest\n\n\ndef pytest_addoption(parser):\n parser.addoption(\"--runslow\", action=\"store_true\", default=False, help=\"run slow tests\")\n parser.addoption(\"--run_single_gpu\", action=\"store_true\", default=False, help=\"run single-gpu tests\")\n parser.addoption(\"--run_torch_mmdet\", action=\"store_true\", default=False, help=\"run object detection tests\")\n\n\ndef pytest_configure(config):\n config.addinivalue_line(\"markers\", \"slow: mark test as slow to run\")\n config.addinivalue_line(\"markers\", \"single_gpu: mark test as single-gpu to run\")\n config.addinivalue_line(\"markers\", \"torch_mmdet: mark test as torch_mmdet to run\")\n\n\ndef pytest_collection_modifyitems(config, items):\n skip_slow = pytest.mark.skip(reason=\"need --runslow option to run\")\n skip_sgpu = pytest.mark.skip(reason=\"need --run_single_gpu option to run\")\n skip_torch_mmdet = pytest.mark.skip(reason=\"need --run_torch_mmdet option to run\")\n custom_markers = dict(slow=skip_slow, single_gpu=skip_sgpu, torch_mmdet=skip_torch_mmdet)\n if config.getoption(\"--runslow\"):\n # --runslow given in cli: do not skip slow tests\n custom_markers.pop(\"slow\", None)\n if config.getoption(\"--run_single_gpu\", None):\n # --run_single_gpu given in cli: do not skip single-gpu tests\n custom_markers.pop(\"single_gpu\", None)\n if config.getoption(\"--run_torch_mmdet\", None):\n # --run_torch_mmdet given in cli: do not skip object detection tests\n custom_markers.pop(\"torch_mmdet\", None)\n for item in items:\n for marker in custom_markers:\n if marker in item.keywords:\n item.add_marker(custom_markers[marker])\n"
},
{
"path": "multimodal/tests/hf_dataset_list.yaml",
"content": "# HuggingFace datasets used in multimodal tests\n# These are cached to S3 and synced before tests to avoid HF Hub rate limiting\n#\n# Format: \"dataset_name\" or \"dataset_name:config\" for datasets with configs\nothers:\n- nyu-mll/glue:mrpc\n- SetFit/stsb\nothers_2: []\npredictor: []\ndocs: []\n"
},
{
"path": "multimodal/tests/hf_model_list.yaml",
"content": "docs:\n- bert-base-german-cased\n- google/electra-small-discriminator\n- hfl/chinese-lert-small\n- sentence-transformers/all-MiniLM-L6-v2\n- google/bert_uncased_L-6_H-768_A-12\n- microsoft/layoutlm-base-uncased\n- facebook/sam-vit-base\nothers:\n- bert-base-chinese\n- bert-base-uncased\n- facebook/bart-base\n- distilbert-base-uncased\n- distilroberta-base\n- google/electra-base-discriminator\n- google/electra-small-discriminator\n- gpt2\n- huawei-noah/TinyBERT_General_4L_312D\n- microsoft/deberta-base\n- microsoft/deberta-v3-base\n- microsoft/deberta-v3-small\n- nlpaueb/legal-bert-small-uncased\n- openai/clip-vit-base-patch32\n- openai/clip-vit-large-patch14\n- openai/clip-vit-large-patch14-336\n- roberta-base\n- sentence-transformers/all-MiniLM-L6-v2\n- sentence-transformers/all-mpnet-base-v2\n- sentence-transformers/msmarco-MiniLM-L-12-v3\n- xlm-roberta-base\n- facebook/sam-vit-base\n- timm/mobilenetv3_small_100.lamb_in1k\n- timm/swin_tiny_patch4_window7_224.ms_in1k\nothers_2:\n- google/electra-small-discriminator\n- google/flan-t5-small\n- hfl/chinese-lert-small\n- microsoft/deberta-v3-small\n- nlpaueb/legal-bert-small-uncased\n- sentence-transformers/all-MiniLM-L6-v2\n- t5-small\n- microsoft/layoutlmv3-base\n- microsoft/layoutlmv2-base-uncased\n- albert-base-v2\n- timm/mobilenetv3_small_100.lamb_in1k\n- timm/swin_tiny_patch4_window7_224.ms_in1k\npredictor:\n- CLTL/MedRoBERTa.nl\n- monsoon-nlp/hindi-bert\n- nlpaueb/legal-bert-small-uncased\n- openai/clip-vit-base-patch32\n- sentence-transformers/all-MiniLM-L6-v2\n- t5-small\n- timm/mobilenetv3_small_100.lamb_in1k\n- timm/swin_tiny_patch4_window7_224.ms_in1k\n"
},
{
"path": "multimodal/tests/test_check_style.py",
"content": "import logging\nimport warnings\nfrom subprocess import PIPE, Popen\n\n\ndef test_check_style():\n logging.getLogger().setLevel(logging.INFO)\n logging.info(\"PEP8 Style check\")\n flake8_proc = Popen([\"flake8\", \"--count\", \"--max-line-length\", \"300\"], stdout=PIPE)\n flake8_out = flake8_proc.communicate()[0]\n lines = flake8_out.splitlines()\n count = int(lines[-1].decode())\n if count > 0:\n warnings.warn(f\"{count} PEP8 warnings remaining\")\n assert count < 1000, \"Too many PEP8 warnings found, improve code quality to pass test.\"\n"
},
{
"path": "multimodal/tests/unittests/__init__.py",
"content": ""
},
{
"path": "multimodal/tests/unittests/others/__init__.py",
"content": ""
},
{
"path": "multimodal/tests/unittests/others/test_backbones.py",
"content": "import os\nimport shutil\n\nimport pytest\nfrom datasets import load_dataset\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import IA3_LORA\nfrom autogluon.multimodal.models import HFAutoModelForTextPrediction, TimmAutoModelForImagePrediction\nfrom autogluon.multimodal.utils import download\n\nfrom ..utils import PetFinderDataset, get_home_dir, verify_no_redundant_model_configs, verify_predictor_save_load\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"distilroberta-base\",\n \"huawei-noah/TinyBERT_General_4L_312D\",\n \"google/electra-base-discriminator\",\n \"microsoft/deberta-v3-base\",\n \"bert-base-uncased\",\n \"xlm-roberta-base\",\n \"microsoft/deberta-base\",\n \"roberta-base\",\n \"distilbert-base-uncased\",\n \"bert-base-chinese\",\n \"gpt2\",\n ],\n)\ndef test_hf_text_init(checkpoint_name):\n model = HFAutoModelForTextPrediction(prefix=\"hf_text\", checkpoint_name=checkpoint_name, num_classes=5)\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"swin_base_patch4_window7_224\",\n \"vit_small_patch16_384\",\n \"resnet18\",\n \"legacy_seresnet18\",\n \"regnety_002\",\n ],\n)\ndef test_timm_image_init(checkpoint_name):\n model = TimmAutoModelForImagePrediction(prefix=\"timm_image\", checkpoint_name=checkpoint_name, num_classes=5)\n\n\n@pytest.mark.parametrize(\"checkpoint_name\", [\"facebook/bart-base\"])\n@pytest.mark.parametrize(\"peft\", [None, IA3_LORA])\ndef test_backbone_bart(checkpoint_name, peft):\n train_data = load_dataset(\"nyu-mll/glue\", \"mrpc\")[\"train\"].to_pandas().drop(\"idx\", axis=1).sample(500)\n test_data = load_dataset(\"nyu-mll/glue\", \"mrpc\")[\"validation\"].to_pandas().drop(\"idx\", axis=1).sample(20)\n predictor = MultiModalPredictor(label=\"label\")\n predictor.fit(\n train_data,\n hyperparameters={\n \"model.hf_text.checkpoint_name\": checkpoint_name,\n \"optim.max_epochs\": 1,\n \"optim.peft\": peft,\n \"optim.top_k\": 1,\n \"optim.top_k_average_method\": \"best\",\n \"env.batch_size\": 2,\n },\n time_limit=20,\n )\n predictor.predict(test_data)\n\n\n@pytest.mark.skip(\n reason=\"Skip this test because the meta-transformer checkpoint needs to be put into the s3 bucket first.\"\n)\ndef test_meta_transformer():\n model_name = \"Meta-Transformer_base_patch16_encoder\"\n model_version = \"base\"\n model_path = f\"./{model_name}\"\n if not os.path.isfile(model_path):\n download(\n url=f\"s3://automl-mm-bench/meta-transformer/{model_name}\",\n path=model_path,\n )\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"meta_transformer\"],\n \"model.meta_transformer.checkpoint_path\": model_path,\n \"model.meta_transformer.model_version\": model_version,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n }\n save_path = os.path.join(get_home_dir(), \"outputs\", \"meta-transformer\")\n\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n verify_no_redundant_model_configs(predictor)\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others/test_data_collators.py",
"content": "import torch\n\nfrom autogluon.multimodal.data.collator import DictCollator, ListCollator, PadCollator, StackCollator, TupleCollator\n\n\ndef test_stack():\n a = [1, 2, 3]\n b = [4, 5, 6]\n c = [7, 8, 9]\n ret = StackCollator()((a, b, c))\n assert torch.all(ret == torch.as_tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]))\n\n\ndef test_pad():\n a = [1, 2, 3]\n b = [4, 5]\n c = [7]\n ret = PadCollator(pad_val=1)((a, b, c))\n assert torch.all(ret == torch.as_tensor([[1, 2, 3], [4, 5, 1], [7, 1, 1]]))\n\n ret = PadCollator(pad_val=1, round_to=2)((a, b, c))\n assert torch.all(ret == torch.as_tensor([[1, 2, 3, 1], [4, 5, 1, 1], [7, 1, 1, 1]]))\n\n ret = PadCollator(pad_val=1, max_length=4)((a, b, c))\n assert torch.all(ret == torch.as_tensor([[1, 2, 3, 1], [4, 5, 1, 1], [7, 1, 1, 1]]))\n\n ret, valid_lengths = PadCollator(pad_val=1, ret_length=True)((a, b, c))\n assert torch.all(valid_lengths == torch.as_tensor([3, 2, 1]))\n\n a = [[1, 4], [2, 5], [3, 6]]\n b = [[7], [8], [9]]\n c = [[10, 11, 12], [13, 14, 15], [16, 17, 18]]\n\n ret = PadCollator(axis=1)((a, b, c))\n assert torch.all(\n ret\n == torch.as_tensor(\n [\n [[1, 4, 0], [2, 5, 0], [3, 6, 0]],\n [[7, 0, 0], [8, 0, 0], [9, 0, 0]],\n [[10, 11, 12], [13, 14, 15], [16, 17, 18]],\n ]\n )\n )\n\n\ndef test_tuple():\n a = ([1, 2, 3, 4], 0)\n b = ([5, 7], 1)\n\n ret_1, ret_2 = TupleCollator(PadCollator(), StackCollator())((a, b))\n\n assert torch.all(ret_1 == torch.as_tensor([[1, 2, 3, 4], [5, 7, 0, 0]]))\n assert torch.all(ret_2 == torch.as_tensor([0, 1]))\n\n\ndef test_list():\n a = ([1, 2, 3, 4], \"id_0\")\n b = ([5, 7, 2, 5], \"id_1\")\n c = ([1, 2, 3, 4], \"id_2\")\n _, l = TupleCollator(StackCollator(), ListCollator())((a, b, c))\n assert l == [\"id_0\", \"id_1\", \"id_2\"]\n\n\ndef test_dict():\n a = {\"data\": [1, 2, 3, 4, 5], \"label\": 0}\n b = {\"data\": [5, 7], \"label\": 1}\n c = {\"data\": [1, 2, 3], \"label\": 0}\n\n collate_fn = DictCollator({\"data\": PadCollator(), \"label\": StackCollator()})\n sample = collate_fn((a, b, c))\n\n assert torch.all(sample[\"data\"] == torch.as_tensor([[1, 2, 3, 4, 5], [5, 7, 0, 0, 0], [1, 2, 3, 0, 0]]))\n assert torch.all(sample[\"label\"] == torch.as_tensor([0, 1, 0]))\n"
},
{
"path": "multimodal/tests/unittests/others/test_deployment_onnx.py",
"content": "import os\nimport shutil\n\nimport numpy as np\nimport numpy.testing\nimport pytest\nfrom packaging import version\nfrom scipy.stats import pearsonr, spearmanr\nfrom sklearn.metrics.pairwise import paired_cosine_distances\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import REGRESSION\nfrom autogluon.multimodal.utils.misc import shopee_dataset\nfrom autogluon.multimodal.utils.onnx import OnnxModule\n\nfrom ..utils import AEDataset, PetFinderDataset\n\nALL_DATASETS = {\n \"petfinder\": PetFinderDataset(),\n \"ae\": AEDataset(),\n}\n\ntry:\n import tensorrt\nexcept ImportError:\n tensorrt = None\n\n\ndef evaluate(predictor, df, onnx_session=None):\n labels = df[\"score\"].to_numpy()\n\n if not onnx_session:\n QEmb = predictor.extract_embedding(df[[\"sentence1\"]])[\"sentence1\"]\n AEmb = predictor.extract_embedding(df[[\"sentence2\"]])[\"sentence2\"]\n else:\n QEmb = onnx_session.run(None, predictor._learner.get_processed_batch_for_deployment(data=df[[\"sentence1\"]]))[0]\n AEmb = onnx_session.run(None, predictor._learner.get_processed_batch_for_deployment(data=df[[\"sentence2\"]]))[0]\n\n cosine_scores = 1 - paired_cosine_distances(QEmb, AEmb)\n eval_pearson_cosine, _ = pearsonr(labels, cosine_scores)\n eval_spearman_cosine, _ = spearmanr(labels, cosine_scores)\n\n return eval_pearson_cosine, eval_spearman_cosine\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\"sentence-transformers/msmarco-MiniLM-L-12-v3\", \"sentence-transformers/all-MiniLM-L6-v2\"],\n)\ndef test_onnx_export_hf_text(checkpoint_name):\n # IMPORTANT: lazy import onnxruntime, otherwise onnxruntime won't be able to compile to tensorrt EP.\n import onnxruntime as ort\n import pandas as pd\n\n # https://huggingface.co/datasets/stsb_multi_mt/viewer/en/test\n data_dict = {\n \"sentence1\": [\n \"A girl is styling her hair.\",\n \"A group of men play soccer on the beach.\",\n \"One woman is measuring another woman's ankle.\",\n \"A man is cutting up a cucumber.\",\n \"A man is playing a harp.\",\n ],\n \"sentence2\": [\n \"A girl is brushing her hair.\",\n \"A group of boys are playing soccer on the beach.\",\n \"A woman measures another woman's ankle.\",\n \"A man is slicing a cucumber.\",\n \"A man is playing a keyboard.\",\n ],\n \"score\": [2.5, 3.6, 5, 4.2, 1.5],\n }\n test_df = pd.DataFrame.from_dict(data_dict)\n\n predictor = MultiModalPredictor(\n problem_type=\"feature_extraction\",\n hyperparameters={\n \"optim.max_epochs\": 1,\n \"model.hf_text.checkpoint_name\": checkpoint_name,\n },\n )\n ag_pearson, ag_spearman = evaluate(predictor, test_df)\n\n model_path = checkpoint_name.replace(\"/\", \"_\")\n onnx_path = predictor.export_onnx(path=model_path, data=test_df)\n\n # TODO: Test with CUDA EP when we upgrade CUDA version to 11.7 (along with pytorch v1.13.1).\n # onnxruntime-gpu v1.13.1 require CUDA version >=11.6\n ort_sess = ort.InferenceSession(onnx_path, providers=[\"CPUExecutionProvider\"])\n onnx_pearson, onnx_spearman = evaluate(predictor, test_df, ort_sess)\n assert pytest.approx(onnx_pearson, 1e-2) == ag_pearson\n assert pytest.approx(onnx_spearman, 1e-2) == ag_spearman\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name,num_gpus\",\n [\n (\"swin_tiny_patch4_window7_224\", -1),\n (\"resnet18\", -1),\n ],\n)\ndef test_onnx_export_timm_image(checkpoint_name, num_gpus):\n model_path = \"./automm_shopee\"\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n image_path_export = test_data.iloc[0][\"image\"]\n image_path_test = test_data.iloc[1][\"image\"]\n\n if os.path.exists(model_path):\n shutil.rmtree(model_path)\n\n # train\n predictor = MultiModalPredictor(\n hyperparameters={\n \"optim.max_epochs\": 1,\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n \"env.num_workers\": 0,\n \"env.strategy\": \"ddp\",\n },\n label=\"label\",\n path=model_path,\n )\n predictor.fit(\n train_data=train_data,\n time_limit=10, # seconds\n )\n predictor.save(path=model_path)\n loaded_predictor = MultiModalPredictor.load(path=model_path)\n\n # predict\n load_proba = loaded_predictor.predict_proba({\"image\": [image_path_test]})\n\n # -------------------------------------------------------\n # convert (no path)\n onnx_path = loaded_predictor.export_onnx({\"image\": [image_path_export]})\n assert isinstance(onnx_path, bytes), (\n f\"export_onnx() method should return onnx bytes directly, if path is not provided.\"\n )\n\n # create onnx module for evaluation\n onnx_module = OnnxModule(onnx_path, providers=[\"CUDAExecutionProvider\"])\n onnx_module.input_keys = loaded_predictor._learner._model.input_keys\n onnx_module.prefix = loaded_predictor._learner._model.prefix\n onnx_module.get_output_dict = loaded_predictor._learner._model.get_output_dict\n\n # simply replace _model in the loaded predictor to predict with onnxruntime\n loaded_predictor._learner._model = onnx_module\n onnx_proba = loaded_predictor.predict_proba({\"image\": [image_path_test]})\n\n # assert allclose\n np.testing.assert_allclose(load_proba, onnx_proba, rtol=1e-2, atol=1e-2)\n\n # -------------------------------------------------------\n # convert (with path)\n loaded_predictor = MultiModalPredictor.load(path=model_path)\n onnx_path = loaded_predictor.export_onnx({\"image\": [image_path_export]}, path=model_path)\n\n # Check existence of the exported onnx model file and tensorrt cache files\n onnx_path_expected = os.path.join(model_path, \"model.onnx\")\n assert isinstance(onnx_path, str), \"export_onnx() method should return a string, if path argument is provided.\"\n assert onnx_path == onnx_path_expected, \"onnx_path mismatch\"\n assert os.path.exists(onnx_path), f\"onnx model file not found at {onnx_path}\"\n\n # create onnx module for evaluation\n onnx_module = OnnxModule(onnx_path, providers=[\"CUDAExecutionProvider\"])\n onnx_module.input_keys = loaded_predictor._learner._model.input_keys\n onnx_module.prefix = loaded_predictor._learner._model.prefix\n onnx_module.get_output_dict = loaded_predictor._learner._model.get_output_dict\n\n # simply replace _model in the loaded predictor to predict with onnxruntime\n loaded_predictor._learner._model = onnx_module\n onnx_proba = loaded_predictor.predict_proba({\"image\": [image_path_test]})\n\n # assert allclose\n np.testing.assert_allclose(load_proba, onnx_proba, rtol=1e-2, atol=1e-2)\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"dataset_name,model_names,text_backbone,image_backbone\",\n [\n (\n \"petfinder\",\n [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"google/electra-small-discriminator\",\n \"mobilenetv3_small_100\",\n ),\n (\n \"ae\",\n [\"numerical_mlp\", \"hf_text\", \"fusion_mlp\"],\n \"google/electra-small-discriminator\",\n None,\n ),\n ],\n)\n@pytest.mark.skipif(\n tensorrt is None or version.parse(tensorrt.__version__) >= version.parse(\"8.5.4\"),\n reason=\"tensorrt above 8.5.4 cause segfault, but is required to support py311\",\n)\ndef test_onnx_optimize_for_inference(dataset_name, model_names, text_backbone, image_backbone):\n dataset = ALL_DATASETS[dataset_name]\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": model_names,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n if text_backbone:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": text_backbone,\n }\n )\n if image_backbone:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": image_backbone,\n }\n )\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0], problem_type=dataset.problem_type, eval_metric=dataset.metric\n )\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n )\n model_path = predictor.path\n predictor.save(path=model_path)\n\n # Use a different subset of the dataset for compilation.\n tail_df = dataset.test_df.tail(2)\n\n # Load a refresh predictor and optimize it for inference\n for providers in [None, [\"TensorrtExecutionProvider\"], [\"CUDAExecutionProvider\"], [\"CPUExecutionProvider\"]]:\n predictor_opt = MultiModalPredictor.load(path=model_path)\n predictor_opt.optimize_for_inference(providers=providers)\n\n # Check module type of optimized predictor\n assert isinstance(predictor_opt._learner._model, OnnxModule), (\n f\"invalid onnx module type, expected to be OnnxModule, but the model type is {type(predictor_opt._learner._model)}\"\n )\n\n # We should support dynamic shape\n for batch_size in [2, 4, 8]:\n test_df = dataset.test_df.head(batch_size)\n if dataset.problem_type == REGRESSION:\n y_pred = predictor.predict(test_df)\n y_pred_trt = predictor_opt.predict(test_df)\n else:\n y_pred = predictor.predict_proba(test_df)\n y_pred_trt = predictor_opt.predict_proba(test_df)\n numpy.testing.assert_allclose(y_pred, y_pred_trt, rtol=0.01, atol=0.01)\n"
},
{
"path": "multimodal/tests/unittests/others/test_dump_model.py",
"content": "import os\n\nimport pytest\nimport timm\nimport transformers\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import AEDataset, PetFinderDataset\n\n\ndef test_dump_timm_image():\n download_dir = \"./\"\n model_dump_path = \"./timm_image_test\"\n base_model_name = \"mobilenetv3_large_100\"\n train_data, _ = shopee_dataset(download_dir=download_dir)\n predictor_1 = MultiModalPredictor(\n label=\"label\",\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"timm_image_1\"],\n \"model.timm_image_1.checkpoint_name\": base_model_name,\n }\n predictor_1.fit(\n train_data=train_data,\n hyperparameters=hyperparameters,\n time_limit=5,\n seed=42,\n )\n predictor_1.dump_model(save_path=model_dump_path)\n model = timm.create_model(\n model_name=base_model_name, checkpoint_path=f\"{model_dump_path}/timm_image_1/pytorch_model.bin\", num_classes=0\n )\n assert isinstance(model, timm.models.mobilenetv3.MobileNetV3)\n predictor_2 = MultiModalPredictor(\n label=\"label\",\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.timm_image.checkpoint_name\": f\"{model_dump_path}/timm_image_1\",\n }\n predictor_2.fit(\n train_data=train_data,\n hyperparameters=hyperparameters,\n time_limit=5,\n seed=42,\n )\n\n\ndef test_dump_hf_text():\n model_dump_path = \"./hf_text_test\"\n base_model_name = \"prajjwal1/bert-tiny\"\n dataset = AEDataset()\n predictor_1 = MultiModalPredictor(\n label=dataset.label_columns[0], problem_type=dataset.problem_type, eval_metric=dataset.metric\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.hf_text.checkpoint_name\": base_model_name,\n \"env.num_workers\": 0, # https://github.com/pytorch/pytorch/issues/33296\n }\n predictor_1.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n seed=42,\n )\n predictor_1.dump_model(save_path=model_dump_path)\n\n model = transformers.AutoModel.from_pretrained(f\"{model_dump_path}/hf_text\")\n assert isinstance(model, transformers.models.bert.modeling_bert.BertModel)\n predictor_2 = MultiModalPredictor(\n label=dataset.label_columns[0], problem_type=dataset.problem_type, eval_metric=dataset.metric\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.hf_text.checkpoint_name\": f\"{model_dump_path}/hf_text\",\n \"env.num_workers\": 0, # https://github.com/pytorch/pytorch/issues/33296\n }\n predictor_2.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n seed=42,\n )\n\n\ndef test_dump_fusion_model():\n model_dump_path = \"./test_fusion_models\"\n dataset = PetFinderDataset()\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0], problem_type=dataset.problem_type, eval_metric=dataset.metric\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"ghostnet_100\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n }\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n seed=42,\n )\n predictor.dump_model(save_path=model_dump_path)\n hf_text_dir = f\"{model_dump_path}/hf_text\"\n timm_image_dir = f\"{model_dump_path}/timm_image\"\n assert os.path.exists(hf_text_dir) and (len(os.listdir(hf_text_dir)) > 2) == True\n assert os.path.exists(timm_image_dir) and (len(os.listdir(timm_image_dir)) == 2) == True\n\n\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\ndef test_mmdet_object_detection_save_and_load():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/object_detection_dataset/tiny_motorbike_coco.zip\"\n download_dir = \"./tiny_motorbike_coco\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"tiny_motorbike\")\n\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n # Init predictor\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": \"yolov3_mobilenetv2_8xb24-320-300e_coco\",\n \"env.num_gpus\": -1,\n },\n problem_type=\"object_detection\",\n )\n\n pred = predictor.predict(test_path)\n\n model_save_dir = predictor.dump_model()\n detection_model_save_subdir = os.path.join(model_save_dir, predictor._learner._model.prefix)\n\n new_predictor = MultiModalPredictor(\n hyperparameters={\"model.mmdet_image.checkpoint_name\": detection_model_save_subdir, \"env.num_gpus\": -1},\n problem_type=\"object_detection\",\n )\n new_pred = new_predictor.predict(test_path)\n\n assert abs(pred[\"bboxes\"][0][0][\"score\"] - new_pred[\"bboxes\"][0][0][\"score\"]) < 1e-4\n"
},
{
"path": "multimodal/tests/unittests/others/test_ensemble.py",
"content": "import os\nimport shutil\n\nfrom autogluon.multimodal import MultiModalPredictor\n\nfrom ..utils import PetFinderDataset, get_home_dir, verify_predictor_save_load\n\n\ndef test_ensemble_from_scratch():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n hyperparameters = {\n \"learner_names\": [\"lf_mlp\", \"lf_clip\"],\n \"lf_mlp\": {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.max_epochs\": 1,\n },\n \"lf_clip\": {\n \"model.names\": [\"ft_transformer\", \"clip_image\", \"clip_text\", \"fusion_mlp\"],\n \"model.clip_image.data_types\": [\"image\"],\n \"model.clip_text.data_types\": [\"text\"],\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.max_epochs\": 1,\n },\n }\n save_path = os.path.join(get_home_dir(), \"outputs\", \"ensemble\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n use_ensemble=True,\n ensemble_mode=\"one_shot\",\n )\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n assert len(predictor._learner._all_learners) == 2\n assert len(predictor._learner._selected_learners) == 2\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df, verify_embedding=False)\n\n\ndef test_ensemble_with_pretrained_predictors():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n hyperparameters_1 = {\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.max_epochs\": 1,\n }\n hyperparameters_2 = {\n \"model.names\": [\"ft_transformer\", \"clip_image\", \"clip_text\", \"fusion_mlp\"],\n \"model.clip_image.data_types\": [\"image\"],\n \"model.clip_text.data_types\": [\"text\"],\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.max_epochs\": 1,\n }\n\n save_path_1 = os.path.join(get_home_dir(), \"outputs\", \"lf_mlp\")\n save_path_2 = os.path.join(get_home_dir(), \"outputs\", \"lf_clip\")\n save_path_ensemble = os.path.join(get_home_dir(), \"outputs\", \"ensemble\")\n\n for per_path in [save_path_1, save_path_2, save_path_ensemble]:\n if os.path.exists(per_path):\n shutil.rmtree(per_path)\n\n predictors = []\n for per_path, per_hparams in zip([save_path_1, save_path_2], [hyperparameters_1, hyperparameters_2]):\n per_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n per_predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=per_hparams,\n time_limit=20,\n save_path=per_path,\n )\n predictors.append(per_predictor.path)\n\n ensemble_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n use_ensemble=True,\n ensemble_mode=\"one_shot\",\n )\n ensemble_predictor.fit(\n train_data=dataset.train_df,\n predictors=predictors,\n )\n assert len(ensemble_predictor._learner._all_learners) == 2\n assert len(ensemble_predictor._learner._selected_learners) == 2\n score = ensemble_predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(ensemble_predictor, dataset.test_df, verify_embedding=False)\n"
},
{
"path": "multimodal/tests/unittests/others/test_extract_features.py",
"content": "import numpy.testing as npt\nimport pandas as pd\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\n@pytest.mark.parametrize(\n \"model_name\", [\"sentence-transformers/all-MiniLM-L6-v2\", \"sentence-transformers/all-mpnet-base-v2\"]\n)\ndef test_sentence_transformer_embedding(model_name):\n predictor = MultiModalPredictor(\n problem_type=\"feature_extraction\", hyperparameters={\"model.hf_text.checkpoint_name\": model_name}\n )\n case1 = {\"sentence\": [\"Hello world\"]}\n case2 = {\"sentence\": [\"Hello world\", \"Test Hello World\"]}\n\n outputs_case1_from_df = predictor.extract_embedding(pd.DataFrame(case1))\n outputs_case1_from_dict = predictor.extract_embedding(case1)\n\n npt.assert_allclose(outputs_case1_from_dict[\"sentence\"], outputs_case1_from_df[\"sentence\"])\n\n outputs_case2_from_df = predictor.extract_embedding(pd.DataFrame(case2))\n outputs_case2_from_dict = predictor.extract_embedding(case2)\n npt.assert_allclose(outputs_case2_from_df[\"sentence\"], outputs_case2_from_dict[\"sentence\"], 1e-3, 1e-3)\n npt.assert_allclose(outputs_case2_from_df[\"sentence\"][:1], outputs_case1_from_df[\"sentence\"], 1e-3, 1e-3)\n"
},
{
"path": "multimodal/tests/unittests/others/test_load.py",
"content": "import os\nimport shutil\n\nimport numpy.testing as npt\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import PetFinderDataset\n\n\ndef test_load_intermediate_ckpt():\n download_dir = \"./\"\n train_data, test_data = shopee_dataset(download_dir=download_dir)\n predictor = MultiModalPredictor(label=\"label\")\n predictor.fit(train_data=train_data, time_limit=20)\n src_file = os.path.join(predictor.path, \"model.ckpt\")\n dest_file = os.path.join(predictor.path, \"epoch=8-step=18.ckpt\")\n shutil.copy(src_file, dest_file)\n loaded_predictor = MultiModalPredictor.load(path=dest_file)\n\n predictions = predictor.predict(test_data, as_pandas=False)\n predictions2 = loaded_predictor.predict(test_data, as_pandas=False)\n npt.assert_equal(predictions, predictions2)\n predictions_prob = predictor.predict_proba(test_data, as_pandas=False)\n predictions2_prob = loaded_predictor.predict_proba(test_data, as_pandas=False)\n npt.assert_equal(predictions_prob, predictions2_prob)\n\n\ndef test_load_fttransformer_pretrained_ckpt():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"model.names\": [\"ft_transformer\"],\n \"model.ft_transformer.checkpoint_name\": \"https://automl-mm-bench.s3.amazonaws.com/ft_transformer_pretrained_ckpt/iter_2k.ckpt\",\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n }\n predictor.fit(\n dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n predictor.evaluate(dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others/test_losses.py",
"content": "import os\nimport shutil\n\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"swin_tiny_patch4_window7_224\",\n ],\n)\ndef test_focal_loss_multiclass(checkpoint_name):\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n\n save_path = f\"./tmp/automm-shopee-focal-loss\"\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor = MultiModalPredictor(label=\"label\", problem_type=\"multiclass\", path=save_path)\n\n predictor.fit(\n hyperparameters={\n \"model.timm_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": -1,\n \"optim.loss_func\": \"focal_loss\",\n \"optim.focal_loss.alpha\": [1, 0.25, 0.35, 0.16], # shopee dataset has 4 classes.\n \"optim.focal_loss.gamma\": 2.5,\n \"optim.focal_loss.reduction\": \"mean\",\n \"optim.max_epochs\": 1,\n },\n train_data=train_data,\n time_limit=30, # seconds\n ) # you can trust the default config, e.g., we use a `swin_base_patch4_window7_224` model\n\n image_path = test_data.iloc[0][\"image\"]\n\n predictions_str = predictor.predict(image_path)\n predictions_list1 = predictor.predict([image_path])\n predictions_list10 = predictor.predict([image_path] * 10)\n"
},
{
"path": "multimodal/tests/unittests/others/test_matcher.py",
"content": "import os\nimport shutil\nimport tempfile\n\nimport numpy.testing as npt\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import convert_data_for_ranking, semantic_search\n\nfrom ..utils import (\n Flickr30kDataset,\n IDChangeDetectionDataset,\n evaluate_matcher_ranking,\n get_home_dir,\n verify_matcher_realtime_inference,\n verify_matcher_save_load,\n)\n\nALL_DATASETS = {\n \"id_change_detection\": IDChangeDetectionDataset,\n \"flickr30k\": Flickr30kDataset,\n}\n\n\n@pytest.mark.parametrize(\n \"dataset_name,query,response,problem_type,text_backbone,image_backbone, is_ranking, symmetric\",\n [\n (\n \"id_change_detection\",\n \"Previous Image\",\n \"Current Image\",\n \"image_similarity\",\n None,\n \"swin_tiny_patch4_window7_224\",\n False,\n False,\n ),\n (\n \"flickr30k\",\n \"caption\",\n \"image\",\n \"image_text_similarity\",\n \"google/electra-small-discriminator\",\n \"swin_tiny_patch4_window7_224\",\n True,\n True,\n ),\n ],\n)\ndef test_matcher_basic(\n dataset_name,\n query,\n response,\n problem_type,\n text_backbone,\n image_backbone,\n is_ranking,\n symmetric,\n):\n dataset = ALL_DATASETS[dataset_name]()\n\n matcher = MultiModalPredictor(\n query=query,\n response=response,\n problem_type=problem_type,\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n )\n\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": \"greedy_soup\",\n }\n\n if text_backbone is not None:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": text_backbone,\n }\n )\n if image_backbone is not None:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": image_backbone,\n }\n )\n\n save_path = os.path.join(get_home_dir(), \"outputs\", dataset_name)\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n matcher.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.val_df if hasattr(dataset, \"val_df\") else None,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n\n if is_ranking:\n evaluate_matcher_ranking(\n matcher=matcher,\n test_df=dataset.test_df,\n query_column=query,\n response_column=response,\n metric_name=dataset.metric,\n symmetric=symmetric,\n )\n text_to_image_hits = semantic_search(\n matcher=matcher,\n query_data={\n query: dataset.test_df[query].tolist()\n }, # need a dict/dataframe instead of a list for a trained matcher\n response_data={\n response: dataset.test_df[response].tolist()\n }, # need a dict/dataframe instead of a list for a trained matcher\n top_k=5,\n )\n image_to_text_hits = semantic_search(\n matcher=matcher,\n query_data={\n response: dataset.test_df[response].tolist()\n }, # need a dict/dataframe instead of a list for a trained matcher\n response_data={\n query: dataset.test_df[query].tolist()\n }, # need a dict/dataframe instead of a list for a trained matcher\n top_k=5,\n )\n else:\n score = matcher.evaluate(dataset.test_df)\n verify_matcher_save_load(matcher, dataset.test_df, cls=MultiModalPredictor)\n\n # Test for continuous fit\n matcher.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.val_df if hasattr(dataset, \"val_df\") else None,\n hyperparameters=hyperparameters,\n time_limit=20,\n )\n verify_matcher_save_load(matcher, dataset.test_df, cls=MultiModalPredictor)\n\n # Saving to folder, loading the saved model and call fit again (continuous fit)\n with tempfile.TemporaryDirectory() as root:\n matcher.save(root)\n matcher = MultiModalPredictor.load(root)\n matcher.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.val_df if hasattr(dataset, \"val_df\") else None,\n hyperparameters=hyperparameters,\n time_limit=20,\n )\n\n\n# Conflicts between realtime inference with dp and lightning strategies\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"dataset_name,query,response,problem_type,text_backbone,image_backbone, is_ranking, symmetric\",\n [\n (\n \"id_change_detection\",\n \"Previous Image\",\n \"Current Image\",\n \"image_similarity\",\n None,\n \"swin_tiny_patch4_window7_224\",\n False,\n False,\n ),\n (\n \"flickr30k\",\n \"caption\",\n \"image\",\n \"image_text_similarity\",\n \"google/electra-small-discriminator\",\n \"swin_tiny_patch4_window7_224\",\n True,\n True,\n ),\n ],\n)\ndef test_matcher_realtime_inference(\n dataset_name,\n query,\n response,\n problem_type,\n text_backbone,\n image_backbone,\n is_ranking,\n symmetric,\n):\n dataset = ALL_DATASETS[dataset_name]()\n\n matcher = MultiModalPredictor(\n query=query,\n response=response,\n problem_type=problem_type,\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n )\n\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": \"greedy_soup\",\n }\n\n if text_backbone is not None:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": text_backbone,\n }\n )\n if image_backbone is not None:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": image_backbone,\n }\n )\n\n save_path = os.path.join(get_home_dir(), \"outputs\", dataset_name)\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n matcher.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.val_df if hasattr(dataset, \"val_df\") else None,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n\n verify_matcher_realtime_inference(matcher, dataset.test_df)\n\n\ndef test_text_semantic_search():\n corpus = [\n \"A man is eating food.\",\n \"A man is eating a piece of bread.\",\n \"The girl is carrying a baby.\",\n \"A man is riding a horse.\",\n \"A woman is playing violin.\",\n \"Two men pushed carts through the woods.\",\n \"A man is riding a white horse on an enclosed ground.\",\n \"A monkey is playing drums.\",\n \"A cheetah is running behind its prey.\",\n ]\n queries = [\n \"A man is eating pasta.\",\n \"Someone in a gorilla costume is playing a set of drums.\",\n \"A cheetah chases prey on across a field.\",\n ]\n\n matcher = MultiModalPredictor(\n problem_type=\"text_similarity\",\n hyperparameters={\"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\"},\n )\n hits = semantic_search(\n matcher=matcher,\n query_data=queries,\n response_data=corpus,\n top_k=5,\n )\n # extract embeddings first and then do semantic search\n query_embeddings = matcher.extract_embedding(queries)\n response_embeddings = matcher.extract_embedding(corpus)\n hits_2 = semantic_search(\n matcher=matcher,\n query_embeddings=query_embeddings,\n response_embeddings=response_embeddings,\n top_k=5,\n )\n\n hits_gt = [\n [\n {\"response_id\": 0, \"score\": 0.7035943269729614},\n {\"response_id\": 1, \"score\": 0.527172327041626},\n {\"response_id\": 3, \"score\": 0.18880528211593628},\n {\"response_id\": 6, \"score\": 0.10461556911468506},\n {\"response_id\": 8, \"score\": 0.09811049699783325},\n ],\n [\n {\"response_id\": 7, \"score\": 0.6432060599327087},\n {\"response_id\": 4, \"score\": 0.2563084363937378},\n {\"response_id\": 3, \"score\": 0.1389961689710617},\n {\"response_id\": 6, \"score\": 0.11907944828271866},\n {\"response_id\": 8, \"score\": 0.1079183965921402},\n ],\n [\n {\"response_id\": 8, \"score\": 0.8252813816070557},\n {\"response_id\": 0, \"score\": 0.13986822962760925},\n {\"response_id\": 7, \"score\": 0.1292111724615097},\n {\"response_id\": 6, \"score\": 0.10977005213499069},\n {\"response_id\": 3, \"score\": 0.06506325304508209},\n ],\n ]\n\n for per_query_hits, per_query_hits_2, per_query_hit_gt in zip(hits, hits_2, hits_gt):\n for per_hit, per_hit_2, per_hit_gt in zip(per_query_hits, per_query_hits_2, per_query_hit_gt):\n assert per_hit[\"response_id\"] == per_hit_2[\"response_id\"] == per_hit_gt[\"response_id\"]\n npt.assert_allclose(per_hit[\"score\"], per_hit_2[\"score\"], 1e-3, 1e-3)\n npt.assert_allclose(per_hit[\"score\"], per_hit_gt[\"score\"], 1e-3, 1e-3)\n\n\ndef test_image_text_semantic_search():\n dataset_name = \"flickr30k\"\n dataset = ALL_DATASETS[dataset_name]()\n image_list = dataset.test_df[\"image\"].tolist()\n text_list = dataset.test_df[\"caption\"].tolist()\n\n matcher = MultiModalPredictor(\n problem_type=\"image_text_similarity\",\n hyperparameters={\"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\"},\n )\n text_to_image_hits = semantic_search(\n matcher=matcher,\n query_data=text_list,\n response_data=image_list,\n top_k=5,\n )\n\n # extract embeddings first and then do semantic search\n query_embeddings = matcher.extract_embedding(text_list, as_tensor=True)\n response_embeddings = matcher.extract_embedding(image_list, as_tensor=True)\n text_to_image_hits_2 = semantic_search(\n matcher=matcher,\n query_embeddings=query_embeddings,\n response_embeddings=response_embeddings,\n top_k=5,\n )\n\n for per_query_hits, per_query_hits_2 in zip(text_to_image_hits, text_to_image_hits_2):\n for per_hit, per_hit_2 in zip(per_query_hits, per_query_hits_2):\n assert per_hit[\"response_id\"] == per_hit_2[\"response_id\"]\n npt.assert_almost_equal(per_hit[\"score\"], per_hit_2[\"score\"])\n\n image_to_text_hits = semantic_search(\n matcher=matcher,\n query_data=image_list,\n response_data=text_list,\n top_k=5,\n )\n\n # extract embeddings first and then do semantic search\n query_embeddings = matcher.extract_embedding(image_list, as_tensor=True)\n response_embeddings = matcher.extract_embedding(text_list, as_tensor=True)\n image_to_text_hits_2 = semantic_search(\n matcher=matcher,\n query_embeddings=query_embeddings,\n response_embeddings=response_embeddings,\n top_k=5,\n )\n\n for per_query_hits, per_query_hits_2 in zip(image_to_text_hits, image_to_text_hits_2):\n for per_hit, per_hit_2 in zip(per_query_hits, per_query_hits_2):\n assert per_hit[\"response_id\"] == per_hit_2[\"response_id\"]\n npt.assert_almost_equal(per_hit[\"score\"], per_hit_2[\"score\"])\n\n\n@pytest.mark.parametrize(\n \"hyperparameters\",\n [\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"mobilenetv3_small_100\",\n },\n ],\n)\ndef test_matcher_hyperparameters_consistency(hyperparameters):\n dataset = IDChangeDetectionDataset()\n\n # pass hyperparameters to init()\n predictor = MultiModalPredictor(\n query=\"Previous Image\",\n response=\"Current Image\",\n problem_type=\"image_similarity\",\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n hyperparameters=hyperparameters,\n )\n predictor.fit(dataset.train_df, time_limit=10)\n\n # pass hyperparameters to fit()\n predictor_2 = MultiModalPredictor(\n query=\"Previous Image\",\n response=\"Current Image\",\n problem_type=\"image_similarity\",\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n )\n predictor_2.fit(\n dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n assert predictor._learner._config == predictor_2._learner._config\n"
},
{
"path": "multimodal/tests/unittests/others/test_metrics.py",
"content": "import os\nimport random\nimport shutil\nimport tempfile\n\nimport numpy as np\nimport pytest\nimport torch\nfrom datasets import load_dataset\nfrom sklearn.metrics import f1_score, log_loss\nfrom torchmetrics import MeanMetric\n\nimport autogluon.core.metrics as ag_metrics\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.optim import CustomHitRate, get_loss_func, get_torchmetric, infer_metrics\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import HatefulMeMesDataset, PetFinderDataset, get_home_dir, ref_symmetric_hit_rate\n\n\n@pytest.mark.parametrize(\n \"metric_name,class_num\",\n [\n (\"log_loss\", 5),\n (\"log_loss\", 10),\n (\"cross_entropy\", 100),\n ],\n)\ndef test_metric_log_loss(metric_name, class_num):\n preds = []\n targets = []\n random.seed(123)\n torch.manual_seed(123)\n\n for i in range(100):\n bs = random.randint(1, 16)\n preds.append(torch.randn(bs, class_num))\n targets.append(torch.randint(0, class_num, (bs,)))\n\n _, custom_metric_func = get_torchmetric(metric_name=metric_name)\n mean_metric = MeanMetric()\n\n for per_pred, per_target in zip(preds, targets):\n mean_metric.update(custom_metric_func(per_pred, per_target))\n\n score1 = mean_metric.compute()\n preds = torch.cat(preds).softmax(dim=1)\n targets = torch.cat(targets)\n score2 = log_loss(\n y_true=targets,\n y_pred=preds,\n )\n assert pytest.approx(score1, 1e-6) == score2\n\n\n@pytest.mark.parametrize(\n \"problem_type,loss_func_name\",\n [\n (\"regression\", \"bcewithlogitsloss\"),\n ],\n)\ndef test_metric_bce_with_logits_loss(problem_type, loss_func_name):\n preds = []\n targets = []\n random.seed(123)\n torch.manual_seed(123)\n\n for i in range(100):\n bs = random.randint(1, 16)\n preds.append(torch.randn(bs, 1))\n targets.append(torch.rand(bs, 1))\n preds = torch.cat(preds)\n targets = torch.cat(targets)\n\n loss_func = get_loss_func(\n problem_type=problem_type,\n mixup_active=False,\n loss_func_name=loss_func_name,\n )\n\n score1 = loss_func(input=preds, target=targets)\n preds = preds.sigmoid()\n bceloss = torch.nn.BCELoss()\n score2 = bceloss(input=preds, target=targets)\n assert pytest.approx(score1, 1e-6) == score2\n\n\n# TODO (1): torchmetrics will give slightly different result under multi GPU runs\n# TODO (2): \"F1\" is not supported for multiclass, will fallback to accuracy\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"eval_metric\",\n [\"f1_macro\", \"f1_micro\", \"f1_weighted\"],\n)\ndef test_f1_metrics_for_multiclass(eval_metric):\n dataset = PetFinderDataset()\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=\"multiclass\",\n eval_metric=eval_metric,\n )\n hyperparameters = {\n \"optim.max_epochs\": 3,\n \"model.names\": [\"ft_transformer\"],\n \"env.num_gpus\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": \"best\",\n \"optim.loss_func\": \"auto\",\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n }\n save_path = os.path.join(get_home_dir(), \"outputs\", eval_metric)\n\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.test_df,\n hyperparameters=hyperparameters,\n time_limit=60,\n save_path=save_path,\n )\n val_score = predictor._learner._best_score\n eval_score = predictor.evaluate(dataset.test_df)[eval_metric]\n assert abs(val_score - eval_score) < 4e-2\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"problem_type, eval_metric_name, validation_metric_name, is_matching, target_eval_metric_name, target_validation_metric_name\",\n [\n (\"binary\", \"acc\", None, False, \"acc\", \"acc\"),\n (\"multiclass\", \"f1_macro\", None, False, \"f1_macro\", \"f1_macro\"),\n (\"regression\", \"f1\", None, False, \"rmse\", \"rmse\"),\n (\"object_detection\", \"map\", \"map\", False, \"map\", \"map\"),\n (\"binary\", None, None, False, \"roc_auc\", \"roc_auc\"),\n (\"multiclass\", None, None, False, \"accuracy\", \"accuracy\"),\n (\"regression\", None, None, False, \"rmse\", \"rmse\"),\n (\"object_detection\", None, None, False, \"map\", \"map\"),\n (\"semantic_segmentation\", None, None, False, \"iou\", \"iou\"),\n (\"ner\", None, None, False, \"overall_f1\", \"ner_token_f1\"),\n (\"few_shot_classification\", None, None, False, \"accuracy\", \"accuracy\"),\n (\"binary\", None, None, True, \"roc_auc\", \"roc_auc\"),\n (\"multiclass\", None, None, True, \"spearmanr\", \"spearmanr\"),\n (\"regression\", None, None, True, \"spearmanr\", \"spearmanr\"),\n (\"regression\", \"pearsonr\", None, True, \"pearsonr\", \"pearsonr\"),\n (None, None, None, True, \"ndcg\", \"recall\"),\n (\"feature_extraction\", None, None, False, None, None),\n (\"feature_extraction\", \"f1\", None, False, None, None),\n ],\n)\ndef test_infer_metrics(\n problem_type,\n eval_metric_name,\n validation_metric_name,\n is_matching,\n target_eval_metric_name,\n target_validation_metric_name,\n):\n validation_metric_name, eval_metric_name = infer_metrics(\n problem_type, eval_metric_name, validation_metric_name, is_matching\n )\n assert eval_metric_name == target_eval_metric_name\n assert validation_metric_name == target_validation_metric_name\n\n\n# Once eval metric is customized, shall not use the fallback eval\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"problem_type, eval_metric, is_matching, target_eval_metric_name, target_validation_metric_name\",\n [\n (\"binary\", ag_metrics.make_scorer(\"dummy\", ag_metrics.get_metric(\"acc\")), False, \"dummy\", \"roc_auc\"),\n (\"regression\", ag_metrics.make_scorer(\"dummy\", ag_metrics.get_metric(\"acc\")), True, \"dummy\", \"spearmanr\"),\n ],\n)\ndef test_infer_metrics_custom(\n problem_type,\n eval_metric,\n is_matching,\n target_eval_metric_name,\n target_validation_metric_name,\n):\n validation_metric_name, eval_metric_name = infer_metrics(problem_type, eval_metric, None, is_matching)\n assert eval_metric_name == target_eval_metric_name\n assert validation_metric_name == target_validation_metric_name\n\n\ndef test_metric_symmetric_hit_rate():\n generator = torch.Generator()\n generator.manual_seed(0)\n for repeat in range(3):\n for top_ks in [[1, 5, 10], [20], [3, 7, 9]]:\n features_a = torch.randn(50, 2, generator=generator)\n features_b = torch.randn(50, 2, generator=generator)\n hit_rate_impl = CustomHitRate.compute_hit_rate(features_a, features_b, logit_scale=1.0, top_ks=top_ks)\n hit_rate_ref = ref_symmetric_hit_rate(features_a, features_b, logit_scale=1.0, top_ks=top_ks)\n assert pytest.approx(hit_rate_impl.item()) == hit_rate_ref.item()\n\n\ndef test_custom_metric():\n dataset = HatefulMeMesDataset()\n metric_name = dataset.metric\n custom_metric_name = \"customized\"\n metric_scorer = ag_metrics.get_metric(metric_name)\n metric_scorer.name = custom_metric_name\n predictor_by_name = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n predictor_by_scorer = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_scorer,\n )\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor_by_name.fit(\n train_data=dataset.train_df,\n time_limit=0,\n save_path=save_path,\n )\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor_by_scorer.fit(\n train_data=dataset.train_df,\n time_limit=0,\n save_path=save_path,\n )\n scores_by_name = predictor_by_name.evaluate(\n data=dataset.test_df,\n metrics=None,\n )\n scores_by_scorer_eval = predictor_by_name.evaluate(\n data=dataset.test_df,\n metrics=[metric_scorer],\n )\n scores_by_scorer_init = predictor_by_scorer.evaluate(\n data=dataset.test_df,\n metrics=None,\n )\n assert scores_by_scorer_eval[custom_metric_name] == scores_by_scorer_init[custom_metric_name]\n assert scores_by_name[custom_metric_name] == scores_by_scorer_eval[custom_metric_name]\n\n\n@pytest.mark.parametrize(\"eval_metric\", [\"spearmanr\", \"pearsonr\"])\ndef test_metric_spearman_and_pearson(eval_metric):\n train_df = load_dataset(\"SetFit/stsb\", split=\"train\").to_pandas()\n predictor = MultiModalPredictor(label=\"label\", eval_metric=eval_metric)\n predictor.fit(train_df, presets=\"medium_quality\", time_limit=5)\n assert predictor.eval_metric == eval_metric\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name,eval_metric\",\n [\n (\"swin_tiny_patch4_window7_224\", \"log_loss\"),\n (\"swin_tiny_patch4_window7_224\", \"f1_micro\"),\n ],\n)\ndef test_metrics_multiclass(checkpoint_name, eval_metric):\n \"\"\"\n Test the MultiModalPredictor's evaluation metrics for multiclass classification.\n\n This test verifies that:\n 1. The predictor correctly implements the specified evaluation metrics (log_loss and f1_micro)\n 2. The manually calculated metrics match the predictor's evaluate() output\n 3. The model training and prediction pipeline works end-to-end\n\n Parameters\n ----------\n checkpoint_name : str\n Name of the model checkpoint to use (e.g., \"swin_tiny_patch4_window7_224\")\n eval_metric : str\n Evaluation metric to test (\"log_loss\" or \"f1_micro\")\n \"\"\"\n # Set up data and model\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, _ = shopee_dataset(download_dir)\n save_path = \"./tmp/automm_shopee\"\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor = MultiModalPredictor(label=\"label\", problem_type=\"multiclass\", eval_metric=eval_metric, path=save_path)\n\n # Train the model\n predictor.fit(\n hyperparameters={\n \"model.timm_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": -1,\n \"optim.max_epochs\": 1,\n },\n train_data=train_data,\n time_limit=30, # seconds\n )\n\n # Get predictions\n if eval_metric == \"log_loss\":\n y_pred = predictor.predict_proba(train_data)\n y_true = train_data[predictor.label].values\n manual_score = log_loss(y_true, y_pred)\n elif eval_metric == \"f1_micro\":\n y_pred = predictor.predict(train_data)\n y_true = train_data[predictor.label].values\n manual_score = f1_score(y_true, y_pred, average=\"micro\")\n else:\n raise NotImplementedError\n\n # Get score from predictor's evaluate method\n predictor_score = predictor.evaluate(train_data)\n\n # Verify metric configuration\n assert predictor.eval_metric == eval_metric\n\n # Verify scores match (within numerical precision)\n np.testing.assert_almost_equal(\n predictor_score[eval_metric],\n manual_score,\n decimal=5,\n err_msg=f\"Predictor's {eval_metric} score doesn't match manual calculation\",\n )\n\n # Verify score is within reasonable bounds\n if eval_metric == \"log_loss\":\n assert predictor_score[eval_metric] > 0, \"Log loss should be positive\"\n else: # f1_micro\n assert 0 <= predictor_score[eval_metric] <= 1, \"F1 score should be between 0 and 1\"\n"
},
{
"path": "multimodal/tests/unittests/others/test_ner.py",
"content": "import json\nimport os\nimport shutil\nimport tempfile\nfrom collections import OrderedDict\nfrom unittest import mock\n\nimport numpy.testing as npt\nimport pandas as pd\nimport pytest\nimport torch\nfrom ray import tune\nfrom transformers import AutoTokenizer\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import (\n IMAGE_PATH,\n NER,\n NER_ANNOTATION,\n TEXT,\n TEXT_NER,\n)\nfrom autogluon.multimodal.data import NerProcessor, infer_ner_column_type\nfrom autogluon.multimodal.utils import merge_bio_format, visualize_ner\n\nfrom ..utils import get_home_dir\n\n\ndef get_data():\n sample = {\n \"text_snippet\": [\n \"EU rejects German call to boycott British lamb .\",\n \"Peter Blackburn\",\n \"He said further scientific study was required and if it was found that action was needed it should be taken by the European Union .\",\n \".\",\n \"It brought in 4,275 tonnes of British mutton , some 10 percent of overall imports .\",\n ],\n \"entity_annotations\": [\n '[{\"entity_group\": \"B-ORG\", \"start\": 0, \"end\": 2}, {\"entity_group\": \"B-MISC\", \"start\": 11, \"end\": 17}, {\"entity_group\": \"B-MISC\", \"start\": 34, \"end\": 41}]',\n '[{\"entity_group\": \"B-PER\", \"start\": 0, \"end\": 5}, {\"entity_group\": \"I-PER\", \"start\": 6, \"end\": 15}]',\n '[{\"entity_group\": \"B-ORG\", \"start\": 115, \"end\": 123}, {\"entity_group\": \"I-ORG\", \"start\": 124, \"end\": 129}]',\n \"[]\",\n '[{\"entity_group\": \"B-MISC\", \"start\": 30, \"end\": 37}]',\n ],\n }\n return pd.DataFrame.from_dict(sample)\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name,searcher,scheduler\",\n [\n (\"google/electra-small-discriminator\", None, None),\n (\"google/electra-small-discriminator\", \"bayes\", \"FIFO\"),\n ],\n)\ndef test_ner(checkpoint_name, searcher, scheduler):\n train_data = get_data()\n label_col = \"entity_annotations\"\n\n if searcher is None:\n lr = 0.0001\n hyperparameter_tune_kwargs = None\n else:\n lr = tune.uniform(0.0001, 0.01)\n hyperparameter_tune_kwargs = {\"num_trials\": 2, \"searcher\": searcher, \"scheduler\": scheduler}\n predictor = MultiModalPredictor(problem_type=\"ner\", label=label_col)\n predictor.fit(\n train_data=train_data,\n time_limit=60,\n hyperparameters={\"model.ner_text.checkpoint_name\": checkpoint_name, \"optim.lr\": lr},\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n\n scores = predictor.evaluate(train_data)\n\n test_predict = train_data.drop(label_col, axis=1)\n test_predict = test_predict.head(2)\n predictions = predictor.predict(test_predict)\n embeddings = predictor.extract_embedding(test_predict)\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [(\"google/electra-small-discriminator\")],\n)\ndef test_multi_column_ner(checkpoint_name):\n train_data = get_data()\n train_data[\"add_text\"] = train_data.text_snippet\n label_col = \"entity_annotations\"\n predictor = MultiModalPredictor(problem_type=\"ner\", label=label_col)\n predictor.fit(\n train_data=train_data,\n time_limit=10,\n column_types={\"text_snippet\": \"text_ner\"},\n hyperparameters={\"model.ner_text.checkpoint_name\": checkpoint_name},\n )\n scores = predictor.evaluate(train_data)\n test_predict = train_data.drop(label_col, axis=1)\n test_predict = test_predict.head(2)\n predictions = predictor.predict(test_predict)\n proba = predictor.predict_proba(test_predict)\n embeddings = predictor.extract_embedding(test_predict)\n\n\ndef test_ner_standalone():\n requests_gag = mock.patch(\n \"requests.Session.request\",\n mock.Mock(side_effect=RuntimeError(\"Please use the `responses` library to mock HTTP in your tests.\")),\n )\n train_data = get_data()\n label_col = \"entity_annotations\"\n test_data = train_data.drop(label_col, axis=1)\n\n predictor = MultiModalPredictor(problem_type=\"ner\", label=label_col)\n\n save_path = os.path.join(get_home_dir(), \"standalone\", \"false\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor.fit(\n train_data=train_data,\n time_limit=40,\n hyperparameters={\"model.ner_text.checkpoint_name\": \"google/electra-small-discriminator\"},\n save_path=save_path,\n )\n save_path_standalone = os.path.join(get_home_dir(), \"standalone\", \"true\")\n\n predictor.save(\n path=save_path_standalone,\n standalone=True,\n )\n\n del predictor\n torch.cuda.empty_cache()\n\n loaded_online_predictor = MultiModalPredictor.load(path=save_path)\n online_predictions = loaded_online_predictor.predict(test_data, as_pandas=False)\n del loaded_online_predictor\n\n with requests_gag:\n # No internet connection here. If any command require internet connection, a RuntimeError will be raised!\n with tempfile.TemporaryDirectory() as tmpdirname:\n torch.hub.set_dir(tmpdirname) # block reading files in `.cache`.\n loaded_offline_predictor = MultiModalPredictor.load(path=save_path_standalone)\n\n offline_predictions = loaded_offline_predictor.predict(test_data, as_pandas=False)\n del loaded_offline_predictor\n npt.assert_equal(online_predictions[0], offline_predictions[0])\n\n\ndef test_merge_bio():\n sentence = \"Game of Thrones is an American fantasy drama television series created by David Benioff\"\n predictions = [\n [\n {\"entity_group\": \"B-TITLE\", \"start\": 0, \"end\": 4},\n {\"entity_group\": \"I-TITLE\", \"start\": 5, \"end\": 7},\n {\"entity_group\": \"I-TITLE\", \"start\": 8, \"end\": 15},\n {\"entity_group\": \"B-GENRE\", \"start\": 22, \"end\": 30},\n {\"entity_group\": \"B-GENRE\", \"start\": 31, \"end\": 38},\n {\"entity_group\": \"I-GENRE\", \"start\": 39, \"end\": 44},\n {\"entity_group\": \"B-DIRECTOR\", \"start\": 74, \"end\": 79},\n {\"entity_group\": \"I-DIRECTOR\", \"start\": 80, \"end\": 87},\n ]\n ]\n res = merge_bio_format([sentence], predictions)\n expected_res = [\n [\n {\"entity_group\": \"TITLE\", \"start\": 0, \"end\": 15},\n {\"entity_group\": \"GENRE\", \"start\": 22, \"end\": 30},\n {\"entity_group\": \"GENRE\", \"start\": 31, \"end\": 44},\n {\"entity_group\": \"DIRECTOR\", \"start\": 74, \"end\": 87},\n ]\n ]\n assert res == expected_res, f\"Wrong results {res} from merge_bio_format!\"\n\n\ndef test_misc_visualize_ner():\n sentence = \"Albert Einstein was born in Germany and is widely acknowledged to be one of the greatest physicists.\"\n annotation = [\n {\"entity_group\": \"PERSON\", \"start\": 0, \"end\": 15},\n {\"entity_group\": \"LOCATION\", \"start\": 28, \"end\": 35},\n ]\n visualize_ner(sentence, annotation)\n\n # Test using string for annotation\n visualize_ner(sentence, json.dumps(annotation))\n\n\ndef test_process_ner_annotations():\n text = \"SwissGear Sion Softside Expandable Roller Luggage, Dark Grey, Checked-Medium 25-Inch\"\n annotation = [((0, 14), \"Brand\"), ((50, 60), \"Color\"), ((70, 85), \"Dimensions\")]\n entity_map = {\n \"X\": 1,\n \"O\": 2,\n \"B-Brand\": 3,\n \"I-Brand\": 4,\n \"B-Color\": 5,\n \"I-Color\": 6,\n \"B-Dimensions\": 7,\n \"I-Dimensions\": 8,\n }\n tokenizer = AutoTokenizer.from_pretrained(\"microsoft/deberta-v3-small\")\n tokenizer.model_max_length = 512\n res = NerProcessor.process_ner_annotations(annotation, text, entity_map, tokenizer, is_eval=True)[0]\n assert res == [3, 4, 1, 1, 1, 1, 1, 5, 6, 1, 1, 1, 7, 8, 8, 8], \"Labelling is wrong!\"\n\n\n@pytest.mark.parametrize(\n \"column_types,gt_column_types\",\n [\n (\n {\n \"abc\": TEXT,\n \"label\": NER_ANNOTATION,\n },\n {\n \"abc\": TEXT_NER,\n \"label\": NER_ANNOTATION,\n },\n ),\n (\n {\n \"abc\": TEXT_NER,\n \"label\": NER_ANNOTATION,\n },\n {\n \"abc\": TEXT_NER,\n \"label\": NER_ANNOTATION,\n },\n ),\n (\n {\n \"abc\": TEXT,\n \"xyz\": TEXT,\n \"label\": NER_ANNOTATION,\n },\n {\n \"abc\": TEXT_NER,\n \"xyz\": TEXT,\n \"label\": NER_ANNOTATION,\n },\n ),\n (\n {\n \"abc\": TEXT,\n \"xyz\": TEXT,\n \"efg\": IMAGE_PATH,\n \"label\": NER_ANNOTATION,\n },\n {\n \"abc\": TEXT_NER,\n \"xyz\": TEXT,\n \"efg\": IMAGE_PATH,\n \"label\": NER_ANNOTATION,\n },\n ),\n ],\n)\ndef test_infer_ner_column_type(column_types, gt_column_types):\n column_types = OrderedDict(column_types)\n gt_column_types = OrderedDict(gt_column_types)\n column_types = infer_ner_column_type(column_types)\n assert column_types == gt_column_types\n"
},
{
"path": "multimodal/tests/unittests/others/test_object_detection.py",
"content": "import json\nimport os\n\nimport numpy as np\nimport pytest\nimport requests\nfrom PIL import Image\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import from_coco_or_voc\n\n\ndef download_sample_images():\n url = \"https://raw.githubusercontent.com/open-mmlab/mmdetection/master/demo/demo.jpg\"\n image = Image.open(requests.get(url, stream=True).raw)\n mmdet_image_name = \"demo.jpg\"\n image.save(mmdet_image_name)\n\n return mmdet_image_name\n\n\ndef download_sample_dataset():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/object_detection_dataset/tiny_motorbike_coco.zip\"\n download_dir = \"./tiny_motorbike_coco\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"tiny_motorbike\")\n\n return data_dir\n\n\n# TODO: Pytest does not support DDP\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolox_s\"])\ndef test_mmdet_object_detection_fit_basics(checkpoint_name):\n mmdet_image_name = download_sample_images()\n data_dir = download_sample_dataset()\n\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n # Init predictor\n predictor = MultiModalPredictor(\n hyperparameters={\"model.mmdet_image.checkpoint_name\": checkpoint_name, \"env.num_gpus\": -1},\n problem_type=\"object_detection\",\n sample_data_path=train_path,\n )\n\n # Fit\n predictor.fit(train_path, hyperparameters={\"optim.lr\": 2e-4, \"env.per_gpu_batch_size\": 2}, time_limit=40)\n\n # Evaluate on COCO format data\n predictor.evaluate(test_path)\n\n # Inference on COCO format data\n pred = predictor.predict(test_path) # test batch inference\n\n # Inference on dictionary format data\n pred = predictor.predict({\"image\": [mmdet_image_name] * 10}) # test batch inference\n\n test_df = from_coco_or_voc(test_path)\n\n # Inference on dataframe format data\n pred = predictor.predict(test_df.iloc[:100])\n\n # Inference on COCO format data without annotations\n test_path_with_images_only = os.path.join(data_dir, \"Annotations\", \"test_cocoformat_nolabel.json\")\n with open(test_path, \"r\") as f:\n test_data = json.load(f)\n test_data_images_only = {}\n test_data_images_only[\"images\"] = test_data[\"images\"][:100]\n with open(test_path_with_images_only, \"w+\") as f_wo_ann:\n json.dump(test_data_images_only, f_wo_ann)\n pred = predictor.predict(test_path_with_images_only)\n\n\n# TODO: Pytest does not support DDP\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolov3_mobilenetv2_8xb24-320-300e_coco\"])\ndef test_mmdet_object_detection_inference_basics(checkpoint_name):\n mmdet_image_name = download_sample_images()\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": -1, # currently mmdet only support single gpu inference\n },\n problem_type=\"object_detection\",\n )\n\n # Inference on list format data\n pred = predictor.predict([mmdet_image_name] * 20) # test batch inference\n assert len(pred) == 20 # test data size is 20\n\n # Inference on single image path\n pred = predictor.predict(mmdet_image_name) # test single entry data inference\n assert len(pred) == 1 # test data size is 1\n\n # Inference on dict format data\n pred = predictor.predict({\"image\": [mmdet_image_name] * 10}) # test batch inference\n assert len(pred) == 10 # test data size is 10\n\n data_dir = download_sample_dataset()\n\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n test_path_with_images_only = os.path.join(data_dir, \"Annotations\", \"test_cocoformat_nolabel.json\")\n with open(test_path, \"r\") as f:\n test_data = json.load(f)\n test_data_images_only = {}\n test_data_images_only[\"images\"] = test_data[\"images\"][:100]\n with open(test_path_with_images_only, \"w+\") as f_wo_ann:\n json.dump(test_data_images_only, f_wo_ann)\n\n # Inference on COCO format data\n test_df = from_coco_or_voc(test_path)\n\n # Inference on dataframe format data\n pred = predictor.predict(test_df.iloc[:100])\n\n # Inference on data without annotations\n pred = predictor.predict(test_path_with_images_only)\n\n # Save inference in COCO on data without annotations\n pred = predictor.predict(test_path_with_images_only, save_results=True)\n\n # Save inference in Pandas Dataframe (.csv) on data without annotations\n pred = predictor.predict(test_path_with_images_only, save_results=True, as_coco=False)\n\n\n# TODO: FIX DDP multi runs!\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolov3_mobilenetv2_8xb24-320-300e_coco\"])\ndef test_mmdet_object_detection_inference_xywh_output(checkpoint_name):\n mmdet_image_name = download_sample_images()\n\n xywh_predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"model.mmdet_image.output_bbox_format\": \"xywh\",\n \"env.num_gpus\": -1, # currently mmdet only support single gpu inference\n },\n problem_type=\"object_detection\",\n )\n xywh_preds = xywh_predictor.predict([mmdet_image_name] * 10, as_pandas=True) # test batch inference\n assert len(xywh_preds) == 10 # test data size is 10\n\n xyxy_predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": -1, # currently mmdet only support single gpu inference\n },\n problem_type=\"object_detection\",\n )\n xyxy_preds = xyxy_predictor.predict([mmdet_image_name] * 10, as_pandas=True) # test batch inference\n assert len(xyxy_preds) == 10 # test data size is 10\n\n xywh_bbox = xywh_preds.iloc[0][\"bboxes\"][0]\n xyxy_bbox = xyxy_preds.iloc[0][\"bboxes\"][0]\n x, y, w, h = xywh_bbox[\"bbox\"]\n x1, y1, x2, y2 = xyxy_bbox[\"bbox\"]\n assert xywh_bbox[\"class\"] == xyxy_bbox[\"class\"]\n assert abs(xywh_bbox[\"score\"] - xyxy_bbox[\"score\"]) < 1e-4\n assert abs(x - x1) < 1e-4\n assert abs(y - y1) < 1e-4\n assert abs(x2 - x1 + 1 - w) < 1e-4\n assert abs(y2 - y1 + 1 - h) < 1e-4\n\n\n# TODO: FIX DDP multi runs!\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolov3_mobilenetv2_8xb24-320-300e_coco\"])\ndef test_mmdet_object_detection_save_and_load(checkpoint_name):\n data_dir = download_sample_dataset()\n\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n # Init predictor\n predictor = MultiModalPredictor(\n hyperparameters={\"model.mmdet_image.checkpoint_name\": checkpoint_name, \"env.num_gpus\": -1},\n problem_type=\"object_detection\",\n )\n\n preds = predictor.predict(test_path)\n\n model_save_subdir = predictor._learner._model.save()\n\n new_predictor = MultiModalPredictor(\n hyperparameters={\"model.mmdet_image.checkpoint_name\": model_save_subdir, \"env.num_gpus\": -1},\n problem_type=\"object_detection\",\n )\n new_preds = new_predictor.predict(test_path)\n\n for batch_idx in range(len(preds)):\n for in_batch_idx in range(len(preds[batch_idx])):\n # Convert tensors to numpy arrays for element-wise comparison\n pred_scores = preds[batch_idx][in_batch_idx][\"scores\"].detach().cpu().numpy()\n new_pred_scores = new_preds[batch_idx][in_batch_idx][\"scores\"].detach().cpu().numpy()\n # Check if all differences are within tolerance\n assert (np.abs(pred_scores - new_pred_scores) < 1e-4).all(), (\n f\"{preds[batch_idx][in_batch_idx]}\\n{new_preds[batch_idx][in_batch_idx]}\"\n )\n\n\n# TODO: FIX DDP multi runs!\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolov3_mobilenetv2_8xb24-320-300e_coco\"])\ndef test_mmdet_object_detection_fit_eval_predict_df(checkpoint_name):\n data_dir = download_sample_dataset()\n\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n # Init predictor\n train_df = from_coco_or_voc(train_path)\n predictor = MultiModalPredictor(\n hyperparameters={\"model.mmdet_image.checkpoint_name\": checkpoint_name, \"env.num_gpus\": -1},\n problem_type=\"object_detection\",\n sample_data_path=train_df,\n )\n\n predictor.fit(train_df, hyperparameters={\"optim.lr\": 2e-4, \"env.per_gpu_batch_size\": 2}, time_limit=30)\n\n test_df = from_coco_or_voc(test_path)\n preds = predictor.predict(data=test_df)\n results = predictor.evaluate(data=test_df)\n\n\n# TODO: Pytest does not support DDP\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\n@pytest.mark.parametrize(\"checkpoint_name\", [\"yolov3_mobilenetv2_8xb24-320-300e_coco\"])\ndef test_mmdet_object_detection_fit_with_freeze_backbone(checkpoint_name):\n data_dir = download_sample_dataset()\n\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n test_path = os.path.join(data_dir, \"Annotations\", \"test_cocoformat.json\")\n # Init predictor\n train_df = from_coco_or_voc(train_path)\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmdet_image.checkpoint_name\": checkpoint_name,\n \"model.mmdet_image.frozen_layers\": [\"backbone\"],\n \"env.num_gpus\": -1,\n },\n problem_type=\"object_detection\",\n sample_data_path=train_df,\n )\n\n predictor.fit(train_df, hyperparameters={\"optim.lr\": 2e-4, \"env.per_gpu_batch_size\": 2}, time_limit=30)\n\n\n# TODO: FIX DDP multi runs!\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\ndef test_detector_hyperparameters_consistency():\n data_dir = download_sample_dataset()\n\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n train_df = from_coco_or_voc(train_path)\n\n hyperparameters = {\n \"model.mmdet_image.checkpoint_name\": \"yolov3_mobilenetv2_8xb24-320-300e_coco\",\n \"env.num_gpus\": -1,\n }\n\n # pass hyperparameters to init()\n predictor = MultiModalPredictor(\n problem_type=\"object_detection\", sample_data_path=train_df, hyperparameters=hyperparameters\n )\n predictor.fit(train_df, time_limit=10)\n\n # pass hyperparameters to fit()\n predictor_2 = MultiModalPredictor(problem_type=\"object_detection\", sample_data_path=train_df)\n predictor_2.fit(train_df, hyperparameters=hyperparameters, time_limit=10)\n assert predictor._learner._config == predictor_2._learner._config\n\n\n# TODO: Issue #4126 Skipping object detection tests due to incompatibility of mmdet with Torch 2.2\n@pytest.mark.torch_mmdet\ndef test_detector_coco_root_setup():\n data_dir = download_sample_dataset()\n train_path = os.path.join(data_dir, \"Annotations\", \"trainval_cocoformat.json\")\n train_df = from_coco_or_voc(train_path)\n\n hyperparameters = {\n \"model.mmdet_image.coco_root\": \"../\",\n \"env.num_gpus\": 1, # no need to test multigpu\n }\n\n # pass hyperparameters to init()\n predictor = MultiModalPredictor(\n problem_type=\"object_detection\",\n sample_data_path=train_df,\n hyperparameters=hyperparameters,\n )\n predictor.fit(train_df, time_limit=10)\n"
},
{
"path": "multimodal/tests/unittests/others/test_preprocess_data.py",
"content": "import pandas as pd\nimport pytest\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom autogluon.multimodal.data import (\n CustomLabelEncoder,\n data_to_df,\n infer_column_types,\n infer_problem_type,\n init_df_preprocessor,\n)\nfrom autogluon.multimodal.utils import get_config, is_url\n\nfrom ..utils import PetFinderDataset\n\n\n@pytest.mark.parametrize(\n \"labels,positive_class\",\n [\n ([1, 2, 2], 2),\n ([1, 2, 2], 1),\n ([1, 0, 1, 0, 0], 0),\n ([1, 0, 1, 0, 0], None),\n ([\"a\", \"e\", \"e\", \"a\"], \"a\"),\n ([\"b\", \"d\", \"b\", \"d\"], \"d\"),\n ([\"a\", \"d\", \"e\", \"b\"], \"d\"),\n ([3, 2, 1, 0], 2),\n ],\n)\ndef test_label_encoder(labels, positive_class):\n label_encoder = CustomLabelEncoder(positive_class=positive_class)\n label_encoder.fit(labels)\n\n # test encoding positive class\n if positive_class:\n assert label_encoder.transform([positive_class]).item() == len(label_encoder.classes_) - 1\n else:\n assert label_encoder.transform([label_encoder.classes_[-1]]).item() == len(label_encoder.classes_) - 1\n\n # test encoding\n sklearn_le = LabelEncoder()\n sklearn_le.fit(labels)\n sk_encoded_labels = sklearn_le.transform(labels)\n our_encoded_labels = label_encoder.transform(labels)\n if positive_class:\n sk_pos_label = sklearn_le.transform([positive_class]).item()\n else:\n sk_pos_label = len(sklearn_le.classes_) - 1\n sk_encoded_labels[sk_encoded_labels == sk_pos_label] = len(sklearn_le.classes_)\n sk_encoded_labels[sk_encoded_labels > sk_pos_label] -= 1\n sk_encoded_labels[sk_encoded_labels < 0] = 0\n assert (sk_encoded_labels == our_encoded_labels).all()\n\n # test inverse encoding\n assert label_encoder.inverse_transform(our_encoded_labels).tolist() == labels\n\n\n@pytest.mark.parametrize(\n \"data,required_columns,all_columns,is_valid_input\",\n [\n ([1, 2, 3], [\"a\"], [\"a\"], True),\n ([1, 2, 3], [\"a\"], [\"a\", \"b\"], False),\n ({\"a\": [1, 2, 3]}, [\"a\"], [\"a\", \"b\"], True),\n ({\"a\": [1, 2, 3]}, [\"b\"], [\"a\", \"b\"], False),\n ({\"a\": [1, 2, 3], \"b\": [4, 5, 6]}, [\"b\", \"a\"], [\"a\", \"b\"], True),\n ([(1, 2), (3, 4)], [\"a\", \"b\"], [\"a\", \"b\"], True),\n ([[1, 2], [3, 4]], [\"a\"], [\"a\", \"b\"], True),\n ([[1, 2], [3, 4]], [\"a\", \"b\"], [\"a\", \"b\", \"c\"], False),\n ([{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 10, \"b\": 20, \"c\": 30}], [\"a\", \"b\", \"c\"], [\"a\", \"b\", \"c\"], True),\n ([{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 10, \"b\": 20, \"c\": 30}], [\"a\", \"c\"], [\"a\", \"b\", \"c\"], True),\n ([{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 10, \"b\": 20, \"c\": 30}], [\"a\", \"b\", \"c\", \"d\"], [\"a\", \"b\", \"c\", \"d\"], False),\n ([{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 10, \"b\": 20, \"c\": 30}], [\"a\", \"b\", \"d\"], [\"a\", \"b\", \"c\", \"d\"], False),\n (pd.DataFrame({\"a\": [1, 2, 3], \"b\": [4, 5, 6]}), [\"b\", \"a\"], [\"b\", \"c\", \"a\"], True),\n ],\n)\ndef test_data_to_df(data, required_columns, all_columns, is_valid_input):\n if is_valid_input:\n df = data_to_df(data=data, required_columns=required_columns, all_columns=all_columns)\n else:\n with pytest.raises(ValueError):\n df = data_to_df(data=data, required_columns=required_columns, all_columns=all_columns)\n\n\n@pytest.mark.parametrize(\n \"path,is_valid_url\",\n [\n (\"/media/data/coco17/annotations/instances_val2017.json\", False),\n (\"This is a test.\", False),\n (\"http://images.cocodataset.org/annotations/annotations_trainval2017.zip\", True),\n (\"http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar\", True),\n (\n \"https://automl-mm-bench.s3.amazonaws.com/voc_script/faster_rcnn_r50_fpn_1x_voc0712_20220320_192712-54bef0f3.pth\",\n True,\n ),\n ],\n)\ndef test_is_url(path, is_valid_url):\n assert is_url(path) == is_valid_url\n\n\n@pytest.mark.parametrize(\n \"convert_categorical_to_text,convert_numerical_to_text\",\n [\n (False, False),\n (True, False),\n (False, True),\n (True, True),\n ],\n)\ndef test_convert_to_text(convert_categorical_to_text, convert_numerical_to_text):\n overrides = {\n \"data.categorical.convert_to_text\": convert_categorical_to_text,\n \"data.numerical.convert_to_text\": convert_numerical_to_text,\n }\n dataset = PetFinderDataset()\n label_column = dataset.label_columns[0]\n train_data = dataset.train_df\n config = get_config(\n problem_type=dataset.problem_type,\n overrides=overrides,\n )\n column_types = infer_column_types(\n data=train_data,\n label_columns=label_column,\n problem_type=dataset.problem_type,\n )\n df_preprocessor = init_df_preprocessor(\n config=config,\n column_types=column_types,\n label_column=label_column,\n train_df_x=train_data.drop(columns=label_column),\n train_df_y=train_data[label_column],\n )\n if convert_categorical_to_text:\n assert len(df_preprocessor.categorical_feature_names) == 0\n assert len(df_preprocessor.text_feature_names) > 0\n if convert_numerical_to_text:\n assert len(df_preprocessor.numerical_feature_names) == 0\n assert len(df_preprocessor.text_feature_names) > 0\n"
},
{
"path": "multimodal/tests/unittests/others/test_presets.py",
"content": "import pytest\nfrom omegaconf import OmegaConf\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import (\n ALL_MODEL_QUALITIES,\n DATA,\n DISTILLER,\n ENV,\n FEATURE_EXTRACTION,\n MODEL,\n OBJECT_DETECTION,\n OPTIM,\n ZERO_SHOT_IMAGE_CLASSIFICATION,\n)\nfrom autogluon.multimodal.utils import PROBLEM_TYPES_REG, get_basic_config, get_config, get_presets, list_presets\n\n\ndef test_get_presets():\n problem_types = list_presets()\n for per_type in problem_types:\n for model_quality in ALL_MODEL_QUALITIES:\n hyperparameters, hyperparameter_tune_kwargs = get_presets(per_type, model_quality)\n\n # test non-existing types\n non_exist_types = [\"hello\", \"haha\"]\n for per_type in non_exist_types:\n for model_quality in ALL_MODEL_QUALITIES:\n with pytest.raises(ValueError):\n hyperparameters, hyperparameter_tune_kwargs = get_presets(per_type, model_quality)\n\n\ndef test_preset_to_config():\n problem_types = list_presets()\n for per_type in problem_types:\n for model_quality in ALL_MODEL_QUALITIES:\n if per_type != \"ensemble\":\n hyperparameters, _ = get_presets(per_type, model_quality)\n config = get_config(\n problem_type=per_type,\n presets=model_quality,\n extra=[\"matcher\", \"distiller\"],\n )\n for k, v in hyperparameters.items():\n assert OmegaConf.select(config, k) == v\n else:\n hyperparameters, _ = get_presets(per_type, model_quality)\n for per_name, per_hparams in hyperparameters.items():\n config = get_config(\n presets=model_quality,\n extra=[\"matcher\", \"distiller\"],\n overrides=per_hparams,\n )\n for k, v in per_hparams.items():\n assert OmegaConf.select(config, k) == v or (\n OmegaConf.select(config, k) is None and v == \"null\"\n )\n\n\n@pytest.mark.parametrize(\"problem_type\", list(PROBLEM_TYPES_REG.list_keys()))\n@pytest.mark.parametrize(\"presets\", ALL_MODEL_QUALITIES)\ndef test_presets_in_init(problem_type, presets):\n if problem_type != OBJECT_DETECTION:\n predictor = MultiModalPredictor(problem_type=problem_type, presets=presets)\n"
},
{
"path": "multimodal/tests/unittests/others/test_process_images.py",
"content": "import PIL\nimport pytest\nimport torch\nfrom torchvision import transforms\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.data.process_image import ImageProcessor\nfrom autogluon.multimodal.models.timm_image import TimmAutoModelForImagePrediction\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import IDChangeDetectionDataset\n\n\n@pytest.mark.parametrize(\n \"augmentations\",\n [\n {\n \"model.timm_image.train_transforms\": [\"resize_to_square\", \"center_crop\"],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\", \"trivial_augment\"],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"random_horizontal_flip\",\n \"random_vertical_flip\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"affine\",\n \"color_jitter\",\n \"randaug\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"affine(15, (0.1, 0.1), (0.9, 1.1))\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"affine({'degrees': 15,'translate': (0.1, 0.1), 'scale': (0.9, 1.1)})\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"color_jitter(0.2, 0.1, 0.1)\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\n \"resize_shorter_side\",\n \"center_crop\",\n \"color_jitter({'brightness': 0.2, 'contrast': 0.1, 'saturation': 0.1})\",\n ],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n {\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\", \"randaug(2, 7)\"],\n \"model.timm_image.val_transforms\": [\"resize_shorter_side\", \"center_crop\"],\n },\n ],\n)\ndef test_train_transforms(augmentations):\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_df, test_df = shopee_dataset(download_dir)\n image = PIL.Image.open(train_df[\"image\"][0]).convert(\"RGB\")\n\n model = TimmAutoModelForImagePrediction(prefix=\"timm_image\", checkpoint_name=\"swin_tiny_patch4_window7_224\")\n image_processor = ImageProcessor(\n model=model,\n train_transforms=augmentations[\"model.timm_image.train_transforms\"],\n val_transforms=augmentations[\"model.timm_image.val_transforms\"],\n )\n\n transformed_image = image_processor.train_processor(image)\n\n assert (\n len(image_processor.train_processor.transforms) == len(augmentations[\"model.timm_image.train_transforms\"]) + 2\n )\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name,provided_size,expected_size\",\n [\n (\"swinv2_tiny_window8_256\", 300, 256), # SwinTransformer\n (\"convnext_nano\", 300, 300), # ConvNext\n (\"resnet50\", 300, 300), # ResNet\n (\"regnety_004\", 300, 300), # RegNet\n (\"fbnetv3_b\", 300, 300), # MobileNetV3\n (\"tf_efficientnet_b2\", 300, 300),\n ],\n) # EfficientNet\ndef test_variable_input_size_backbone(checkpoint_name, provided_size, expected_size):\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_df, test_df = shopee_dataset(download_dir)\n\n train_transforms = [\"resize_shorter_side\", \"center_crop\", \"trivial_augment\"]\n val_transforms = [\"resize_shorter_side\", \"center_crop\"]\n\n timm_model = TimmAutoModelForImagePrediction(\n prefix=\"timm_image\",\n checkpoint_name=checkpoint_name,\n pretrained=False,\n image_size=provided_size,\n )\n\n image_processor = ImageProcessor(\n model=timm_model,\n train_transforms=train_transforms,\n val_transforms=val_transforms,\n )\n ret = image_processor.process_one_sample(\n {\"image\": train_df[\"image\"].tolist()}, sub_dtypes={\"image\": \"image_path\"}, is_training=True\n )\n assert ret[\"timm_image_image\"].shape[-1] == expected_size\n out = timm_model(\n torch.unsqueeze(ret[timm_model.image_key], dim=0),\n torch.unsqueeze(torch.Tensor(ret[timm_model.image_valid_num_key]), dim=0),\n )\n\n\n@pytest.mark.parametrize(\n \"train_transforms,val_transforms\",\n [\n (\n [\"resize_shorter_side\", \"center_crop\", \"random_horizontal_flip\", \"color_jitter\"],\n [\"resize_shorter_side\", \"center_crop\"],\n ),\n (\n [transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip()],\n [transforms.Resize(256), transforms.CenterCrop(224)],\n ),\n ],\n)\ndef test_customize_predictor_image_aug(train_transforms, val_transforms):\n download_dir = \"./\"\n train_data, test_data = shopee_dataset(download_dir)\n predictor = MultiModalPredictor(label=\"label\", verbosity=4)\n predictor.fit(\n train_data=train_data,\n hyperparameters={\n \"model.timm_image.train_transforms\": train_transforms,\n \"model.timm_image.val_transforms\": val_transforms,\n },\n time_limit=10, # seconds\n )\n\n assert str(train_transforms) == str(predictor._learner._data_processors[\"image\"][0].train_transforms)\n assert str(val_transforms) == str(predictor._learner._data_processors[\"image\"][0].val_transforms)\n assert len(predictor._learner._data_processors[\"image\"][0].train_processor.transforms) == len(train_transforms) + 2\n assert len(predictor._learner._data_processors[\"image\"][0].val_processor.transforms) == len(val_transforms) + 2\n\n\n@pytest.mark.parametrize(\n \"train_transforms,val_transforms\",\n [\n (\n [\"resize_shorter_side\", \"center_crop\", \"random_horizontal_flip\", \"color_jitter\"],\n [\"resize_shorter_side\", \"center_crop\"],\n ),\n (\n [transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip()],\n [transforms.Resize(256), transforms.CenterCrop(224)],\n ),\n ],\n)\ndef test_customize_matcher_image_aug(train_transforms, val_transforms):\n dataset = IDChangeDetectionDataset()\n\n matcher = MultiModalPredictor(\n query=\"Previous Image\",\n response=\"Current Image\",\n problem_type=\"image_similarity\",\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n hyperparameters={\n \"model.timm_image.train_transforms\": train_transforms,\n \"model.timm_image.val_transforms\": val_transforms,\n },\n verbosity=4,\n )\n\n matcher.fit(\n train_data=dataset.train_df,\n tuning_data=dataset.val_df if hasattr(dataset, \"val_df\") else None,\n time_limit=10, # seconds\n )\n\n assert str(train_transforms) == str(matcher._learner._query_processors[\"image\"][0].train_transforms)\n assert str(train_transforms) == str(matcher._learner._response_processors[\"image\"][0].train_transforms)\n assert str(val_transforms) == str(matcher._learner._query_processors[\"image\"][0].val_transforms)\n assert str(val_transforms) == str(matcher._learner._response_processors[\"image\"][0].val_transforms)\n assert len(matcher._learner._query_processors[\"image\"][0].train_processor.transforms) == len(train_transforms) + 2\n assert len(matcher._learner._query_processors[\"image\"][0].val_processor.transforms) == len(val_transforms) + 2\n"
},
{
"path": "multimodal/tests/unittests/others/test_process_multimodal.py",
"content": "import os\nimport shutil\nimport tempfile\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import BEST\n\nfrom ..utils import PetFinderDataset, verify_predictor_save_load\n\n\ndef test_mixup():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"optim.top_k_average_method\": BEST,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"data.mixup.turn_on\": True,\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n \"model.timm_image.checkpoint_name\": \"mobilenetv3_small_100\",\n }\n\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n time_limit=10,\n save_path=save_path,\n hyperparameters=hyperparameters,\n )\n\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n\ndef test_trivialaugment():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"optim.top_k_average_method\": BEST,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"data.mixup.turn_on\": True,\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n \"model.hf_text.text_trivial_aug_maxscale\": 0.1,\n \"model.hf_text.text_aug_detect_length\": 10,\n \"model.timm_image.checkpoint_name\": \"mobilenetv3_small_100\",\n \"model.timm_image.train_transforms\": [\"resize_shorter_side\", \"center_crop\", \"trivial_augment\"],\n }\n\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n time_limit=30,\n save_path=save_path,\n hyperparameters=hyperparameters,\n )\n\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others/test_process_text.py",
"content": "import copy\nimport os\nimport pickle\nimport shutil\nimport tempfile\n\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.data.process_text import TextProcessor\n\nfrom ..utils import AEDataset\n\n\ndef test_text_processor_deepcopy_and_dump():\n dataset = AEDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n \"model.hf_text.text_trivial_aug_maxscale\": 0.05,\n \"model.hf_text.text_train_augment_types\": [\"identity\"],\n \"optim.top_k_average_method\": \"uniform_soup\",\n }\n\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n time_limit=10,\n save_path=save_path,\n hyperparameters=hyperparameters,\n )\n\n # Deepcopy data processors\n predictor._learner._data_processors = copy.deepcopy(predictor._learner._data_processors)\n\n # Test copied data processors\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n\n # Copy data processors via pickle + load\n predictor._learner._data_processors = pickle.loads(pickle.dumps(predictor._learner._data_processors))\n\n # Test copied data processors\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n\n\n@pytest.mark.parametrize(\n \"lengths,max_length,do_merge,gt_trimmed_lengths\",\n [\n ([7, 8, 9], 29, True, [7, 8, 9]),\n ([7, 8, 9], 24, True, [7, 8, 9]),\n ([7, 8, 9], 23, True, [7, 8, 8]),\n ([7, 8, 9], 22, True, [7, 8, 7]),\n ([7, 8, 9], 21, True, [7, 7, 7]),\n ([7, 8, 9], 20, True, [7, 7, 6]),\n ([7, 8, 9], 19, True, [7, 6, 6]),\n ([7, 8, 9], 18, True, [6, 6, 6]),\n ([7, 8, 9], 10, False, [7, 8, 9]),\n ([7, 8, 9], 9, False, [7, 8, 9]),\n ([7, 8, 9], 8, False, [7, 8, 8]),\n ([7, 8, 9], 7, False, [7, 7, 7]),\n ([7, 8, 9], 6, False, [6, 6, 6]),\n ],\n)\ndef test_trim_token_sequence(lengths, max_length, do_merge, gt_trimmed_lengths):\n trimmed_lengths = TextProcessor.get_trimmed_lengths(\n lengths=lengths,\n max_length=max_length,\n do_merge=do_merge,\n )\n assert trimmed_lengths == gt_trimmed_lengths\n"
},
{
"path": "multimodal/tests/unittests/others/test_registry.py",
"content": "from autogluon.multimodal.utils.registry import Registry\n\n\ndef test_registry():\n MODEL_REGISTRY = Registry(\"MODEL\")\n\n @MODEL_REGISTRY.register()\n class MyModel:\n def __init__(self, a, b):\n self.a = a\n self.b = b\n\n @MODEL_REGISTRY.register()\n def my_model():\n return\n\n @MODEL_REGISTRY.register(\"test_class\")\n class MyModelWithNickName:\n def __init__(self, a, b, c):\n self.a = a\n self.b = b\n self.c = c\n\n @MODEL_REGISTRY.register(\"test_function\")\n def my_model_with_nick_name():\n return\n\n class MyModel2:\n pass\n\n MODEL_REGISTRY.register(MyModel2)\n MODEL_REGISTRY.register(\"my_model2\", MyModel2)\n assert MODEL_REGISTRY.list_keys() == [\n \"MyModel\",\n \"my_model\",\n \"test_class\",\n \"test_function\",\n \"MyModel2\",\n \"my_model2\",\n ]\n model = MODEL_REGISTRY.create(\"MyModel\", 1, 2)\n assert model.a == 1 and model.b == 2\n model = MODEL_REGISTRY.create(\"MyModel\", a=2, b=3)\n assert model.a == 2 and model.b == 3\n model = MODEL_REGISTRY.create_with_json(\"MyModel\", \"[4, 5]\")\n assert model.a == 4 and model.b == 5\n model = MODEL_REGISTRY.create_with_json(\"test_class\", '{\"a\": 100, \"b\": 200, \"c\": 300}')\n assert model.a == 100 and model.b == 200 and model.c == 300\n assert MODEL_REGISTRY.get(\"test_class\") == MyModelWithNickName\n"
},
{
"path": "multimodal/tests/unittests/others/test_save.py",
"content": "import os\nimport shutil\n\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\n\nfrom ..utils import PetFinderDataset\n\n\n@pytest.mark.parametrize(\n \"save_path\",\n [\n \"an_empty_existing_path\",\n \"~/an_empty_existing_path\",\n os.path.join(os.path.expanduser(\"~\"), \"an_empty_existing_path\"),\n ],\n)\ndef test_existing_save_path_but_empty_folder(save_path):\n dataset = PetFinderDataset()\n hyperparameters = {\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"model.names\": [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n }\n\n abs_path = os.path.abspath(os.path.expanduser(save_path))\n if os.path.exists(abs_path):\n shutil.rmtree(abs_path)\n os.makedirs(abs_path, exist_ok=True)\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n path=save_path,\n )\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n\n shutil.rmtree(abs_path)\n os.makedirs(abs_path)\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n save_path=save_path,\n time_limit=10,\n )\n\n shutil.rmtree(abs_path)\n\n\n@pytest.mark.parametrize(\n \"save_path\",\n [\n \"an_existing_path\",\n \"~/an_existing_path\",\n os.path.join(os.path.expanduser(\"~\"), \"an_existing_path\"),\n ],\n)\ndef test_existing_save_path_with_content_inside(save_path):\n dataset = PetFinderDataset()\n hyperparameters = {\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"model.names\": [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n }\n\n abs_path = os.path.abspath(os.path.expanduser(save_path))\n if os.path.exists(abs_path):\n shutil.rmtree(abs_path)\n os.makedirs(abs_path, exist_ok=True)\n dummy_file_path = os.path.join(abs_path, \"dummy.txt\")\n with open(dummy_file_path, \"w\") as f:\n f.write(\"dummy\")\n\n predictor = MultiModalPredictor(path=save_path)\n with pytest.raises(ValueError):\n predictor.fit(train_data=dataset.train_df, hyperparameters=hyperparameters)\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n with pytest.raises(ValueError):\n predictor.fit(train_data=dataset.train_df, hyperparameters=hyperparameters, save_path=save_path)\n\n shutil.rmtree(abs_path)\n\n\ndef test_continuous_training_save_path():\n save_path = \"a_tmp_path\"\n abs_path = os.path.abspath(os.path.expanduser(save_path))\n if os.path.exists(abs_path):\n shutil.rmtree(abs_path)\n\n dataset = PetFinderDataset()\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n hyperparameters = {\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"model.names\": [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n }\n predictor.fit(train_data=dataset.train_df, save_path=save_path, hyperparameters=hyperparameters, time_limit=10)\n\n assert predictor.path == abs_path\n\n with pytest.raises(ValueError):\n predictor.fit(train_data=dataset.train_df, save_path=save_path, time_limit=10)\n\n # continue training\n predictor.fit(train_data=dataset.train_df, time_limit=10)\n assert predictor.path != abs_path and os.path.join(\"AutogluonModels\", \"ag-\") in predictor.path\n\n # load a saved predictor and continue training\n predictor_loaded = MultiModalPredictor.load(save_path)\n predictor_loaded.fit(train_data=dataset.train_df, time_limit=10)\n assert predictor_loaded.path != abs_path and os.path.join(\"AutogluonModels\", \"ag-\") in predictor.path\n"
},
{
"path": "multimodal/tests/unittests/others/test_semantic_segmentation.py",
"content": "import os\nimport shutil\nimport uuid\n\nimport numpy as np\nimport numpy.testing as npt\nimport pandas as pd\nimport pytest\nimport torch\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\n\n\n# modify the load_sem_seg function in detectron2\ndef file2id(folder_path, file_path, split_str=\"_\"):\n image_id = os.path.normpath(os.path.relpath(file_path, start=folder_path))\n if split_str in image_id:\n image_id = os.path.splitext(image_id)[0].split(split_str)[0]\n else:\n image_id = os.path.splitext(image_id)[0]\n return image_id\n\n\ndef get_file_paths(directory, split_str=\"_\"):\n file_paths = sorted(os.listdir(directory), key=lambda file_path: file2id(directory, file_path, split_str))\n return [os.path.join(directory, file_path) for file_path in file_paths]\n\n\ndef download_binary_semantic_seg_sample_dataset():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/unit-tests/tiny_isic2017.zip\"\n download_dir = \"./tiny_isic2017\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"tiny_isic2017\")\n\n return data_dir\n\n\ndef download_multi_semantic_seg_sample_dataset():\n zip_file = \"https://automl-mm-bench.s3.amazonaws.com/unit-tests/tiny_trans10kcls12.zip\"\n download_dir = \"./tiny_trans10kcls12\"\n\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n data_dir = os.path.join(download_dir, \"tiny_trans10kcls12\")\n\n return data_dir\n\n\ndef get_file_df_binary_semantic_seg(need_test_gt=False):\n data_dir = download_binary_semantic_seg_sample_dataset()\n train_img_files = get_file_paths(os.path.join(data_dir, \"train/ISIC-2017_Train\"))\n train_gt_files = get_file_paths(os.path.join(data_dir, \"train/ISIC-2017_Training_Part1_GroundTruth\"))\n val_img_files = get_file_paths(os.path.join(data_dir, \"val/ISIC-2017_Val\"))\n val_gt_files = get_file_paths(os.path.join(data_dir, \"val/ISIC-2017_Validation_Part1_GroundTruth\"))\n test_img_files = get_file_paths(os.path.join(data_dir, \"test/ISIC-2017_Test\"))\n test_gt_files = get_file_paths(os.path.join(data_dir, \"test/ISIC-2017_Test_v2_Part1_GroundTruth\"))\n\n train_df = pd.DataFrame({\"image\": train_img_files, \"label\": train_gt_files})\n val_df = pd.DataFrame({\"image\": val_img_files, \"label\": val_gt_files})\n if need_test_gt:\n test_df = pd.DataFrame({\"image\": test_img_files, \"label\": test_gt_files})\n else:\n test_df = pd.DataFrame({\"image\": test_img_files})\n return train_df, val_df, test_df\n\n\ndef get_file_df_multi_semantic_seg(need_test_gt=False):\n data_dir = download_multi_semantic_seg_sample_dataset()\n train_img_files = get_file_paths(os.path.join(data_dir, \"train/images\"))\n train_gt_files = get_file_paths(os.path.join(data_dir, \"train/masks_12\"))\n val_img_files = get_file_paths(os.path.join(data_dir, \"validation/images\"))\n val_gt_files = get_file_paths(os.path.join(data_dir, \"validation/masks_12\"))\n test_img_files = get_file_paths(os.path.join(data_dir, \"test/images\"))\n test_gt_files = get_file_paths(os.path.join(data_dir, \"test/masks_12\"))\n\n train_df = pd.DataFrame({\"image\": train_img_files, \"label\": train_gt_files})\n val_df = pd.DataFrame({\"image\": val_img_files, \"label\": val_gt_files})\n if need_test_gt:\n test_df = pd.DataFrame({\"image\": test_img_files, \"label\": test_gt_files})\n else:\n test_df = pd.DataFrame({\"image\": test_img_files})\n return train_df, val_df, test_df\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name,peft\", [(\"facebook/sam-vit-base\", \"conv_lora\"), (\"facebook/sam-vit-base\", \"lora\")]\n)\ndef test_sam_semantic_segmentation_isic_fit_eval_predict_save_load(checkpoint_name, peft):\n # Binary semantic segmentation\n train_df, val_df, test_df = get_file_df_binary_semantic_seg(need_test_gt=True)\n\n validation_metric = \"iou\"\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters={\n \"model.sam.checkpoint_name\": checkpoint_name,\n \"optim.peft\": peft,\n },\n label=\"label\",\n sample_data_path=train_df,\n )\n\n # Fit\n predictor.fit(train_data=train_df, tuning_data=val_df, time_limit=20)\n\n # Evaluation\n predictor.evaluate(test_df, metrics=[validation_metric])\n\n # Predict, save and load\n verify_predictor_save_load_for_semantic_seg(predictor, test_df, as_multiclass=False)\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"facebook/sam-vit-base\",\n ],\n)\ndef test_sam_semantic_segmentation_zero_shot_evaluate_predict(checkpoint_name):\n _, _, test_df = get_file_df_binary_semantic_seg(need_test_gt=True)\n\n validation_metric = \"iou\"\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters={\n \"model.sam.checkpoint_name\": checkpoint_name,\n },\n label=\"label\",\n sample_data_path=test_df,\n )\n\n # Evaluate\n predictor.evaluate(test_df, metrics=[\"iou\"])\n\n # Predict\n predictor.predict(test_df, save_results=False)\n\n # Predict without ground truth\n _, _, test_df = get_file_df_binary_semantic_seg(need_test_gt=False)\n predictor.predict(test_df, save_results=False)\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name,peft\", [(\"facebook/sam-vit-base\", \"lora\"), (\"facebook/sam-vit-base\", \"conv_lora\")]\n)\ndef test_sam_semantic_segmentation_trans10k_fit_eval_predict_save_load(checkpoint_name, peft):\n # Multi-class semantic segmentation\n train_df, val_df, test_df = get_file_df_multi_semantic_seg(need_test_gt=True)\n\n validation_metric = \"iou\"\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters={\n \"env.precision\": 32,\n \"model.sam.checkpoint_name\": checkpoint_name,\n \"optim.loss_func\": \"mask2former_loss\",\n \"optim.peft\": peft,\n \"model.sam.num_mask_tokens\": 10,\n },\n label=\"label\",\n sample_data_path=train_df,\n )\n\n # Fit\n predictor.fit(train_data=train_df, tuning_data=val_df, time_limit=20)\n\n # Evaluation\n predictor.evaluate(test_df, metrics=[validation_metric])\n\n # Predict, save and load\n verify_predictor_save_load_for_semantic_seg(predictor, test_df, as_multiclass=True)\n\n\ndef verify_predictor_save_load_for_semantic_seg(predictor, df, as_multiclass, cls=MultiModalPredictor):\n root = str(uuid.uuid4())\n os.makedirs(root, exist_ok=True)\n predictor.save(root)\n predictions = predictor.predict(df, as_pandas=False)\n # Test fit_summary()\n predictor.fit_summary()\n\n loaded_predictor = cls.load(root)\n # Test fit_summary()\n loaded_predictor.fit_summary()\n\n predictions2 = loaded_predictor.predict(df, as_pandas=False)\n for prediction, prediction2 in zip(predictions, predictions2):\n npt.assert_equal(prediction, prediction2)\n\n predictions_prob = predictor.predict_proba(df, as_pandas=False, as_multiclass=as_multiclass)\n predictions2_prob = loaded_predictor.predict_proba(df, as_pandas=False, as_multiclass=as_multiclass)\n for prediction_prob, prediction2_prob in zip(predictions_prob, predictions2_prob):\n npt.assert_equal(prediction_prob, prediction2_prob)\n\n shutil.rmtree(root)\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"facebook/sam-vit-base\",\n ],\n)\ndef test_sam_semantic_segmentation_get_class_num_func(checkpoint_name):\n train_df, _, _ = get_file_df_multi_semantic_seg(need_test_gt=True)\n\n validation_metric = \"iou\"\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters={\n \"env.precision\": 32,\n \"model.sam.checkpoint_name\": checkpoint_name,\n \"optim.loss_func\": \"mask2former_loss\",\n \"model.sam.num_mask_tokens\": 10,\n },\n label=\"label\",\n )\n\n get_class_num_func = predictor._learner.get_semantic_segmentation_class_num\n num_classes = 11 # the true number of classes within the provided data\n\n # pd.DataFrame as input\n assert num_classes == get_class_num_func(train_df)\n # file path as input\n assert num_classes == get_class_num_func(train_df[\"label\"][2]) # tiny_trans10kcls12/train/masks_12/2492_mask.png\n # file directory path as input\n assert num_classes == get_class_num_func(os.path.dirname(train_df[\"label\"][0]))\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"frozen_layers\",\n [\n [\"prompt_encoder\"],\n [\"vision_encoder\"],\n ],\n)\ndef test_sam_semantic_segmentation_lora_insert(frozen_layers):\n _, _, test_df = get_file_df_binary_semantic_seg(need_test_gt=True)\n # SAM's vision encoder has query and value linear layers, while the prompt encoder does not.\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n hyperparameters={\n \"model.sam.checkpoint_name\": \"facebook/sam-vit-base\",\n \"model.sam.frozen_layers\": frozen_layers,\n },\n label=\"label\",\n )\n # Evaluate\n predictor.evaluate(test_df, metrics=[\"iou\"])\n model = predictor._learner._model\n assert hasattr(model, \"frozen_layers\") and len(model.frozen_layers) > 0\n for k, v in model.named_parameters():\n for filter_layer in model.frozen_layers:\n if filter_layer in k:\n assert \"lora\" not in k\n\n\n# TODO: Pytest does not support DDP\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"peft_method\",\n [\n \"bit_fit\",\n \"norm_fit\",\n ],\n)\ndef test_sam_semantic_segmentation_non_additive_peft_methods(peft_method):\n # Binary semantic segmentation\n train_df, val_df, test_df = get_file_df_binary_semantic_seg(need_test_gt=True)\n\n validation_metric = \"iou\"\n predictor = MultiModalPredictor(\n problem_type=\"semantic_segmentation\",\n validation_metric=validation_metric,\n eval_metric=validation_metric,\n hyperparameters={\n \"model.sam.checkpoint_name\": \"facebook/sam-vit-base\",\n \"optim.peft\": peft_method,\n },\n label=\"label\",\n sample_data_path=train_df,\n )\n\n # Fit\n predictor.fit(train_data=train_df, tuning_data=val_df, time_limit=20)\n\n # Evaluation\n predictor.evaluate(test_df, metrics=[validation_metric])\n\n # Predict, save and load\n verify_predictor_save_load_for_semantic_seg(predictor, test_df, as_multiclass=False)\n"
},
{
"path": "multimodal/tests/unittests/others_2/__init__.py",
"content": ""
},
{
"path": "multimodal/tests/unittests/others_2/test_classify_doc.py",
"content": "import os\nimport shutil\nimport tempfile\nfrom unittest import mock\n\nimport numpy.testing as npt\nimport pandas as pd\nimport pytest\nimport torch\n\nfrom autogluon.core.utils.loaders import load_zip\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import path_expander\n\nfrom ..utils import get_home_dir\n\nDOC_PATH_COL = \"doc_path\"\n\n\ndef get_rvl_cdip_sample_data(\n zip_file=\"https://automl-mm-bench.s3.amazonaws.com/doc_classification/rvl_cdip_sample.zip\",\n dataset_folder=\"rvl_cdip_sample\",\n dataset_file_name=\"rvl_cdip_train_data.csv\",\n download_dir=\"./ag_automm_doc_image\",\n):\n if os.path.exists(download_dir):\n shutil.rmtree(download_dir)\n load_zip.unzip(zip_file, unzip_dir=download_dir)\n\n dataset_path = os.path.join(download_dir, dataset_folder)\n rvl_cdip_data = pd.read_csv(f\"{dataset_path}/{dataset_file_name}\")\n train_data = rvl_cdip_data.sample(frac=0.8, random_state=200)\n test_data = rvl_cdip_data.drop(train_data.index)\n\n train_data[DOC_PATH_COL] = train_data[DOC_PATH_COL].apply(lambda ele: path_expander(ele, base_folder=download_dir))\n test_data[DOC_PATH_COL] = test_data[DOC_PATH_COL].apply(lambda ele: path_expander(ele, base_folder=download_dir))\n return train_data, test_data\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [(\"google/electra-small-discriminator\"), (\"microsoft/layoutlmv3-base\")],\n)\ndef test_doc_classification(checkpoint_name):\n predictor = MultiModalPredictor(label=\"label\")\n train_data, test_data = get_rvl_cdip_sample_data()\n predictor.fit(\n train_data=train_data,\n hyperparameters={\n \"model.document_transformer.checkpoint_name\": checkpoint_name,\n \"env.num_workers\": 0,\n },\n time_limit=30,\n )\n\n predictor.evaluate(test_data)\n\n doc_path = test_data.iloc[0][DOC_PATH_COL]\n # make predictions\n predictions = predictor.predict({DOC_PATH_COL: [doc_path]})\n # output probability\n proba = predictor.predict_proba({DOC_PATH_COL: [doc_path]})\n # extract embeddings\n feature = predictor.extract_embedding({DOC_PATH_COL: [doc_path]})\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [(\"google/electra-small-discriminator\"), (\"microsoft/layoutlmv3-base\")],\n)\ndef test_pdf_doc_classification(checkpoint_name):\n predictor = MultiModalPredictor(label=\"label\")\n train_data, test_data = get_rvl_cdip_sample_data(\n zip_file=\"https://automl-mm-bench.s3.amazonaws.com/doc_classification/rvl_cdip_sample_pdf.zip\",\n dataset_folder=\"rvl_cdip_sample_pdf\",\n dataset_file_name=\"rvl_cdip_pdf.csv\",\n download_dir=\"./ag_automm_doc_pdf\",\n )\n predictor.fit(\n train_data=train_data,\n hyperparameters={\n \"model.document_transformer.checkpoint_name\": checkpoint_name,\n \"env.per_gpu_batch_size\": 2,\n \"env.num_workers\": 0,\n },\n time_limit=30,\n )\n\n predictor.evaluate(test_data)\n\n doc_path = test_data.iloc[0][DOC_PATH_COL]\n # make predictions\n predictions = predictor.predict({DOC_PATH_COL: [doc_path]})\n # output probability\n proba = predictor.predict_proba({DOC_PATH_COL: [doc_path]})\n # extract embeddings\n feature = predictor.extract_embedding({DOC_PATH_COL: [doc_path]})\n\n\ndef test_doc_classifier_standalone():\n requests_gag = mock.patch(\n \"requests.Session.request\",\n mock.Mock(side_effect=RuntimeError(\"Please use the `responses` library to mock HTTP in your tests.\")),\n )\n predictor = MultiModalPredictor(label=\"label\", verbosity=5)\n train_data, test_data = get_rvl_cdip_sample_data()\n\n save_path = os.path.join(get_home_dir(), \"standalone_doc\", \"false\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor.fit(\n train_data=train_data,\n hyperparameters={\n \"model.document_transformer.checkpoint_name\": \"microsoft/layoutlm-base-uncased\",\n \"env.num_workers\": 0,\n },\n time_limit=150,\n save_path=save_path,\n )\n\n save_path_standalone = os.path.join(get_home_dir(), \"standalone_doc\", \"true\")\n\n predictor.save(\n path=save_path_standalone,\n standalone=True,\n )\n\n del predictor\n torch.cuda.empty_cache()\n\n loaded_online_predictor = MultiModalPredictor.load(path=save_path)\n online_predictions = loaded_online_predictor.predict(test_data, as_pandas=False)\n del loaded_online_predictor\n\n with requests_gag:\n # No internet connection here. If any command require internet connection, a RuntimeError will be raised!\n with tempfile.TemporaryDirectory() as tmpdirname:\n torch.hub.set_dir(tmpdirname) # block reading files in `.cache`.\n loaded_offline_predictor = MultiModalPredictor.load(path=save_path_standalone)\n\n offline_predictions = loaded_offline_predictor.predict(test_data, as_pandas=False)\n del loaded_offline_predictor\n npt.assert_equal(online_predictions[0], offline_predictions[0])\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_config.py",
"content": "import os\n\nimport pytest\nfrom omegaconf import OmegaConf\n\nfrom autogluon.multimodal.constants import DATA, ENV, MODEL, OPTIM\nfrom autogluon.multimodal.utils import apply_omegaconf_overrides, get_basic_config, get_config, parse_dotlist_conf\n\n\ndef test_basic_config():\n basic_config = get_basic_config()\n assert list(basic_config.keys()).sort() == [MODEL, DATA, OPTIM, ENV].sort()\n\n basic_config = get_basic_config()\n assert list(basic_config.keys()).sort() == [MODEL, DATA, OPTIM, ENV].sort()\n\n\ndef test_get_config():\n cur_path = os.path.dirname(os.path.abspath(__file__))\n model_config_path = os.path.join(cur_path, \"../../../src/autogluon/multimodal/configs/model/default.yaml\")\n model_config = OmegaConf.load(model_config_path)\n data_config_path = os.path.join(cur_path, \"../../../src/autogluon/multimodal/configs/data/default.yaml\")\n data_config = OmegaConf.load(data_config_path)\n optim_config_path = os.path.join(cur_path, \"../../../src/autogluon/multimodal/configs/optim/default.yaml\")\n optim_config = OmegaConf.load(optim_config_path)\n environemnt_config_path = os.path.join(cur_path, \"../../../src/autogluon/multimodal/configs/env/default.yaml\")\n environemnt_config = OmegaConf.load(environemnt_config_path)\n config_gt = OmegaConf.merge(model_config, data_config, optim_config, environemnt_config)\n\n # test yaml path\n config = {\n MODEL: model_config_path,\n DATA: data_config_path,\n OPTIM: optim_config_path,\n ENV: environemnt_config_path,\n }\n config = get_config(config=config)\n assert config == config_gt\n\n # test DictConfg\n config = {\n MODEL: model_config,\n DATA: data_config,\n OPTIM: optim_config,\n ENV: environemnt_config,\n }\n config = get_config(config=config)\n assert config == config_gt\n\n # test dict\n model_config = OmegaConf.to_container(model_config)\n assert isinstance(model_config, dict)\n\n data_config = OmegaConf.to_container(data_config)\n assert isinstance(data_config, dict)\n\n optim_config = OmegaConf.to_container(optim_config)\n assert isinstance(optim_config, dict)\n\n environemnt_config = OmegaConf.to_container(environemnt_config)\n assert isinstance(environemnt_config, dict)\n\n config = {\n MODEL: model_config,\n DATA: data_config,\n OPTIM: optim_config,\n ENV: environemnt_config,\n }\n config = get_config(config=config)\n assert config == config_gt\n\n # test default string\n config = {\n MODEL: f\"default\",\n DATA: \"default\",\n OPTIM: \"default\",\n ENV: \"default\",\n }\n config = get_config(config=config)\n assert config == config_gt\n\n\n@pytest.mark.parametrize(\n \"model_names,\",\n [\n [\"timm_image\"],\n [\"hf_text\"],\n [\"clip\"],\n [\"timm_image\", \"hf_text\", \"clip\", \"fusion_mlp\"],\n [\"numerical_mlp\", \"categorical_mlp\", \"hf_text\", \"fusion_mlp\"],\n [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"fusion_mlp\"],\n [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"hf_text\", \"clip\", \"fusion_mlp\"],\n ],\n)\ndef test_model_config_selection(model_names):\n overrides = {\"model.names\": model_names}\n config = get_config(overrides=overrides)\n assert sorted(config.model.names) == sorted(model_names)\n names2 = list(config.model.keys())\n names2.remove(\"names\")\n assert sorted(config.model.names) == sorted(names2)\n\n\n@pytest.mark.parametrize(\n \"model_names,\",\n [\n [\"image\"],\n [\"text\"],\n [\"numerical\"],\n [\"categorical\"],\n ],\n)\ndef test_invalid_model_config_selection(model_names):\n overrides = {\"model.names\": model_names}\n\n with pytest.raises(ValueError):\n config = get_config(overrides=overrides)\n\n\n@pytest.mark.parametrize(\n \"data,expected\",\n [\n (\"aaa=a bbb=b ccc=c\", {\"aaa\": \"a\", \"bbb\": \"b\", \"ccc\": \"c\"}),\n (\"a.a.aa=b b.b.bb=c\", {\"a.a.aa\": \"b\", \"b.b.bb\": \"c\"}),\n (\"a.a.aa=1 b.b.bb=100\", {\"a.a.aa\": \"1\", \"b.b.bb\": \"100\"}),\n ([\"a.a.aa=1\", \"b.b.bb=100\"], {\"a.a.aa\": \"1\", \"b.b.bb\": \"100\"}),\n ],\n)\ndef test_parse_dotlist_conf(data, expected):\n assert parse_dotlist_conf(data) == expected\n\n\ndef test_apply_omegaconf_overrides():\n conf = OmegaConf.from_dotlist([\"a.aa.aaa=[1, 2, 3, 4]\", \"a.aa.bbb=2\", \"a.bb.aaa='100'\", \"a.bb.bbb=4\"])\n overrides = \"a.aa.aaa=[1, 3, 5] a.aa.bbb=3\"\n new_conf = apply_omegaconf_overrides(conf, overrides.split())\n assert new_conf.a.aa.aaa == [1, 3, 5]\n assert new_conf.a.aa.bbb == 3\n new_conf2 = apply_omegaconf_overrides(conf, {\"a.aa.aaa\": [1, 3, 5, 7], \"a.aa.bbb\": 4})\n assert new_conf2.a.aa.aaa == [1, 3, 5, 7]\n assert new_conf2.a.aa.bbb == 4\n\n with pytest.raises(KeyError):\n new_conf3 = apply_omegaconf_overrides(conf, {\"a.aa.aaaaaa\": [1, 3, 5, 7], \"a.aa.bbb\": 4})\n\n\n@pytest.mark.parametrize(\n \"overrides\",\n [\n {\"optim.peft\": \"None\"},\n {\"optim.peft\": \"none\"},\n {\"optim.peft\": \"nOne\"},\n {\"optim.peft\": None},\n {\"data.label.numerical_preprocessing\": None},\n {\"data.label.numerical_preprocessing\": \"none\"},\n ],\n)\ndef test_none_str_config(overrides):\n config = get_config(overrides=overrides)\n if \"optim.peft\" in overrides:\n assert config.optim.peft is None\n if \"data.label.numerical_preprocessing\" in overrides:\n assert config.data.label.numerical_preprocessing is None\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_custom_hparams.py",
"content": "import copy\nimport os\nimport shutil\nimport tempfile\n\nimport pytest\nfrom omegaconf import OmegaConf\nfrom torchvision import transforms\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import (\n BINARY,\n CATEGORICAL,\n CLASSIFICATION,\n DATA,\n IMAGE_PATH,\n MODEL,\n MULTICLASS,\n NUMERICAL,\n REGRESSION,\n TEXT,\n)\nfrom autogluon.multimodal.utils import (\n filter_hyperparameters,\n get_default_config,\n shopee_dataset,\n split_hyperparameters,\n update_ensemble_hyperparameters,\n)\n\nfrom ..utils import PetFinderDataset, get_home_dir, verify_predictor_save_load\n\n\ndef test_hyperparameters_in_terminal_format():\n download_dir = \"./\"\n train_df, tune_df = shopee_dataset(download_dir=download_dir)\n predictor = MultiModalPredictor(label=\"label\")\n hyperparameters = [\n \"model.names=[timm_image]\",\n \"model.timm_image.checkpoint_name=ghostnet_100\",\n \"env.num_workers=0\",\n \"env.num_workers_inference=0\",\n \"optim.top_k_average_method=best\",\n \"optim.val_check_interval=1.0\",\n ]\n predictor.fit(\n train_data=train_df,\n tuning_data=tune_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n )\n\n\n@pytest.mark.parametrize(\n \"hyperparameters\",\n [\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"mobilenetv3_small_100\",\n },\n {\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n },\n {\n \"model.names\": [\"timm_image_haha\", \"hf_text_hello\", \"numerical_mlp_456\", \"fusion_mlp\"],\n \"model.timm_image_haha.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text_hello.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"data.numerical.convert_to_text\": False,\n },\n ],\n)\ndef test_hyperparameters_consistency(hyperparameters):\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n # pass hyperparameters to init()\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n hyperparameters=hyperparameters,\n )\n predictor.fit(dataset.train_df, time_limit=0)\n\n # pass hyperparameters to fit()\n predictor_2 = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n predictor_2.fit(\n dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=0,\n )\n assert predictor._learner._config == predictor_2._learner._config\n\n\n@pytest.mark.parametrize(\n \"hyperparameters\",\n [\n {\n \"model.names\": [\"timm_image_0\", \"timm_image_1\", \"fusion_mlp\"],\n \"model.timm_image_0.checkpoint_name\": \"mobilenetv3_small_100\",\n \"model.timm_image_1.checkpoint_name\": \"mobilenetv3_small_100\",\n },\n {\n \"model.names\": [\"hf_text_abc\", \"hf_text_def\", \"hf_text_xyz\", \"fusion_mlp_123\"],\n \"model.hf_text_def.checkpoint_name\": \"monsoon-nlp/hindi-bert\",\n \"model.hf_text_xyz.checkpoint_name\": \"distilroberta-base\",\n \"model.hf_text_abc.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n },\n {\n \"model.names\": [\"timm_image_haha\", \"hf_text_hello\", \"numerical_mlp_456\", \"fusion_mlp\"],\n \"model.timm_image_haha.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text_hello.checkpoint_name\": \"distilroberta-base\",\n \"data.numerical.convert_to_text\": False,\n },\n ],\n)\ndef test_customize_model_names(\n hyperparameters,\n):\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters.update(\n {\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n )\n hyperparameters_gt = copy.deepcopy(hyperparameters)\n if isinstance(hyperparameters_gt[\"model.names\"], str):\n hyperparameters_gt[\"model.names\"] = OmegaConf.from_dotlist([f\"names={hyperparameters['model.names']}\"]).names\n\n save_path = os.path.join(get_home_dir(), \"outputs\", \"petfinder\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=0,\n save_path=save_path,\n )\n\n assert sorted(predictor._learner._config.model.names) == sorted(hyperparameters_gt[\"model.names\"])\n for per_name in hyperparameters_gt[\"model.names\"]:\n assert hasattr(predictor._learner._config.model, per_name)\n\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # Test for continuous fit\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=0,\n )\n assert sorted(predictor._learner._config.model.names) == sorted(hyperparameters_gt[\"model.names\"])\n for per_name in hyperparameters_gt[\"model.names\"]:\n assert hasattr(predictor._learner._config.model, per_name)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # Saving to folder, loading the saved model and call fit again (continuous fit)\n with tempfile.TemporaryDirectory() as root:\n predictor.save(root)\n predictor = MultiModalPredictor.load(root)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=0,\n )\n assert sorted(predictor._learner._config.model.names) == sorted(hyperparameters_gt[\"model.names\"])\n for per_name in hyperparameters_gt[\"model.names\"]:\n assert hasattr(predictor._learner._config.model, per_name)\n\n\n@pytest.mark.parametrize(\n \"hyperparameters,column_types,model_in_config,fit_called,result\",\n [\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH, \"b\": TEXT, \"c\": NUMERICAL, \"d\": CATEGORICAL},\n None,\n False,\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH},\n None,\n False,\n {\n \"model.names\": [\"timm_image\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n },\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"b\": TEXT},\n None,\n False,\n {\n \"model.names\": [\"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"b\": TEXT, \"c\": NUMERICAL},\n None,\n False,\n {\n \"model.names\": [\"numerical_mlp\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n ),\n (\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH},\n None,\n False,\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n },\n ),\n (\n {\n \"model.names\": [\"hf_text\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"b\": TEXT, \"c\": NUMERICAL, \"d\": CATEGORICAL},\n None,\n False,\n {\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n ),\n (\n {\n \"model.names\": [\"hf_text\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"c\": NUMERICAL, \"d\": CATEGORICAL},\n None,\n False,\n AssertionError,\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH, \"b\": TEXT, \"c\": NUMERICAL, \"d\": CATEGORICAL},\n \"timm_image\",\n False,\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n },\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH, \"b\": TEXT, \"c\": NUMERICAL, \"d\": CATEGORICAL},\n \"numerical_mlp\",\n False,\n {\n \"model.names\": [\"numerical_mlp\"],\n },\n ),\n (\n {\n \"model.names\": [\"categorical_mlp\", \"numerical_mlp\", \"fusion_mlp\"],\n },\n {\"a\": IMAGE_PATH, \"b\": TEXT, \"c\": NUMERICAL, \"d\": CATEGORICAL},\n \"hf_text\",\n False,\n AssertionError,\n ),\n (\n {\n \"model.names\": [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n },\n {\"a\": IMAGE_PATH, \"b\": TEXT},\n None,\n True,\n {},\n ),\n ],\n)\ndef test_filter_hyperparameters(hyperparameters, column_types, model_in_config, fit_called, result):\n if model_in_config:\n config = get_default_config()\n config.model.names = [model_in_config]\n model_keys = list(config.model.keys())\n for key in model_keys:\n if key != \"names\" and key != model_in_config:\n delattr(config.model, key)\n else:\n config = None\n\n if result == AssertionError:\n with pytest.raises(AssertionError):\n filtered_hyperparameters = filter_hyperparameters(\n hyperparameters=hyperparameters,\n column_types=column_types,\n config=config,\n fit_called=fit_called,\n )\n else:\n filtered_hyperparameters = filter_hyperparameters(\n hyperparameters=hyperparameters,\n column_types=column_types,\n config=config,\n fit_called=fit_called,\n )\n\n assert filtered_hyperparameters == result\n\n\n@pytest.mark.parametrize(\n \"train_transforms,val_transforms,empty_advanced_hyperparameters\",\n [\n (\n [\"resize_shorter_side\", \"center_crop\", \"random_horizontal_flip\", \"color_jitter\"],\n [\"resize_shorter_side\", \"center_crop\"],\n True,\n ),\n (\n [transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip()],\n [transforms.Resize(256), transforms.CenterCrop(224)],\n False,\n ),\n ],\n)\ndef test_split_hyperparameters(train_transforms, val_transforms, empty_advanced_hyperparameters):\n hyperparameters = {\n \"model.timm_image.train_transforms\": train_transforms,\n \"model.timm_image.val_transforms\": val_transforms,\n }\n hyperparameters, advanced_hyperparameters = split_hyperparameters(hyperparameters)\n if empty_advanced_hyperparameters:\n assert not advanced_hyperparameters\n else:\n assert advanced_hyperparameters\n\n\n@pytest.mark.parametrize(\n \"provided_hyperparameters\",\n [\n {\n \"learner_names\": [\"early_fusion\", \"lf_mlp\"],\n \"early_fusion\": {\n \"model.meta_transformer.checkpoint_path\": \"local_meta_transformer_path\",\n },\n },\n {\n \"early_fusion\": {\n \"model.meta_transformer.checkpoint_path\": \"local_meta_transformer_path\",\n }\n },\n ],\n)\ndef test_ensemble_hyperparameters(provided_hyperparameters):\n hyperparameters = update_ensemble_hyperparameters(\n presets=None,\n provided_hyperparameters=provided_hyperparameters,\n )\n provided_hyperparameters.pop(\"learner_names\", None)\n for k, v in provided_hyperparameters.items():\n for kk, vv in provided_hyperparameters[k].items():\n assert hyperparameters[k][kk] == provided_hyperparameters[k][kk]\n\n\n@pytest.mark.parametrize(\n \"hyperparameters,key_mappings\",\n [\n (\n {\n \"optimization.learning_rate\": 1e-3,\n \"optimization.efficient_finetune\": \"bit_fit\",\n \"optimization.loss_function\": \"cross_entropy\",\n },\n {\n \"optimization.learning_rate\": \"optim.lr\",\n \"optimization.efficient_finetune\": \"optim.peft\",\n \"optimization.loss_function\": \"optim.loss_func\",\n },\n ),\n (\n {\n \"env.num_workers_evaluation\": 100,\n \"env.eval_batch_size_ratio\": 5,\n },\n {\n \"env.num_workers_evaluation\": \"env.num_workers_inference\",\n \"env.eval_batch_size_ratio\": \"env.inference_batch_size_ratio\",\n },\n ),\n (\n {\n \"data.label.numerical_label_preprocessing\": \"minmaxscaler\",\n },\n {\n \"data.label.numerical_label_preprocessing\": \"data.label.numerical_preprocessing\",\n },\n ),\n (\n {\n \"model.names\": [\"timm_image\", \"numerical_mlp\", \"categorical_mlp\", \"fusion_mlp\"],\n \"model.timm_image.max_img_num_per_col\": 3,\n \"model.categorical_mlp.drop_rate\": 0.5,\n \"model.numerical_mlp.drop_rate\": 0.5,\n \"model.numerical_mlp.d_token\": 16,\n \"model.fusion_mlp.weight\": 0.1,\n \"model.fusion_mlp.drop_rate\": 0.5,\n },\n {\n \"model.names\": \"model.names\",\n \"model.timm_image.max_img_num_per_col\": \"model.timm_image.max_image_num_per_column\",\n \"model.categorical_mlp.drop_rate\": \"model.categorical_mlp.dropout\",\n \"model.numerical_mlp.drop_rate\": \"model.numerical_mlp.dropout\",\n \"model.numerical_mlp.d_token\": \"model.numerical_mlp.token_dim\",\n \"model.fusion_mlp.weight\": \"model.fusion_mlp.aux_loss_weight\",\n \"model.fusion_mlp.drop_rate\": \"model.fusion_mlp.dropout\",\n },\n ),\n (\n {\n \"model.names\": [\"ft_transformer\", \"clip_image\", \"clip_text\", \"fusion_transformer\"],\n \"model.clip_image.data_types\": [\"image\"],\n \"model.clip_text.data_types\": [\"text\"],\n \"model.clip_image.max_img_num_per_col\": 3,\n \"model.fusion_transformer.n_blocks\": 6,\n \"model.fusion_transformer.attention_n_heads\": 16,\n \"model.fusion_transformer.ffn_d_hidden\": 256,\n \"model.ft_transformer.attention_n_heads\": 16,\n },\n {\n \"model.names\": \"model.names\",\n \"model.clip_image.data_types\": \"model.clip_image.data_types\",\n \"model.clip_text.data_types\": \"model.clip_text.data_types\",\n \"model.clip_image.max_img_num_per_col\": \"model.clip_image.max_image_num_per_column\",\n \"model.fusion_transformer.n_blocks\": \"model.fusion_transformer.num_blocks\",\n \"model.fusion_transformer.attention_n_heads\": \"model.fusion_transformer.attention_num_heads\",\n \"model.fusion_transformer.ffn_d_hidden\": \"model.fusion_transformer.ffn_hidden_size\",\n \"model.ft_transformer.attention_n_heads\": \"model.ft_transformer.attention_num_heads\",\n },\n ),\n ],\n)\ndef test_hyperparameter_backward_compatibility(hyperparameters, key_mappings):\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n predictor.fit(\n dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=0,\n )\n for k, v in hyperparameters.items():\n if k == \"model.names\":\n assert sorted(OmegaConf.select(predictor._learner._config, key_mappings[k])) == sorted(v)\n else:\n assert OmegaConf.select(predictor._learner._config, key_mappings[k]) == v\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_custom_training.py",
"content": "import os\nimport shutil\n\nimport numpy.testing as npt\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import BIT_FIT, IA3, IA3_BIAS, IA3_LORA, LORA, LORA_BIAS, LORA_NORM, NORM_FIT\nfrom autogluon.multimodal.models.timm_image import TimmAutoModelForImagePrediction\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import AmazonReviewSentimentCrossLingualDataset, PetFinderDataset\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"backbone,peft,pooling_mode,precision,expected_ratio,standalone\",\n [\n (\"t5-small\", LORA_NORM, \"mean\", \"bf16-mixed\", 0.00557, True),\n (\"google/flan-t5-small\", IA3_LORA, \"mean\", \"bf16-mixed\", 0.006865, True),\n (\"google/flan-t5-small\", IA3, \"cls\", \"bf16-mixed\", 0.0004201, False),\n (\"microsoft/deberta-v3-small\", LORA_BIAS, \"mean\", \"16-mixed\", 0.001422, True),\n (\"microsoft/deberta-v3-small\", LORA, \"cls\", \"16-mixed\", 0.0010533, False),\n ],\n)\ndef test_gradient_checkpointing(backbone, peft, pooling_mode, precision, expected_ratio, standalone):\n dataset = AmazonReviewSentimentCrossLingualDataset()\n train_data = dataset.train_df.sample(200)\n test_data = dataset.test_df.sample(50)\n save_path = f\"gradient_checkpointing_{backbone}_{peft}_{pooling_mode}_{precision}\"\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(label=dataset.label_columns[0], path=save_path)\n predictor.fit(\n train_data,\n standalone=standalone,\n hyperparameters={\n \"model.names\": [\"hf_text\"],\n \"model.hf_text.checkpoint_name\": backbone,\n \"model.hf_text.pooling_mode\": pooling_mode,\n \"model.hf_text.gradient_checkpointing\": True,\n \"optim.peft\": peft,\n \"optim.lr_decay\": 1.0,\n \"optim.lr\": 1e-03,\n \"optim.max_epochs\": 1,\n \"env.precision\": precision,\n \"env.per_gpu_batch_size\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"env.num_gpus\": -1,\n },\n time_limit=30,\n )\n predictions = predictor.predict(test_data, as_pandas=False)\n tunable_ratio = predictor.trainable_parameters / predictor.total_parameters\n npt.assert_allclose(tunable_ratio, expected_ratio, 2e-05, 2e-05)\n save_path = save_path + \"_new\"\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor.save(save_path, standalone=standalone)\n new_predictor = MultiModalPredictor.load(save_path)\n new_predictions = new_predictor.predict(test_data, as_pandas=False)\n npt.assert_allclose(new_predictions, predictions)\n\n\ndef test_skip_final_val():\n download_dir = \"./\"\n save_path = \"petfinder_checkpoint\"\n train_df, tune_df = shopee_dataset(download_dir=download_dir)\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(label=\"label\", path=save_path)\n hyperparameters = {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"ghostnet_100\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": \"best\",\n \"optim.val_check_interval\": 1.0,\n \"optim.skip_final_val\": True,\n }\n predictor.fit(\n train_data=train_df,\n tuning_data=tune_df,\n hyperparameters=hyperparameters,\n time_limit=5,\n )\n predictor_new = MultiModalPredictor.load(path=save_path)\n assert isinstance(predictor_new._learner._model, TimmAutoModelForImagePrediction)\n\n\ndef test_fit_with_data_path():\n download_dir = \"./\"\n train_csv_file = \"shopee_train_data.csv\"\n train_data, _ = shopee_dataset(download_dir=download_dir)\n train_data.to_csv(train_csv_file)\n predictor = MultiModalPredictor(label=\"label\")\n predictor.fit(train_data=train_csv_file, time_limit=0)\n predictor.fit(train_data=train_csv_file, tuning_data=train_csv_file, time_limit=0)\n\n\ndef test_train_with_cpu_only():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"ft_transformer\"],\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n \"env.accelerator\": \"cpu\",\n }\n predictor.fit(\n dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n predictor.evaluate(dataset.test_df)\n predictor.predict(dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_distiller.py",
"content": "import os\nimport shutil\n\nfrom autogluon.multimodal import MultiModalPredictor\n\nfrom ..utils import PetFinderDataset, get_home_dir, verify_predictor_save_load\n\n\ndef test_distillation():\n dataset = PetFinderDataset()\n\n teacher_hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"hf_text\", \"timm_image\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n # test for distillation with different model structures\n student_hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"hf_text\", \"timm_image\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"distiller.temperature\": 5.0,\n \"distiller.hard_label_weight\": 0.1,\n \"distiller.soft_label_weight\": 1.0,\n \"distiller.output_feature_loss_weight\": 0.01,\n \"distiller.rkd_distance_loss_weight\": 0.01,\n \"distiller.rkd_angle_loss_weight\": 0.02,\n \"distiller.output_feature_loss_type\": \"mse\",\n }\n\n teacher_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n teacher_save_path = os.path.join(get_home_dir(), \"output\", \"petfinder\", \"teacher\")\n if os.path.exists(teacher_save_path):\n shutil.rmtree(teacher_save_path)\n\n teacher_predictor = teacher_predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=teacher_hyperparameters,\n time_limit=10,\n save_path=teacher_save_path,\n )\n\n # test for distillation\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n student_save_path = os.path.join(get_home_dir(), \"output\", \"petfinder\", \"student\")\n if os.path.exists(student_save_path):\n shutil.rmtree(student_save_path)\n\n predictor = predictor.fit(\n train_data=dataset.train_df,\n teacher_predictor=teacher_predictor,\n hyperparameters=student_hyperparameters,\n time_limit=10,\n save_path=student_save_path,\n )\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # test for distillation with teacher predictor path\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n student_save_path = os.path.join(get_home_dir(), \"output\", \"petfinder\", \"student\")\n if os.path.exists(student_save_path):\n shutil.rmtree(student_save_path)\n\n predictor = predictor.fit(\n train_data=dataset.train_df,\n teacher_predictor=teacher_predictor.path,\n hyperparameters=student_hyperparameters,\n time_limit=10,\n save_path=student_save_path,\n )\n\n verify_predictor_save_load(predictor, dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_few_shot.py",
"content": "import os\nimport shutil\nimport tempfile\nfrom unittest import mock\n\nimport numpy.testing as npt\nimport pandas as pd\nimport pytest\nimport torch\nfrom omegaconf import OmegaConf\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import BINARY, FEW_SHOT_CLASSIFICATION, MULTICLASS\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import (\n get_home_dir,\n verify_predict_and_predict_proba,\n verify_predict_as_pandas_and_multiclass,\n verify_predict_without_label_column,\n verify_predictor_realtime_inference,\n verify_predictor_save_load,\n)\n\n\n@pytest.mark.single_gpu\ndef test_few_shot_svm_fit_predict():\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n save_path = f\"./tmp/automm_stanfordcars-8shot-en\"\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters={\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"model.clip.image_size\": 224,\n },\n path=save_path,\n )\n predictor.fit(train_data)\n verify_predictor_save_load(predictor, test_data, verify_embedding=True)\n verify_predictor_realtime_inference(predictor, test_data, verify_embedding=True)\n verify_predict_without_label_column(test_data, predictor)\n verify_predict_and_predict_proba(test_data, predictor)\n verify_predict_as_pandas_and_multiclass(test_data, predictor)\n\n\ndef test_few_shot_svm_save_load():\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n save_path = f\"./tmp/automm_stanfordcars-8shot-en\"\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters={\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"model.clip.image_size\": 224,\n },\n path=save_path,\n )\n predictor.fit(train_data)\n results = predictor.evaluate(test_data)\n preds = predictor.predict(test_data.drop(columns=[\"label\"], axis=1))\n predictor2 = MultiModalPredictor.load(save_path)\n results2 = predictor2.evaluate(test_data)\n preds2 = predictor2.predict(test_data.drop(columns=[\"label\"], axis=1))\n assert results == results2\n assert (preds == preds2).all()\n predictor2.fit(train_data)\n\n\n@pytest.mark.parametrize(\n \"hyperparameters,gt_ckpt_name,gt_model_name\",\n [\n (\n {\n \"model.names\": [\"hf_text\", \"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n },\n \"swin_small_patch4_window7_224\",\n [\"timm_image\"],\n ),\n (\n {\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n },\n \"swin_small_patch4_window7_224\",\n [\"timm_image\"],\n ),\n (\n {\n \"model.names\": [\"clip\", \"timm_image\"],\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n },\n \"swin_small_patch4_window7_224\",\n [\"timm_image\"],\n ),\n (\n {\n \"model.names\": [\"clip\", \"hf_text\"],\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n },\n \"openai/clip-vit-base-patch32\",\n [\"clip\"],\n ),\n ],\n)\ndef test_few_shot_customize_models(hyperparameters, gt_ckpt_name, gt_model_name):\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters=hyperparameters,\n )\n predictor.fit(train_data)\n assert predictor._learner._config.model.names == gt_model_name\n assert OmegaConf.select(predictor._learner._config.model, f\"{gt_model_name[0]}.checkpoint_name\") == gt_ckpt_name\n\n\ndef test_one_shot_two_classes():\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters={\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"model.clip.image_size\": 224,\n },\n )\n train_data = train_data.groupby(\"label\").sample(n=1)[:2]\n test_data = test_data.groupby(\"label\").sample(n=1)[:2]\n assert len(train_data) == 2 and len(test_data) == 2\n assert len(train_data[\"label\"].unique()) == 2 and len(test_data[\"label\"].unique()) == 2\n predictor.fit(train_data)\n score = predictor.evaluate(test_data)\n pred = predictor.predict(test_data.drop(columns=[\"label\"], axis=1))\n prob = predictor.predict_proba(test_data.drop(columns=[\"label\"], axis=1))\n embedding = predictor.extract_embedding(test_data.drop(columns=[\"label\"], axis=1))\n\n\n@pytest.mark.parametrize(\n \"column_features_pooling_mode\",\n [\"concat\", \"mean\"],\n)\ndef test_few_shot_multi_columns(column_features_pooling_mode):\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n train_data = pd.concat([train_data[\"image\"]] * 3 + [train_data[\"label\"]], axis=1, ignore_index=True)\n train_data.rename(\n dict(zip(train_data.columns, [\"image_1\", \"image_2\", \"image_3\", \"label\"])),\n axis=1,\n inplace=True,\n )\n test_data = pd.concat([test_data[\"image\"]] * 3 + [test_data[\"label\"]], axis=1, ignore_index=True)\n test_data.rename(\n dict(zip(test_data.columns, [\"image_1\", \"image_2\", \"image_3\", \"label\"])),\n axis=1,\n inplace=True,\n )\n assert len(train_data.columns) == 4 and len(test_data.columns) == 4\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters={\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"model.clip.image_size\": 224,\n \"data.column_features_pooling_mode\": column_features_pooling_mode,\n },\n )\n predictor.fit(train_data)\n score = predictor.evaluate(test_data)\n pred = predictor.predict(test_data.drop(columns=[\"label\"], axis=1))\n proba = predictor.predict_proba(test_data.drop(columns=[\"label\"], axis=1))\n embedding = predictor.extract_embedding(test_data.drop(columns=[\"label\"], axis=1))\n\n\ndef test_few_shot_standalone(): # test standalone feature in MultiModalPredictor.save()\n requests_gag = mock.patch(\n \"requests.Session.request\",\n mock.Mock(side_effect=RuntimeError(\"Please use the `responses` library to mock HTTP in your tests.\")),\n )\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n predictor = MultiModalPredictor(\n label=\"label\",\n problem_type=FEW_SHOT_CLASSIFICATION,\n hyperparameters={\n \"model.clip.checkpoint_name\": \"openai/clip-vit-base-patch32\",\n \"model.clip.image_size\": 224,\n },\n )\n save_path = os.path.join(get_home_dir(), \"outputs\", \"few_shot_standalone\", \"true\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=train_data,\n save_path=save_path,\n standalone=True,\n )\n predictions = predictor.predict(test_data, as_pandas=False)\n torch.cuda.empty_cache()\n # Check if the predictor can be loaded from an offline environment.\n with requests_gag:\n # No internet connection here. If any command require internet connection, a RuntimeError will be raised.\n with tempfile.TemporaryDirectory() as tmpdirname:\n torch.hub.set_dir(tmpdirname) # block reading files in `.cache`\n loaded_offline_predictor = MultiModalPredictor.load(path=save_path)\n\n offline_predictions = loaded_offline_predictor.predict(test_data, as_pandas=False)\n npt.assert_equal(predictions, offline_predictions)\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_hpo.py",
"content": "import os\nimport shutil\n\nimport pytest\nfrom ray import tune\nfrom torch import nn\n\nfrom autogluon.core.hpo.ray_tune_constants import SCHEDULER_PRESETS, SEARCHER_PRESETS\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import ALL_MODEL_QUALITIES\nfrom autogluon.multimodal.models import modify_duplicate_model_names\nfrom autogluon.multimodal.utils import filter_search_space\n\nfrom ..utils import (\n IDChangeDetectionDataset,\n PetFinderDataset,\n get_home_dir,\n verify_matcher_save_load,\n verify_predictor_save_load,\n)\n\n\ndef predictor_hpo(searcher, scheduler, presets=None):\n dataset = PetFinderDataset()\n\n hyperparameters = {\n \"optim.lr\": tune.uniform(0.0001, 0.01),\n \"optim.max_epochs\": 1,\n \"model.names\": [\"numerical_mlp\", \"categorical_mlp\", \"fusion_mlp\"],\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n hyperparameter_tune_kwargs = {\n \"searcher\": searcher,\n \"scheduler\": scheduler,\n \"num_trials\": 2,\n }\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n presets=presets,\n )\n\n save_path = os.path.join(get_home_dir(), \"outputs\", \"hpo\", f\"_{searcher}\", f\"_{scheduler}\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor_hpo = predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n save_path=save_path,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n assert predictor == predictor_hpo\n\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # test for continuous training\n predictor = predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n\n\ndef matcher_hpo(searcher, scheduler, presets=None):\n dataset = IDChangeDetectionDataset()\n\n hyperparameters = {\n \"optim.lr\": tune.uniform(0.0001, 0.001),\n \"optim.max_epochs\": 1,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": \"greedy_soup\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n }\n\n hyperparameter_tune_kwargs = {\n \"searcher\": searcher,\n \"scheduler\": scheduler,\n \"num_trials\": 2,\n }\n\n matcher = MultiModalPredictor(\n query=\"Previous Image\",\n response=\"Current Image\",\n problem_type=\"image_similarity\",\n label=dataset.label_columns[0] if dataset.label_columns else None,\n match_label=dataset.match_label,\n eval_metric=dataset.metric,\n presets=presets,\n )\n\n save_path = os.path.join(get_home_dir(), \"outputs\", \"hpo\", f\"_{searcher}\", f\"_{scheduler}\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n matcher_hpo = matcher.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n save_path=save_path,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n assert matcher_hpo == matcher\n\n score = matcher.evaluate(dataset.test_df)\n verify_matcher_save_load(matcher, dataset.test_df)\n\n # test for continuous training\n predictor = matcher.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n )\n\n\n@pytest.mark.parametrize(\n \"hyperparameters, keys_to_filter, expected\",\n [\n ({\"model.abc\": tune.choice([\"a\", \"b\"])}, [\"model\"], {}),\n ({\"model.abc\": tune.choice([\"a\", \"b\"])}, [\"data\"], {\"model.abc\": tune.choice([\"a\", \"b\"])}),\n ({\"model.abc\": \"def\"}, [\"model\"], {\"model.abc\": \"def\"}),\n (\n {\n \"data.abc.def\": tune.choice([\"a\", \"b\"]),\n \"model.abc\": \"def\",\n \"env.abc.def\": tune.choice([\"a\", \"b\"]),\n },\n [\"data\"],\n {\"model.abc\": \"def\", \"env.abc.def\": tune.choice([\"a\", \"b\"])},\n ),\n ],\n)\ndef test_filter_search_space(hyperparameters, keys_to_filter, expected):\n # We test keys here because the object might be copied and hence direct comparison will fail\n assert filter_search_space(hyperparameters, keys_to_filter).keys() == expected.keys()\n\n\n@pytest.mark.parametrize(\"hyperparameters, keys_to_filter\", [({\"model.abc\": tune.choice([\"a\", \"b\"])}, [\"abc\"])])\ndef test_invalid_filter_search_space(hyperparameters, keys_to_filter):\n with pytest.raises(Exception) as e_info:\n filter_search_space(hyperparameters, keys_to_filter)\n\n\n@pytest.mark.parametrize(\"searcher\", list(SEARCHER_PRESETS.keys()))\n@pytest.mark.parametrize(\"scheduler\", list(SCHEDULER_PRESETS.keys()))\ndef test_predictor_hpo_searchers_schedulers(searcher, scheduler):\n predictor_hpo(searcher, scheduler)\n\n\n@pytest.mark.parametrize(\"presets\", [f\"{quality}_hpo\" for quality in ALL_MODEL_QUALITIES])\ndef test_predictor_hpo_presets(presets):\n predictor_hpo(\"random\", \"FIFO\", presets)\n\n\n@pytest.mark.parametrize(\"searcher\", list(SEARCHER_PRESETS.keys()))\n@pytest.mark.parametrize(\"scheduler\", list(SCHEDULER_PRESETS.keys()))\ndef test_matcher_hpo_searchers_schedulers(searcher, scheduler):\n matcher_hpo(searcher, scheduler)\n\n\n@pytest.mark.parametrize(\"presets\", [f\"{quality}_hpo\" for quality in ALL_MODEL_QUALITIES])\ndef test_matcher_hpo_presets(presets):\n matcher_hpo(\"random\", \"FIFO\", presets)\n\n\n@pytest.mark.single_gpu\ndef test_hpo_distillation():\n searcher = \"random\"\n scheduler = \"FIFO\"\n dataset = PetFinderDataset()\n\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"numerical_mlp\", \"categorical_mlp\", \"fusion_mlp\"],\n \"data.categorical.convert_to_text\": False,\n \"data.numerical.convert_to_text\": False,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n hyperparameter_tune_kwargs = {\n \"searcher\": searcher,\n \"scheduler\": scheduler,\n \"num_trials\": 2,\n }\n\n teacher_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n teacher_save_path = os.path.join(\n get_home_dir(), \"outputs\", \"hpo_distillation_teacher\", f\"_{searcher}\", f\"_{scheduler}\"\n )\n if os.path.exists(teacher_save_path):\n shutil.rmtree(teacher_save_path)\n\n teacher_predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n save_path=teacher_save_path,\n )\n\n hyperparameters = {\n \"optim.lr\": tune.uniform(0.0001, 0.01),\n \"optim.max_epochs\": 1,\n \"model.names\": [\"numerical_mlp\"],\n \"data.numerical.convert_to_text\": False,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n # test for distillation\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n student_save_path = os.path.join(\n get_home_dir(), \"outputs\", \"hpo_distillation_student\", f\"_{searcher}\", f\"_{scheduler}\"\n )\n if os.path.exists(student_save_path):\n shutil.rmtree(student_save_path)\n\n predictor.fit(\n train_data=dataset.train_df,\n teacher_predictor=teacher_save_path,\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n save_path=student_save_path,\n )\n\n\ndef test_modifying_duplicate_model_names():\n dataset = PetFinderDataset()\n metric_name = dataset.metric\n\n teacher_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n teacher_predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=1,\n )\n student_predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n student_predictor.fit(\n train_data=dataset.train_df,\n time_limit=0,\n )\n\n teacher_predictor._learner = modify_duplicate_model_names(\n learner=teacher_predictor._learner,\n postfix=\"teacher\",\n blacklist=student_predictor._learner._config.model.names,\n )\n\n # verify teacher and student have no duplicate model names\n assert all(\n [\n n not in teacher_predictor._learner._config.model.names\n for n in student_predictor._learner._config.model.names\n ]\n ), (\n f\"teacher model names {teacher_predictor._learner._config.model.names} and\"\n f\" student model names {student_predictor._learner._config.model.names} have duplicates.\"\n )\n\n # verify each model name prefix is valid\n assert teacher_predictor._learner._model.prefix in teacher_predictor._learner._config.model.names\n if isinstance(teacher_predictor._learner._model.model, nn.ModuleList):\n for per_model in teacher_predictor._learner._model.model:\n assert per_model.prefix in teacher_predictor._learner._config.model.names\n\n # verify each data processor's prefix is valid\n for per_modality_processors in teacher_predictor._learner._data_processors.values():\n for per_processor in per_modality_processors:\n assert per_processor.prefix in teacher_predictor._learner._config.model.names\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_image_formats.py",
"content": "import os\nimport shutil\nimport tempfile\n\nimport numpy.testing as npt\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import IMAGE_BASE64_STR, IMAGE_BYTEARRAY\nfrom autogluon.multimodal.utils.misc import shopee_dataset\n\nfrom ..utils import PetFinderDataset, verify_predictor_save_load\n\n\n@pytest.mark.parametrize(\"image_type\", [IMAGE_BYTEARRAY, IMAGE_BASE64_STR])\ndef test_image_bytearray_or_base64_str(image_type):\n download_dir = \"./\"\n train_data_1, test_data_1 = shopee_dataset(download_dir=download_dir)\n if image_type == IMAGE_BYTEARRAY:\n train_data_2, test_data_2 = shopee_dataset(download_dir=download_dir, is_bytearray=True)\n elif image_type == IMAGE_BASE64_STR:\n train_data_2, test_data_2 = shopee_dataset(download_dir=download_dir, is_base64str=True)\n\n predictor_1 = MultiModalPredictor(\n label=\"label\",\n )\n predictor_2 = MultiModalPredictor(\n label=\"label\",\n )\n model_names = [\"timm_image\"]\n hyperparameters = {\n \"optim.max_epochs\": 2,\n \"model.names\": model_names,\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n }\n predictor_1.fit(\n train_data=train_data_1,\n hyperparameters=hyperparameters,\n seed=42,\n )\n predictor_2.fit(\n train_data=train_data_2,\n hyperparameters=hyperparameters,\n seed=42,\n )\n\n score_1 = predictor_1.evaluate(test_data_1)\n score_2 = predictor_2.evaluate(test_data_2)\n # train and predict using different image types\n score_3 = predictor_1.evaluate(test_data_2)\n score_4 = predictor_2.evaluate(test_data_1)\n\n prediction_1 = predictor_1.predict(test_data_1, as_pandas=False)\n prediction_2 = predictor_2.predict(test_data_2, as_pandas=False)\n prediction_3 = predictor_1.predict(test_data_2, as_pandas=False)\n prediction_4 = predictor_2.predict(test_data_1, as_pandas=False)\n\n prediction_prob_1 = predictor_1.predict_proba(test_data_1, as_pandas=False)\n prediction_prob_2 = predictor_2.predict_proba(test_data_2, as_pandas=False)\n prediction_prob_3 = predictor_1.predict_proba(test_data_2, as_pandas=False)\n prediction_prob_4 = predictor_1.predict_proba(test_data_1, as_pandas=False)\n\n npt.assert_array_equal([score_1, score_2, score_3, score_4], [score_1] * 4)\n npt.assert_array_equal([prediction_1, prediction_2, prediction_3, prediction_4], [prediction_1] * 4)\n npt.assert_array_equal(\n [prediction_prob_1, prediction_prob_2, prediction_prob_3, prediction_prob_4], [prediction_prob_1] * 4\n )\n\n\ndef test_predict_with_image_str_or_list():\n download_dir = \"./ag_automm_tutorial_imgcls\"\n train_data, test_data = shopee_dataset(download_dir)\n\n save_path = f\"./tmp/automm_shopee\"\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor = MultiModalPredictor(label=\"label\", path=save_path)\n predictor.fit(\n train_data=train_data,\n time_limit=0, # seconds\n ) # you can trust the default config, e.g., we use a `swin_base_patch4_window7_224` model\n\n image_path = test_data.iloc[0][\"image\"]\n\n predictions_str = predictor.predict(image_path)\n predictions_list1 = predictor.predict([image_path])\n predictions_list10 = predictor.predict([image_path] * 10)\n\n\n@pytest.mark.parametrize(\"invalid_value\", [None, \"invalid/image/path\"])\ndef test_fit_with_invalid_images(invalid_value):\n dataset = PetFinderDataset()\n train_df = dataset.train_df\n invalid_num = int(0.5 * len(train_df))\n train_df.loc[0:invalid_num, \"Images\"] = invalid_value\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n hyperparameters = {\n \"model.names\": [\n \"categorical_mlp\",\n \"numerical_mlp\",\n \"timm_image\",\n \"hf_text\",\n \"fusion_mlp\",\n ],\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n \"model.timm_image.checkpoint_name\": \"swin_small_patch4_window7_224\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n\n predictor.fit(\n train_data=train_df,\n time_limit=10,\n save_path=save_path,\n hyperparameters=hyperparameters,\n )\n\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_ner_chinese.py",
"content": "import json\nimport os\n\nimport pandas as pd\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.utils import download\n\nfrom ..utils import get_home_dir\n\n\ndef download_ecommerce():\n download(\n \"https://raw.githubusercontent.com/allanj/ner_incomplete_annotation/master/data/ecommerce/train.txt\",\n os.path.join(get_home_dir(), \"train.txt\"),\n )\n download(\n \"https://raw.githubusercontent.com/allanj/ner_incomplete_annotation/master/data/ecommerce/dev.txt\",\n os.path.join(get_home_dir(), \"dev.txt\"),\n )\n\n\ndef read_bio(sample):\n raw_string = \"\"\n entities = []\n new_entity = None\n prev_bio_type = None\n ptr = 0\n\n # Parse the data from the BIO format\n for ele in sample.split(\"\\n\"):\n dat = ele.split()\n if len(dat) != 2:\n continue\n token, bio_label = dat\n raw_string += token\n new_ptr = ptr + len(token)\n if bio_label.startswith(\"O\"):\n bio_type, label = \"O\", None\n else:\n bio_type, label = bio_label.split(\"-\")\n if bio_type == \"O\":\n if new_entity:\n entities.append(new_entity)\n new_entity = None\n elif bio_type == \"B\":\n if new_entity:\n entities.append(new_entity)\n new_entity = {\"entity_group\": label, \"start\": ptr, \"end\": new_ptr}\n elif bio_type == \"I\":\n if prev_bio_type in [\"B\", \"I\"]:\n # Keep update the new_entity\n assert new_entity[\"entity_group\"] == label\n new_entity[\"end\"] = new_ptr\n else:\n new_entity = {\"entity_group\": label, \"start\": ptr, \"end\": new_ptr}\n else:\n raise NotImplementedError\n ptr = new_ptr\n prev_bio_type = bio_type\n if new_entity:\n entities.append(new_entity)\n return raw_string, entities\n\n\ndef bio_samples_to_df(samples):\n raw_strings = []\n entity_list = []\n for sample in samples:\n raw_string, entities = read_bio(sample)\n raw_strings.append(raw_string)\n entity_list.append(json.dumps(entities))\n df = pd.DataFrame({\"text_snippet\": raw_strings, \"entity_annotations\": entity_list})\n return df\n\n\ndef get_data():\n train_data = open(os.path.join(get_home_dir(), \"dev.txt\"), encoding=\"utf-8\").read()\n train_df = bio_samples_to_df(train_data.split(\"\\n\\n\"))\n\n dev_data = open(os.path.join(get_home_dir(), \"dev.txt\"), encoding=\"utf-8\").read()\n dev_df = bio_samples_to_df(dev_data.split(\"\\n\\n\"))\n\n return train_df, dev_df\n\n\ndef test_ner_chinese():\n download_ecommerce()\n train_df, dev_df = get_data()\n predictor = MultiModalPredictor(\n problem_type=\"ner\",\n label=\"entity_annotations\",\n )\n predictor.fit(\n train_data=train_df,\n tuning_data=dev_df,\n hyperparameters={\n \"model.ner_text.checkpoint_name\": \"hfl/chinese-lert-small\",\n \"optim.top_k\": 1,\n \"env.num_gpus\": -1,\n \"optim.max_epochs\": 1,\n },\n )\n predictor.evaluate(dev_df)\n predictor.predict(dev_df.head(2))\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_problem_types.py",
"content": "import numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import BINARY, CLASSIFICATION, MULTICLASS, NER, OBJECT_DETECTION, REGRESSION\nfrom autogluon.multimodal.data import infer_problem_type\nfrom autogluon.multimodal.utils.problem_types import PROBLEM_TYPES_REG\n\n\n@pytest.mark.parametrize(\n \"name\",\n [\n \"classification\",\n \"multiclass\",\n \"binary\",\n \"regression\",\n \"ner\",\n \"named_entity_recognition\",\n \"object_detection\",\n \"text_similarity\",\n \"image_similarity\",\n \"image_text_similarity\",\n \"feature_extraction\",\n \"zero_shot_image_classification\",\n \"few_shot_classification\",\n ],\n)\ndef test_get_problem_type(name):\n problem_prop = PROBLEM_TYPES_REG.get(name)\n assert problem_prop.name == PROBLEM_TYPES_REG.get(problem_prop.name).name\n\n\n@pytest.mark.parametrize(\"name\", PROBLEM_TYPES_REG.list_keys())\ndef test_problem_type_in_init(name):\n if name != OBJECT_DETECTION:\n predictor = MultiModalPredictor(problem_type=name)\n assert predictor.problem_type == PROBLEM_TYPES_REG.get(name).name\n\n\n@pytest.mark.parametrize(\n \"y_data,provided_problem_type,gt_problem_type\",\n [\n (pd.Series([0, 1, 0, 1, 1, 0]), None, BINARY),\n (pd.Series([\"a\", \"b\", \"c\"]), None, MULTICLASS),\n (pd.Series([\"a\", \"b\", \"c\"]), CLASSIFICATION, MULTICLASS),\n (pd.Series(np.linspace(0.0, 1.0, 100)), None, REGRESSION),\n (pd.Series([\"0\", \"1\", \"2\", 3, 4, 5, 5, 5, 0]), None, MULTICLASS),\n (None, NER, NER),\n (None, OBJECT_DETECTION, OBJECT_DETECTION),\n ],\n)\ndef test_infer_problem_type(y_data, provided_problem_type, gt_problem_type):\n inferred_problem_type = infer_problem_type(\n y_train_data=y_data,\n provided_problem_type=provided_problem_type,\n )\n assert inferred_problem_type == gt_problem_type\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_text_detection.py",
"content": "import numpy as np\nimport pytest\nimport requests\nfrom mim.commands.download import download\n\ntry:\n import mmocr\n from mmocr.utils.ocr import MMOCR\nexcept (ImportError, ModuleNotFoundError):\n pytest.skip(\"MMOCR is not installed. Skip this test.\", allow_module_level=True)\n\nfrom PIL import Image\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef download_sample_images():\n url = \"https://raw.githubusercontent.com/open-mmlab/mmocr/main/demo/demo_text_det.jpg\"\n image = Image.open(requests.get(url, stream=True).raw)\n mmocr_image_name = \"demo.jpg\"\n image.save(mmocr_image_name)\n\n return mmocr_image_name\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\"textsnake_r50_fpn_unet_1200e_ctw1500\"],\n)\n@pytest.mark.skip(\n reason=\"Output format of OCR shall be changed to match with Object Detection. Since they both have ret_type=BBOX\"\n)\ndef test_mmocr_text_detection_inference(checkpoint_name):\n mmocr_image_name = download_sample_images()\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmocr_text_detection.checkpoint_name\": checkpoint_name,\n },\n problem_type=\"ocr_text_detection\",\n )\n\n # two dimensions, (num of text lines, 2 * num of coordinate points)\n pred = predictor.predict({\"image\": [mmocr_image_name]})\n\n # original MMOCR model's output\n checkpoints = download(package=\"mmocr\", configs=[checkpoint_name], dest_root=\".\")\n checkpoint = checkpoints[0]\n config_file = checkpoint_name + \".py\"\n ocr = MMOCR(det_ckpt=checkpoint, det_config=config_file, recog=None)\n MMOCR_res = ocr.readtext(mmocr_image_name, output=None)\n\n # compare the outputs of original model's output and our model\n assert len(pred) == len(MMOCR_res[0][\"boundary_result\"]) # num of text lines\n\n for i in range(len(pred)):\n p = pred[i]\n m = MMOCR_res[0][\"boundary_result\"][i]\n assert len(p) == len(m) # 2 * num of coordinate points\n\n for j in range(len(p)):\n assert abs(p[j] - m[j]) <= 1e-6\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_text_recognition.py",
"content": "import numpy as np\nimport pytest\nimport requests\nfrom mim.commands.download import download\n\ntry:\n from mmocr.utils.ocr import MMOCR\nexcept ImportError:\n pytest.skip(\n 'Skip the OCR test because there is no mmocr installed. Try to install it via mim install \"mmocr<1.0\"',\n allow_module_level=True,\n )\n\nfrom PIL import Image\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef download_sample_images():\n url = \"https://raw.githubusercontent.com/open-mmlab/mmocr/main/demo/demo_text_recog.jpg\"\n image = Image.open(requests.get(url, stream=True).raw)\n mmocr_image_name = \"demo.jpg\"\n image.save(mmocr_image_name)\n\n return mmocr_image_name\n\n\n# TODO: when using crnn checkpoint, the results are wrong.\n@pytest.mark.parametrize(\n \"checkpoint_name\",\n [\n \"abinet_academic\",\n \"sar_r31_parallel_decoder_academic\",\n \"seg_r31_1by16_fpnocr_academic\",\n \"nrtr_r31_1by16_1by8_academic\",\n ],\n)\n@pytest.mark.skip(\n reason=\"Output format of OCR shall be changed to match with Object Detection. Since they both have ret_type=BBOX\"\n)\ndef test_mmocr_text_recognition_inference(checkpoint_name):\n mmocr_image_name = download_sample_images()\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.mmocr_text_recognition.checkpoint_name\": checkpoint_name,\n },\n problem_type=\"ocr_text_recognition\",\n )\n\n pred = predictor.predict({\"image\": [mmocr_image_name]})\n\n # original MMOCR model's output\n checkpoints = download(package=\"mmocr\", configs=[checkpoint_name], dest_root=\".\")\n checkpoint = checkpoints[0]\n config_file = checkpoint_name + \".py\"\n ocr = MMOCR(recog_ckpt=checkpoint, recog_config=config_file, det=None)\n MMOCR_res = ocr.readtext(mmocr_image_name, output=None)\n\n assert pred[0][0] == MMOCR_res[0][\"text\"]\n assert pred[1][0] == MMOCR_res[0][\"score\"]\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_tokenizers.py",
"content": "import os\nimport shutil\nimport tempfile\n\nimport pytest\nfrom transformers import AlbertTokenizer, AlbertTokenizerFast\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import TEXT\n\nfrom ..utils import AEDataset\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name,use_fast,tokenizer_type\",\n [\n (\n \"albert-base-v2\",\n None,\n AlbertTokenizerFast,\n ),\n (\n \"albert-base-v2\",\n True,\n AlbertTokenizerFast,\n ),\n (\n \"albert-base-v2\",\n False,\n AlbertTokenizer,\n ),\n ],\n)\ndef test_tokenizer_use_fast(checkpoint_name, use_fast, tokenizer_type):\n dataset = AEDataset()\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"data.categorical.convert_to_text\": True,\n \"data.numerical.convert_to_text\": True,\n \"model.hf_text.checkpoint_name\": checkpoint_name,\n }\n if use_fast is not None:\n hyperparameters[\"model.hf_text.use_fast\"] = use_fast\n\n with tempfile.TemporaryDirectory() as save_path:\n if os.path.isdir(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n time_limit=5,\n save_path=save_path,\n hyperparameters=hyperparameters,\n )\n assert isinstance(predictor._learner._data_processors[TEXT][0].tokenizer, tokenizer_type)\n"
},
{
"path": "multimodal/tests/unittests/others_2/test_zero_shot.py",
"content": "import numpy as np\nimport pytest\nimport requests\nfrom PIL import Image\n\nfrom autogluon.multimodal import MultiModalPredictor\n\n\ndef download_sample_images():\n url = \"https://autogluon.s3.us-west-2.amazonaws.com/images/automm_test_images/cats.jpg\"\n image = Image.open(requests.get(url, stream=True).raw)\n cat_image_name = \"cat.jpg\"\n image.save(cat_image_name)\n\n url = \"https://autogluon.s3.us-west-2.amazonaws.com/images/automm_test_images/dog.jpg\"\n image = Image.open(requests.get(url, stream=True).raw)\n dog_image_name = \"dog.jpg\"\n image.save(dog_image_name)\n\n return cat_image_name, dog_image_name\n\n\ndef test_clip_zero_shot():\n cat_image_name, dog_image_name = download_sample_images()\n\n cat_text = \"a photo of a cat\"\n dog_text = \"a photo of a dog\"\n bird_text = \"a photo of a bird\"\n\n predictor = MultiModalPredictor(\n problem_type=\"zero_shot_image_classification\",\n )\n\n # compute the cosine similarity of one image-text pair.\n pred = predictor.predict({\"image\": [cat_image_name], \"text\": [cat_text]})\n assert pred.shape == (1,)\n\n # compute the cosine similarities of more image and text pairs.\n pred = predictor.predict({\"image\": [cat_image_name, dog_image_name], \"text\": [cat_text, dog_text]})\n assert pred.shape == (2,)\n\n # match images in a given text pool and output the matched text index (starting from 0) for each image.\n pred = predictor.predict({\"image0\": [cat_image_name, dog_image_name]}, {\"names\": [dog_text, cat_text, bird_text]})\n assert pred.shape == (2,)\n\n # match texts in a given image pool and output the matched image index (starting from 0) for each text.\n pred = predictor.predict(\n {\"query\": [dog_text, cat_text, bird_text]}, {\"candidates\": [cat_image_name, dog_image_name]}\n )\n assert pred.shape == (3,)\n\n # predict the probabilities of one image matching several texts.\n prob = predictor.predict_proba({\"image\": [cat_image_name]}, {\"text\": [cat_text, dog_text, bird_text]})\n assert prob.shape == (1, 3)\n for per_row_prob in prob:\n assert pytest.approx(sum(per_row_prob), 1e-6) == 1\n\n # given two or more images, we can get the probabilities of matching each image with a pool of texts.\n prob = predictor.predict_proba(\n {\"image\": [dog_image_name, cat_image_name]}, {\"text\": [bird_text, cat_text, dog_text]}\n )\n assert prob.shape == (2, 3)\n for per_row_prob in prob:\n assert pytest.approx(sum(per_row_prob), 1e-6) == 1\n\n # predict the probabilities of one text matching several images.\n prob = predictor.predict_proba({\"x\": [cat_text]}, {\"y\": [dog_image_name, cat_image_name]})\n assert prob.shape == (1, 2)\n for per_row_prob in prob:\n assert pytest.approx(sum(per_row_prob), 1e-6) == 1\n\n # given two or more texts, we can get the probabilities of matching each text with a pool of images.\n prob = predictor.predict_proba({\"a\": [bird_text, cat_text, dog_text]}, {\"b\": [dog_image_name, cat_image_name]})\n assert prob.shape == (3, 2)\n for per_row_prob in prob:\n assert pytest.approx(sum(per_row_prob), 1e-6) == 1\n\n # extract image embeddings.\n embedding = predictor.extract_embedding({\"123\": [dog_image_name, cat_image_name]})\n assert list(embedding.keys()) == [\"123\"]\n for v in embedding.values():\n assert v.shape == (2, 512) or v.shape == (2, 768)\n\n # extract text embeddings.\n embedding = predictor.extract_embedding({\"xyz\": [bird_text, dog_text, cat_text]})\n assert list(embedding.keys()) == [\"xyz\"]\n for v in embedding.values():\n assert v.shape == (3, 512) or v.shape == (3, 768)\n\n # extract embeddings for both images and texts.\n embedding = predictor.extract_embedding({\"image\": [cat_image_name, dog_image_name], \"text\": [bird_text, dog_text]})\n assert list(embedding.keys()).sort() == [\"image\", \"text\"].sort()\n for v in embedding.values():\n assert v.shape == (2, 512) or v.shape == (2, 768)\n\n # invalid API usage 1: passing one dictionary, but different keys have inconsistent list lengths.\n with pytest.raises(ValueError):\n pred = predictor.predict({\"image\": [cat_image_name, dog_image_name], \"text\": [cat_text]})\n\n with pytest.raises(ValueError):\n embedding = predictor.extract_embedding({\"image\": [cat_image_name], \"text\": [cat_text, bird_text]})\n\n\n@pytest.mark.parametrize(\n \"checkpoint_name,num_gpus\",\n [\n (\"swin_tiny_patch4_window7_224\", -1),\n (\"vit_tiny_patch16_224\", -1),\n (\"resnet18\", -1),\n (\"legacy_seresnet18\", -1),\n ],\n)\ndef test_timm_zero_shot(checkpoint_name, num_gpus):\n cat_image_name, dog_image_name = download_sample_images()\n\n predictor = MultiModalPredictor(\n hyperparameters={\n \"model.names\": [\"timm_image\"],\n \"model.timm_image.checkpoint_name\": checkpoint_name,\n \"env.num_gpus\": num_gpus,\n },\n problem_type=\"zero_shot_image_classification\",\n )\n\n pred = predictor.predict({\"image\": [cat_image_name, dog_image_name]})\n assert pred.shape == (2,)\n\n prob = predictor.predict_proba({\"image\": [cat_image_name, dog_image_name]})\n assert prob.shape == (2, 1000)\n\n features = predictor.extract_embedding({\"abc\": [cat_image_name, dog_image_name]})\n assert features[\"abc\"].ndim == 2 and features[\"abc\"].shape[0] == 2\n\n features, masks = predictor.extract_embedding({\"abc\": [cat_image_name, dog_image_name]}, return_masks=True)\n assert features[\"abc\"].ndim == 2 and features[\"abc\"].shape[0] == 2\n assert np.all(masks[\"abc\"] == np.array([1, 1]))\n\n features, masks = predictor.extract_embedding(\n {\"abc\": [cat_image_name], \"123\": [dog_image_name]}, return_masks=True\n )\n assert features[\"abc\"].ndim == 2 and features[\"abc\"].shape[0] == 1\n assert features[\"123\"].ndim == 2 and features[\"123\"].shape[0] == 1\n assert np.all(masks[\"abc\"] == np.array([1]))\n assert np.all(masks[\"123\"] == np.array([1]))\n\n\ndef test_matcher_text_similarity():\n matcher = MultiModalPredictor(\n problem_type=\"text_similarity\",\n hyperparameters={\"model.hf_text.checkpoint_name\": \"sentence-transformers/all-MiniLM-L6-v2\"},\n )\n corpus = [\n \"A man is eating food.\",\n \"A man is eating a piece of bread.\",\n \"The girl is carrying a baby.\",\n \"A man is riding a horse.\",\n \"A woman is playing violin.\",\n \"Two men pushed carts through the woods.\",\n \"A man is riding a white horse on an enclosed ground.\",\n \"A monkey is playing drums.\",\n \"A cheetah is running behind its prey.\",\n ]\n queries = [\n \"A man is eating pasta.\",\n \"Someone in a gorilla costume is playing a set of drums.\",\n \"A cheetah chases prey on across a field.\",\n ]\n query_embeddings = matcher.extract_embedding(queries)\n corpus_embeddings = matcher.extract_embedding(corpus)\n assert len(query_embeddings) == len(queries)\n assert len(corpus_embeddings) == len(corpus)\n\n query_embeddings = matcher.extract_embedding(queries, signature=\"query\")\n corpus_embeddings = matcher.extract_embedding(corpus, signature=\"response\")\n assert len(query_embeddings) == len(queries)\n assert len(corpus_embeddings) == len(corpus)\n\n query_embeddings = matcher.extract_embedding({\"abc\": queries})\n corpus_embeddings = matcher.extract_embedding({\"abc\": corpus})\n assert len(query_embeddings) == len(queries)\n assert len(corpus_embeddings) == len(corpus)\n\n query_embeddings = matcher.extract_embedding({\"abc\": queries}, signature=\"query\")\n corpus_embeddings = matcher.extract_embedding({\"abc\": corpus}, signature=\"response\")\n assert len(query_embeddings) == len(queries)\n assert len(corpus_embeddings) == len(corpus)\n"
},
{
"path": "multimodal/tests/unittests/predictor/__init__.py",
"content": ""
},
{
"path": "multimodal/tests/unittests/predictor/test_predictor.py",
"content": "import os\nimport shutil\nimport tempfile\n\nimport numpy.testing as npt\nimport pytest\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import (\n BEST,\n BINARY,\n BIT_FIT,\n DATA,\n DISTILLER,\n ENV,\n GREEDY_SOUP,\n IA3,\n IMAGE_BASE64_STR,\n IMAGE_BYTEARRAY,\n LORA,\n LORA_BIAS,\n LORA_NORM,\n MODEL,\n MULTICLASS,\n NORM_FIT,\n OPTIM,\n UNIFORM_SOUP,\n)\n\nfrom ..utils import (\n AEDataset,\n HatefulMeMesDataset,\n PetFinderDataset,\n get_home_dir,\n verify_no_redundant_model_configs,\n verify_predictor_realtime_inference,\n verify_predictor_save_load,\n)\n\nALL_DATASETS = {\n \"petfinder\": PetFinderDataset(),\n \"petfinder_bytearray\": PetFinderDataset(is_bytearray=True),\n \"hateful_memes\": HatefulMeMesDataset(),\n \"hateful_memes_bytearray\": HatefulMeMesDataset(is_bytearray=True),\n \"ae\": AEDataset(),\n}\n\n\n@pytest.mark.parametrize(\n \"dataset_name,model_names,text_backbone,image_backbone,top_k_average_method,peft,loss_func\",\n [\n (\n \"petfinder\",\n [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"sentence-transformers/all-MiniLM-L6-v2\",\n \"swin_tiny_patch4_window7_224\",\n GREEDY_SOUP,\n None,\n \"auto\",\n ),\n (\n \"petfinder\",\n [\"ft_transformer\", \"clip_image\", \"clip_text\", \"fusion_mlp\"],\n \"openai/clip-vit-base-patch32\",\n \"openai/clip-vit-base-patch32\",\n GREEDY_SOUP,\n LORA,\n \"auto\",\n ),\n (\n \"petfinder\",\n [\"t_few\"],\n \"t5-small\",\n None,\n BEST,\n IA3,\n \"auto\",\n ),\n (\n \"hateful_memes\",\n [\"timm_image\", \"t_few\", \"fusion_mlp\"],\n \"t5-small\",\n \"mobilenetv3_small_100\",\n BEST,\n IA3,\n \"auto\",\n ),\n (\n \"hateful_memes_bytearray\",\n [\"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"monsoon-nlp/hindi-bert\",\n \"mobilenetv3_small_100\",\n UNIFORM_SOUP,\n LORA_NORM,\n \"auto\",\n ),\n (\n \"petfinder_bytearray\",\n [\"ft_transformer\", \"timm_image\", \"fusion_mlp\"],\n None,\n \"mobilenetv3_small_100\",\n GREEDY_SOUP,\n None,\n \"auto\",\n ),\n (\n \"petfinder\",\n [\"ft_transformer\", \"hf_text\", \"fusion_transformer\"],\n \"sentence-transformers/all-MiniLM-L6-v2\",\n None,\n UNIFORM_SOUP,\n None,\n \"auto\",\n ),\n (\n \"petfinder\",\n [\"ft_transformer\"],\n None,\n None,\n BEST,\n BIT_FIT,\n \"auto\",\n ),\n (\n \"hateful_memes\",\n [\"timm_image\"],\n None,\n \"mobilenetv3_small_100\",\n UNIFORM_SOUP,\n NORM_FIT,\n \"auto\",\n ),\n (\n \"ae\",\n [\"hf_text\"],\n \"nlpaueb/legal-bert-small-uncased\",\n None,\n BEST,\n LORA_BIAS,\n \"bcewithlogitsloss\",\n ),\n (\n \"ae\",\n [\"hf_text\"],\n \"CLTL/MedRoBERTa.nl\",\n None,\n BEST,\n None,\n \"auto\",\n ),\n (\n \"hateful_memes\",\n [\"clip\"],\n None,\n None,\n BEST,\n NORM_FIT,\n \"auto\",\n ),\n ],\n)\ndef test_predictor_basic(\n dataset_name,\n model_names,\n text_backbone,\n image_backbone,\n top_k_average_method,\n peft,\n loss_func,\n):\n dataset = ALL_DATASETS[dataset_name]\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": model_names,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": top_k_average_method,\n \"optim.peft\": peft,\n \"optim.loss_func\": loss_func,\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n }\n if text_backbone is not None:\n if \"t_few\" in model_names:\n hyperparameters.update(\n {\n \"model.t_few.checkpoint_name\": \"t5-small\",\n \"model.t_few.gradient_checkpointing\": False,\n }\n )\n elif \"hf_text\" in model_names:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": text_backbone,\n }\n )\n elif \"clip_text\" in model_names:\n hyperparameters.update(\n {\n \"model.clip_text.checkpoint_name\": text_backbone,\n }\n )\n if image_backbone is not None:\n if \"timm_image\" in model_names:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": image_backbone,\n }\n )\n elif \"clip_image\" in model_names:\n hyperparameters.update(\n {\n \"model.clip_image.checkpoint_name\": image_backbone,\n }\n )\n save_path = os.path.join(get_home_dir(), \"outputs\", dataset_name)\n if text_backbone is not None:\n save_path = os.path.join(save_path, text_backbone)\n if image_backbone is not None:\n save_path = os.path.join(save_path, image_backbone)\n\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n verify_no_redundant_model_configs(predictor)\n score = predictor.evaluate(dataset.test_df)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # Test for continuous fit\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=20,\n )\n verify_no_redundant_model_configs(predictor)\n verify_predictor_save_load(predictor, dataset.test_df)\n\n # Saving to folder, loading the saved model and call fit again (continuous fit)\n with tempfile.TemporaryDirectory() as root:\n predictor.save(root)\n predictor = MultiModalPredictor.load(root)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=10,\n )\n\n\n@pytest.mark.single_gpu\n@pytest.mark.parametrize(\n \"dataset_name,model_names,text_backbone,image_backbone,top_k_average_method,peft,loss_func\",\n [\n (\n \"petfinder\",\n [\"numerical_mlp\", \"categorical_mlp\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"nlpaueb/legal-bert-small-uncased\",\n \"swin_tiny_patch4_window7_224\",\n GREEDY_SOUP,\n LORA,\n \"auto\",\n ),\n (\n \"hateful_memes\",\n [\"timm_image\", \"t_few\", \"fusion_mlp\"],\n \"t5-small\",\n \"mobilenetv3_small_100\",\n BEST,\n IA3,\n \"auto\",\n ),\n (\n \"hateful_memes\",\n [\"clip\"],\n None,\n None,\n BEST,\n NORM_FIT,\n \"auto\",\n ),\n ],\n)\ndef test_predictor_realtime_inference(\n dataset_name,\n model_names,\n text_backbone,\n image_backbone,\n top_k_average_method,\n peft,\n loss_func,\n):\n dataset = ALL_DATASETS[dataset_name]\n metric_name = dataset.metric\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=metric_name,\n )\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": model_names,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.top_k_average_method\": top_k_average_method,\n \"optim.peft\": peft,\n \"optim.loss_func\": loss_func,\n \"data.categorical.convert_to_text\": False, # ensure the categorical model is used.\n \"data.numerical.convert_to_text\": False, # ensure the numerical model is used.\n }\n if text_backbone is not None:\n if \"t_few\" in model_names:\n hyperparameters.update(\n {\n \"model.t_few.checkpoint_name\": \"t5-small\",\n \"model.t_few.gradient_checkpointing\": False,\n }\n )\n else:\n hyperparameters.update(\n {\n \"model.hf_text.checkpoint_name\": text_backbone,\n }\n )\n if image_backbone is not None:\n hyperparameters.update(\n {\n \"model.timm_image.checkpoint_name\": image_backbone,\n }\n )\n save_path = os.path.join(get_home_dir(), \"outputs\", dataset_name)\n if text_backbone is not None:\n save_path = os.path.join(save_path, text_backbone)\n if image_backbone is not None:\n save_path = os.path.join(save_path, image_backbone)\n\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=20,\n save_path=save_path,\n )\n verify_predictor_realtime_inference(predictor, dataset.test_df)\n\n\ndef test_predictor_standalone(): # test standalone feature in MultiModalPredictor.save()\n from unittest import mock\n\n import torch\n\n requests_gag = mock.patch(\n \"requests.Session.request\",\n mock.Mock(side_effect=RuntimeError(\"Please use the `responses` library to mock HTTP in your tests.\")),\n )\n\n dataset = ALL_DATASETS[\"petfinder\"]\n\n hyperparameters = {\n \"optim.max_epochs\": 1,\n \"model.names\": [\"ft_transformer\", \"timm_image\", \"hf_text\", \"fusion_mlp\"],\n \"model.hf_text.checkpoint_name\": \"nlpaueb/legal-bert-small-uncased\",\n \"model.timm_image.checkpoint_name\": \"swin_tiny_patch4_window7_224\",\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n }\n\n predictor = MultiModalPredictor(\n label=dataset.label_columns[0],\n problem_type=dataset.problem_type,\n eval_metric=dataset.metric,\n )\n\n save_path = os.path.join(get_home_dir(), \"outputs\", \"standalone\", \"false\")\n if os.path.exists(save_path):\n shutil.rmtree(save_path)\n\n predictor.fit(\n train_data=dataset.train_df,\n hyperparameters=hyperparameters,\n time_limit=30,\n save_path=save_path,\n standalone=False,\n )\n\n save_path_standalone = os.path.join(get_home_dir(), \"outputs\", \"standalone\", \"true\")\n if os.path.exists(save_path_standalone):\n shutil.rmtree(save_path_standalone)\n\n predictor.save(\n path=save_path_standalone,\n standalone=True,\n )\n\n # make sure the dumping doesn't affect predictor loading\n predictor.dump_model()\n\n del predictor\n torch.cuda.empty_cache()\n\n loaded_online_predictor = MultiModalPredictor.load(path=save_path)\n online_predictions = loaded_online_predictor.predict(dataset.test_df, as_pandas=False)\n del loaded_online_predictor\n\n # Check if the predictor can be loaded from an offline environment.\n with requests_gag:\n # No internet connection here. If any command require internet connection, a RuntimeError will be raised.\n with tempfile.TemporaryDirectory() as tmpdirname:\n torch.hub.set_dir(tmpdirname) # block reading files in `.cache`\n loaded_offline_predictor = MultiModalPredictor.load(path=save_path_standalone)\n\n offline_predictions = loaded_offline_predictor.predict(dataset.test_df, as_pandas=False)\n del loaded_offline_predictor\n\n # check if save with standalone=True coincide with standalone=False\n npt.assert_equal(online_predictions, offline_predictions)\n"
},
{
"path": "multimodal/tests/unittests/utils/__init__.py",
"content": "from .unittest_datasets import (\n AEDataset,\n AmazonReviewSentimentCrossLingualDataset,\n Flickr30kDataset,\n HatefulMeMesDataset,\n IDChangeDetectionDataset,\n PetFinderDataset,\n)\nfrom .utils import (\n evaluate_matcher_ranking,\n get_data_home_dir,\n get_home_dir,\n get_repo_url,\n ref_symmetric_hit_rate,\n verify_matcher_realtime_inference,\n verify_matcher_save_load,\n verify_no_redundant_model_configs,\n verify_predict_and_predict_proba,\n verify_predict_as_pandas_and_multiclass,\n verify_predict_without_label_column,\n verify_predictor_realtime_inference,\n verify_predictor_save_load,\n)\n"
},
{
"path": "multimodal/tests/unittests/utils/unittest_datasets.py",
"content": "import os\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom autogluon.multimodal.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.multimodal.utils import download, path_expander, path_to_bytearray_expander, protected_zip_extraction\n\nfrom .utils import get_data_home_dir, get_repo_url\n\n\nclass PetFinderDataset:\n def __init__(\n self,\n is_bytearray=False,\n ):\n sha1sum_id = \"72cb19612318bb304d4a169804f525f88dc3f0d0\"\n dataset = \"petfinder\"\n file_name = f\"{dataset}_for_unit_tests.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=self._path,\n )\n self._train_df = pd.read_csv(os.path.join(self._path, \"train.csv\"), index_col=0)\n self._test_df = pd.read_csv(os.path.join(self._path, \"test.csv\"), index_col=0)\n expander = path_to_bytearray_expander if is_bytearray else path_expander\n for img_col in self.image_columns:\n self._train_df[img_col] = self._train_df[img_col].apply(\n lambda ele: expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._test_df[img_col] = self._test_df[img_col].apply(\n lambda ele: expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n\n _, self._train_df = train_test_split(\n self._train_df,\n test_size=0.3,\n random_state=np.random.RandomState(123),\n stratify=self._train_df[self.label_columns[0]],\n )\n _, self._test_df = train_test_split(\n self._test_df,\n test_size=0.3,\n random_state=np.random.RandomState(123),\n stratify=self._test_df[self.label_columns[0]],\n )\n self._train_df.reset_index(drop=True, inplace=True)\n self._test_df.reset_index(drop=True, inplace=True)\n\n print(f\"train sample num: {len(self._train_df)}\")\n print(f\"test sample num: {len(self._test_df)}\")\n\n @property\n def path(self):\n return self._path\n\n @property\n def feature_columns(self):\n return [\n \"Type\",\n \"Name\",\n \"Age\",\n \"Breed1\",\n \"Breed2\",\n \"Gender\",\n \"Color1\",\n \"Color2\",\n \"Color3\",\n \"MaturitySize\",\n \"FurLength\",\n \"Vaccinated\",\n \"Dewormed\",\n \"Sterilized\",\n \"Health\",\n \"Quantity\",\n \"Fee\",\n \"State\",\n \"VideoAmt\",\n \"Description\",\n \"PhotoAmt\",\n \"Images\",\n ]\n\n @property\n def label_columns(self):\n return [\"AdoptionSpeed\"]\n\n @property\n def train_df(self):\n return self._train_df\n\n @property\n def test_df(self):\n return self._test_df\n\n @property\n def image_columns(self):\n return [\"Images\"]\n\n @property\n def metric(self):\n return \"quadratic_kappa\"\n\n @property\n def problem_type(self):\n return MULTICLASS\n\n\nclass HatefulMeMesDataset:\n def __init__(\n self,\n is_bytearray=False,\n ):\n sha1sum_id = \"2aae657b786f505004ac2922b66097d60a540a58\"\n dataset = \"hateful_memes\"\n file_name = f\"{dataset}_for_unit_tests.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=self._path,\n )\n self._train_df = pd.read_csv(os.path.join(self._path, \"train.csv\"), index_col=0)\n self._test_df = pd.read_csv(os.path.join(self._path, \"test.csv\"), index_col=0)\n expander = path_to_bytearray_expander if is_bytearray else path_expander\n for img_col in self.image_columns:\n self._train_df[img_col] = self._train_df[img_col].apply(\n lambda ele: expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._test_df[img_col] = self._test_df[img_col].apply(\n lambda ele: expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._train_df.reset_index(drop=True, inplace=True)\n self._test_df.reset_index(drop=True, inplace=True)\n\n print(f\"train sample num: {len(self._train_df)}\")\n print(f\"test sample num: {len(self._test_df)}\")\n\n @property\n def path(self):\n return self._path\n\n @property\n def feature_columns(self):\n return [\"img\", \"text\"]\n\n @property\n def label_columns(self):\n return [\"label\"]\n\n @property\n def train_df(self):\n return self._train_df\n\n @property\n def test_df(self):\n return self._test_df\n\n @property\n def image_columns(self):\n return [\"img\"]\n\n @property\n def metric(self):\n return \"roc_auc\"\n\n @property\n def problem_type(self):\n return BINARY\n\n\nclass AEDataset:\n def __init__(\n self,\n ):\n sha1sum_id = \"8c2a25555c49ef2b30545004488022465808d03f\"\n dataset = \"ae\"\n file_name = f\"{dataset}_for_unit_tests.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=self._path,\n )\n self._train_df = pd.read_csv(os.path.join(self._path, \"train.csv\"), index_col=0)\n self._test_df = pd.read_csv(os.path.join(self._path, \"test.csv\"), index_col=0)\n self._train_df.reset_index(drop=True, inplace=True)\n self._test_df.reset_index(drop=True, inplace=True)\n\n print(f\"train sample num: {len(self._train_df)}\")\n print(f\"test sample num: {len(self._test_df)}\")\n\n @property\n def path(self):\n return self._path\n\n @property\n def feature_columns(self):\n return [\n \"product_name\",\n \"brand_name\",\n \"product_category\",\n \"retailer\",\n \"description\",\n \"rating\",\n \"review_count\",\n \"style_attributes\",\n \"total_sizes\",\n \"available_size\",\n \"color\",\n ]\n\n @property\n def label_columns(self):\n return [\"price\"]\n\n @property\n def train_df(self):\n return self._train_df\n\n @property\n def test_df(self):\n return self._test_df\n\n @property\n def metric(self):\n return \"r2\"\n\n @property\n def problem_type(self):\n return REGRESSION\n\n\nclass AmazonReviewSentimentCrossLingualDataset:\n def __init__(\n self,\n ):\n sha1sum_id = \"9c701aa6fc42ec3fe429bfe85a8dac4532ab9fcd\"\n dataset = \"amazon_review_sentiment_cross_lingual\"\n file_name = f\"{dataset}.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=get_data_home_dir(),\n )\n self._train_en_df = pd.read_csv(\n os.path.join(self._path, \"en_train.tsv\"),\n sep=\"\\t\",\n header=None,\n names=[\"label\", \"text\"],\n ).sample(1000, random_state=123)\n\n self._test_en_df = pd.read_csv(\n os.path.join(self._path, \"en_test.tsv\"),\n sep=\"\\t\",\n header=None,\n names=[\"label\", \"text\"],\n ).sample(200, random_state=123)\n\n self._train_en_df.reset_index(drop=True, inplace=True)\n self._test_en_df.reset_index(drop=True, inplace=True)\n\n print(f\"train sample num: {len(self._train_en_df)}\")\n print(f\"test sample num: {len(self._test_en_df)}\")\n\n @property\n def path(self):\n return self._path\n\n @property\n def label_columns(self):\n return [\"label\"]\n\n @property\n def train_df(self):\n return self._train_en_df\n\n @property\n def test_df(self):\n return self._test_en_df\n\n\nclass IDChangeDetectionDataset:\n def __init__(self):\n sha1sum_id = \"b4a7f3ad12778d65e2ff9a2e4e7bd002c91a0458\"\n dataset = \"id_change_detection\"\n file_name = f\"{dataset}_for_unit_tests.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=self._path,\n )\n # Extract\n self._train_df = pd.read_csv(os.path.join(self._path, \"train.csv\"), index_col=0)\n\n self._test_df = pd.read_csv(os.path.join(self._path, \"test.csv\"), index_col=0)\n\n for img_col in self.image_columns:\n self._train_df[img_col] = self._train_df[img_col].apply(\n lambda ele: path_expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._test_df[img_col] = self._test_df[img_col].apply(\n lambda ele: path_expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n\n @property\n def feature_columns(self):\n return [\"Previous Image\", \"Current Image\", \"Product Title\", \"Product Type\"]\n\n @property\n def label_columns(self):\n return [\"Is Identity Changed?\"]\n\n @property\n def train_df(self):\n return self._train_df\n\n @property\n def test_df(self):\n return self._test_df\n\n @property\n def image_columns(self):\n return [\"Previous Image\", \"Current Image\"]\n\n @property\n def metric(self):\n return \"roc_auc\"\n\n @property\n def problem_type(self):\n return BINARY\n\n @property\n def match_label(self):\n return 0\n\n\nclass Flickr30kDataset:\n def __init__(self):\n sha1sum_id = \"9f748e009f51ce4013a4244861813220fa1eb517\"\n dataset = \"flickr30k\"\n file_name = f\"{dataset}_for_unit_tests.zip\"\n url = get_repo_url() + file_name\n save_path = os.path.join(get_data_home_dir(), file_name)\n self._path = os.path.join(get_data_home_dir(), dataset)\n download(\n url=url,\n path=save_path,\n sha1_hash=sha1sum_id,\n )\n protected_zip_extraction(\n save_path,\n sha1_hash=sha1sum_id,\n folder=self._path,\n )\n # Extract\n self._train_df = pd.read_csv(os.path.join(self._path, \"train.csv\"), index_col=0)\n\n self._val_df = pd.read_csv(os.path.join(self._path, \"val.csv\"), index_col=0)\n\n self._test_df = pd.read_csv(os.path.join(self._path, \"test.csv\"), index_col=0)\n\n for img_col in self.image_columns:\n self._train_df[img_col] = self._train_df[img_col].apply(\n lambda ele: path_expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._val_df[img_col] = self._val_df[img_col].apply(\n lambda ele: path_expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n self._test_df[img_col] = self._test_df[img_col].apply(\n lambda ele: path_expander(ele, base_folder=os.path.join(self._path, \"images\"))\n )\n\n @property\n def feature_columns(self):\n return [\"image\", \"caption\"]\n\n @property\n def label_columns(self):\n return None\n\n @property\n def train_df(self):\n return self._train_df\n\n @property\n def val_df(self):\n return self._val_df\n\n @property\n def test_df(self):\n return self._test_df\n\n @property\n def image_columns(self):\n return [\"image\"]\n\n @property\n def metric(self):\n return \"ndcg\"\n\n @property\n def problem_type(self):\n return None\n\n @property\n def match_label(self):\n return None\n"
},
{
"path": "multimodal/tests/unittests/utils/utils.py",
"content": "import os\nimport shutil\nimport tempfile\nimport uuid\n\nimport numpy.testing as npt\nimport torch\nfrom torchmetrics import RetrievalHitRate\n\nfrom autogluon.multimodal import MultiModalPredictor\nfrom autogluon.multimodal.constants import BINARY, MULTICLASS, QUERY, RESPONSE, UNIFORM_SOUP\nfrom autogluon.multimodal.utils import convert_data_for_ranking\n\n\ndef get_home_dir():\n \"\"\"Get home directory\"\"\"\n _home_dir = os.path.join(\"~\", \".automm_unit_tests\")\n # expand ~ to actual path\n _home_dir = os.path.expanduser(_home_dir)\n return _home_dir\n\n\ndef get_data_home_dir():\n \"\"\"Get home directory for storing the datasets\"\"\"\n home_dir = get_home_dir()\n return os.path.join(home_dir, \"datasets\")\n\n\ndef get_repo_url():\n \"\"\"Return the base URL for Gluon dataset and model repository\"\"\"\n repo_url = \"https://automl-mm-bench.s3.us-east-1.amazonaws.com/unit-tests-0.4/datasets/\"\n if repo_url[-1] != \"/\":\n repo_url = repo_url + \"/\"\n return repo_url\n\n\ndef verify_predictor_save_load(predictor, df, verify_embedding=True, cls=MultiModalPredictor):\n root = str(uuid.uuid4())\n os.makedirs(root, exist_ok=True)\n predictor.save(root)\n predictions = predictor.predict(df, as_pandas=False)\n # Test fit_summary()\n predictor.fit_summary()\n\n loaded_predictor = cls.load(root)\n # Test fit_summary()\n loaded_predictor.fit_summary()\n\n predictions2 = loaded_predictor.predict(df, as_pandas=False)\n predictions2_df = loaded_predictor.predict(df, as_pandas=True)\n npt.assert_equal(predictions, predictions2)\n npt.assert_equal(predictions2, predictions2_df.to_numpy())\n if predictor.problem_type in [BINARY, MULTICLASS]:\n predictions_prob = predictor.predict_proba(df, as_pandas=False)\n predictions2_prob = loaded_predictor.predict_proba(df, as_pandas=False)\n predictions2_prob_df = loaded_predictor.predict_proba(df, as_pandas=True)\n npt.assert_equal(predictions_prob, predictions2_prob)\n npt.assert_equal(predictions2_prob, predictions2_prob_df.to_numpy())\n if verify_embedding:\n embeddings = predictor.extract_embedding(df)\n assert embeddings.shape[0] == len(df)\n shutil.rmtree(root)\n\n\ndef verify_predictor_realtime_inference(predictor, df, verify_embedding=True):\n for i in range(1, 3):\n df_small = df.head(i)\n predictions_default = predictor.predict(df_small, as_pandas=False, realtime=False)\n predictions_realtime = predictor.predict(df_small, as_pandas=False, realtime=True)\n npt.assert_equal(predictions_default, predictions_realtime)\n if predictor.problem_type in [BINARY, MULTICLASS]:\n predictions_prob_default = predictor.predict_proba(df_small, as_pandas=False, realtime=False)\n predictions_prob_realtime = predictor.predict_proba(df_small, as_pandas=False, realtime=True)\n npt.assert_equal(predictions_prob_default, predictions_prob_realtime)\n if verify_embedding:\n embeddings_default = predictor.extract_embedding(df_small, realtime=False)\n embeddings_realtime = predictor.extract_embedding(df_small, realtime=True)\n npt.assert_equal(embeddings_default, embeddings_realtime)\n\n\ndef verify_no_redundant_model_configs(predictor):\n model_names = list(predictor._learner._config.model.keys())\n model_names.remove(\"names\")\n assert sorted(predictor._learner._config.model.names) == sorted(model_names)\n\n\ndef verify_predict_and_predict_proba(test_data, predictor):\n preds = predictor.predict(test_data)\n proba = predictor.predict_proba(test_data, as_pandas=False)\n assert len(proba) == len(test_data)\n assert (proba.argmax(axis=1) == preds).all()\n\n\ndef verify_predict_as_pandas_and_multiclass(test_data, predictor):\n pandas_pred = predictor.predict(test_data, as_pandas=True)\n pandas_proba = predictor.predict_proba(test_data, as_pandas=True)\n pandas_proba_as_multiclass = predictor.predict_proba(test_data, as_pandas=True, as_multiclass=True)\n pandas_proba_no_multiclass = predictor.predict_proba(test_data, as_pandas=True, as_multiclass=False)\n\n proba_as_multiclass = predictor.predict_proba(test_data, as_pandas=False, as_multiclass=True)\n proba_as_multiclass = predictor.predict_proba(test_data, as_pandas=False, as_multiclass=True)\n\n\ndef verify_predict_without_label_column(test_data, predictor, label_col=\"label\"):\n test_data_no_label_col = test_data.drop(columns=[label_col], axis=1)\n preds = predictor.predict(test_data_no_label_col)\n assert len(preds) == len(test_data)\n preds2 = predictor.predict(test_data)\n assert len(preds2) == len(test_data)\n assert (preds == preds2).all()\n return preds\n\n\ndef verify_matcher_save_load(matcher, df, verify_embedding=True, cls=MultiModalPredictor):\n with tempfile.TemporaryDirectory() as root:\n matcher.save(root)\n predictions = matcher.predict(df, as_pandas=False)\n loaded_matcher = cls.load(root)\n predictions2 = loaded_matcher.predict(df, as_pandas=False)\n predictions2_df = loaded_matcher.predict(df, as_pandas=True)\n npt.assert_equal(predictions, predictions2)\n npt.assert_equal(predictions2, predictions2_df.to_numpy())\n if matcher.problem_type.endswith((BINARY, MULTICLASS)):\n print(\"\\nverifying predict and predict_proba...\\n\")\n predictions_prob = matcher.predict_proba(df, as_pandas=False)\n predictions2_prob = loaded_matcher.predict_proba(df, as_pandas=False)\n predictions2_prob_df = loaded_matcher.predict_proba(df, as_pandas=True)\n npt.assert_equal(predictions_prob, predictions2_prob)\n npt.assert_equal(predictions2_prob, predictions2_prob_df.to_numpy())\n if verify_embedding:\n query_embeddings = matcher.extract_embedding(df, signature=QUERY)\n response_embeddings = matcher.extract_embedding(df, signature=RESPONSE)\n assert query_embeddings.shape[0] == len(df)\n assert response_embeddings.shape[0] == len(df)\n\n\ndef verify_matcher_realtime_inference(matcher, df, verify_embedding=True):\n for i in range(1, 3):\n df_small = df.head(i)\n predictions_default = matcher.predict(df_small, as_pandas=False, realtime=False)\n predictions_realtime = matcher.predict(df_small, as_pandas=False, realtime=True)\n npt.assert_equal(predictions_default, predictions_realtime)\n if matcher.problem_type.endswith((BINARY, MULTICLASS)):\n predictions_prob_default = matcher.predict_proba(df_small, as_pandas=False, realtime=False)\n predictions_prob_realtime = matcher.predict_proba(df_small, as_pandas=False, realtime=True)\n npt.assert_almost_equal(predictions_prob_default, predictions_prob_realtime, decimal=5)\n if verify_embedding:\n embeddings_default = matcher.extract_embedding(df_small, signature=QUERY, realtime=False)\n embeddings_realtime = matcher.extract_embedding(df_small, signature=QUERY, realtime=True)\n npt.assert_equal(embeddings_default, embeddings_realtime)\n embeddings_default = matcher.extract_embedding(df_small, signature=RESPONSE, realtime=False)\n embeddings_realtime = matcher.extract_embedding(df_small, signature=RESPONSE, realtime=True)\n npt.assert_equal(embeddings_default, embeddings_realtime)\n\n\ndef evaluate_matcher_ranking(matcher, test_df, query_column, response_column, metric_name, symmetric=False):\n test_df_with_label, test_query_text_data, test_response_image_data, test_label_column = convert_data_for_ranking(\n data=test_df,\n query_column=query_column,\n response_column=response_column,\n )\n socre_1 = matcher.evaluate(\n data=test_df_with_label,\n query_data=test_query_text_data,\n response_data=test_response_image_data,\n metrics=[metric_name],\n label=test_label_column,\n cutoffs=[1, 5, 10],\n )\n\n if symmetric:\n (\n test_df_with_label,\n test_query_image_data,\n test_response_text_data,\n test_label_column,\n ) = convert_data_for_ranking(\n data=test_df,\n query_column=response_column,\n response_column=query_column,\n )\n socre_2 = matcher.evaluate(\n data=test_df_with_label,\n query_data=test_query_image_data,\n response_data=test_response_text_data,\n metrics=[metric_name],\n label=test_label_column,\n cutoffs=[1, 5, 10],\n )\n\n\ndef ref_symmetric_hit_rate(features_a, features_b, logit_scale, top_ks=[1, 5, 10]):\n assert len(features_a) == len(features_b)\n hit_rate = 0\n logits_per_a = (logit_scale * features_a @ features_b.t()).detach().cpu()\n logits_per_b = logits_per_a.t().detach().cpu()\n num_elements = len(features_a)\n for logits in [logits_per_a, logits_per_b]:\n preds = logits.reshape(-1)\n indexes = torch.broadcast_to(torch.arange(num_elements).reshape(-1, 1), (num_elements, num_elements)).reshape(\n -1\n )\n target = torch.eye(num_elements, dtype=bool).reshape(-1)\n for k in top_ks:\n hr_k = RetrievalHitRate(top_k=k)\n hit_rate += hr_k(preds, target, indexes=indexes)\n return hit_rate / (2 * len(top_ks))\n"
},
{
"path": "pyproject.toml",
"content": "[tool.ruff]\nline-length = 119\ntarget-version = \"py310\"\nlint.ignore = [\n \"E501\", # Line too long\n \"E731\", # Do not assign a `lambda` expression, use a `def`\n \"E722\", # Do not use bare `except`\n]\nlint.isort.known-first-party = [\"autogluon\"]\nlint.isort.known-third-party = [\n \"gluonts\",\n \"mxnet\",\n \"networkx\",\n \"numpy\",\n \"pandas\",\n \"psutil\",\n \"pytest\",\n \"scipy\",\n \"sklearn\",\n \"sktime\",\n \"statsmodels\",\n \"tqdm\",\n \"Pillow\",\n \"boto3\",\n \"requests\",\n \"timm\",\n \"torch\",\n \"torchvision\",\n \"fairscale\",\n \"scikit-image\",\n \"smart_open\",\n \"lightning\",\n \"torchmetrics\",\n \"transformers\",\n \"nptyping\",\n \"omegaconf\",\n \"pytorch-metric-learning\",\n \"nlpaug\",\n \"nltk\",\n]\n[tool.ruff.lint.per-file-ignores]\n\"__init__.py\" = [\"F401\"] # Unused imports in __init__.py files\n\n\n[tool.codespell]\nskip = '.git,*.pdf,*.svg,*.ipynb,*.csv,kaggle_feedback_prize'\n#\nignore-words-list = 'mape,ans,2st,fo,nd,te,fpr,coo,rouge,SME,NoES'\n\n\n[tool.pyright]\npythonVersion = \"3.10\"\ninclude = [\n \"timeseries/src/\"\n]\nextraPaths = [\n \"common/src\",\n \"core/src\",\n \"tabular/src\",\n \"timeseries/src\",\n]\n"
},
{
"path": "release_instructions/ReleaseInstructions.md",
"content": "# Release process\n\n## Prior to release: 1 week out\n\n* Ensure the version specified in `docs/conf.py` and `VERSION` align with the intended release version.\n* Check all dependency version ranges.\n * Ensure all dependencies are not capped by major version, unless the reason is documented inline.\n * Example of major version cap: `scikit-learn<2`\n * Ensure all dependencies have an upper cap, unless the reason is documented inline.\n * Example of no upper cap: `scikit-learn>=1.0`\n * Ensure no dependency is pinned to an exact version unless the reason is documented inline.\n * Example: `scikit-learn==1.1.3`. This is very fragile and overly strict.\n * Ensure all dependencies are capped by minor version and not micro version unless the reason is documented inline.\n * Minor version capping would be `=1.0,<1.2`\n * Ensure dependencies shared across multiple AutoGluon modules have the same version range in each module, unless the reason is documented inline.\n * If the same version range is used in multiple modules, ensure the version range is defined only once by defining it in `core/_setup_utils.py`\n * Ensure all upper caps are using `<` and not `<=`.\n * Ensure all dependencies whose ranges are obtained via `core/_setup_utils.py` have the following inline comment:\n * \"\"\"# version range defined in `core/_setup_utils.py`\"\"\"\n* Try upgrading all dependency version range upper caps to include the latest stable release.\n * Note: For micro releases such as AutoGluon 0.6.2, this is optional.\n * If increasing the range causes an error, either:\n 1. Fix the error.\n 2. Avoid the range change and add an inline comment in setup.py that an error occurred and why we didn't fix.\n * If increasing the range causes a warning, either:\n 1. Fix the warning.\n 2. Suppress the warning for the user + provide justification and appropriate TODOs.\n 3. Avoid the range change and add an inline comment in setup.py that a warning occurred and why we didn't fix.\n * Ensure CI passes, potentially benchmark to catch performance regressions / more complex bugs.\n * Note: To truly catch potential errors, you will need to use the latest supported Python version, since some packages may only support the newer Python version in their latest releases.\n* Make final call for which in-progress PRs are release critical.\n* Communicate with in-progress PR owners that code freeze is in effect, no PRs will be merged that are not release critical.\n* Wait 1 day after code-freeze for pre-release to be published.\n* Ensure latest pre-release is working via `pip install --pre autogluon` and testing to get an idea of how the actual release will function (Ideally with fresh venv). DO NOT RELEASE if the pre-release does not work.\n* Ensure pip install instructions are working correctly for both CPU and GPU.\n * Ensure explicit torch installs have the correct version range and are not overwritten during `pip install autogluon`.\n* Ensure each sub-module is working IN ISOLATION via `pip install --pre autogluon.{submodule}`.\n * Ensure a fresh venv is used for each submodule.\n * Doing this will avoid issues like in v0.4 release with `autogluon.text` crashing when installed standalone due to missing setup.py dependencies\n * https://github.com/autogluon/autogluon/issues/1607\n* Fix any broken website links in the dev branch by referring to the following table (updated daily): https://github.com/autogluon/autogluon-brokenlinks/blob/master/Broken%20Links%20Dev.csv\n* If minor fixes are needed, create PRs and merge them as necessary if they are low risk. Ensure fixes are tested manually.\n* If major fixes are needed, consider the severity and if they are release critical. If they are, consider delaying release to ensure the issue is fixed (and tested).\n\n## Prior to release: 1 day out\n\n* Ensure that the mainline code you are planning to release is stable: Benchmark, ensure CI passes, check with team, etc.\n* Prepare the release notes located in `docs/whats_new/vX.Y.Z.md`:\n * This will be copy-pasted into GitHub when you release.\n * Include all merged PRs into the notes and mention all PR authors / contributors (refer to past releases for examples).\n * Prioritize major features before minor features when ordering, otherwise order by merge date.\n * Run the script `release_instructions/add_links_to_release_notes.py` to add links to all pull requests and GitHub users mentioned in the release notes.\n * Review with at least 2 core maintainers to ensure release notes are correct.\n * Merge a PR that adds the new `docs/whats_new/vX.Y.Z.md` file. Ensure you also update the `docs/whats_new/index.md` in the same PR.\n * DO NOT commit the `docs/whats_new/vX.Y.Z_paste_to_github.md` file that is created. This is only used for pasting the GitHub release notes.\n* Cut a release branch with format `X.Y.Z` (no v) - this branch is required to publish docs to versioned path\n * Clone from master branch\n * Add 1 commit to the release branch to remove pre-release warnings and update install instructions to remove `--pre`: [Old diff](https://github.com/autogluon/autogluon/commit/1d66194d4685b06e884bbf15dcb97580cbfb9261)\n * Add 1 commit that converts notebook links from `master` to `stable` by running this command from the root project directory: `LC_ALL=C find docs/tutorials/ -type f -exec sed -i '' 's#blob/master/docs#blob/stable/docs#' {} +`\n * Update links to AG sub-modules website to be stable ones, i.e. cloud\n * Push release branch\n * Build the release branch docs in [CI](https://ci.gluon.ai/job/autogluon/).\n * Once CI passes, verify it's available at `https://auto.gluon.ai/0.x.y/index.html`\n\n## Release\n\n* Update the `stable` documentation to the new release:\n * Delete the `stable` branch.\n * Create new `stable` branch from `vX.Y.Z` branch (They should be identical).\n * Add and push any change in `docs/README.md` (i.e. space) to ensure `stable` branch is different from `0.x.y`. \n * This is required for GH Action to execute CI continuous integration step if `vX.Y.Z` and `stable` hashes are matching.\n * Wait for CI build of the `stable` branch to pass\n * Check that website has updated to align with the release docs.\n* Perform version release by going to https://github.com/autogluon/autogluon/releases and click 'Draft a new release' in top right.\n * Tag release with format `vX.Y.Z`\n * Name the release identically to the tag (ex: `vX.Y.Z`)\n * Select `master` branch as a target\n * Note: we generally use master unless there are certain commits there we don't want to add to the release\n * DO NOT use the 'Save draft' option during the creation of the release. This breaks GitHub pipelines.\n * Copy-paste the content of `docs/whats_new/vX.Y.Z_paste_to_github.md` into the release notes box.\n * If this file doesn't exist, run `release_instructions/add_links_to_release_notes.py` to generate it.\n * DO NOT use `docs/whats_new/vX.Y.Z.md` -> This will break GitHub's contributor detection logic due to the URLs present around the GitHub aliases. This is why we need to use the `_paste_to_github.md` variant.\n * Ensure release notes look correct and make any final formatting fixes.\n * Click 'Publish release' and the release will go live.\n* Wait ~10 minutes and then locally test that the PyPi package is available and working with the latest release version, ask team members to also independently verify.\n\n## Conda-Forge Release\n\nAfter GitHub & PyPi release, conduct release on Conda-Forge\n* Please refer to [conda release instructions](update-conda-recipes.md) for details.\n\n## Release Cheatsheet\n\n* If intending to create a new cheatsheet for the release, refer to [autogluon-doc-utils README.md](https://github.com/Innixma/autogluon-doc-utils) for instructions on creating a new cheatsheet.\n* If a cheatsheet exists for `0.x.y` (or `0.x`), update the `docs/cheatsheet.md` url paths ([example](https://github.com/autogluon/autogluon/blob/0.4.1/docs/cheatsheet.rst)) in branch `0.x.y` to the correct location ([example for v0.4.0 and v0.4.1](https://github.com/Innixma/autogluon-doc-utils/tree/main/docs/cheatsheets/v0.4.0)).\n * Example urls: [JPEG](https://raw.githubusercontent.com/Innixma/autogluon-doc-utils/main/docs/cheatsheets/v0.4.0/autogluon-cheat-sheet.jpeg), [PDF](https://nbviewer.org/github/Innixma/autogluon-doc-utils/blob/main/docs/cheatsheets/v0.4.0/autogluon-cheat-sheet.pdf)\n * Do NOT do this for `stable` branch or `master` branch, instead have them continue pointing to the [stable cheatsheet files](https://github.com/Innixma/autogluon-doc-utils/tree/main/docs/cheatsheets/stable). This is to ensure that as we release new versions of the cheatsheet, old docs will still refer to the correct cheatsheet for their version.\n * Finally, update the stable files [here](https://github.com/Innixma/autogluon-doc-utils/tree/main/docs/cheatsheets/stable) to reflect the latest released version of the cheatsheet.\n\n## Post Release\n\n* IF THERE IS A MAJOR ISSUE: Do an emergency hot-fix and a new release ASAP. Releases cannot be deleted, so a new release will have to be done.\n\nAfter release is published, on the mainline branch:\n* Update `release` in `docs/conf.py`\n* Increment version in the `VERSION` file and `SECURITY.md`\n* Update doc links in `docs/versions.rst`\n* Update `README.md` sample code with new release version.\n* Send release update to internal and external slack channels and mailing lists\n* Publish any blogs / talks planned for release to generate interest.\n\n## Post Release Conda-Forge Patching\n\nConda-Forge releases are mutable and can be changed post-release to fix breaking bugs without releasing a new version.\n\n* Create a new branch in your forked `autogluon.{module}-feedstock` repo\n* Make necessary updates on packages for patching\n* Increment the `number` field under `build` by 1 and keep the rest of `package` and `source` information unchanged\n* Refer to [conda release instructions](update-conda-recipes.md) for more details\n"
},
{
"path": "release_instructions/add_links_to_release_notes.py",
"content": "import re\nfrom pathlib import Path\n\n\ndef linkify_pull_requests(text: str) -> str:\n \"\"\"\n Replace instances of # with markdown links to the corresponding\n AutoGluon GitHub pull request.\n \"\"\"\n # Pattern matches # followed by 3 to 5 digits, ensures not already inside a markdown link\n pattern = r'(? str:\n \"\"\"\n Replace instances of @username with markdown links to the corresponding\n GitHub user profile, unless already inside a markdown link.\n \"\"\"\n # Match @username not already inside a markdown link\n pattern = r'(? str:\n \"\"\"\n Revert GitHub user profile markdown links back to plain @username mentions.\n Example: [@Innixma](https://github.com/Innixma) -> @Innixma\n \"\"\"\n pattern = r'\\[@([A-Za-z0-9-]+)\\]\\(https://github\\.com/\\1\\)'\n return re.sub(pattern, r'@\\1', text)\n\n\ndef unlinkify_pull_requests(text: str) -> str:\n \"\"\"\n Reverts GitHub pull request markdown links back to plain #1234 format.\n Example: [#5020](https://github.com/autogluon/autogluon/pull/5020) -> #5020\n \"\"\"\n pattern = r'\\[#(\\d{3,5})\\]\\(https://github\\.com/autogluon/autogluon/pull/\\1\\)'\n return re.sub(pattern, r'#\\1', text)\n\n\ndef transform_changelog(file_path: str, strip_links_for_github_release: bool = False) -> None:\n \"\"\"\n Reads a text file, applies GitHub pull request and user profile link transformations,\n and saves the updated text back to the original file.\n \"\"\"\n path = Path(file_path)\n if not path.is_file():\n raise FileNotFoundError(f\"File not found: {file_path}\")\n\n text = path.read_text(encoding='utf-8')\n\n text_w_pr_urls = linkify_pull_requests(text)\n text_w_user_urls = linkify_user_mentions(text_w_pr_urls)\n\n path.write_text(text_w_user_urls, encoding='utf-8')\n print(f\"Updated file saved: {file_path}\")\n\n if strip_links_for_github_release:\n path_stem = Path(file_path).stem\n path_suffix = Path(file_path).suffix\n path_wo_urls = Path(file_path).parent / f\"{path_stem}_paste_to_github{path_suffix}\"\n text_wo_pr_urls = unlinkify_pull_requests(text_w_user_urls)\n text_wo_user_urls = unlinkify_user_mentions(text_wo_pr_urls)\n path_wo_urls.write_text(text_wo_user_urls)\n print(f\"Saved file for GitHub release notes pasting: {path_wo_urls}\")\n\n\nif __name__ == '__main__':\n \"\"\"\n Run this to add urls for all pull requests and GitHub users in the `whats_new` markdown files.\n Uncomment the files you wish to update.\n \n Use the `vX.Y.Z_paste_to_github.md` file generating from running this script to paste the release notes into GitHub.\n This ensures that the URL links for PRs and GitHub users are removed,\n as having them breaks GitHub's contributor detection logic, and they are automatically added by GitHub.\n \"\"\"\n file_prefix = \"../docs/whats_new/\"\n\n files = [\n # \"v0.4.0.md\",\n # \"v0.4.1.md\",\n # \"v0.4.2.md\",\n # \"v0.4.3.md\",\n # \"v0.5.1.md\",\n # \"v0.5.2.md\",\n # \"v0.6.0.md\",\n # \"v0.6.1.md\",\n # \"v0.6.2.md\",\n # \"v0.7.0.md\",\n # \"v0.8.0.md\",\n # \"v0.8.1.md\",\n # \"v0.8.2.md\",\n # \"v0.8.3.md\",\n # \"v1.0.0.md\",\n # \"v1.1.0.md\",\n # \"v1.1.1.md\",\n # \"v1.2.0.md\",\n # \"v1.3.0.md\",\n ]\n\n files = [file_prefix + f for f in files]\n\n for f in files:\n transform_changelog(f, strip_links_for_github_release=True)\n"
},
{
"path": "release_instructions/autogluon-conda-upgrade/SKILL.md",
"content": "---\nname: autogluon-conda-upgrade\ndescription: Automate AutoGluon conda-forge feedstock version upgrades. Use when the user wants to upgrade AutoGluon to a new version in conda-forge, create PRs for AutoGluon conda feedstocks, or update autogluon.common, autogluon.core, autogluon.features, autogluon.tabular, autogluon.multimodal, autogluon.timeseries, or autogluon meta-package feedstocks.\n---\n\n# AutoGluon Conda Feedstock Upgrade Workflow\n\n**CRITICAL: DO NOT MERGE PULL REQUESTS.** Only create PRs. The user will review and merge them manually.\n\n## Step 1: Prerequisites Check\n\n### 1.1 Check GitHub CLI\n\n```bash\ngh --version\n```\n\nIf not installed, stop and tell the user to install from https://github.com/cli/cli#installation and run `gh auth login`.\n\n### 1.2 Check Authentication\n\n```bash\ngh auth status\n```\n\nIf not authenticated, ask user to run `gh auth login`.\n\n### 1.3 Gather User Input\n\nAsk for:\n- **New AutoGluon version number** (e.g., `1.5.0`)\n- **Working directory** (default: `~/autogluon-feedstock-upgrade`)\n\n## Step 2: Setup Working Directory and Fork/Clone Repos\n\n```bash\nmkdir -p {WORKING_DIR}\ncd {WORKING_DIR}\n\ngh repo fork conda-forge/autogluon.common-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon.features-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon.core-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon.tabular-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon.multimodal-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon.timeseries-feedstock --clone=true --remote=true\ngh repo fork conda-forge/autogluon-feedstock --clone=true --remote=true\n```\n\n## Step 3: Compute SHA256 Hash\n\n```bash\ncurl -sL \"https://github.com/autogluon/autogluon/archive/refs/tags/v{NEW_VERSION}.tar.gz\" -o /tmp/autogluon-{NEW_VERSION}.tar.gz\nopenssl sha256 /tmp/autogluon-{NEW_VERSION}.tar.gz | awk '{print $2}'\nrm /tmp/autogluon-{NEW_VERSION}.tar.gz\n```\n\nIf curl fails with 404, ask user to verify the version number.\n\n## Step 4: Fetch and Analyze Dependencies\n\n### 4.1 Get Current (Old) Version\n\nRead from `{WORKING_DIR}/autogluon.common-feedstock/recipe/meta.yaml`:\n```\n{% set version = \"X.Y.Z\" %}\n```\n\n### 4.2 Fetch Version Bounds\n\nFetch `_setup_utils.py` for both versions:\n- New: `https://raw.githubusercontent.com/autogluon/autogluon/refs/tags/v{NEW_VERSION}/core/src/autogluon/core/_setup_utils.py`\n- Old: `https://raw.githubusercontent.com/autogluon/autogluon/refs/tags/v{OLD_VERSION}/core/src/autogluon/core/_setup_utils.py`\n\nExtract `DEPENDENT_PACKAGES` dictionary and `PYTHON_REQUIRES` string.\n\n### 4.3 Fetch Package-Specific Setup Files\n\nFor each subpackage (common, features, core, tabular, multimodal, timeseries, autogluon), fetch:\n`https://raw.githubusercontent.com/autogluon/autogluon/refs/tags/v{NEW_VERSION}/{SUBPACKAGE}/setup.py`\n\nThe `install_requires` shows which `DEPENDENT_PACKAGES` each subpackage needs.\n\n### 4.4 Create Dependency Change Summary\n\nCompare old vs new. Summarize:\n1. Changed version bounds\n2. Added dependencies\n3. Removed dependencies\n4. Python version changes\n\n**Present summary to user and ask for confirmation before proceeding.**\n\n## Step 5: Update Each Feedstock\n\nProcess in **dependency order**:\n\n| Order | Feedstock | Dependencies |\n|-------|-----------|--------------|\n| 1 | autogluon.common-feedstock | (none) |\n| 2 | autogluon.features-feedstock | common |\n| 3 | autogluon.core-feedstock | common |\n| 4 | autogluon.tabular-feedstock | core, features |\n| 5 | autogluon.multimodal-feedstock | core |\n| 6 | autogluon.timeseries-feedstock | core, tabular |\n| 7 | autogluon-feedstock | all subpackages |\n\n### For Each Feedstock:\n\n#### 5.1 Sync Fork and Create Branch (DO THIS FIRST)\n\n```bash\ncd {WORKING_DIR}/{FEEDSTOCK_NAME}\ngit fetch upstream\ngit checkout main\ngit reset --hard upstream/main\ngit checkout -b {NEW_VERSION}\n```\n\n#### 5.2 Read Current meta.yaml\n\nAfter creating branch, read `recipe/meta.yaml` to understand current structure.\n\n#### 5.3 Update meta.yaml\n\n1. **Update version:** `{% set version = \"{NEW_VERSION}\" %}`\n2. **Update sha256:** Use computed hash\n3. **Reset build number:** `number: 0`\n4. **Update Python version** (if changed): `python >={{ python_min }},<{NEW_PYTHON_MAX}`\n5. **Update dependency version bounds:** Match `DEPENDENT_PACKAGES`\n\n**Rules:**\n- Keep `autogluon.*` dependencies as `=={{ version }}`\n- Only include dependencies from that package's `setup.py`\n- Preserve existing comments\n- Use conda naming (see Package Name Mappings below)\n\n#### 5.4 Handle python_min Changes\n\nIf minimum Python changed, update `.ci_support/linux_64_.yaml`:\n```yaml\npython_min:\n- '{NEW_PYTHON_MIN}'\n```\n\n## Step 6: Commit and Push\n\nFor each feedstock:\n```bash\ncd {WORKING_DIR}/{FEEDSTOCK_NAME}\ngit add recipe/meta.yaml\ngit commit -m \"Update to v{NEW_VERSION}\"\ngit push -u origin {NEW_VERSION}\n```\n\n## Step 7: Create Pull Requests\n\nFor each feedstock:\n```bash\ncd {WORKING_DIR}/{FEEDSTOCK_NAME}\ngh pr create \\\n --repo conda-forge/{FEEDSTOCK_NAME} \\\n --title \"Update to v{NEW_VERSION}\" \\\n --body \"$(cat <<'EOF'\n## Summary\n- Update {PACKAGE_NAME} to version {NEW_VERSION}\n- Updated dependency version bounds from upstream\n\n## Dependency Changes\n{LIST_RELEVANT_CHANGES}\n\n## Checklist\n* [x] Used a personal fork of the feedstock to propose changes\n* [x] Reset the build number to `0`\n* [ ] Re-rendered (Use `@conda-forge-admin, please rerender` in a comment)\nEOF\n)\"\n```\n\n## Step 8: Final Summary\n\n### 8.1 Provide PR Links\n\nList all 7 created PRs with clickable links.\n\n### 8.2 Merge Order Reminder\n\n> **Merge PRs in dependency order:**\n> 1. `autogluon.common` (no dependencies)\n> 2. `autogluon.features` and `autogluon.core` (parallel)\n> 3. `autogluon.tabular` and `autogluon.multimodal` (parallel)\n> 4. `autogluon.timeseries`\n> 5. `autogluon` (meta-package)\n\n### 8.3 Post-Merge Instructions\n\n> After each PR's CI passes:\n> 1. Comment: `@conda-forge-admin, please rerender`\n> 2. Wait for rerender bot to update\n> 3. Once CI passes again, merge\n> 4. Wait for package to be published before merging dependent PRs\n\n---\n\n## Appendix A: Dependency Tree\n\n```\nautogluon.common (base - no AG deps)\n â\n âââ autogluon.features (depends: common)\n â\n âââ autogluon.core (depends: common)\n â â\n â âââ autogluon.tabular (depends: core, features)\n â â\n â âââ autogluon.multimodal (depends: core)\n â â\n â âââ autogluon.timeseries (depends: core, tabular)\n â\n âââ autogluon [meta-package] (depends: all subpackages)\n```\n\n## Appendix B: URL Patterns\n\n| Resource | URL |\n|----------|-----|\n| Release tarball | `https://github.com/autogluon/autogluon/archive/refs/tags/v{VERSION}.tar.gz` |\n| Version bounds file | `https://raw.githubusercontent.com/autogluon/autogluon/refs/tags/v{VERSION}/core/src/autogluon/core/_setup_utils.py` |\n| Package setup.py | `https://raw.githubusercontent.com/autogluon/autogluon/refs/tags/v{VERSION}/{SUBPACKAGE}/setup.py` |\n\n## Appendix C: Sample PRs\n\n- [autogluon.common PR #6](https://github.com/conda-forge/autogluon.common-feedstock/pull/6/files)\n- [autogluon.features PR #5](https://github.com/conda-forge/autogluon.features-feedstock/pull/5/files)\n- [autogluon.core PR #8](https://github.com/conda-forge/autogluon.core-feedstock/pull/8/files)\n- [autogluon.tabular PR #15](https://github.com/conda-forge/autogluon.tabular-feedstock/pull/15/files)\n- [autogluon.multimodal PR #16](https://github.com/conda-forge/autogluon.multimodal-feedstock/pull/16/files)\n- [autogluon.timeseries PR #7](https://github.com/conda-forge/autogluon.timeseries-feedstock/pull/7/files)\n- [autogluon PR #6](https://github.com/conda-forge/autogluon-feedstock/pull/6/files)\n\n## Appendix D: Package Name Mappings\n\n| PyPI Name | Conda-Forge Name |\n|-----------|------------------|\n| torch | pytorch |\n| Pillow | pillow |\n| scikit-learn | scikit-learn |\n| PyYAML | pyyaml |\n| opencv-python | opencv |\n| tensorflow | tensorflow |\n"
},
{
"path": "release_instructions/update-conda-recipes.md",
"content": "# How to update the conda-forge recipes for AutoGluon\n\n## Conda-forge packages\n\n- [autogluon.common-feedstock](https://anaconda.org/conda-forge/autogluon.common)\n- [autogluon.features-feedstock](https://anaconda.org/conda-forge/autogluon.features)\n- [autogluon.core-feedstock](https://anaconda.org/conda-forge/autogluon.core)\n- [autogluon.tabular-feedstock](https://anaconda.org/conda-forge/autogluon.tabular)\n- [autogluon.multimodal-feedstock](https://anaconda.org/conda-forge/autogluon.multimodal)\n- [autogluon.timeseries-feedstock](https://anaconda.org/conda-forge/autogluon.timeseries)\n- [autogluon-feedstock](https://anaconda.org/conda-forge/autogluon)\n\n## Conda-forge recipes\n\n- [autogluon.common-feedstock](https://github.com/conda-forge/autogluon.common-feedstock)\n- [autogluon.features-feedstock](https://github.com/conda-forge/autogluon.features-feedstock/)\n- [autogluon.core-feedstock](https://github.com/conda-forge/autogluon.core-feedstock)\n- [autogluon.tabular-feedstock](https://github.com/conda-forge/autogluon.tabular-feedstock)\n- [autogluon.multimodal-feedstock](https://github.com/conda-forge/autogluon.multimodal-feedstock)\n- [autogluon.timeseries-feedstock](https://github.com/conda-forge/autogluon.timeseries-feedstock)\n- [autogluon-feedstock](https://github.com/conda-forge/autogluon-feedstock)\n\n## Sample PRs\n\n- [autogluon.common-feedstock](https://github.com/conda-forge/autogluon.common-feedstock/pull/6/files)\n- [autogluon.features-feedstock](https://github.com/conda-forge/autogluon.features-feedstock/pull/5/files)\n- [autogluon.core-feedstock](https://github.com/conda-forge/autogluon.core-feedstock/pull/8/files)\n- [autogluon.tabular-feedstock](https://github.com/conda-forge/autogluon.tabular-feedstock/pull/15/files)\n- [autogluon.multimodal-feedstock](https://github.com/conda-forge/autogluon.multimodal-feedstock/pull/16/files)\n- [autogluon.timeseries-feedstock](https://github.com/conda-forge/autogluon.timeseries-feedstock/pull/7/files)\n- [autogluon-feedstock](https://github.com/conda-forge/autogluon-feedstock/pull/6/files)\n\n## Steps to update the conda-forge recipes\n\n1. Go to the [AutoGluon](https://github.com/autogluon/autogluon/releases) release page on GitHub. Under the `Assets` section, right click on `Source code (tar.gz)` to copy the link address. It should be something like this:\n\n ```text\n https://github.com/autogluon/autogluon/archive/refs/tags/v0.7.0.tar.gz\n ```\n\n2. Click on the link above to download the source code. The file should be named something like `autogluon-0.7.0.tar.gz`.\n3. Use `openssl` to generate the sha256 hash of the downloaded file, e.g.,\n\n ```bash\n openssl sha256 autogluon-0.7.0.tar.gz\n ```\n\n The output should be something like this:\n\n ```text\n 455831de3c9de8fbe11b100054b8f150661d0651212fcfa4ec2e42417fdac355\n ```\n\n4. Fork the [autogluon.common-feedstock](https://github.com/conda-forge/autogluon.common-feedstock) repo to your GitHub account.\n5. Clone the forked repo to your local machine.\n6. Create a new branch, e.g., `v0.7.0`\n7. Open the `recipe/meta.yaml` file in your favorite text editor. Update the `version`, `sha256` and `number` fields. The `version` field should be the version number of the new release. For example, if the new release is `v0.7.0`, then the `version` field should be `0.7.0`. The `sha256` field should be the hash generated in step 3. The `number` field should be reset to `0` for a new release. If the `version` number stays the same, then the `number` field should be incremented by 1. This is usually the case when you are updating the dependencies of the package but not updating the package version.\n\n\n\n8. Update the package dependencies and version bounds for each recipe based on the release. For `autogluon.common`, the dependency list can be found at [`common/setup.py`](https://github.com/autogluon/autogluon/blob/master/common/setup.py#L19), but the version bounds can be found at [`core/_setup_utils.py`](https://github.com/autogluon/autogluon/blob/master/core/src/autogluon/core/_setup_utils.py#L20)\n\n\n\n9. Commit the changes and push to your forked repo. Then create a pull request to the `autogluon.common-feedstock` repo.\n10. Comment on the pull request with the following text to trigger the CI build:\n\n ```text\n @conda-forge-admin, please rerender\n ```\n\n11. Once the CI build is successful, merge the pull request.\n12. Repeat steps above for the other six packages.\n\n## Steps to build the conda-forge packages locally\n\nOptionally, you can build the conda-forge packages locally to test if the recipes are correct. This is especially useful when you are updating the dependencies of the package. The steps are as follows:\n\n1. Install docker on your machine. See [here](https://docs.docker.com/get-docker/) for instructions.\n2. Install Anaconda or Miniconda on your machine. See [here](https://docs.conda.io/en/latest/miniconda.html) for instructions.\n3. Install `mamba` in the base environment of your conda installation. This is a faster version of `conda` that is recommended for conda-forge builds.\n\n ```bash\n conda install -n base mamba -c conda-forge\n ```\n\n4. Create a new conda environment named `ag` with Python 3.10 or higher that's supported by AutoGluon.\n\n ```bash\n mamba create -n ag python=3.10\n ```\n\n5. Navigate to the root directory of the cloned repo of the package you want to build. For example, if you want to build `autogluon.multimodal`, then you should be in the root directory of the `autogluon.multimodal-feedstock` repo.\n\n ```bash\n chmod 777 -R autogluon.multimodal-feedstock\n cd autogluon.multimodal-feedstock\n python build-locally.py\n ```\n\n6. Choose an option that you want to build that matches your computer configuration. For example, if you want to build the `linux_64_cuda` package with Python 3.10, then choose option `3`.\n\n\n\n7. If the build is successful, you should find the built packages in the `build_artifacts` directory under the root directory of the cloned repo.\n8. Install the built packages in the `ag` environment you created in step 4.\n\n ```bash\n mamba install -n ag -c \"file://${PWD}/build_artifacts\" -c conda-forge autogluon.multimodal\n ```\n\n Make sure to check the `pytorch` version in the list of packages to install. If you see `cuda` in the version number, that means the package is built with CUDA support. Otherwise, it's built without CUDA support.\n\n\n\n## How to add maintainers to the conda-forge recipes\n\n1. Please ask the existing maintainers if you want to be added as a maintainer. Only the existing maintainers can add new maintainers.\n2. The existing maintainer needs to go to the feedstock repo, e.g., [autogluon.common-feedstock](https://github.com/conda-forge/autogluon-feedstock/issues/new/choose).\n3. Open a new issue and choose the `Bot commands` template. Click on `Get started` to open the issue.\n4. Enter the following text in the title of the issue. Be sure to replace `@username` with the GitHub username of the maintainer you want to add.\n\n```bash\n@conda-forge-admin, please add user @username\n```\n\n5. Click on `Submit new issue` to submit the issue.\n6. Once the issue is submitted and the CI build is successful, merge the pull request.\n"
},
{
"path": "setup.cfg",
"content": "[pycodestyle]\nmax_line_length = 160\n\n[flake8]\nmax-line-length = 160\nper-file-ignores =\n */__init__.py: F401\n"
},
{
"path": "tabular/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport importlib.util\nimport os\nimport platform\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec)\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag)\n###########################\n\nimport copy\nimport sys\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = \"tabular\"\ninstall_requires = [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"scipy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"scikit-learn\", # version range defined in `core/_setup_utils.py`\n \"networkx\", # version range defined in `core/_setup_utils.py`\n f\"{ag.PACKAGE_NAME}.core=={version}\",\n f\"{ag.PACKAGE_NAME}.features=={version}\",\n]\n\nextras_require = {\n \"lightgbm\": [\n \"lightgbm>=4.0,<4.7\", # <{N+1} upper cap, where N is the latest released minor version\n ],\n \"catboost\": [\n \"catboost>=1.2,<1.3\",\n ],\n \"xgboost\": [\n \"xgboost>=2.0,<3.2\", # <{N+1} upper cap, where N is the latest released minor version\n ],\n \"realmlp\": [\n \"pytabkit>=1.7.2,<1.8\",\n ],\n \"interpret\": [\n \"interpret-core>=0.7.2,<0.8\",\n ],\n \"fastai\": [\n \"spacy<3.9\",\n \"torch\", # version range defined in `core/_setup_utils.py`\n \"fastai>=2.3.1,<2.8.6\", # Cap due to dependency conflict in fastai-2.8.6 https://github.com/autogluon/autogluon/issues/5521\n ],\n \"tabm\": [\n \"torch\", # version range defined in `core/_setup_utils.py`\n ],\n \"tabpfn\": [\n \"tabpfn>=6.2.0,<6.2.1\", # <{N+1} upper cap, where N is the latest released minor version\n ],\n \"tabdpt\": [\n \"tabdpt>=1.1.11,<1.2\",\n ],\n \"tabpfnmix\": [\n \"torch\", # version range defined in `core/_setup_utils.py`\n \"huggingface_hub[torch]\", # version range defined in `core/_setup_utils.py`\n \"einops>=0.7,<0.9\",\n ],\n \"mitra\": [\n \"loguru\",\n \"einx\",\n \"omegaconf\",\n \"torch\",\n \"transformers\",\n \"huggingface_hub[torch]\", # version range defined in `core/_setup_utils.py`\n \"einops>=0.7,<0.9\",\n ],\n \"tabicl\": [\n \"tabicl>=2.0,<2.1\",\n ],\n \"ray\": [\n f\"{ag.PACKAGE_NAME}.core[all]=={version}\",\n ],\n \"skex\": [\n \"scikit-learn-intelex>=2025.0,<2025.10\", # <{N+1} upper cap, where N is the latest released minor version\n ],\n \"imodels\": [\n \"imodels>=1.3.10,<2.1.0\", # 1.3.8/1.3.9 either remove/renamed attribute `complexity_` causing failures. https://github.com/csinva/imodels/issues/147\n ],\n}\n\nextras_require[\"skl2onnx\"] = [\n \"skl2onnx>=1.15.0,<1.20.0\",\n # Sync ONNX requirements with multimodal/setup.py\n \"onnx>=1.13.0,!=1.16.2,<1.21.0;platform_system=='Windows'\", # exclude 1.16.2 for issue https://github.com/onnx/onnx/issues/6267\n \"onnx>=1.13.0,<1.21.0;platform_system!='Windows'\",\n # For macOS, there isn't a onnxruntime-gpu package installed with skl2onnx.\n # Therefore, we install onnxruntime explicitly here just for macOS.\n \"onnxruntime>=1.17.0,<1.24.0\",\n \"onnxruntime-gpu>=1.17.0,<1.24.0; platform_system != 'Darwin' and platform_machine != 'aarch64'\",\n]\n\n# TODO: v1.0: Rename `all` to `core`, make `all` contain everything.\nall_requires = []\nfor extra_package in [\n \"lightgbm\",\n \"catboost\",\n \"xgboost\",\n \"fastai\",\n \"tabm\",\n \"mitra\",\n \"ray\",\n]:\n all_requires += extras_require[extra_package]\nall_requires = list(set(all_requires))\nextras_require[\"all\"] = all_requires\n\ntabarena_requires = copy.deepcopy(all_requires)\nfor extra_package in [\n \"interpret\",\n \"tabdpt\",\n \"tabicl\",\n \"tabpfn\",\n \"realmlp\",\n]:\n tabarena_requires += extras_require[extra_package]\ntabarena_requires = list(set(tabarena_requires))\nextras_require[\"tabarena\"] = tabarena_requires\n\ntest_requires = []\nfor test_package in [\n \"interpret\",\n \"tabdpt\",\n \"tabicl\", # Currently has unnecessary extra dependencies such as xgboost and wandb\n \"tabpfn\",\n \"realmlp\", # Will consider to put as part of `all_requires` once part of a portfolio\n \"tabpfnmix\", # Refer to `mitra`, which is an improved version of `tabpfnmix`\n \"imodels\",\n \"skl2onnx\",\n]:\n test_requires += extras_require[test_package]\nextras_require[\"tests\"] = test_requires\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\nextras_require = {key: ag.get_dependency_version_ranges(value) for key, value in extras_require.items()}\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/__init__.py",
"content": "# noinspection PyUnresolvedReferences\nfrom autogluon.common.dataset import TabularDataset\n\n# noinspection PyUnresolvedReferences\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.utils.log_utils import _add_stream_handler\n\ntry:\n from .version import __version__\nexcept ImportError:\n pass\n\nfrom .predictor import TabularPredictor\n\n_add_stream_handler()\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/configs/config_helper.py",
"content": "from __future__ import annotations\n\nimport copy\nfrom typing import Union\n\nfrom sklearn.base import BaseEstimator\n\nfrom autogluon.core.scheduler import scheduler_factory\nfrom autogluon.features import AutoMLPipelineFeatureGenerator\nfrom autogluon.tabular.configs.hyperparameter_configs import hyperparameter_config_dict\nfrom autogluon.tabular.configs.presets_configs import tabular_presets_dict\nfrom autogluon.tabular.registry import ag_model_registry\n\n\nclass FeatureGeneratorBuilder:\n def __init__(self, parent=None):\n self.parent = parent\n self.config = {}\n\n def enable_numeric_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to keep features of 'int' and 'float' raw types.\n These features are passed without alteration to the models.\n Appends IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=['int', 'float']))) to the generator group.\n \"\"\"\n self.config[\"enable_numeric_features\"] = value\n return self\n\n def enable_categorical_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to keep features of 'object' and 'category' raw types.\n These features are processed into memory optimized 'category' features.\n Appends CategoryFeatureGenerator() to the generator group.\n \"\"\"\n self.config[\"enable_categorical_features\"] = value\n return self\n\n def enable_datetime_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to keep features of 'datetime' raw type and 'object' features identified as 'datetime_as_object' features.\n These features will be converted to 'int' features representing milliseconds since epoch.\n Appends DatetimeFeatureGenerator() to the generator group.\n \"\"\"\n self.config[\"enable_datetime_features\"] = value\n return self\n\n def enable_text_special_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to use 'object' features identified as 'text' features to generate 'text_special' features such as word count, capital letter ratio, and symbol counts.\n Appends TextSpecialFeatureGenerator() to the generator group.\n \"\"\"\n self.config[\"enable_text_special_features\"] = value\n return self\n\n def enable_text_ngram_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to use 'object' features identified as 'text' features to generate 'text_ngram' features.\n Appends TextNgramFeatureGenerator(vectorizer=vectorizer) to the generator group.\n \"\"\"\n self.config[\"enable_text_ngram_features\"] = value\n return self\n\n def enable_raw_text_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Whether to keep the raw text features.\n Appends IdentityFeatureGenerator(infer_features_in_args=dict(required_special_types=['text'])) to the generator group.\n \"\"\"\n self.config[\"enable_raw_text_features\"] = value\n return self\n\n def enable_vision_features(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n [Experimental]\n Whether to keep 'object' features identified as 'image_path' special type. Features of this form should have a string path to an image file as their value.\n Only vision models can leverage these features, and these features will not be treated as categorical.\n Note: 'image_path' features will not be automatically inferred. These features must be explicitly specified as such in a custom FeatureMetadata object.\n Note: It is recommended that the string paths use absolute paths rather than relative, as it will likely be more stable.\n \"\"\"\n self.config[\"enable_vision_features\"] = value\n return self\n\n def vectorizer(self, value: BaseEstimator) -> FeatureGeneratorBuilder:\n \"\"\"\n sklearn CountVectorizer object to use in TextNgramFeatureGenerator.\n Only used if `enable_text_ngram_features=True`.\n \"\"\"\n self.config[\"vectorizer\"] = value\n return self\n\n def text_ngram_params(self, value: bool = True) -> FeatureGeneratorBuilder:\n \"\"\"\n Appends TextNgramFeatureGenerator(vectorizer=vectorizer, text_ngram_params) to the generator group. See text_ngram.py for valid parameters.\n \"\"\"\n self.config[\"text_ngram_params\"] = value\n return self\n\n def build(self) -> Union[ConfigBuilder, AutoMLPipelineFeatureGenerator]:\n generator = AutoMLPipelineFeatureGenerator(**self.config)\n if self.parent:\n self.parent.config[\"feature_generator\"] = generator\n return self.parent\n else:\n return generator\n\n\nclass ConfigBuilder:\n def __init__(self):\n self.config = {}\n\n def _valid_keys(self):\n valid_keys = [m for m in ag_model_registry.keys if m not in [\"ENS_WEIGHTED\", \"SIMPLE_ENS_WEIGHTED\"]]\n return valid_keys\n\n def presets(self, presets: Union[str, list, dict]) -> ConfigBuilder:\n \"\"\"\n List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`.\n It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon.\n Available Presets: ['best_quality', 'high_quality', 'good_quality', 'medium_quality', 'optimize_for_deployment', 'ignore_text']\n It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other.\n If there is an overlap in presets keys, the latter presets will override the earlier ones.\n \"\"\"\n valid_keys = list(tabular_presets_dict.keys())\n if isinstance(presets, str):\n presets = [presets]\n\n if isinstance(presets, list):\n unknown_keys = [k for k in presets if k not in valid_keys]\n assert len(unknown_keys) == 0, (\n f\"The following presets are not recognized: {unknown_keys} - use one of the valid presets: {valid_keys}\"\n )\n\n self.config[\"presets\"] = presets\n return self\n\n def time_limit(self, time_limit: int) -> ConfigBuilder:\n \"\"\"\n Approximately how long `fit()` should run for (wallclock time in seconds).\n If not specified, `fit()` will run until all models have completed training, but will not repeatedly bag models unless `num_bag_sets` is specified.\n \"\"\"\n if time_limit is not None:\n assert time_limit > 0, \"time_limit must be greater than zero\"\n self.config[\"time_limit\"] = time_limit\n return self\n\n def hyperparameters(self, hyperparameters: Union[str, dict]) -> ConfigBuilder:\n valid_keys = self._valid_keys()\n valid_str_values = list(hyperparameter_config_dict.keys())\n if isinstance(hyperparameters, str):\n assert hyperparameters in hyperparameter_config_dict, (\n f\"{hyperparameters} is not one of the valid presets {valid_str_values}\"\n )\n elif isinstance(hyperparameters, dict):\n unknown_keys = [k for k in hyperparameters.keys() if isinstance(k, str) and (k not in valid_keys)]\n assert len(unknown_keys) == 0, (\n f\"The following model types are not recognized: {unknown_keys} - use one of the valid models: {valid_keys}\"\n )\n else:\n raise ValueError(\n f\"hyperparameters must be either str: {valid_str_values} or dict with keys of {valid_keys}\"\n )\n self.config[\"hyperparameters\"] = hyperparameters\n return self\n\n def auto_stack(self, auto_stack: bool = True) -> ConfigBuilder:\n \"\"\"\n Whether AutoGluon should automatically utilize bagging and multi-layer stack ensembling to boost predictive accuracy.\n Set this = True if you are willing to tolerate longer training times in order to maximize predictive accuracy!\n Automatically sets `num_bag_folds` and `num_stack_levels` arguments based on dataset properties.\n Note: Setting `num_bag_folds` and `num_stack_levels` arguments will override `auto_stack`.\n Note: This can increase training time (and inference time) by up to 20x, but can greatly improve predictive performance.\n \"\"\"\n self.config[\"auto_stack\"] = auto_stack\n return self\n\n def num_bag_folds(self, num_bag_folds: int) -> ConfigBuilder:\n \"\"\"\n Number of folds used for bagging of models. When `num_bag_folds = k`, training time is roughly increased by a factor of `k` (set = 0 to disable bagging).\n Disabled by default (0), but we recommend values between 5-10 to maximize predictive performance.\n Increasing num_bag_folds will result in models with lower bias but that are more prone to overfitting.\n `num_bag_folds = 1` is an invalid value, and will raise a ValueError.\n Values > 10 may produce diminishing returns, and can even harm overall results due to overfitting.\n To further improve predictions, avoid increasing `num_bag_folds` much beyond 10 and instead increase `num_bag_sets`.\n \"\"\"\n assert num_bag_folds >= 0, \"num_bag_folds must be greater or equal than zero\"\n self.config[\"num_bag_folds\"] = num_bag_folds\n return self\n\n def num_bag_sets(self, num_bag_sets: int) -> ConfigBuilder:\n \"\"\"\n Number of repeats of kfold bagging to perform (values must be >= 1). Total number of models trained during bagging = `num_bag_folds * num_bag_sets`.\n Defaults to 1 if `time_limit` is not specified, otherwise 20 (always disabled if `num_bag_folds` is not specified).\n Values greater than 1 will result in superior predictive performance, especially on smaller problems and with stacking enabled (reduces overall variance).\n \"\"\"\n assert num_bag_sets > 0, \"num_bag_sets must be greater than zero\"\n self.config[\"num_bag_sets\"] = num_bag_sets\n return self\n\n def num_stack_levels(self, num_stack_levels: int) -> ConfigBuilder:\n \"\"\"\n Number of stacking levels to use in stack ensemble. Roughly increases model training time by factor of `num_stack_levels+1` (set = 0 to disable stack ensembling).\n Disabled by default (0), but we recommend values between 1-3 to maximize predictive performance.\n To prevent overfitting, `num_bag_folds >= 2` must also be set or else a ValueError will be raised.\n \"\"\"\n assert num_stack_levels >= 0, \"num_stack_levels must be greater or equal than zero\"\n self.config[\"num_stack_levels\"] = num_stack_levels\n return self\n\n def holdout_frac(self, holdout_frac: float) -> ConfigBuilder:\n \"\"\"\n Fraction of train_data to holdout as tuning data for optimizing hyperparameters (ignored unless `tuning_data = None`, ignored if `num_bag_folds != 0` unless `use_bag_holdout == True`).\n Default value (if None) is selected based on the number of rows in the training data. Default values range from 0.2 at 2,500 rows to 0.01 at 250,000 rows.\n Default value is doubled if `hyperparameter_tune_kwargs` is set, up to a maximum of 0.2.\n Disabled if `num_bag_folds >= 2` unless `use_bag_holdout == True`.\n \"\"\"\n assert (holdout_frac >= 0) & (holdout_frac <= 1), \"holdout_frac must be between 0 and 1\"\n self.config[\"holdout_frac\"] = holdout_frac\n return self\n\n def use_bag_holdout(self, use_bag_holdout: bool = True) -> ConfigBuilder:\n \"\"\"\n If True, a `holdout_frac` portion of the data is held-out from model bagging.\n This held-out data is only used to score models and determine weighted ensemble weights.\n Enable this if there is a large gap between score_val and score_test in stack models.\n Note: If `tuning_data` was specified, `tuning_data` is used as the holdout data.\n Disabled if not bagging.\n \"\"\"\n self.config[\"use_bag_holdout\"] = use_bag_holdout\n return self\n\n def hyperparameter_tune_kwargs(self, hyperparameter_tune_kwargs: Union[str, dict]) -> ConfigBuilder:\n \"\"\"\n Hyperparameter tuning strategy and kwargs (for example, how many HPO trials to run).\n If None, then hyperparameter tuning will not be performed.\n Valid preset values:\n 'auto': Uses the 'random' preset.\n 'random': Performs HPO via random search using local scheduler.\n The 'searcher' key is required when providing a dict.\n \"\"\"\n valid_str_values = scheduler_factory._scheduler_presets.keys()\n if isinstance(hyperparameter_tune_kwargs, str):\n assert hyperparameter_tune_kwargs in valid_str_values, (\n f\"{hyperparameter_tune_kwargs} string must be one of {valid_str_values}\"\n )\n elif not isinstance(hyperparameter_tune_kwargs, dict):\n raise ValueError(f\"hyperparameter_tune_kwargs must be either str: {valid_str_values} or dict\")\n self.config[\"hyperparameter_tune_kwargs\"] = hyperparameter_tune_kwargs\n\n return self\n\n def ag_args(self, ag_args: dict) -> ConfigBuilder:\n \"\"\"\n Keyword arguments to pass to all models (i.e. common hyperparameters shared by all AutoGluon models).\n See the `ag_args` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args` parameter for all models in `hyperparameters`.\n If a key in `ag_args` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n \"\"\"\n self.config[\"ag_args\"] = ag_args\n return self\n\n def ag_args_fit(self, ag_args_fit: dict) -> ConfigBuilder:\n \"\"\"\n Keyword arguments to pass to all models.\n See the `ag_args_fit` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args_fit` parameter for all models in `hyperparameters`.\n If a key in `ag_args_fit` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n \"\"\"\n self.config[\"ag_args_fit\"] = ag_args_fit\n return self\n\n def ag_args_ensemble(self, ag_args_ensemble: dict) -> ConfigBuilder:\n \"\"\"\n Keyword arguments to pass to all models.\n See the `ag_args_ensemble` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args_ensemble` parameter for all models in `hyperparameters`.\n If a key in `ag_args_ensemble` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n \"\"\"\n self.config[\"ag_args_ensemble\"] = ag_args_ensemble\n return self\n\n def excluded_model_types(self, models: Union[str, list]) -> ConfigBuilder:\n \"\"\"\n Banned subset of model types to avoid training during `fit()`, even if present in `hyperparameters`.\n Reference `hyperparameters` documentation for what models correspond to each value.\n Useful when a particular model type such as 'KNN' or 'custom' is not desired but altering the `hyperparameters` dictionary is difficult or time-consuming.\n Example: To exclude both 'KNN' and 'custom' models, specify `excluded_model_types=['KNN', 'custom']`.\n \"\"\"\n valid_keys = self._valid_keys()\n if not isinstance(models, list):\n models = [models]\n for model in models:\n assert model in valid_keys, f\"{model} is not one of the valid models {valid_keys}\"\n self.config[\"excluded_model_types\"] = sorted(list(set(models)))\n return self\n\n def included_model_types(self, models: Union[str, list]) -> ConfigBuilder:\n \"\"\"\n Subset of model types to train during `fit()`.\n Reference `hyperparameters` documentation for what models correspond to each value.\n Useful when only the particular models should be trained such as 'KNN' or 'custom', but altering the `hyperparameters` dictionary is difficult or time-consuming.\n Example: To keep only 'KNN' and 'custom' models, specify `included_model_types=['KNN', 'custom']`.\n \"\"\"\n valid_keys = self._valid_keys()\n if not isinstance(models, list):\n models = [models]\n\n unknown_keys = [k for k in models if isinstance(k, str) and (k not in valid_keys)]\n assert len(unknown_keys) == 0, (\n f\"The following model types are not recognized: {unknown_keys} - use one of the valid models: {valid_keys}\"\n )\n\n models = [m for m in valid_keys if m not in models]\n self.config[\"excluded_model_types\"] = models\n return self\n\n def refit_full(self, refit_full: Union[bool, str] = True) -> ConfigBuilder:\n \"\"\"\n Whether to retrain all models on all of the data (training + validation) after the normal training procedure.\n This is equivalent to calling `predictor.refit_full(model=refit_full)` after fit.\n If `refit_full=True`, it will be treated as `refit_full='all'`.\n If `refit_full=False`, refitting will not occur.\n Valid str values:\n `all`: refits all models.\n `best`: refits only the best model (and its ancestors if it is a stacker model).\n `{model_name}`: refits only the specified model (and its ancestors if it is a stacker model).\n \"\"\"\n self.config[\"refit_full\"] = refit_full\n return self\n\n def set_best_to_refit_full(self, set_best_to_refit_full=True) -> ConfigBuilder:\n \"\"\"\n If True, will change the default model that Predictor uses for prediction when model is not specified to the refit_full version of the model that exhibited the highest validation score.\n Only valid if `refit_full` is set.\n \"\"\"\n self.config[\"set_best_to_refit_full\"] = set_best_to_refit_full\n return self\n\n def keep_only_best(self, keep_only_best=True) -> ConfigBuilder:\n \"\"\"\n If True, only the best model and its ancestor models are saved in the outputted `predictor`. All other models are deleted.\n If you only care about deploying the most accurate predictor with the smallest file-size and no longer need any of the other trained models or functionality beyond prediction on new data, then set: `keep_only_best=True`, `save_space=True`.\n This is equivalent to calling `predictor.delete_models(models_to_keep='best')` directly after `fit()`.\n If used with `refit_full` and `set_best_to_refit_full`, the best model will be the refit_full model, and the original bagged best model will be deleted.\n `refit_full` will be automatically set to 'best' in this case to avoid training models which will be later deleted.\n \"\"\"\n self.config[\"keep_only_best\"] = keep_only_best\n return self\n\n def save_space(self, save_space=True) -> ConfigBuilder:\n \"\"\"\n If True, reduces the memory and disk size of predictor by deleting auxiliary model files that aren't needed for prediction on new data.\n This is equivalent to calling `predictor.save_space()` directly after `fit()`.\n This has NO impact on inference accuracy.\n It is recommended if the only goal is to use the trained model for prediction.\n Certain advanced functionality may no longer be available if `save_space=True`. Refer to `predictor.save_space()` documentation for more details.\n \"\"\"\n self.config[\"save_space\"] = save_space\n return self\n\n def feature_generator(self) -> FeatureGeneratorBuilder:\n \"\"\"\n The feature generator used by AutoGluon to process the input data to the form sent to the models. This often includes automated feature generation and data cleaning.\n It is generally recommended to keep the default feature generator unless handling an advanced use-case.\n \"\"\"\n return FeatureGeneratorBuilder(self)\n\n def calibrate(self, calibrate=True) -> ConfigBuilder:\n \"\"\"\n If True and the problem_type is classification, temperature scaling will be used to calibrate the Predictor's estimated class probabilities\n (which may improve metrics like log_loss) and will train a scalar parameter on the validation set.\n If True and the problem_type is quantile regression, conformalization will be used to calibrate the Predictor's estimated quantiles\n (which may improve the prediction interval coverage, and bagging could further improve it) and will compute a set of scalar parameters on the validation set.\n \"\"\"\n self.config[\"calibrate\"] = calibrate\n return self\n\n def build(self) -> dict:\n \"\"\"\n Build the config.\n \"\"\"\n return copy.deepcopy(self.config)\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/feature_generator_presets.py",
"content": "import copy\n\nfrom autogluon.features.generators import (\n AutoMLInterpretablePipelineFeatureGenerator,\n AutoMLPipelineFeatureGenerator,\n IdentityFeatureGenerator,\n)\n\n\ndef get_default_feature_generator(feature_generator, feature_metadata=None, init_kwargs=None):\n if init_kwargs is None:\n init_kwargs = dict()\n if feature_generator is None:\n feature_generator = IdentityFeatureGenerator()\n elif isinstance(feature_generator, str):\n if feature_generator == \"auto\":\n feature_generator = AutoMLPipelineFeatureGenerator(**init_kwargs)\n elif feature_generator == \"interpretable\":\n feature_generator = AutoMLInterpretablePipelineFeatureGenerator(**init_kwargs)\n else:\n raise ValueError(\n f\"Unknown feature_generator preset: '{feature_generator}', valid presets: {['auto', 'interpretable']}\"\n )\n if feature_metadata is not None:\n if feature_generator.feature_metadata_in is None and not feature_generator.is_fit():\n feature_generator.feature_metadata_in = copy.deepcopy(feature_metadata)\n else:\n raise AssertionError(\"`feature_metadata_in` already exists in `feature_generator`.\")\n return feature_generator\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/hyperparameter_configs.py",
"content": "import copy\n\nfrom .zeroshot.zeroshot_portfolio_2023 import hyperparameter_portfolio_zeroshot_2023\nfrom .zeroshot.zeroshot_portfolio_2025 import hyperparameter_portfolio_zeroshot_2025_small\nfrom .zeroshot.zeroshot_portfolio_cpu_2025_12_18 import hyperparameter_portfolio_zeroshot_cpu_2025_12_18\nfrom .zeroshot.zeroshot_portfolio_gpu_2025_12_18 import hyperparameter_portfolio_zeroshot_gpu_2025_12_18\n\n# Dictionary of preset hyperparameter configurations.\nhyperparameter_config_dict = dict(\n # Default AutoGluon hyperparameters intended to maximize accuracy without significant regard to inference time or disk usage.\n default={\n \"NN_TORCH\": {},\n \"GBM\": [\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n \"CAT\": {},\n \"XGB\": {},\n \"FASTAI\": {},\n \"RF\": [\n {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\n \"criterion\": \"squared_error\",\n \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]},\n },\n ],\n \"XT\": [\n {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\n \"criterion\": \"squared_error\",\n \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]},\n },\n ],\n },\n # Results in smaller models. Generally will make inference speed much faster and disk usage much lower, but with worse accuracy.\n light={\n \"NN_TORCH\": {},\n \"GBM\": [\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n \"CAT\": {},\n \"XGB\": {},\n \"FASTAI\": {},\n \"RF\": [\n {\n \"criterion\": \"gini\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]},\n },\n {\n \"criterion\": \"entropy\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]},\n },\n {\n \"criterion\": \"squared_error\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]},\n },\n ],\n \"XT\": [\n {\n \"criterion\": \"gini\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]},\n },\n {\n \"criterion\": \"entropy\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]},\n },\n {\n \"criterion\": \"squared_error\",\n \"max_depth\": 15,\n \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]},\n },\n ],\n },\n # Results in much smaller models. Behaves similarly to 'light', but in many cases with over 10x less disk usage and a further reduction in accuracy.\n very_light={\n \"NN_TORCH\": {},\n \"GBM\": [\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n \"CAT\": {},\n \"XGB\": {},\n \"FASTAI\": {},\n },\n # Results in extremely quick to train models. Only use this when prototyping, as the model accuracy will be severely reduced.\n toy={\n \"NN_TORCH\": {\"num_epochs\": 5},\n \"GBM\": {\"num_boost_round\": 10},\n \"CAT\": {\"iterations\": 10},\n \"XGB\": {\"n_estimators\": 10},\n },\n # Default AutoGluon hyperparameters intended to maximize accuracy in multimodal tabular + text datasets. Requires GPU.\n multimodal={\n \"NN_TORCH\": {},\n \"GBM\": [\n {},\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n \"CAT\": {},\n \"XGB\": {},\n # 'FASTAI': {}, # FastAI gets killed if the dataset is large (400K rows).\n \"AG_AUTOMM\": {},\n },\n # Hyperparameters intended to find an interpretable model which doesn't sacrifice predictive accuracy\n interpretable={\n \"IM_RULEFIT\": [{\"max_rules\": 7}, {\"max_rules\": 12}, {\"max_rules\": 18}],\n \"IM_FIGS\": [{\"max_rules\": 6}, {\"max_rules\": 10}, {\"max_rules\": 15}],\n # Note: Below are commented out because they are not meaningfully interpretable via the existing API\n # 'IM_GREEDYTREE': [{'max_leaf_nodes': 7, 'max_leaf_nodes': 18}],\n # 'IM_BOOSTEDRULES': [{'n_estimators': 5}, {'n_estimators': 10}],\n # 'IM_HSTREE': [{'max_rules': 6}, {'max_rules': 12}, {'max_rules': 18}],\n },\n zeroshot=hyperparameter_portfolio_zeroshot_2023,\n zeroshot_2023=hyperparameter_portfolio_zeroshot_2023,\n zeroshot_2025_tabfm=hyperparameter_portfolio_zeroshot_2025_small,\n zeroshot_2025_12_18_gpu=hyperparameter_portfolio_zeroshot_gpu_2025_12_18,\n zeroshot_2025_12_18_cpu=hyperparameter_portfolio_zeroshot_cpu_2025_12_18,\n)\n\ntabpfnmix_default = {\n \"model_path_classifier\": \"autogluon/tabpfn-mix-1.0-classifier\",\n \"model_path_regressor\": \"autogluon/tabpfn-mix-1.0-regressor\",\n \"n_ensembles\": 1,\n \"max_epochs\": 30,\n \"ag.sample_rows_val\": 5000, # Beyond 5k val rows fine-tuning becomes very slow\n \"ag.max_rows\": 50000, # Beyond 50k rows, the time taken is longer than most users would like (hours), while the model is very weak at this size\n \"ag_args\": {\"name_suffix\": \"_v1\"},\n}\n\nhyperparameter_config_dict[\"experimental_2024\"] = {\"TABPFNMIX\": tabpfnmix_default}\nhyperparameter_config_dict[\"experimental_2024\"].update(hyperparameter_config_dict[\"zeroshot_2023\"])\nhyperparameter_config_dict[\"experimental\"] = hyperparameter_config_dict[\"experimental_2024\"]\n\n\ndef get_hyperparameter_config_options():\n return list(hyperparameter_config_dict.keys())\n\n\ndef get_hyperparameter_config(config_name):\n config_options = get_hyperparameter_config_options()\n if config_name not in config_options:\n raise ValueError(\n f\"Valid hyperparameter config names are: {config_options}, but '{config_name}' was given instead.\"\n )\n return copy.deepcopy(hyperparameter_config_dict[config_name])\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/pipeline_presets.py",
"content": "from __future__ import annotations\n\nimport math\n\nfrom autogluon.core.constants import BINARY, PROBLEM_TYPES\nfrom autogluon.core.utils.utils import default_holdout_frac\n\nUSE_BAG_HOLDOUT_AUTO_THRESHOLD = 1_000_000\n\n\ndef _get_validation_preset(num_train_rows: int, hpo_enabled: bool) -> dict[str, int | float]:\n \"\"\"Recommended validation preset manually defined by the AutoGluon developers.\"\"\"\n\n # -- Default recommendation\n # max 8 due to 8 cores per CPU being very common.\n # down to 5 folds for small datasets to have enough samples for a representative validation set.\n num_bag_folds = min(8, max(5, math.floor(num_train_rows / 10)))\n\n num_bag_sets = 1 # More repeats do not seem to help due to overfitting on val data.\n use_bag_holdout = num_train_rows >= USE_BAG_HOLDOUT_AUTO_THRESHOLD\n holdout_frac = round(default_holdout_frac(num_train_rows=num_train_rows, hyperparameter_tune=hpo_enabled), 4)\n\n return dict(\n num_bag_sets=num_bag_sets,\n num_bag_folds=num_bag_folds,\n use_bag_holdout=use_bag_holdout,\n holdout_frac=holdout_frac,\n )\n\n\n# TODO(refactor): use a data class for the config of the validation method.\n# TODO(improvement): Implement a more sophisticated solution.\n# Could also use more metadata such as num_features, num_models,\n# or time_limit for a heuristic.\n# num_features: The number of features in the dataset.\n# num_models: The number of models in the portfolio to fit.\n# time_limit: The time limit for fitting models.\n# Pointer for non-heuristic approach:\n# -> meta-learning like Auto-Sklearn 2.0, needs a lot of metadata\ndef get_validation_and_stacking_method(\n # Validation parameters\n num_bag_folds: int | None,\n num_bag_sets: int | None,\n use_bag_holdout: bool | None,\n holdout_frac: float | None,\n # Stacking/Pipeline parameters\n auto_stack: bool,\n num_stack_levels: int | None,\n dynamic_stacking: bool | None,\n refit_full: bool | None,\n # Metadata\n num_train_rows: int,\n problem_type: PROBLEM_TYPES,\n hpo_enabled: bool,\n) -> tuple[int, int, int, bool, bool, float, bool]:\n \"\"\"Get the validation method for AutoGluon via a heuristic.\n\n Input variables are `None` if they were not specified by the user or have an explicit default.\n\n Parameters\n ----------\n num_bag_folds: int | None\n The number of folds for cross-validation.\n num_bag_sets: int | None\n The number of repeats for cross-validation.\n use_bag_holdout: bool | None\n Whether to use (additional) holdout validation.\n holdout_frac: float | None\n The fraction of data to holdout for validation.\n auto_stack: bool\n Whether to automatically determine the stacking method.\n num_stack_levels: int | None\n The number of stacking levels.\n dynamic_stacking: bool | None\n Whether to use dynamic stacking.\n refit_full: bool\n Whether to refit the full training dataset.\n num_train_rows: int\n The number of rows in the training dataset.\n problem_type: PROBLEM_TYPES\n The type of problem to solve.\n hpo_enabled: bool\n If True, HPO is enabled during the run of AutoGluon.\n\n Returns:\n --------\n Returns all variables needed to define the validation method.\n \"\"\"\n\n cv_preset = _get_validation_preset(num_train_rows=num_train_rows, hpo_enabled=hpo_enabled)\n\n # Independent of `auto_stack`\n if use_bag_holdout is None:\n use_bag_holdout = cv_preset[\"use_bag_holdout\"]\n if holdout_frac is None:\n holdout_frac = cv_preset[\"holdout_frac\"]\n if dynamic_stacking is None:\n dynamic_stacking = not use_bag_holdout\n if refit_full is None:\n refit_full = False\n\n # Changed by `auto_stack`\n if num_bag_folds is None:\n # `num_bag_folds == 0` -> only use holdout validation\n num_bag_folds = cv_preset[\"num_bag_folds\"] if auto_stack else 0\n if num_bag_sets is None:\n # `num_bag_sets == 1` -> no repeats\n num_bag_sets = cv_preset[\"num_bag_sets\"] if auto_stack else 1\n if num_stack_levels is None:\n # Disable multi-layer stacking by default\n num_stack_levels = 0\n\n # Activate multi-layer stacking for `auto_stack` if\n if auto_stack and (\n dynamic_stacking # -> We use dynamic stacking\n or\n # -> We have holdout validation or a non-binary problem with more than 750 training rows\n ((use_bag_holdout or (problem_type != BINARY)) and (num_train_rows >= 750))\n ):\n num_stack_levels = 1\n\n return (\n num_bag_folds,\n num_bag_sets,\n num_stack_levels,\n dynamic_stacking,\n use_bag_holdout,\n holdout_frac,\n refit_full,\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/presets_configs.py",
"content": "# Dictionary of preset fit() parameter configurations.\ntabular_presets_dict = dict(\n # Best predictive accuracy with little consideration to inference time or disk usage. Achieve even better results by specifying a large time_limit value.\n # Recommended for applications that benefit from the best possible model accuracy.\n # Aliases: best\n best_quality={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"hyperparameters\": \"zeroshot\",\n \"time_limit\": 3600,\n },\n best_quality_v150={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_stack_levels\": 0,\n \"hyperparameters\": \"zeroshot_2025_12_18_cpu\",\n \"time_limit\": 3600,\n \"callbacks\": [\n [\n \"EarlyStoppingCountCallback\",\n {\"patience\": [[100, 4], [500, 8], [2500, 15], [10000, 40], [100000, 100], None]},\n ]\n ],\n },\n # High predictive accuracy with fast inference. ~8x faster inference and ~8x lower disk usage than `best_quality`.\n # Recommended for applications that require fast inference speed and/or small model size.\n # Aliases: high\n high_quality={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"hyperparameters\": \"zeroshot\",\n \"time_limit\": 3600,\n \"refit_full\": True,\n \"set_best_to_refit_full\": True,\n \"save_bag_folds\": False,\n },\n high_quality_v150={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_stack_levels\": 0,\n \"hyperparameters\": \"zeroshot_2025_12_18_cpu\",\n \"time_limit\": 3600,\n \"callbacks\": [\n [\n \"EarlyStoppingCountCallback\",\n {\"patience\": [[100, 4], [500, 8], [2500, 15], [10000, 40], [100000, 100], None]},\n ]\n ],\n \"refit_full\": True,\n \"set_best_to_refit_full\": True,\n \"save_bag_folds\": False,\n },\n # Good predictive accuracy with very fast inference. ~4x faster training, ~8x faster inference and ~8x lower disk usage than `high_quality`.\n # Recommended for applications that require very fast inference speed.\n # Aliases: good\n good_quality={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"hyperparameters\": \"light\",\n \"time_limit\": 3600,\n \"refit_full\": True,\n \"set_best_to_refit_full\": True,\n \"save_bag_folds\": False,\n },\n # Medium predictive accuracy with very fast inference and very fast training time. ~20x faster training than `good_quality`.\n # This is the default preset in AutoGluon, but should generally only be used for quick prototyping, as `good_quality` results in significantly better predictive accuracy with similar inference time.\n # Aliases: medium, medium_quality_faster_train\n medium_quality={\"auto_stack\": False},\n # Optimizes result immediately for deployment by deleting unused models and removing training artifacts.\n # Often can reduce disk usage by ~2-4x with no negatives to model accuracy or inference speed.\n # This will disable numerous advanced functionality, but has no impact on inference.\n # Recommended for applications where the inner details of AutoGluon's training is not important and there is no intention of manually choosing between the final models.\n # This preset pairs well with the other presets such as `good_quality` to make a very compact final model.\n # Identical to calling `predictor.delete_models(models_to_keep='best', dry_run=False)` and `predictor.save_space()` directly after `fit()`.\n optimize_for_deployment={\"keep_only_best\": True, \"save_space\": True},\n # Disables automated feature generation when text features are detected.\n # This is useful to determine how beneficial text features are to the end result, as well as to ensure features are not mistaken for text when they are not.\n ignore_text={\n \"_feature_generator_kwargs\": {\n \"enable_text_ngram_features\": False,\n \"enable_text_special_features\": False,\n \"enable_raw_text_features\": False,\n }\n },\n ignore_text_ngrams={\"_feature_generator_kwargs\": {\"enable_text_ngram_features\": False}},\n # Fit only interpretable models.\n interpretable={\n \"auto_stack\": False,\n \"hyperparameters\": \"interpretable\",\n \"feature_generator\": \"interpretable\",\n \"fit_weighted_ensemble\": False,\n \"calibrate\": False,\n },\n # ------------------------------------------\n # ------------------------------------------\n # Legacy presets\n # Best predictive accuracy with little consideration to inference time or disk usage. Achieve even better results by specifying a large time_limit value.\n # Recommended for applications that benefit from the best possible model accuracy.\n best_quality_v082={\"auto_stack\": True},\n # High predictive accuracy with fast inference. ~10x-200x faster inference and ~10x-200x lower disk usage than `best_quality`.\n # Recommended for applications that require reasonable inference speed and/or model size.\n high_quality_v082={\n \"auto_stack\": True,\n \"refit_full\": True,\n \"set_best_to_refit_full\": True,\n \"save_bag_folds\": False,\n },\n # Good predictive accuracy with very fast inference. ~4x faster inference and ~4x lower disk usage than `high_quality`.\n # Recommended for applications that require fast inference speed.\n good_quality_v082={\n \"auto_stack\": True,\n \"refit_full\": True,\n \"set_best_to_refit_full\": True,\n \"save_bag_folds\": False,\n \"hyperparameters\": \"light\",\n },\n # ------------------------------------------\n # Experimental presets. Only use these presets if you are ok with unstable and potentially poor performing presets.\n # Experimental presets can be removed or changed without warning.\n # [EXPERIMENTAL PRESET] The `extreme` preset may be changed or removed without warning.\n # This preset acts as a testing ground for cutting edge features and models which could later be added to the `best_quality` preset in future releases.\n # Using this preset can lead to unexpected crashes, as it hasn't been as thoroughly tested as other presets.\n # Absolute best predictive accuracy with **zero** consideration to inference time or disk usage.\n # Recommended for applications that benefit from the best possible model accuracy and **do not** care about inference speed.\n # Significantly stronger than `best_quality`, but can be over 10x slower in inference.\n # Uses pre-trained tabular foundation models, which add a minimum of 100 MB to the predictor artifact's size.\n # For best results, use as large of an instance as possible with a GPU and as many CPU cores as possible (ideally 64+ cores)\n # Aliases: extreme, experimental, experimental_quality\n # GPU STRONGLY RECOMMENDED\n extreme_quality={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_stack_levels\": 0,\n \"hyperparameters\": \"zeroshot_2025_12_18_gpu\",\n \"time_limit\": 3600,\n \"callbacks\": [\n [\n \"EarlyStoppingCountCallback\",\n {\"patience\": [[100, 4], [500, 8], [2500, 15], [10000, 40], [100000, 100], None]},\n ]\n ],\n },\n extreme_quality_v140={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_bag_sets\": 1,\n \"_experimental_dynamic_hyperparameters\": True,\n \"hyperparameters\": None,\n \"time_limit\": 3600,\n },\n # Preset with a portfolio learned from TabArena v0.1: https://tabarena.ai/\n # Uses tabular foundation models: TabPFNv2, TabICL, Mitra\n # Uses deep learning model: TabM\n # Uses tree models: LightGBM, CatBoost, XGBoost\n # Extremely powerful on small datasets with <= 10000 training samples.\n # Requires a GPU for best results.\n tabarena={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_bag_sets\": 1,\n \"num_stack_levels\": 0,\n \"hyperparameters\": \"zeroshot_2025_tabfm\",\n \"time_limit\": 3600,\n },\n # DOES NOT SUPPORT GPU.\n experimental_quality_v120={\n \"auto_stack\": True,\n \"dynamic_stacking\": \"auto\",\n \"num_bag_sets\": 1,\n \"hyperparameters\": \"experimental\",\n \"fit_strategy\": \"parallel\",\n \"num_gpus\": 0,\n \"time_limit\": 3600,\n },\n # ------------------------------------------\n # ------------------------------------------\n # ------------------------------------------\n)\n\n\n# Alias preset name alternatives\ntabular_presets_alias = dict(\n extreme=\"extreme_quality\",\n best=\"best_quality\",\n high=\"high_quality\",\n high_quality_fast_inference_only_refit=\"high_quality\",\n good=\"good_quality\",\n good_quality_faster_inference_only_refit=\"good_quality\",\n medium=\"medium_quality\",\n medium_quality_faster_train=\"medium_quality\",\n eq=\"extreme_quality\",\n bq=\"best_quality\",\n hq=\"high_quality\",\n gq=\"good_quality\",\n mq=\"medium_quality\",\n experimental=\"extreme_quality\",\n experimental_quality=\"extreme_quality\",\n experimental_quality_v140=\"extreme_quality_v140\",\n best_v140=\"best_quality\",\n best_v150=\"best_quality_v150\",\n best_quality_v140=\"best_quality\",\n high_v150=\"high_quality_v150\",\n extreme_v140=\"extreme_quality_v140\",\n extreme_v150=\"extreme_quality\",\n)\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/zeroshot/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2023.py",
"content": "# Portfolio learned from zeroshot-HPO using TabRepo: https://github.com/autogluon/tabrepo\n# Contains the default AutoGluon-Tabular hyperparameters as well as up to 100 learned model configs.\nhyperparameter_portfolio_zeroshot_2023 = {\n \"NN_TORCH\": [\n {},\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.10077639529843717,\n \"hidden_size\": 108,\n \"learning_rate\": 0.002735937344002146,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.356433327634438e-12,\n \"ag_args\": {\"name_suffix\": \"_r79\", \"priority\": -2},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.11897478034205347,\n \"hidden_size\": 213,\n \"learning_rate\": 0.0010474382260641949,\n \"num_layers\": 4,\n \"use_batchnorm\": False,\n \"weight_decay\": 5.594471067786272e-10,\n \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": -7},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.24622382571353768,\n \"hidden_size\": 159,\n \"learning_rate\": 0.008507536855608535,\n \"num_layers\": 5,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.8201539594953562e-06,\n \"ag_args\": {\"name_suffix\": \"_r30\", \"priority\": -17},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.09976801642258049,\n \"hidden_size\": 135,\n \"learning_rate\": 0.001631450730978947,\n \"num_layers\": 5,\n \"use_batchnorm\": False,\n \"weight_decay\": 3.867683394425807e-05,\n \"ag_args\": {\"name_suffix\": \"_r86\", \"priority\": -19},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.3905837860053583,\n \"hidden_size\": 106,\n \"learning_rate\": 0.0018297905295930797,\n \"num_layers\": 1,\n \"use_batchnorm\": True,\n \"weight_decay\": 9.178069874232892e-08,\n \"ag_args\": {\"name_suffix\": \"_r14\", \"priority\": -26},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.05488816803887784,\n \"hidden_size\": 32,\n \"learning_rate\": 0.0075612897834015985,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.652353009917866e-08,\n \"ag_args\": {\"name_suffix\": \"_r41\", \"priority\": -35},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.01030258381183309,\n \"hidden_size\": 111,\n \"learning_rate\": 0.01845979186513771,\n \"num_layers\": 5,\n \"use_batchnorm\": True,\n \"weight_decay\": 0.00020238017476912164,\n \"ag_args\": {\"name_suffix\": \"_r158\", \"priority\": -38},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.18109219857068798,\n \"hidden_size\": 250,\n \"learning_rate\": 0.00634181748507711,\n \"num_layers\": 1,\n \"use_batchnorm\": False,\n \"weight_decay\": 5.3861175580695396e-08,\n \"ag_args\": {\"name_suffix\": \"_r197\", \"priority\": -41},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.1703783780377607,\n \"hidden_size\": 212,\n \"learning_rate\": 0.0004107199833213839,\n \"num_layers\": 5,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.105439140660822e-07,\n \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -49},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.013288954106470907,\n \"hidden_size\": 81,\n \"learning_rate\": 0.005340914647396154,\n \"num_layers\": 4,\n \"use_batchnorm\": False,\n \"weight_decay\": 8.762168370775353e-05,\n \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -52},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.36669080773207274,\n \"hidden_size\": 95,\n \"learning_rate\": 0.015280159186761077,\n \"num_layers\": 3,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.3082489374636015e-08,\n \"ag_args\": {\"name_suffix\": \"_r87\", \"priority\": -59},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.3027114570947557,\n \"hidden_size\": 196,\n \"learning_rate\": 0.006482759295309238,\n \"num_layers\": 1,\n \"use_batchnorm\": False,\n \"weight_decay\": 1.2806509958776e-12,\n \"ag_args\": {\"name_suffix\": \"_r71\", \"priority\": -60},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.12166942295569863,\n \"hidden_size\": 151,\n \"learning_rate\": 0.0018866871631794007,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 9.190843763153802e-05,\n \"ag_args\": {\"name_suffix\": \"_r185\", \"priority\": -65},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.006531401073483156,\n \"hidden_size\": 192,\n \"learning_rate\": 0.012418052210914356,\n \"num_layers\": 1,\n \"use_batchnorm\": True,\n \"weight_decay\": 3.0406866089493607e-05,\n \"ag_args\": {\"name_suffix\": \"_r76\", \"priority\": -77},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.33926015213879396,\n \"hidden_size\": 247,\n \"learning_rate\": 0.0029983839090226075,\n \"num_layers\": 5,\n \"use_batchnorm\": False,\n \"weight_decay\": 0.00038926240517691234,\n \"ag_args\": {\"name_suffix\": \"_r121\", \"priority\": -79},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.06134755114373829,\n \"hidden_size\": 144,\n \"learning_rate\": 0.005834535148903801,\n \"num_layers\": 5,\n \"use_batchnorm\": True,\n \"weight_decay\": 2.0826540090463355e-09,\n \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -84},\n },\n {\n \"activation\": \"elu\",\n \"dropout_prob\": 0.3457125770744979,\n \"hidden_size\": 37,\n \"learning_rate\": 0.006435774191713849,\n \"num_layers\": 3,\n \"use_batchnorm\": True,\n \"weight_decay\": 2.4012185204155345e-08,\n \"ag_args\": {\"name_suffix\": \"_r36\", \"priority\": -87},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.2211285919550286,\n \"hidden_size\": 196,\n \"learning_rate\": 0.011307978270179143,\n \"num_layers\": 1,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.8441764217351068e-06,\n \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -92},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.23713784729000734,\n \"hidden_size\": 200,\n \"learning_rate\": 0.00311256170909018,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 4.573016756474468e-08,\n \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -96},\n },\n {\n \"activation\": \"relu\",\n \"dropout_prob\": 0.33567564890346097,\n \"hidden_size\": 245,\n \"learning_rate\": 0.006746560197328548,\n \"num_layers\": 3,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.6470047305392933e-10,\n \"ag_args\": {\"name_suffix\": \"_r89\", \"priority\": -97},\n },\n ],\n \"GBM\": [\n {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.7023601671276614,\n \"learning_rate\": 0.012144796373999013,\n \"min_data_in_leaf\": 14,\n \"num_leaves\": 53,\n \"ag_args\": {\"name_suffix\": \"_r131\", \"priority\": -3},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.5636931414546802,\n \"learning_rate\": 0.01518660230385841,\n \"min_data_in_leaf\": 48,\n \"num_leaves\": 16,\n \"ag_args\": {\"name_suffix\": \"_r96\", \"priority\": -6},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.8282601210460099,\n \"learning_rate\": 0.033929021353492905,\n \"min_data_in_leaf\": 6,\n \"num_leaves\": 127,\n \"ag_args\": {\"name_suffix\": \"_r188\", \"priority\": -14},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.6245777099925497,\n \"learning_rate\": 0.04711573688184715,\n \"min_data_in_leaf\": 56,\n \"num_leaves\": 89,\n \"ag_args\": {\"name_suffix\": \"_r130\", \"priority\": -18},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.5898927512279213,\n \"learning_rate\": 0.010464516487486093,\n \"min_data_in_leaf\": 11,\n \"num_leaves\": 252,\n \"ag_args\": {\"name_suffix\": \"_r161\", \"priority\": -27},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.5143401489640409,\n \"learning_rate\": 0.00529479887023554,\n \"min_data_in_leaf\": 6,\n \"num_leaves\": 133,\n \"ag_args\": {\"name_suffix\": \"_r196\", \"priority\": -31},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.7421180622507277,\n \"learning_rate\": 0.018603888565740096,\n \"min_data_in_leaf\": 6,\n \"num_leaves\": 22,\n \"ag_args\": {\"name_suffix\": \"_r15\", \"priority\": -37},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.9408897917880529,\n \"learning_rate\": 0.01343464462043561,\n \"min_data_in_leaf\": 21,\n \"num_leaves\": 178,\n \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -44},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.4341088458599442,\n \"learning_rate\": 0.04034449862560467,\n \"min_data_in_leaf\": 33,\n \"num_leaves\": 16,\n \"ag_args\": {\"name_suffix\": \"_r94\", \"priority\": -48},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.9773131270704629,\n \"learning_rate\": 0.010534290864227067,\n \"min_data_in_leaf\": 21,\n \"num_leaves\": 111,\n \"ag_args\": {\"name_suffix\": \"_r30\", \"priority\": -56},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.8254432681390782,\n \"learning_rate\": 0.031251656439648626,\n \"min_data_in_leaf\": 50,\n \"num_leaves\": 210,\n \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -69},\n },\n {\n \"extra_trees\": False,\n \"feature_fraction\": 0.5730390983988963,\n \"learning_rate\": 0.010305352949119608,\n \"min_data_in_leaf\": 10,\n \"num_leaves\": 215,\n \"ag_args\": {\"name_suffix\": \"_r121\", \"priority\": -74},\n },\n {\n \"extra_trees\": True,\n \"feature_fraction\": 0.4601361323873807,\n \"learning_rate\": 0.07856777698860955,\n \"min_data_in_leaf\": 12,\n \"num_leaves\": 198,\n \"ag_args\": {\"name_suffix\": \"_r42\", \"priority\": -95},\n },\n ],\n \"CAT\": [\n {},\n {\n \"depth\": 6,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 2.1542798306067823,\n \"learning_rate\": 0.06864209415792857,\n \"max_ctr_complexity\": 4,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r177\", \"priority\": -1},\n },\n {\n \"depth\": 8,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 2.7997999596449104,\n \"learning_rate\": 0.031375015734637225,\n \"max_ctr_complexity\": 2,\n \"one_hot_max_size\": 3,\n \"ag_args\": {\"name_suffix\": \"_r9\", \"priority\": -5},\n },\n {\n \"depth\": 4,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 4.559174625782161,\n \"learning_rate\": 0.04939557741379516,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 3,\n \"ag_args\": {\"name_suffix\": \"_r137\", \"priority\": -10},\n },\n {\n \"depth\": 8,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 3.3274013177541373,\n \"learning_rate\": 0.017301189655111057,\n \"max_ctr_complexity\": 5,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -12},\n },\n {\n \"depth\": 4,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 2.7018061518087038,\n \"learning_rate\": 0.07092851311746352,\n \"max_ctr_complexity\": 1,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r50\", \"priority\": -20},\n },\n {\n \"depth\": 5,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 1.0457098345001241,\n \"learning_rate\": 0.050294288910022224,\n \"max_ctr_complexity\": 5,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r69\", \"priority\": -24},\n },\n {\n \"depth\": 6,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 1.3584121369544215,\n \"learning_rate\": 0.03743901034980473,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r70\", \"priority\": -29},\n },\n {\n \"depth\": 7,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 4.522712492188319,\n \"learning_rate\": 0.08481607830570326,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r167\", \"priority\": -33},\n },\n {\n \"depth\": 8,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 1.6376578537958237,\n \"learning_rate\": 0.032899230324940465,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r86\", \"priority\": -39},\n },\n {\n \"depth\": 4,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 3.353268454214423,\n \"learning_rate\": 0.06028218319511302,\n \"max_ctr_complexity\": 1,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -42},\n },\n {\n \"depth\": 8,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 1.640921865280573,\n \"learning_rate\": 0.036232951900213306,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 5,\n \"ag_args\": {\"name_suffix\": \"_r128\", \"priority\": -50},\n },\n {\n \"depth\": 4,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 2.894432181094842,\n \"learning_rate\": 0.055078095725390575,\n \"max_ctr_complexity\": 4,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r5\", \"priority\": -58},\n },\n {\n \"depth\": 7,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 1.6761016245166451,\n \"learning_rate\": 0.06566144806528762,\n \"max_ctr_complexity\": 2,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -61},\n },\n {\n \"depth\": 5,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 3.3217885487525205,\n \"learning_rate\": 0.05291587380674719,\n \"max_ctr_complexity\": 5,\n \"one_hot_max_size\": 3,\n \"ag_args\": {\"name_suffix\": \"_r60\", \"priority\": -67},\n },\n {\n \"depth\": 4,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 1.5734131496361856,\n \"learning_rate\": 0.08472519974533015,\n \"max_ctr_complexity\": 3,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r6\", \"priority\": -72},\n },\n {\n \"depth\": 7,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 4.43335055453705,\n \"learning_rate\": 0.055406199833457785,\n \"max_ctr_complexity\": 5,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r180\", \"priority\": -76},\n },\n {\n \"depth\": 7,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 4.835797074498082,\n \"learning_rate\": 0.03534026385152556,\n \"max_ctr_complexity\": 5,\n \"one_hot_max_size\": 10,\n \"ag_args\": {\"name_suffix\": \"_r12\", \"priority\": -83},\n },\n {\n \"depth\": 5,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 3.7454481983750014,\n \"learning_rate\": 0.09328642499990342,\n \"max_ctr_complexity\": 1,\n \"one_hot_max_size\": 2,\n \"ag_args\": {\"name_suffix\": \"_r163\", \"priority\": -89},\n },\n {\n \"depth\": 6,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 3.637071465711953,\n \"learning_rate\": 0.04387418552563314,\n \"max_ctr_complexity\": 4,\n \"one_hot_max_size\": 5,\n \"ag_args\": {\"name_suffix\": \"_r198\", \"priority\": -90},\n },\n ],\n \"XGB\": [\n {},\n {\n \"colsample_bytree\": 0.6917311125174739,\n \"enable_categorical\": False,\n \"learning_rate\": 0.018063876087523967,\n \"max_depth\": 10,\n \"min_child_weight\": 0.6028633586934382,\n \"ag_args\": {\"name_suffix\": \"_r33\", \"priority\": -8},\n },\n {\n \"colsample_bytree\": 0.6628423832084077,\n \"enable_categorical\": False,\n \"learning_rate\": 0.08775715546881824,\n \"max_depth\": 5,\n \"min_child_weight\": 0.6294123374222513,\n \"ag_args\": {\"name_suffix\": \"_r89\", \"priority\": -16},\n },\n {\n \"colsample_bytree\": 0.9090166528779192,\n \"enable_categorical\": True,\n \"learning_rate\": 0.09290221350439203,\n \"max_depth\": 7,\n \"min_child_weight\": 0.8041986915994078,\n \"ag_args\": {\"name_suffix\": \"_r194\", \"priority\": -22},\n },\n {\n \"colsample_bytree\": 0.516652313273348,\n \"enable_categorical\": True,\n \"learning_rate\": 0.007158072983547058,\n \"max_depth\": 9,\n \"min_child_weight\": 0.8567068904025429,\n \"ag_args\": {\"name_suffix\": \"_r98\", \"priority\": -36},\n },\n {\n \"colsample_bytree\": 0.7452294043087835,\n \"enable_categorical\": False,\n \"learning_rate\": 0.038404229910104046,\n \"max_depth\": 7,\n \"min_child_weight\": 0.5564183327139662,\n \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -57},\n },\n {\n \"colsample_bytree\": 0.7506621909633511,\n \"enable_categorical\": False,\n \"learning_rate\": 0.009974712407899168,\n \"max_depth\": 4,\n \"min_child_weight\": 0.9238550485581797,\n \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -64},\n },\n {\n \"colsample_bytree\": 0.6326947454697227,\n \"enable_categorical\": False,\n \"learning_rate\": 0.07792091886639502,\n \"max_depth\": 6,\n \"min_child_weight\": 1.0759464955561793,\n \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": -70},\n },\n {\n \"colsample_bytree\": 0.975937238416368,\n \"enable_categorical\": False,\n \"learning_rate\": 0.06634196266155237,\n \"max_depth\": 5,\n \"min_child_weight\": 1.4088437184127383,\n \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -93},\n },\n {\n \"colsample_bytree\": 0.546186944730449,\n \"enable_categorical\": False,\n \"learning_rate\": 0.029357102578825213,\n \"max_depth\": 10,\n \"min_child_weight\": 1.1532008198571337,\n \"ag_args\": {\"name_suffix\": \"_r34\", \"priority\": -94},\n },\n ],\n \"FASTAI\": [\n {},\n {\n \"bs\": 256,\n \"emb_drop\": 0.5411770367537934,\n \"epochs\": 43,\n \"layers\": [800, 400],\n \"lr\": 0.01519848858318159,\n \"ps\": 0.23782946566604385,\n \"ag_args\": {\"name_suffix\": \"_r191\", \"priority\": -4},\n },\n {\n \"bs\": 2048,\n \"emb_drop\": 0.05070411322605811,\n \"epochs\": 29,\n \"layers\": [200, 100],\n \"lr\": 0.08974235041576624,\n \"ps\": 0.10393466140748028,\n \"ag_args\": {\"name_suffix\": \"_r102\", \"priority\": -11},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.44339037504795686,\n \"epochs\": 31,\n \"layers\": [400, 200, 100],\n \"lr\": 0.008615195908919904,\n \"ps\": 0.19220253419114286,\n \"ag_args\": {\"name_suffix\": \"_r145\", \"priority\": -15},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.026897798530914306,\n \"epochs\": 31,\n \"layers\": [800, 400],\n \"lr\": 0.08045277634470181,\n \"ps\": 0.4569532219038436,\n \"ag_args\": {\"name_suffix\": \"_r11\", \"priority\": -21},\n },\n {\n \"bs\": 256,\n \"emb_drop\": 0.1508701680951814,\n \"epochs\": 46,\n \"layers\": [400, 200],\n \"lr\": 0.08794353125787312,\n \"ps\": 0.19110623090573325,\n \"ag_args\": {\"name_suffix\": \"_r103\", \"priority\": -25},\n },\n {\n \"bs\": 1024,\n \"emb_drop\": 0.6239200452002372,\n \"epochs\": 39,\n \"layers\": [200, 100, 50],\n \"lr\": 0.07170321592506483,\n \"ps\": 0.670815151683455,\n \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -28},\n },\n {\n \"bs\": 2048,\n \"emb_drop\": 0.5055288166864152,\n \"epochs\": 44,\n \"layers\": [400],\n \"lr\": 0.0047762208542912405,\n \"ps\": 0.06572612802222005,\n \"ag_args\": {\"name_suffix\": \"_r156\", \"priority\": -30},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.6656668277387758,\n \"epochs\": 32,\n \"layers\": [400, 200, 100],\n \"lr\": 0.019326244622675428,\n \"ps\": 0.04084945128641206,\n \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -34},\n },\n {\n \"bs\": 512,\n \"emb_drop\": 0.1567472816422661,\n \"epochs\": 41,\n \"layers\": [400, 200, 100],\n \"lr\": 0.06831450078222204,\n \"ps\": 0.4930900813464729,\n \"ag_args\": {\"name_suffix\": \"_r37\", \"priority\": -40},\n },\n {\n \"bs\": 2048,\n \"emb_drop\": 0.006251885504130949,\n \"epochs\": 47,\n \"layers\": [800, 400],\n \"lr\": 0.01329622020483052,\n \"ps\": 0.2677080696008348,\n \"ag_args\": {\"name_suffix\": \"_r134\", \"priority\": -46},\n },\n {\n \"bs\": 2048,\n \"emb_drop\": 0.6343202884164582,\n \"epochs\": 21,\n \"layers\": [400, 200],\n \"lr\": 0.08479209380262258,\n \"ps\": 0.48362560779595565,\n \"ag_args\": {\"name_suffix\": \"_r111\", \"priority\": -51},\n },\n {\n \"bs\": 1024,\n \"emb_drop\": 0.22771721361129746,\n \"epochs\": 38,\n \"layers\": [400],\n \"lr\": 0.0005383511954451698,\n \"ps\": 0.3734259772256502,\n \"ag_args\": {\"name_suffix\": \"_r65\", \"priority\": -54},\n },\n {\n \"bs\": 1024,\n \"emb_drop\": 0.4329361816589235,\n \"epochs\": 50,\n \"layers\": [400],\n \"lr\": 0.09501311551121323,\n \"ps\": 0.2863378667611431,\n \"ag_args\": {\"name_suffix\": \"_r88\", \"priority\": -55},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.3171659718142149,\n \"epochs\": 20,\n \"layers\": [400, 200, 100],\n \"lr\": 0.03087210106068273,\n \"ps\": 0.5909644730871169,\n \"ag_args\": {\"name_suffix\": \"_r160\", \"priority\": -66},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.3209601865656554,\n \"epochs\": 21,\n \"layers\": [200, 100, 50],\n \"lr\": 0.019935403046870463,\n \"ps\": 0.19846319260751663,\n \"ag_args\": {\"name_suffix\": \"_r69\", \"priority\": -71},\n },\n {\n \"bs\": 128,\n \"emb_drop\": 0.08669109226243704,\n \"epochs\": 45,\n \"layers\": [800, 400],\n \"lr\": 0.0041554361714983635,\n \"ps\": 0.2669780074016213,\n \"ag_args\": {\"name_suffix\": \"_r138\", \"priority\": -73},\n },\n {\n \"bs\": 512,\n \"emb_drop\": 0.05604276533830355,\n \"epochs\": 32,\n \"layers\": [400],\n \"lr\": 0.027320709383189166,\n \"ps\": 0.022591301744255762,\n \"ag_args\": {\"name_suffix\": \"_r172\", \"priority\": -75},\n },\n {\n \"bs\": 1024,\n \"emb_drop\": 0.31956392388385874,\n \"epochs\": 25,\n \"layers\": [200, 100],\n \"lr\": 0.08552736732040143,\n \"ps\": 0.0934076022219228,\n \"ag_args\": {\"name_suffix\": \"_r127\", \"priority\": -80},\n },\n {\n \"bs\": 256,\n \"emb_drop\": 0.5117456464220826,\n \"epochs\": 21,\n \"layers\": [400, 200, 100],\n \"lr\": 0.007212882302137526,\n \"ps\": 0.2747013981281539,\n \"ag_args\": {\"name_suffix\": \"_r194\", \"priority\": -82},\n },\n {\n \"bs\": 256,\n \"emb_drop\": 0.06099050979107849,\n \"epochs\": 39,\n \"layers\": [200],\n \"lr\": 0.04119582873110387,\n \"ps\": 0.5447097256648953,\n \"ag_args\": {\"name_suffix\": \"_r4\", \"priority\": -85},\n },\n {\n \"bs\": 2048,\n \"emb_drop\": 0.6960805527533755,\n \"epochs\": 38,\n \"layers\": [800, 400],\n \"lr\": 0.0007278526871749883,\n \"ps\": 0.20495582200836318,\n \"ag_args\": {\"name_suffix\": \"_r100\", \"priority\": -88},\n },\n {\n \"bs\": 1024,\n \"emb_drop\": 0.5074958658302495,\n \"epochs\": 42,\n \"layers\": [200, 100, 50],\n \"lr\": 0.026342427824862867,\n \"ps\": 0.34814978753283593,\n \"ag_args\": {\"name_suffix\": \"_r187\", \"priority\": -91},\n },\n ],\n \"RF\": [\n {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]}},\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 37308,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r195\", \"priority\": -13},\n },\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 28310,\n \"min_samples_leaf\": 2,\n \"ag_args\": {\"name_suffix\": \"_r39\", \"priority\": -32},\n },\n {\n \"max_features\": 1.0,\n \"max_leaf_nodes\": 38572,\n \"min_samples_leaf\": 5,\n \"ag_args\": {\"name_suffix\": \"_r127\", \"priority\": -45},\n },\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 18242,\n \"min_samples_leaf\": 40,\n \"ag_args\": {\"name_suffix\": \"_r34\", \"priority\": -47},\n },\n {\n \"max_features\": \"log2\",\n \"max_leaf_nodes\": 42644,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r166\", \"priority\": -63},\n },\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 36230,\n \"min_samples_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"_r15\", \"priority\": -68},\n },\n {\n \"max_features\": 1.0,\n \"max_leaf_nodes\": 48136,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r16\", \"priority\": -81},\n },\n ],\n \"XT\": [\n {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]}},\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 18392,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r42\", \"priority\": -9},\n },\n {\n \"max_features\": 1.0,\n \"max_leaf_nodes\": 12845,\n \"min_samples_leaf\": 4,\n \"ag_args\": {\"name_suffix\": \"_r172\", \"priority\": -23},\n },\n {\n \"max_features\": \"sqrt\",\n \"max_leaf_nodes\": 28532,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -43},\n },\n {\n \"max_features\": 1.0,\n \"max_leaf_nodes\": 19935,\n \"min_samples_leaf\": 20,\n \"ag_args\": {\"name_suffix\": \"_r4\", \"priority\": -53},\n },\n {\n \"max_features\": 0.75,\n \"max_leaf_nodes\": 29813,\n \"min_samples_leaf\": 4,\n \"ag_args\": {\"name_suffix\": \"_r178\", \"priority\": -62},\n },\n {\n \"max_features\": 1.0,\n \"max_leaf_nodes\": 40459,\n \"min_samples_leaf\": 1,\n \"ag_args\": {\"name_suffix\": \"_r197\", \"priority\": -78},\n },\n {\n \"max_features\": \"sqrt\",\n \"max_leaf_nodes\": 29702,\n \"min_samples_leaf\": 2,\n \"ag_args\": {\"name_suffix\": \"_r126\", \"priority\": -86},\n },\n ],\n}\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_2025.py",
"content": "# optimized for <=10000 samples and <=500 features, with a GPU present\nhyperparameter_portfolio_zeroshot_2025_small = {\n \"REALTABPFN-V2\": [\n {\n \"ag_args\": {\"priority\": -1},\n },\n ],\n \"GBM\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r33\", \"priority\": -2},\n \"bagging_fraction\": 0.9625293420216,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1236875455555,\n \"cat_smooth\": 68.8584757332856,\n \"extra_trees\": False,\n \"feature_fraction\": 0.6189215809382,\n \"lambda_l1\": 0.1641757352921,\n \"lambda_l2\": 0.6937755557881,\n \"learning_rate\": 0.0154031028561,\n \"max_cat_to_onehot\": 17,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 30,\n \"num_leaves\": 68,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r21\", \"priority\": -16},\n \"bagging_fraction\": 0.7218730663234,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0296205152578,\n \"cat_smooth\": 0.0010255271303,\n \"extra_trees\": False,\n \"feature_fraction\": 0.4557131604374,\n \"lambda_l1\": 0.5219704038237,\n \"lambda_l2\": 0.1070959487853,\n \"learning_rate\": 0.0055891584996,\n \"max_cat_to_onehot\": 71,\n \"min_data_in_leaf\": 50,\n \"min_data_per_group\": 10,\n \"num_leaves\": 30,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r11\", \"priority\": -19},\n \"bagging_fraction\": 0.775784726514,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.3888471449178,\n \"cat_smooth\": 0.0057144748021,\n \"extra_trees\": True,\n \"feature_fraction\": 0.7732354787904,\n \"lambda_l1\": 0.2211002452568,\n \"lambda_l2\": 1.1318405980187,\n \"learning_rate\": 0.0090151778542,\n \"max_cat_to_onehot\": 15,\n \"min_data_in_leaf\": 4,\n \"min_data_per_group\": 15,\n \"num_leaves\": 2,\n },\n ],\n \"CAT\": [\n {\n \"ag_args\": {\"priority\": -5},\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r51\", \"priority\": -10},\n \"boosting_type\": \"Plain\",\n \"bootstrap_type\": \"Bernoulli\",\n \"colsample_bylevel\": 0.8771035272558,\n \"depth\": 7,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 2.0107286863021,\n \"leaf_estimation_iterations\": 2,\n \"learning_rate\": 0.0058424016622,\n \"max_bin\": 254,\n \"max_ctr_complexity\": 4,\n \"model_size_reg\": 0.1307400355809,\n \"one_hot_max_size\": 23,\n \"subsample\": 0.809527841437,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r10\", \"priority\": -12},\n \"boosting_type\": \"Plain\",\n \"bootstrap_type\": \"Bernoulli\",\n \"colsample_bylevel\": 0.8994502668431,\n \"depth\": 6,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 1.8187025215896,\n \"leaf_estimation_iterations\": 7,\n \"learning_rate\": 0.005177304142,\n \"max_bin\": 254,\n \"max_ctr_complexity\": 4,\n \"model_size_reg\": 0.5247386875068,\n \"one_hot_max_size\": 53,\n \"subsample\": 0.8705228845742,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r24\", \"priority\": -15},\n \"boosting_type\": \"Plain\",\n \"bootstrap_type\": \"Bernoulli\",\n \"colsample_bylevel\": 0.8597809376276,\n \"depth\": 8,\n \"grow_policy\": \"Depthwise\",\n \"l2_leaf_reg\": 0.3628261923976,\n \"leaf_estimation_iterations\": 5,\n \"learning_rate\": 0.016851077771,\n \"max_bin\": 254,\n \"max_ctr_complexity\": 4,\n \"model_size_reg\": 0.1253820547902,\n \"one_hot_max_size\": 20,\n \"subsample\": 0.8120271122061,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r91\", \"priority\": -17},\n \"boosting_type\": \"Plain\",\n \"bootstrap_type\": \"Bernoulli\",\n \"colsample_bylevel\": 0.8959275863514,\n \"depth\": 4,\n \"grow_policy\": \"SymmetricTree\",\n \"l2_leaf_reg\": 0.0026915894253,\n \"leaf_estimation_iterations\": 12,\n \"learning_rate\": 0.0475233791203,\n \"max_bin\": 254,\n \"max_ctr_complexity\": 5,\n \"model_size_reg\": 0.1633175256924,\n \"one_hot_max_size\": 11,\n \"subsample\": 0.798554178926,\n },\n ],\n \"TABM\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r184\", \"priority\": -6},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 864,\n \"d_embedding\": 24,\n \"dropout\": 0.0,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0019256819924656217,\n \"n_blocks\": 3,\n \"num_emb_n_bins\": 3,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r69\", \"priority\": -7},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 848,\n \"d_embedding\": 28,\n \"dropout\": 0.40215621636031007,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0010413640454559532,\n \"n_blocks\": 3,\n \"num_emb_n_bins\": 18,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r52\", \"priority\": -11},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 1024,\n \"d_embedding\": 32,\n \"dropout\": 0.0,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0006297851297842611,\n \"n_blocks\": 4,\n \"num_emb_n_bins\": 22,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.06900108498839816,\n },\n {\n \"ag_args\": {\"priority\": -13},\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r191\", \"priority\": -14},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 864,\n \"d_embedding\": 8,\n \"dropout\": 0.45321529282058803,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0003781238075322413,\n \"n_blocks\": 4,\n \"num_emb_n_bins\": 27,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.01766851962579851,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -20},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 640,\n \"d_embedding\": 28,\n \"dropout\": 0.15296207419190627,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.002277678490593717,\n \"n_blocks\": 3,\n \"num_emb_n_bins\": 48,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0578159148243893,\n },\n ],\n \"TABICL\": [\n {\n \"ag_args\": {\"priority\": -8},\n },\n ],\n \"XGB\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r171\", \"priority\": -9},\n \"colsample_bylevel\": 0.9213705632288,\n \"colsample_bynode\": 0.6443385965381,\n \"enable_categorical\": True,\n \"grow_policy\": \"lossguide\",\n \"learning_rate\": 0.0068171645251,\n \"max_cat_to_onehot\": 8,\n \"max_depth\": 6,\n \"max_leaves\": 10,\n \"min_child_weight\": 0.0507304250576,\n \"reg_alpha\": 4.2446346389037,\n \"reg_lambda\": 1.4800570021253,\n \"subsample\": 0.9656290596647,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r40\", \"priority\": -18},\n \"colsample_bylevel\": 0.6377491713202,\n \"colsample_bynode\": 0.9237625621103,\n \"enable_categorical\": True,\n \"grow_policy\": \"lossguide\",\n \"learning_rate\": 0.0112462621131,\n \"max_cat_to_onehot\": 33,\n \"max_depth\": 10,\n \"max_leaves\": 35,\n \"min_child_weight\": 0.1403464856034,\n \"reg_alpha\": 3.4960653958503,\n \"reg_lambda\": 1.3062320805235,\n \"subsample\": 0.6948898835178,\n },\n ],\n \"MITRA\": [\n {\n \"n_estimators\": 1,\n \"fine_tune\": True,\n \"fine_tune_steps\": 50,\n \"ag.num_gpus\": 1,\n \"ag_args\": {\"priority\": -21},\n },\n ],\n}\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_cpu_2025_12_18.py",
"content": "# On par with `best_quality` while being much faster for smaller datasets. Runs on CPU.\nhyperparameter_portfolio_zeroshot_cpu_2025_12_18 = {\n \"CAT\": [{\"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -1}}],\n \"GBM_PREP\": [\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -2},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9923026236907,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.014290368488,\n \"cat_smooth\": 1.8662939903973,\n \"extra_trees\": True,\n \"feature_fraction\": 0.5533919718605,\n \"lambda_l1\": 0.914411672958,\n \"lambda_l2\": 1.90439560009,\n \"learning_rate\": 0.0193225778401,\n \"max_cat_to_onehot\": 18,\n \"min_data_in_leaf\": 28,\n \"min_data_per_group\": 54,\n \"num_leaves\": 64,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r41\", \"priority\": -7},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7215411996558,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.887369154362,\n \"cat_smooth\": 0.0278693980873,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4247583287144,\n \"lambda_l1\": 0.1129800247772,\n \"lambda_l2\": 0.2623265718536,\n \"learning_rate\": 0.0074201920651,\n \"max_cat_to_onehot\": 9,\n \"min_data_in_leaf\": 15,\n \"min_data_per_group\": 10,\n \"num_leaves\": 8,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -10},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9591526242875,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.8962346412823,\n \"cat_smooth\": 0.0215219089995,\n \"extra_trees\": False,\n \"feature_fraction\": 0.5791844062459,\n \"lambda_l1\": 0.938461750637,\n \"lambda_l2\": 0.9899852075056,\n \"learning_rate\": 0.0397613094741,\n \"max_cat_to_onehot\": 27,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 39,\n \"num_leaves\": 16,\n },\n {\n \"ag.prep_params\": [],\n \"ag_args\": {\"name_suffix\": \"_r21\", \"priority\": -12},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7111549514262,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.8679131150136,\n \"cat_smooth\": 48.7244965504817,\n \"extra_trees\": False,\n \"feature_fraction\": 0.425140839263,\n \"lambda_l1\": 0.5140528525242,\n \"lambda_l2\": 0.5134051978198,\n \"learning_rate\": 0.0134375321277,\n \"max_cat_to_onehot\": 16,\n \"min_data_in_leaf\": 2,\n \"min_data_per_group\": 32,\n \"num_leaves\": 20,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r17\", \"priority\": -17},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9277474245702,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0731876168104,\n \"cat_smooth\": 0.1369210915339,\n \"extra_trees\": False,\n \"feature_fraction\": 0.6680440910385,\n \"lambda_l1\": 0.0125057410295,\n \"lambda_l2\": 0.7157181359874,\n \"learning_rate\": 0.0351342879995,\n \"max_cat_to_onehot\": 20,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 2,\n \"num_leaves\": 64,\n },\n {\n \"ag.prep_params\": [[[[\"ArithmeticFeatureGenerator\", {}]]]],\n \"ag_args\": {\"name_suffix\": \"_r47\", \"priority\": -18},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9918048278435,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.984162386723,\n \"cat_smooth\": 0.0049687445294,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4974006116018,\n \"lambda_l1\": 0.7970644065518,\n \"lambda_l2\": 1.2179933810825,\n \"learning_rate\": 0.0537072755122,\n \"max_cat_to_onehot\": 13,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 4,\n \"num_leaves\": 32,\n },\n {\n \"ag.prep_params\": [\n [\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ]\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -19},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.8836335684032,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.6608043016307,\n \"cat_smooth\": 0.0451936212097,\n \"extra_trees\": True,\n \"feature_fraction\": 0.6189315903408,\n \"lambda_l1\": 0.6514130054123,\n \"lambda_l2\": 1.7382678663835,\n \"learning_rate\": 0.0412716109215,\n \"max_cat_to_onehot\": 9,\n \"min_data_in_leaf\": 9,\n \"min_data_per_group\": 3,\n \"num_leaves\": 128,\n },\n {\n \"ag.prep_params\": [\n [\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ]\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -26},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7106002663401,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1559746777257,\n \"cat_smooth\": 0.0036366126697,\n \"extra_trees\": False,\n \"feature_fraction\": 0.688233104808,\n \"lambda_l1\": 0.8732887427372,\n \"lambda_l2\": 0.446716114323,\n \"learning_rate\": 0.0815946452855,\n \"max_cat_to_onehot\": 78,\n \"min_data_in_leaf\": 12,\n \"min_data_per_group\": 2,\n \"num_leaves\": 16,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r34\", \"priority\": -32},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.8453534561545,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0321580936847,\n \"cat_smooth\": 0.0011470238114,\n \"extra_trees\": True,\n \"feature_fraction\": 0.8611499511087,\n \"lambda_l1\": 0.910743969343,\n \"lambda_l2\": 1.2750027607225,\n \"learning_rate\": 0.0151455176168,\n \"max_cat_to_onehot\": 8,\n \"min_data_in_leaf\": 60,\n \"min_data_per_group\": 4,\n \"num_leaves\": 32,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r32\", \"priority\": -37},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.927947070297,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0082294539727,\n \"cat_smooth\": 0.0671878797989,\n \"extra_trees\": True,\n \"feature_fraction\": 0.9169657691675,\n \"lambda_l1\": 0.9386485912678,\n \"lambda_l2\": 1.619775689786,\n \"learning_rate\": 0.0056864355547,\n \"max_cat_to_onehot\": 11,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 10,\n \"num_leaves\": 32,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r7\", \"priority\": -38},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.8984634022103,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0053608956358,\n \"cat_smooth\": 89.7168790664636,\n \"extra_trees\": False,\n \"feature_fraction\": 0.847638045482,\n \"lambda_l1\": 0.5684527742857,\n \"lambda_l2\": 1.0738026980295,\n \"learning_rate\": 0.0417108779005,\n \"max_cat_to_onehot\": 8,\n \"min_data_in_leaf\": 2,\n \"min_data_per_group\": 7,\n \"num_leaves\": 128,\n },\n {\n \"ag.prep_params\": [\n [\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ]\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r14\", \"priority\": -40},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9318953983366,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.065532200068,\n \"cat_smooth\": 0.0696287198368,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4649868965096,\n \"lambda_l1\": 0.6586569196642,\n \"lambda_l2\": 1.7799375779553,\n \"learning_rate\": 0.072046289471,\n \"max_cat_to_onehot\": 72,\n \"min_data_in_leaf\": 26,\n \"min_data_per_group\": 32,\n \"num_leaves\": 32,\n },\n {\n \"ag.prep_params\": [[[[\"ArithmeticFeatureGenerator\", {}]]]],\n \"ag_args\": {\"name_suffix\": \"_r27\", \"priority\": -42},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.811983527375,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0255048028385,\n \"cat_smooth\": 1.5339379274002,\n \"extra_trees\": True,\n \"feature_fraction\": 0.5246746068724,\n \"lambda_l1\": 0.9737915306165,\n \"lambda_l2\": 1.929596568261,\n \"learning_rate\": 0.0172284745143,\n \"max_cat_to_onehot\": 9,\n \"min_data_in_leaf\": 8,\n \"min_data_per_group\": 51,\n \"num_leaves\": 20,\n },\n {\n \"ag.prep_params\": [[[[\"ArithmeticFeatureGenerator\", {}]]]],\n \"ag_args\": {\"name_suffix\": \"_r37\", \"priority\": -46},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7853761603489,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.2934796127084,\n \"cat_smooth\": 10.1721684646257,\n \"extra_trees\": False,\n \"feature_fraction\": 0.4813265290277,\n \"lambda_l1\": 0.9744837697365,\n \"lambda_l2\": 0.6058665958153,\n \"learning_rate\": 0.0371000014124,\n \"max_cat_to_onehot\": 85,\n \"min_data_in_leaf\": 22,\n \"min_data_per_group\": 3,\n \"num_leaves\": 32,\n },\n ],\n \"GBM\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r177\", \"priority\": -3},\n \"bagging_fraction\": 0.8769107816033,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.3418014393813,\n \"cat_smooth\": 15.4304556649114,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4622189821941,\n \"lambda_l1\": 0.2375070586896,\n \"lambda_l2\": 0.3551561351804,\n \"learning_rate\": 0.0178593900218,\n \"max_cat_to_onehot\": 16,\n \"min_data_in_leaf\": 3,\n \"min_data_per_group\": 9,\n \"num_leaves\": 39,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r163\", \"priority\": -5},\n \"bagging_fraction\": 0.9783898288461,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1553395260142,\n \"cat_smooth\": 0.0093122749318,\n \"extra_trees\": False,\n \"feature_fraction\": 0.5279825611461,\n \"lambda_l1\": 0.0269274915833,\n \"lambda_l2\": 0.8375250972309,\n \"learning_rate\": 0.0113913650333,\n \"max_cat_to_onehot\": 42,\n \"min_data_in_leaf\": 3,\n \"min_data_per_group\": 75,\n \"num_leaves\": 84,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r72\", \"priority\": -8},\n \"bagging_fraction\": 0.950146543918,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.2159137242663,\n \"cat_smooth\": 0.0638204395719,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4044759649281,\n \"lambda_l1\": 0.7661581500422,\n \"lambda_l2\": 1.6041759693902,\n \"learning_rate\": 0.0179845918984,\n \"max_cat_to_onehot\": 11,\n \"min_data_in_leaf\": 12,\n \"min_data_per_group\": 3,\n \"num_leaves\": 180,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r120\", \"priority\": -13},\n \"bagging_fraction\": 0.8541333332514,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0110343197541,\n \"cat_smooth\": 5.0905236124522,\n \"extra_trees\": True,\n \"feature_fraction\": 0.7334718346252,\n \"lambda_l1\": 0.241338427726,\n \"lambda_l2\": 0.298107723769,\n \"learning_rate\": 0.0126654490778,\n \"max_cat_to_onehot\": 67,\n \"min_data_in_leaf\": 12,\n \"min_data_per_group\": 93,\n \"num_leaves\": 5,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r6\", \"priority\": -16},\n \"bagging_fraction\": 0.8148132107231,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0058363329714,\n \"cat_smooth\": 0.0289414318324,\n \"extra_trees\": False,\n \"feature_fraction\": 0.939979116902,\n \"lambda_l1\": 0.4369494828584,\n \"lambda_l2\": 0.2997524486083,\n \"learning_rate\": 0.0078971749764,\n \"max_cat_to_onehot\": 28,\n \"min_data_in_leaf\": 24,\n \"min_data_per_group\": 3,\n \"num_leaves\": 8,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r184\", \"priority\": -21},\n \"bagging_fraction\": 0.8406256713136,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.9284921901786,\n \"cat_smooth\": 0.0898191451684,\n \"extra_trees\": False,\n \"feature_fraction\": 0.5876132298377,\n \"lambda_l1\": 0.078943697912,\n \"lambda_l2\": 0.7713118402478,\n \"learning_rate\": 0.0090676429159,\n \"max_cat_to_onehot\": 16,\n \"min_data_in_leaf\": 17,\n \"min_data_per_group\": 11,\n \"num_leaves\": 2,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r46\", \"priority\": -23},\n \"bagging_fraction\": 0.999426150416,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0076879104679,\n \"cat_smooth\": 89.4599055435924,\n \"extra_trees\": False,\n \"feature_fraction\": 0.8588138897928,\n \"lambda_l1\": 0.0413597548025,\n \"lambda_l2\": 0.2258713386858,\n \"learning_rate\": 0.0074056102479,\n \"max_cat_to_onehot\": 11,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 26,\n \"num_leaves\": 14,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r68\", \"priority\": -24},\n \"bagging_fraction\": 0.7199080522958,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.9369509319667,\n \"cat_smooth\": 11.0984745216942,\n \"extra_trees\": False,\n \"feature_fraction\": 0.9550596478029,\n \"lambda_l1\": 0.1109843723892,\n \"lambda_l2\": 0.5969094177111,\n \"learning_rate\": 0.0079480499426,\n \"max_cat_to_onehot\": 8,\n \"min_data_in_leaf\": 3,\n \"min_data_per_group\": 8,\n \"num_leaves\": 111,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r47\", \"priority\": -29},\n \"bagging_fraction\": 0.8831228358892,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1402622388062,\n \"cat_smooth\": 3.3545774392409,\n \"extra_trees\": True,\n \"feature_fraction\": 0.6155890374887,\n \"lambda_l1\": 0.1749502746898,\n \"lambda_l2\": 0.8761391715812,\n \"learning_rate\": 0.00891978331,\n \"max_cat_to_onehot\": 84,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 21,\n \"num_leaves\": 55,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r63\", \"priority\": -31},\n \"bagging_fraction\": 0.7801003412553,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0071438335269,\n \"cat_smooth\": 0.1338043459574,\n \"extra_trees\": False,\n \"feature_fraction\": 0.490455360592,\n \"lambda_l1\": 0.6420805635778,\n \"lambda_l2\": 0.5813319300456,\n \"learning_rate\": 0.0308746408751,\n \"max_cat_to_onehot\": 38,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 83,\n \"num_leaves\": 24,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r39\", \"priority\": -36},\n \"bagging_fraction\": 0.7035743460186,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0134845084619,\n \"cat_smooth\": 56.4934757686511,\n \"extra_trees\": True,\n \"feature_fraction\": 0.7824899527144,\n \"lambda_l1\": 0.3700115211248,\n \"lambda_l2\": 0.0341499593689,\n \"learning_rate\": 0.094652390088,\n \"max_cat_to_onehot\": 13,\n \"min_data_in_leaf\": 13,\n \"min_data_per_group\": 4,\n \"num_leaves\": 23,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r18\", \"priority\": -43},\n \"bagging_fraction\": 0.7041134150362,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1139031650222,\n \"cat_smooth\": 41.8937939300815,\n \"extra_trees\": True,\n \"feature_fraction\": 0.5028791565785,\n \"lambda_l1\": 0.1031941284118,\n \"lambda_l2\": 1.2554010747358,\n \"learning_rate\": 0.0186530122901,\n \"max_cat_to_onehot\": 29,\n \"min_data_in_leaf\": 5,\n \"min_data_per_group\": 74,\n \"num_leaves\": 5,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r50\", \"priority\": -45},\n \"bagging_fraction\": 0.9673434664048,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.7662226703416,\n \"cat_smooth\": 0.0097667848046,\n \"extra_trees\": True,\n \"feature_fraction\": 0.9286299570284,\n \"lambda_l1\": 0.0448644389135,\n \"lambda_l2\": 1.7322446850205,\n \"learning_rate\": 0.0507909494543,\n \"max_cat_to_onehot\": 11,\n \"min_data_in_leaf\": 4,\n \"min_data_per_group\": 2,\n \"num_leaves\": 106,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r104\", \"priority\": -48},\n \"bagging_fraction\": 0.9327643671568,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0067636494662,\n \"cat_smooth\": 29.2351010915576,\n \"extra_trees\": False,\n \"feature_fraction\": 0.660864035482,\n \"lambda_l1\": 0.556745328417,\n \"lambda_l2\": 1.2717605868201,\n \"learning_rate\": 0.0433336000175,\n \"max_cat_to_onehot\": 42,\n \"min_data_in_leaf\": 18,\n \"min_data_per_group\": 6,\n \"num_leaves\": 19,\n },\n ],\n \"NN_TORCH\": [\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r37\", \"priority\": -4},\n \"dropout_prob\": 0.0889772897547275,\n \"hidden_size\": 109,\n \"learning_rate\": 0.02184363543226557,\n \"num_layers\": 3,\n \"use_batchnorm\": True,\n \"weight_decay\": 3.1736637236578543e-10,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -9},\n \"dropout_prob\": 0.013288954106470907,\n \"hidden_size\": 81,\n \"learning_rate\": 0.005340914647396153,\n \"num_layers\": 4,\n \"use_batchnorm\": False,\n \"weight_decay\": 8.76216837077536e-05,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r193\", \"priority\": -14},\n \"dropout_prob\": 0.2976404923811552,\n \"hidden_size\": 131,\n \"learning_rate\": 0.0038408014156739775,\n \"num_layers\": 3,\n \"use_batchnorm\": False,\n \"weight_decay\": 0.01745189206113213,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r144\", \"priority\": -15},\n \"dropout_prob\": 0.2670859555485912,\n \"hidden_size\": 52,\n \"learning_rate\": 0.015189605588375421,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 2.8013784883244263e-08,\n },\n {\n \"activation\": \"relu\",\n \"ag_args\": {\"name_suffix\": \"_r82\", \"priority\": -22},\n \"dropout_prob\": 0.27342918414623907,\n \"hidden_size\": 207,\n \"learning_rate\": 0.0004069380929899853,\n \"num_layers\": 4,\n \"use_batchnorm\": False,\n \"weight_decay\": 0.002473667327700422,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r39\", \"priority\": -27},\n \"dropout_prob\": 0.21699951000415899,\n \"hidden_size\": 182,\n \"learning_rate\": 0.00014675249427915203,\n \"num_layers\": 2,\n \"use_batchnorm\": False,\n \"weight_decay\": 9.787353852692089e-08,\n },\n {\n \"activation\": \"relu\",\n \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -30},\n \"dropout_prob\": 0.23713784729000734,\n \"hidden_size\": 200,\n \"learning_rate\": 0.0031125617090901805,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 4.57301675647447e-08,\n },\n {\n \"activation\": \"relu\",\n \"ag_args\": {\"name_suffix\": \"_r48\", \"priority\": -34},\n \"dropout_prob\": 0.14224509513998226,\n \"hidden_size\": 26,\n \"learning_rate\": 0.007085904739869829,\n \"num_layers\": 2,\n \"use_batchnorm\": False,\n \"weight_decay\": 2.465786211798467e-10,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -39},\n \"dropout_prob\": 0.06134755114373829,\n \"hidden_size\": 144,\n \"learning_rate\": 0.005834535148903802,\n \"num_layers\": 5,\n \"use_batchnorm\": True,\n \"weight_decay\": 2.0826540090463376e-09,\n },\n {\n \"activation\": \"elu\",\n \"ag_args\": {\"name_suffix\": \"_r24\", \"priority\": -49},\n \"dropout_prob\": 0.257596079691855,\n \"hidden_size\": 168,\n \"learning_rate\": 0.0034108596383714608,\n \"num_layers\": 4,\n \"use_batchnorm\": True,\n \"weight_decay\": 1.4840689603685264e-07,\n },\n {\n \"activation\": \"relu\",\n \"ag_args\": {\"name_suffix\": \"_r159\", \"priority\": -50},\n \"dropout_prob\": 0.16724368469920037,\n \"hidden_size\": 44,\n \"learning_rate\": 0.011043937174833164,\n \"num_layers\": 4,\n \"use_batchnorm\": False,\n \"weight_decay\": 0.007265742373924609,\n },\n ],\n \"FASTAI\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r25\", \"priority\": -6},\n \"bs\": 1024,\n \"emb_drop\": 0.6167722379778131,\n \"epochs\": 44,\n \"layers\": [200, 100, 50],\n \"lr\": 0.05344037785562929,\n \"ps\": 0.48477211305443607,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r162\", \"priority\": -11},\n \"bs\": 2048,\n \"emb_drop\": 0.5474625640581479,\n \"epochs\": 45,\n \"layers\": [400, 200],\n \"lr\": 0.0047438648957706655,\n \"ps\": 0.07533239360470734,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r147\", \"priority\": -20},\n \"bs\": 128,\n \"emb_drop\": 0.6378380130337095,\n \"epochs\": 48,\n \"layers\": [200],\n \"lr\": 0.058027179860229344,\n \"ps\": 0.23253362133888375,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r192\", \"priority\": -25},\n \"bs\": 1024,\n \"emb_drop\": 0.0698130630643278,\n \"epochs\": 37,\n \"layers\": [400, 200],\n \"lr\": 0.0018949411343821322,\n \"ps\": 0.6526067160491229,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r109\", \"priority\": -28},\n \"bs\": 128,\n \"emb_drop\": 0.1978897556618756,\n \"epochs\": 49,\n \"layers\": [400, 200, 100],\n \"lr\": 0.02155144303508465,\n \"ps\": 0.005518872455908264,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r78\", \"priority\": -33},\n \"bs\": 512,\n \"emb_drop\": 0.4897354379753617,\n \"epochs\": 26,\n \"layers\": [400, 200, 100],\n \"lr\": 0.027563880686468895,\n \"ps\": 0.44524273881299886,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r150\", \"priority\": -35},\n \"bs\": 2048,\n \"emb_drop\": 0.6148607467659958,\n \"epochs\": 27,\n \"layers\": [400, 200],\n \"lr\": 0.09351668652547614,\n \"ps\": 0.5314977162016676,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r133\", \"priority\": -41},\n \"bs\": 256,\n \"emb_drop\": 0.6242606757570891,\n \"epochs\": 43,\n \"layers\": [200, 100, 50],\n \"lr\": 0.001533613235987637,\n \"ps\": 0.5354961132962562,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r99\", \"priority\": -44},\n \"bs\": 512,\n \"emb_drop\": 0.6071025838237253,\n \"epochs\": 49,\n \"layers\": [400, 200],\n \"lr\": 0.02669945959641021,\n \"ps\": 0.4897025421573259,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r197\", \"priority\": -47},\n \"bs\": 256,\n \"emb_drop\": 0.5277230463737563,\n \"epochs\": 45,\n \"layers\": [400, 200],\n \"lr\": 0.006908743712130657,\n \"ps\": 0.08262909528632323,\n },\n ],\n}\n"
},
{
"path": "tabular/src/autogluon/tabular/configs/zeroshot/zeroshot_portfolio_gpu_2025_12_18.py",
"content": "# State-of-the-art for datasets < 100k samples. Requires a GPU with at least 20 GB VRAM.\nhyperparameter_portfolio_zeroshot_gpu_2025_12_18 = {\n \"TABDPT\": [\n {\n \"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -3},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": False},\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r20\", \"priority\": -5},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": False},\n \"clip_sigma\": 8,\n \"feature_reduction\": \"subsample\",\n \"missing_indicators\": False,\n \"normalizer\": \"quantile-uniform\",\n \"permute_classes\": False,\n \"temperature\": 0.5,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -7},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": False},\n \"clip_sigma\": 16,\n \"feature_reduction\": \"subsample\",\n \"missing_indicators\": False,\n \"normalizer\": \"log1p\",\n \"permute_classes\": False,\n \"temperature\": 0.5,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r15\", \"priority\": -9},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": False},\n \"clip_sigma\": 16,\n \"feature_reduction\": \"subsample\",\n \"missing_indicators\": False,\n \"normalizer\": \"standard\",\n \"permute_classes\": True,\n \"temperature\": 0.7,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": -11},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": False},\n \"clip_sigma\": 8,\n \"feature_reduction\": \"pca\",\n \"missing_indicators\": True,\n \"normalizer\": \"robust\",\n \"permute_classes\": False,\n \"temperature\": 0.5,\n },\n ],\n \"TABICL\": [{\"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -4}, \"ag_args_ensemble\": {\"refit_folds\": True}}],\n \"MITRA\": [\n {\n \"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -12},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n }\n ],\n \"TABM\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r99\", \"priority\": -13},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 880,\n \"d_embedding\": 24,\n \"dropout\": 0.10792355695428629,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0013641856391615784,\n \"n_blocks\": 5,\n \"num_emb_n_bins\": 16,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r124\", \"priority\": -17},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 208,\n \"d_embedding\": 16,\n \"dropout\": 0.0,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.00042152744054701374,\n \"n_blocks\": 2,\n \"num_emb_n_bins\": 109,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.00014007839435474664,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r69\", \"priority\": -21},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 848,\n \"d_embedding\": 28,\n \"dropout\": 0.40215621636031007,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0010413640454559532,\n \"n_blocks\": 3,\n \"num_emb_n_bins\": 18,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r184\", \"priority\": -24},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 864,\n \"d_embedding\": 24,\n \"dropout\": 0.0,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.0019256819924656217,\n \"n_blocks\": 3,\n \"num_emb_n_bins\": 3,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r34\", \"priority\": -26},\n \"amp\": False,\n \"arch_type\": \"tabm-mini\",\n \"batch_size\": \"auto\",\n \"d_block\": 896,\n \"d_embedding\": 8,\n \"dropout\": 0.0,\n \"gradient_clipping_norm\": 1.0,\n \"lr\": 0.002459175026451607,\n \"n_blocks\": 4,\n \"num_emb_n_bins\": 104,\n \"num_emb_type\": \"pwl\",\n \"patience\": 16,\n \"share_training_batches\": False,\n \"tabm_k\": 32,\n \"weight_decay\": 0.0006299584388562901,\n },\n ],\n \"GBM_PREP\": [\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -14},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9923026236907,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.014290368488,\n \"cat_smooth\": 1.8662939903973,\n \"extra_trees\": True,\n \"feature_fraction\": 0.5533919718605,\n \"lambda_l1\": 0.914411672958,\n \"lambda_l2\": 1.90439560009,\n \"learning_rate\": 0.0193225778401,\n \"max_cat_to_onehot\": 18,\n \"min_data_in_leaf\": 28,\n \"min_data_per_group\": 54,\n \"num_leaves\": 64,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r41\", \"priority\": -16},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7215411996558,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.887369154362,\n \"cat_smooth\": 0.0278693980873,\n \"extra_trees\": True,\n \"feature_fraction\": 0.4247583287144,\n \"lambda_l1\": 0.1129800247772,\n \"lambda_l2\": 0.2623265718536,\n \"learning_rate\": 0.0074201920651,\n \"max_cat_to_onehot\": 9,\n \"min_data_in_leaf\": 15,\n \"min_data_per_group\": 10,\n \"num_leaves\": 8,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -18},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9591526242875,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.8962346412823,\n \"cat_smooth\": 0.0215219089995,\n \"extra_trees\": False,\n \"feature_fraction\": 0.5791844062459,\n \"lambda_l1\": 0.938461750637,\n \"lambda_l2\": 0.9899852075056,\n \"learning_rate\": 0.0397613094741,\n \"max_cat_to_onehot\": 27,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 39,\n \"num_leaves\": 16,\n },\n {\n \"ag.prep_params\": [],\n \"ag_args\": {\"name_suffix\": \"_r21\", \"priority\": -20},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.7111549514262,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.8679131150136,\n \"cat_smooth\": 48.7244965504817,\n \"extra_trees\": False,\n \"feature_fraction\": 0.425140839263,\n \"lambda_l1\": 0.5140528525242,\n \"lambda_l2\": 0.5134051978198,\n \"learning_rate\": 0.0134375321277,\n \"max_cat_to_onehot\": 16,\n \"min_data_in_leaf\": 2,\n \"min_data_per_group\": 32,\n \"num_leaves\": 20,\n },\n {\n \"ag.prep_params\": [\n [\n [[\"ArithmeticFeatureGenerator\", {}]],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ]\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n \"ag_args\": {\"name_suffix\": \"_r17\", \"priority\": -23},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"bagging_fraction\": 0.9277474245702,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0731876168104,\n \"cat_smooth\": 0.1369210915339,\n \"extra_trees\": False,\n \"feature_fraction\": 0.6680440910385,\n \"lambda_l1\": 0.0125057410295,\n \"lambda_l2\": 0.7157181359874,\n \"learning_rate\": 0.0351342879995,\n \"max_cat_to_onehot\": 20,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 2,\n \"num_leaves\": 64,\n },\n ],\n \"CAT\": [{\"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -15}}],\n \"GBM\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r73\", \"priority\": -19},\n \"bagging_fraction\": 0.7295548973583,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.8025485263237,\n \"cat_smooth\": 59.6178463268351,\n \"extra_trees\": False,\n \"feature_fraction\": 0.8242607305914,\n \"lambda_l1\": 0.7265522905459,\n \"lambda_l2\": 0.3492160682092,\n \"learning_rate\": 0.0068803786367,\n \"max_cat_to_onehot\": 16,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 10,\n \"num_leaves\": 24,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r37\", \"priority\": -22},\n \"bagging_fraction\": 0.8096374561947,\n \"bagging_freq\": 1,\n \"cat_l2\": 1.6385754694703,\n \"cat_smooth\": 16.1922506671724,\n \"extra_trees\": True,\n \"feature_fraction\": 0.885927003286,\n \"lambda_l1\": 0.0430386950502,\n \"lambda_l2\": 0.2507506811761,\n \"learning_rate\": 0.0079622660542,\n \"max_cat_to_onehot\": 23,\n \"min_data_in_leaf\": 7,\n \"min_data_per_group\": 49,\n \"num_leaves\": 6,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r162\", \"priority\": -25},\n \"bagging_fraction\": 0.7552878818396,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.0081083103544,\n \"cat_smooth\": 75.7373446363438,\n \"extra_trees\": False,\n \"feature_fraction\": 0.6171258454584,\n \"lambda_l1\": 0.1071522383181,\n \"lambda_l2\": 1.7882554584069,\n \"learning_rate\": 0.0229328987255,\n \"max_cat_to_onehot\": 24,\n \"min_data_in_leaf\": 23,\n \"min_data_per_group\": 2,\n \"num_leaves\": 125,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r57\", \"priority\": -27},\n \"bagging_fraction\": 0.8515739264605,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.2263901847144,\n \"cat_smooth\": 1.7397457971767,\n \"extra_trees\": True,\n \"feature_fraction\": 0.6284015946887,\n \"lambda_l1\": 0.6935431676756,\n \"lambda_l2\": 1.7605230133162,\n \"learning_rate\": 0.0294830579218,\n \"max_cat_to_onehot\": 52,\n \"min_data_in_leaf\": 8,\n \"min_data_per_group\": 3,\n \"num_leaves\": 43,\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r33\", \"priority\": -28},\n \"bagging_fraction\": 0.9625293420216,\n \"bagging_freq\": 1,\n \"cat_l2\": 0.1236875455555,\n \"cat_smooth\": 68.8584757332856,\n \"extra_trees\": False,\n \"feature_fraction\": 0.6189215809382,\n \"lambda_l1\": 0.1641757352921,\n \"lambda_l2\": 0.6937755557881,\n \"learning_rate\": 0.0154031028561,\n \"max_cat_to_onehot\": 17,\n \"min_data_in_leaf\": 1,\n \"min_data_per_group\": 30,\n \"num_leaves\": 68,\n },\n ],\n \"REALTABPFN-V2\": [\n {\n \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -1},\n \"ag_args_ensemble\": {\"model_random_seed\": 104, \"vary_seed_across_folds\": True},\n \"balance_probabilities\": False,\n \"inference_config/OUTLIER_REMOVAL_STD\": 6,\n \"inference_config/POLYNOMIAL_FEATURES\": \"no\",\n \"inference_config/REGRESSION_Y_PREPROCESS_TRANSFORMS\": [None, \"safepower\"],\n \"preprocessing/append_original\": False,\n \"preprocessing/categoricals\": \"numeric\",\n \"preprocessing/global\": None,\n \"preprocessing/scaling\": [\"squashing_scaler_default\", \"quantile_uni_coarse\"],\n \"softmax_temperature\": 1.0,\n \"zip_model_path\": [\"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\", \"tabpfn-v2-regressor-v2_default.ckpt\"],\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r106\", \"priority\": -2},\n \"ag_args_ensemble\": {\"model_random_seed\": 848, \"vary_seed_across_folds\": True},\n \"balance_probabilities\": False,\n \"inference_config/OUTLIER_REMOVAL_STD\": 6,\n \"inference_config/POLYNOMIAL_FEATURES\": \"no\",\n \"inference_config/REGRESSION_Y_PREPROCESS_TRANSFORMS\": [None],\n \"preprocessing/append_original\": True,\n \"preprocessing/categoricals\": \"numeric\",\n \"preprocessing/global\": \"svd_quarter_components\",\n \"preprocessing/scaling\": [\"quantile_uni_coarse\"],\n \"softmax_temperature\": 0.8,\n \"zip_model_path\": [\"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\", \"tabpfn-v2-regressor-v2_default.ckpt\"],\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r11\", \"priority\": -6},\n \"ag_args_ensemble\": {\"model_random_seed\": 88, \"vary_seed_across_folds\": True},\n \"balance_probabilities\": True,\n \"inference_config/OUTLIER_REMOVAL_STD\": 6,\n \"inference_config/POLYNOMIAL_FEATURES\": 25,\n \"inference_config/REGRESSION_Y_PREPROCESS_TRANSFORMS\": [None],\n \"preprocessing/append_original\": True,\n \"preprocessing/categoricals\": \"onehot\",\n \"preprocessing/global\": \"svd_quarter_components\",\n \"preprocessing/scaling\": [\"safepower\", \"quantile_uni\"],\n \"softmax_temperature\": 0.7,\n \"zip_model_path\": [\"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\", \"tabpfn-v2-regressor-v2_default.ckpt\"],\n },\n {\n \"ag_args\": {\"name_suffix\": \"_c1\", \"priority\": -8},\n \"ag_args_ensemble\": {\"model_random_seed\": 0, \"vary_seed_across_folds\": True},\n \"zip_model_path\": [\"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\", \"tabpfn-v2-regressor-v2_default.ckpt\"],\n },\n {\n \"ag_args\": {\"name_suffix\": \"_r196\", \"priority\": -10},\n \"ag_args_ensemble\": {\"model_random_seed\": 1568, \"vary_seed_across_folds\": True},\n \"balance_probabilities\": False,\n \"inference_config/OUTLIER_REMOVAL_STD\": 12,\n \"inference_config/POLYNOMIAL_FEATURES\": \"no\",\n \"inference_config/REGRESSION_Y_PREPROCESS_TRANSFORMS\": [\"kdi_alpha_1.0\"],\n \"preprocessing/append_original\": False,\n \"preprocessing/categoricals\": \"numeric\",\n \"preprocessing/global\": None,\n \"preprocessing/scaling\": [\"squashing_scaler_default\"],\n \"softmax_temperature\": 1.25,\n \"zip_model_path\": [\"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\", \"tabpfn-v2-regressor-v2_default.ckpt\"],\n },\n ],\n}\n"
},
{
"path": "tabular/src/autogluon/tabular/experimental/__init__.py",
"content": "from ._tabular_classifier import TabularClassifier\nfrom ._tabular_regressor import TabularRegressor\n"
},
{
"path": "tabular/src/autogluon/tabular/experimental/_scikit_mixin.py",
"content": "from __future__ import annotations\n\nimport pandas as pd\nfrom sklearn.utils.validation import check_array, check_is_fitted\n\n\nclass ScikitMixin:\n def _get_init_args(self, problem_type: str) -> dict:\n init_args = self.init_args\n if init_args is None:\n init_args = dict()\n init_args = init_args.copy()\n if \"label\" not in init_args:\n init_args[\"label\"] = \"_target_\"\n if \"problem_type\" not in init_args:\n init_args[\"problem_type\"] = problem_type\n if \"eval_metric\" not in init_args:\n init_args[\"eval_metric\"] = self.eval_metric\n if \"path\" not in init_args:\n init_args[\"path\"] = self.path\n if \"verbosity\" not in init_args:\n init_args[\"verbosity\"] = self.verbosity\n return init_args\n\n def _get_fit_args(self) -> dict:\n fit_args = self.fit_args\n if fit_args is None:\n fit_args = dict()\n fit_args = fit_args.copy()\n\n if \"time_limit\" not in fit_args:\n fit_args[\"time_limit\"] = self.time_limit\n if \"presets\" not in fit_args:\n fit_args[\"presets\"] = self.presets\n if \"hyperparameters\" not in fit_args:\n fit_args[\"hyperparameters\"] = self.hyperparameters\n if fit_args[\"time_limit\"] is None:\n # TODO: This isn't technically right if the user specified `None`. Can fix in future by setting Predictor's default `time_limit=\"auto\"`\n fit_args.pop(\"time_limit\")\n return fit_args\n\n def _validate_input(self, X):\n check_is_fitted(self)\n # Input validation\n X = check_array(X)\n if X.shape[1] != self.n_features_in_:\n raise ValueError(\n f\"Inconsistent number of features between fit and predict calls: ({self.n_features_in_}, {X.shape[1]})\"\n )\n return X\n\n def _combine_X_y(self, X, y) -> pd.DataFrame:\n label = self.predictor_.label\n X = pd.DataFrame(X)\n assert label not in list(X.columns), (\n f\"Cannot have column named {label}. Please rename the column to a different value.\"\n )\n X[label] = y\n return X\n\n def leaderboard(self, X, y, **kwargs) -> pd.DataFrame:\n data = self._combine_X_y(X=X, y=y)\n return self.predictor_.leaderboard(data=data, **kwargs)\n"
},
{
"path": "tabular/src/autogluon/tabular/experimental/_tabular_classifier.py",
"content": "from __future__ import annotations\n\nimport pandas as pd\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.utils.multiclass import unique_labels\nfrom sklearn.utils.validation import check_array, check_is_fitted, check_X_y\n\nfrom autogluon.core.metrics import Scorer\n\nfrom .. import TabularPredictor\nfrom ._scikit_mixin import ScikitMixin\n\n\nclass TabularClassifier(BaseEstimator, ClassifierMixin, ScikitMixin):\n def __init__(\n self,\n eval_metric: str | Scorer = None,\n time_limit: float = None,\n presets: list[str] | str = None,\n hyperparameters: dict | str = None,\n path: str = None,\n verbosity: int = 2,\n init_args: dict = None,\n fit_args: dict = None,\n ):\n self.eval_metric = eval_metric\n self.time_limit = time_limit\n self.presets = presets\n self.hyperparameters = hyperparameters\n self.path = path\n self.verbosity = verbosity\n self.init_args = init_args\n self.fit_args = fit_args\n\n def fit(self, X, y):\n # Check that X and y have correct shape\n # X, y = check_X_y(X, y) # Commented out to allow for object dtypes\n\n # Store the classes seen during fit\n self.n_features_in_ = X.shape[1]\n self.classes_ = unique_labels(y)\n\n if len(self.classes_) == 1:\n raise ValueError(\"Classifier can't train when only one class is present.\")\n if len(self.classes_) == 2:\n problem_type = \"binary\"\n else:\n problem_type = \"multiclass\"\n\n init_args = self._get_init_args(problem_type=problem_type)\n fit_args = self._get_fit_args()\n\n self.predictor_ = TabularPredictor(**init_args)\n\n train_data = self._combine_X_y(X=X, y=y)\n\n self.predictor_.fit(train_data, **fit_args)\n\n # Return the classifier\n return self\n\n def predict(self, X):\n # Check if fit has been called\n check_is_fitted(self)\n # Input validation\n X = check_array(X)\n if X.shape[1] != self.n_features_in_:\n raise ValueError(\n f\"Inconsistent number of features between fit and predict calls: ({self.n_features_in_}, {X.shape[1]})\"\n )\n\n data = pd.DataFrame(X)\n y_pred = self.predictor_.predict(data=data).to_numpy()\n return y_pred\n\n def predict_proba(self, X):\n X = self._validate_input(X=X)\n data = pd.DataFrame(X)\n y_pred_proba = self.predictor_.predict_proba(data=data).to_numpy()\n return y_pred_proba\n"
},
{
"path": "tabular/src/autogluon/tabular/experimental/_tabular_regressor.py",
"content": "from __future__ import annotations\n\nimport pandas as pd\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.utils.validation import check_array, check_is_fitted, check_X_y\n\nfrom autogluon.core.metrics import Scorer\n\nfrom .. import TabularPredictor\nfrom ._scikit_mixin import ScikitMixin\n\n\nclass TabularRegressor(BaseEstimator, RegressorMixin, ScikitMixin):\n def __init__(\n self,\n eval_metric: str | Scorer = None,\n time_limit: float = None,\n presets: list[str] | str = None,\n hyperparameters: dict | str = None,\n path: str = None,\n verbosity: int = 2,\n init_args: dict = None,\n fit_args: dict = None,\n ):\n self.eval_metric = eval_metric\n self.time_limit = time_limit\n self.presets = presets\n self.hyperparameters = hyperparameters\n self.path = path\n self.verbosity = verbosity\n self.init_args = init_args\n self.fit_args = fit_args\n\n def fit(self, X, y):\n # Check that X and y have correct shape\n # X, y = check_X_y(X, y) # Commented out to allow for object dtypes\n\n self.n_features_in_ = X.shape[1]\n\n init_args = self._get_init_args(problem_type=\"regression\")\n fit_args = self._get_fit_args()\n\n self.predictor_ = TabularPredictor(**init_args)\n\n train_data = self._combine_X_y(X=X, y=y)\n\n self.predictor_.fit(train_data, **fit_args)\n\n # Return the regressor\n return self\n\n def predict(self, X):\n # Check if fit has been called\n check_is_fitted(self)\n # Input validation\n X = check_array(X)\n if X.shape[1] != self.n_features_in_:\n raise ValueError(\n f\"Inconsistent number of features between fit and predict calls: ({self.n_features_in_}, {X.shape[1]})\"\n )\n\n data = pd.DataFrame(X)\n y_pred = self.predictor_.predict(data=data).to_numpy()\n return y_pred\n"
},
{
"path": "tabular/src/autogluon/tabular/experimental/plot_leaderboard.py",
"content": "from __future__ import annotations\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom matplotlib.figure import Figure\n\nfrom autogluon.tabular import TabularPredictor\n\n\ndef _cumulative_min_idx(series: pd.Series) -> pd.Series:\n \"\"\"\n\n Parameters\n ----------\n series: pd.Series\n\n Returns\n -------\n pd.Series\n The index of the cumulative min of the series values.\n\n \"\"\"\n min_val = float(\"inf\")\n min_index = -1\n result = []\n for i, val in enumerate(series):\n if pd.isna(val):\n result.append(min_index)\n elif val < min_val:\n min_val = val\n min_index = i\n result.append(min_index)\n else:\n result.append(min_index)\n return pd.Series(series.index[result], index=series.index)\n\n\ndef compute_cumulative_leaderboard_stats(leaderboard: pd.DataFrame) -> pd.DataFrame:\n \"\"\"\n\n Parameters\n ----------\n leaderboard: pd.DataFrame\n\n Returns\n -------\n leaderboard_stats: pd.DataFrame\n\n \"\"\"\n leaderboard = leaderboard.copy(deep=True)\n leaderboard = leaderboard.sort_values(by=[\"fit_order\"]).set_index(\"model\")\n leaderboard[\"best_model_so_far\"] = _cumulative_min_idx(leaderboard[\"metric_error_val\"])\n leaderboard[\"best_idx_so_far\"] = leaderboard[\"best_model_so_far\"].map(leaderboard[\"fit_order\"])\n leaderboard[\"time_so_far\"] = leaderboard[\"fit_time_marginal\"].cumsum()\n leaderboard[\"metric_error_val_so_far\"] = leaderboard[\"best_model_so_far\"].map(leaderboard[\"metric_error_val\"])\n if \"metric_error_test\" in leaderboard:\n leaderboard[\"metric_error_test_so_far\"] = leaderboard[\"best_model_so_far\"].map(\n leaderboard[\"metric_error_test\"]\n )\n leaderboard = leaderboard.reset_index(drop=False).set_index(\"fit_order\")\n return leaderboard\n\n\n# TODO: Include constraints as options:\n# infer_limit\n# disk_usage\n# TODO: Avoid calling leaderboard on the original models again\n# TODO: Calibration?\ndef compute_cumulative_leaderboard_stats_ensemble(\n leaderboard: pd.DataFrame,\n predictor: TabularPredictor,\n test_data: pd.DataFrame | None = None,\n cleanup_ensembles: bool = True,\n) -> pd.DataFrame:\n \"\"\"\n\n Parameters\n ----------\n leaderboard: pd.DataFrame\n predictor: TabularPredictor\n test_data: pd.DataFrame | None, default None\n cleanup_ensembles: bool, default True\n\n Returns\n -------\n leaderboard_stats: pd.DataFrame\n\n \"\"\"\n leaderboard_stats = compute_cumulative_leaderboard_stats(leaderboard)\n model_fit_order = list(leaderboard_stats[\"model\"])\n ens_names = []\n for i in range(len(model_fit_order)):\n models_to_ens = model_fit_order[: i + 1]\n ens_name = predictor.fit_weighted_ensemble(base_models=models_to_ens, name_suffix=f\"_fit_{i + 1}\")[0]\n ens_names.append(ens_name)\n\n leaderboard_stats_ens = predictor.leaderboard(test_data, score_format=\"error\", display=False)\n leaderboard_stats_ens = leaderboard_stats_ens[leaderboard_stats_ens[\"model\"].isin(ens_names)]\n leaderboard_stats_ens = leaderboard_stats_ens.set_index(\"model\").reindex(ens_names).reset_index()\n leaderboard_stats_ens[\"fit_order\"] = leaderboard_stats.index\n leaderboard_stats_ens[\"model\"] = leaderboard_stats[\"model\"].values\n leaderboard_stats_ens = compute_cumulative_leaderboard_stats(leaderboard_stats_ens)\n\n leaderboard_stats[\"metric_error_val_so_far_ens\"] = leaderboard_stats_ens[\"metric_error_val_so_far\"]\n if test_data is not None:\n leaderboard_stats[\"metric_error_test_so_far_ens\"] = leaderboard_stats_ens[\"metric_error_test_so_far\"]\n leaderboard_stats[\"best_idx_so_far_ens\"] = leaderboard_stats_ens[\"best_idx_so_far\"]\n leaderboard_stats[\"best_model_so_far_ens\"] = leaderboard_stats_ens[\"best_model_so_far\"]\n if cleanup_ensembles:\n predictor.delete_models(models_to_delete=ens_names, dry_run=False)\n\n return leaderboard_stats\n\n\ndef plot_leaderboard_from_predictor(\n predictor: TabularPredictor,\n test_data: pd.DataFrame | None = None,\n ensemble: bool = False,\n include_val: bool = True,\n) -> tuple[Figure, pd.DataFrame]:\n \"\"\"\n\n Parameters\n ----------\n predictor: TabularPredictor\n test_data: pd.DataFrame | None, default None\n If specified, plots the test error.\n ensemble: bool, default False\n If True, additionally plots the results of cumulatively ensembling models at each step.\n include_val: bool, default True\n If True, plots the validation error.\n\n Returns\n -------\n fig: Figure\n leaderboard_stats: pd.DataFrame\n\n Examples\n --------\n >>> data_root = 'https://autogluon.s3.amazonaws.com/datasets/Inc/'\n >>> predictor_example = TabularPredictor(label=\"class\").fit(train_data=data_root + \"train.csv\", time_limit=60)\n >>> figure, lb = plot_leaderboard_from_predictor(predictor=predictor_example, test_data=data_root + \"test.csv\", ensemble=True)\n >>> with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n >>> print(lb)\n >>> figure.savefig(\"example_leaderboard_plot.png\")\n \"\"\"\n leaderboard = predictor.leaderboard(test_data, score_format=\"error\", display=False)\n if ensemble:\n leaderboard_order_sorted = compute_cumulative_leaderboard_stats_ensemble(\n leaderboard=leaderboard, test_data=test_data, predictor=predictor\n )\n else:\n leaderboard_order_sorted = compute_cumulative_leaderboard_stats(leaderboard=leaderboard)\n return plot_leaderboard(\n leaderboard=leaderboard_order_sorted, preprocess=False, ensemble=ensemble, include_val=include_val\n )\n\n\ndef plot_leaderboard(\n leaderboard: pd.DataFrame,\n preprocess: bool = True,\n ensemble: bool = False,\n include_val: bool = True,\n include_test: bool | None = None,\n) -> tuple[Figure, pd.DataFrame]:\n \"\"\"\n\n Parameters\n ----------\n leaderboard: pd.DataFrame\n Either the raw leaderboard output of `predictor.leaderboard(..., score_format=\"error\")` or the output of `compute_cumulative_leaderboard_stats`.\n preprocess: bool, default True\n Whether to preprocess the leaderboard to obtain leaderboard_stats.\n Set to False if `leaderboard` has already been transformed\n via `compute_cumulative_leaderboard_stats` or `compute_cumulative_leaderboard_stats_ensemble`.\n ensemble: bool, default False\n If True, additionally plots the results of cumulatively ensembling models at each step.\n Can only be set to True if ensemble columns are present in the leaderboard,\n which are generated by first calling `compute_cumulative_leaderboard_stats_ensemble`.\n include_val: bool, default True\n If True, plots the validation error.\n include_test: bool | None, default None\n If True, plots the test error.\n If None, infers based on the existence of the test error column in `leaderboard`.\n\n Returns\n -------\n fig: Figure\n leaderboard_stats: pd.DataFrame\n\n \"\"\"\n leaderboard_order_sorted = leaderboard\n if preprocess:\n if ensemble:\n raise AssertionError(\n f\"Cannot have both `preprocess=True` and `ensemble=True`.\"\n f\"Instead call `plot_leaderboard_from_predictor(..., ensemble=True)`\"\n )\n leaderboard_order_sorted = compute_cumulative_leaderboard_stats(leaderboard=leaderboard_order_sorted)\n\n eval_metric = leaderboard_order_sorted[\"eval_metric\"].iloc[0]\n if include_test is None:\n include_test = \"metric_error_test_so_far\" in leaderboard_order_sorted\n\n # TODO: View on inference time, can take from ensemble model, 3rd dimension, color?\n fig, axes = plt.subplots(1, 2, sharey=True)\n fig.suptitle(\"AutoGluon Metric Error Over Time\")\n\n ax = axes[0]\n\n if include_test:\n ax.plot(\n leaderboard_order_sorted.index,\n leaderboard_order_sorted[\"metric_error_test_so_far\"].values,\n \"-\",\n color=\"b\",\n label=\"test\",\n )\n if include_val:\n ax.plot(\n leaderboard_order_sorted.index,\n leaderboard_order_sorted[\"metric_error_val_so_far\"].values,\n \"-\",\n color=\"orange\",\n label=\"val\",\n )\n if ensemble:\n if include_test:\n ax.plot(\n leaderboard_order_sorted.index,\n leaderboard_order_sorted[\"metric_error_test_so_far_ens\"].values,\n \"--\",\n color=\"b\",\n label=\"test (ens)\",\n )\n if include_val:\n ax.plot(\n leaderboard_order_sorted.index,\n leaderboard_order_sorted[\"metric_error_val_so_far_ens\"].values,\n \"--\",\n color=\"orange\",\n label=\"val (ens)\",\n )\n ax.set_xlim(left=1, right=leaderboard_order_sorted.index.max())\n ax.set_xlabel(\"# Models Fit\")\n ax.set_ylabel(f\"Metric Error ({eval_metric})\")\n ax.grid()\n\n ax = axes[1]\n\n if include_test:\n ax.plot(\n leaderboard_order_sorted[\"time_so_far\"].values,\n leaderboard_order_sorted[\"metric_error_test_so_far\"].values,\n \"-\",\n color=\"b\",\n label=\"test\",\n )\n if include_val:\n ax.plot(\n leaderboard_order_sorted[\"time_so_far\"].values,\n leaderboard_order_sorted[\"metric_error_val_so_far\"].values,\n \"-\",\n color=\"orange\",\n label=\"val\",\n )\n if ensemble:\n if include_test:\n ax.plot(\n leaderboard_order_sorted[\"time_so_far\"].values,\n leaderboard_order_sorted[\"metric_error_test_so_far_ens\"].values,\n \"--\",\n color=\"b\",\n label=\"test (ens)\",\n )\n if include_val:\n ax.plot(\n leaderboard_order_sorted[\"time_so_far\"].values,\n leaderboard_order_sorted[\"metric_error_val_so_far_ens\"].values,\n \"--\",\n color=\"orange\",\n label=\"val (ens)\",\n )\n ax.set_xlabel(\"Time Elapsed (s)\")\n ax.grid()\n ax.legend()\n\n return fig, leaderboard_order_sorted\n"
},
{
"path": "tabular/src/autogluon/tabular/learner/__init__.py",
"content": "from .abstract_learner import *\nfrom .default_learner import *\n"
},
{
"path": "tabular/src/autogluon/tabular/learner/abstract_learner.py",
"content": "from __future__ import annotations\n\nimport copy\nimport json\nimport logging\nimport time\nfrom collections.abc import Iterable\nfrom typing import List\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\nfrom sklearn.metrics import classification_report\n\nfrom autogluon.core.constants import AUTO_WEIGHT, BALANCE_WEIGHT, BINARY, MULTICLASS, QUANTILE, REGRESSION\nfrom autogluon.core.data.label_cleaner import LabelCleaner, LabelCleanerMulticlass, LabelCleanerMulticlassToBinary\nfrom autogluon.core.learner import AbstractLearner\nfrom autogluon.core.metrics import Scorer, compute_metric, confusion_matrix, get_metric\nfrom autogluon.core.models.greedy_ensemble.ensemble_selection import EnsembleSelection\nfrom autogluon.core.utils import (\n augment_rare_classes,\n extract_column,\n get_leaderboard_pareto_frontier,\n get_pred_from_proba,\n get_pred_from_proba_df,\n infer_problem_type,\n)\nfrom autogluon.features.generators import PipelineFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: - Semi-supervised learning\n# TODO: - Minimize memory usage of DataFrames (convert int64 -> uint8 when possible etc.)\n# Learner encompasses full problem, loading initial data, feature generation, model training, model prediction\n# TODO: Loading learner from S3 on Windows may cause issues due to os.path.sep\nclass AbstractTabularLearner(AbstractLearner):\n def __init__(\n self,\n path_context: str,\n label: str,\n feature_generator: PipelineFeatureGenerator | None = None,\n ignored_columns: list = None,\n label_count_threshold: int = 10,\n problem_type: str | None = None,\n quantile_levels: list[float] | None = None,\n eval_metric: Scorer | None = None,\n positive_class: str | None = None,\n cache_data: bool = True,\n is_trainer_present: bool = False,\n random_state: int = 0,\n sample_weight: str | None = None,\n weight_evaluation: bool = False,\n groups: str | None = None,\n ):\n super().__init__(path_context=path_context, random_state=random_state)\n self.label = label\n self.ignored_columns = ignored_columns\n if self.ignored_columns is None:\n self.ignored_columns = []\n self.threshold = label_count_threshold\n self.problem_type = problem_type\n self._eval_metric_was_str = eval_metric is not None and isinstance(eval_metric, str)\n self.eval_metric = get_metric(eval_metric, self.problem_type, \"eval_metric\")\n\n if self.problem_type == QUANTILE and quantile_levels is None:\n raise ValueError(\"if `problem_type='quantile'`, `quantile_levels` has to be specified\")\n if isinstance(quantile_levels, float):\n quantile_levels = [quantile_levels]\n if isinstance(quantile_levels, Iterable):\n for quantile in quantile_levels:\n if quantile <= 0.0 or quantile >= 1.0:\n raise ValueError(\n \"quantile values have to be non-negative and less than 1.0 (0.0 < q < 1.0). \"\n \"For example, 0.95 quantile = 95 percentile\"\n )\n quantile_levels = np.sort(np.array(quantile_levels))\n self.quantile_levels = quantile_levels\n\n self.cache_data = cache_data\n if not self.cache_data:\n logger.log(\n 30,\n \"Warning: `cache_data=False` will disable or limit advanced functionality after training such as feature importance calculations. It is recommended to set `cache_data=True` unless you explicitly wish to not have the data saved to disk.\",\n )\n self.is_trainer_present = is_trainer_present\n\n self.cleaner = None\n self.label_cleaner: LabelCleaner = None\n self.feature_generator: PipelineFeatureGenerator = feature_generator\n\n self._original_features = None\n self._pre_X_rows = None\n self._post_X_rows = None\n self._positive_class = positive_class\n self.sample_weight = sample_weight\n self.weight_evaluation = weight_evaluation\n self.groups = groups\n if sample_weight is not None and not isinstance(sample_weight, str):\n raise ValueError(\n \"sample_weight must be a string indicating the name of the column that contains sample weights. If you have a vector of sample weights, first add these as an extra column to your data.\"\n )\n if weight_evaluation and sample_weight is None:\n raise ValueError(\"Must specify sample_weight column if you specify weight_evaluation=True\")\n if groups is not None and not isinstance(groups, str):\n raise ValueError(\n \"groups must be a string indicating the name of the column that contains the split groups. If you have a vector of split groups, first add these as an extra column to your data.\"\n )\n\n @property\n def original_features(self) -> List[str]:\n \"\"\"Original features user passed in before autogluon doing any processing\"\"\"\n return self._original_features\n\n # TODO: Possibly rename to features_in or consider refactoring all feature_generators features_in -> features\n @property\n def features(self):\n return self.feature_generator.features_in\n\n @property\n def feature_metadata_in(self):\n return self.feature_generator.feature_metadata_in\n\n @property\n def feature_generators(self):\n return [self.feature_generator]\n\n @property\n def class_labels(self):\n return self.label_cleaner.ordered_class_labels\n\n @property\n def class_labels_transformed(self):\n return self.label_cleaner.ordered_class_labels_transformed\n\n @property\n def positive_class(self):\n \"\"\"\n Returns the positive class name in binary classification. Useful for computing metrics such as F1 which require a positive and negative class.\n In binary classification, :class:`TabularPredictor.predict_proba()` returns the estimated probability that each row belongs to the positive class.\n Will print a warning and return None if called when `predictor.problem_type != 'binary'`.\n\n Returns\n -------\n The positive class name in binary classification or None if the problem is not binary classification.\n \"\"\"\n if not self.is_fit:\n if self._positive_class is not None:\n return self._positive_class\n raise AssertionError(\"Predictor must be fit to return positive_class.\")\n if self.problem_type != BINARY:\n logger.warning(\n f\"Warning: Attempted to retrieve positive class label in a non-binary problem. Positive class labels only exist in binary classification. Returning None instead. self.problem_type is '{self.problem_type}' but positive_class only exists for '{BINARY}'.\"\n )\n return None\n return self.label_cleaner.cat_mappings_dependent_var[1]\n\n def fit(self, X: DataFrame, X_val: DataFrame = None, **kwargs):\n if self.is_fit:\n raise AssertionError(\"Learner is already fit.\")\n self._validate_fit_input(X=X, X_val=X_val, **kwargs)\n return self._fit(X=X, X_val=X_val, **kwargs)\n\n def _fit(\n self,\n X: DataFrame,\n X_val: DataFrame = None,\n scheduler_options=None,\n hyperparameter_tune=False,\n feature_prune=False,\n holdout_frac=0.1,\n hyperparameters=None,\n verbosity=2,\n ):\n raise NotImplementedError\n\n def predict_proba(\n self,\n X: DataFrame,\n model: str | None = None,\n as_pandas: bool = True,\n as_multiclass: bool = True,\n inverse_transform: bool = True,\n transform_features: bool = True,\n ):\n X_index = copy.deepcopy(X.index) if as_pandas else None\n if X.empty:\n y_pred_proba = np.array([])\n else:\n if transform_features:\n X = self.transform_features(X)\n y_pred_proba = self.load_trainer().predict_proba(X, model=model)\n y_pred_proba = self._post_process_predict_proba(\n y_pred_proba=y_pred_proba,\n as_pandas=as_pandas,\n index=X_index,\n as_multiclass=as_multiclass,\n inverse_transform=inverse_transform,\n )\n return y_pred_proba\n\n def predict(\n self,\n X: DataFrame,\n model: str | None = None,\n as_pandas: bool = True,\n inverse_transform: bool = True,\n transform_features: bool = True,\n *,\n decision_threshold: float | None = None,\n ):\n if decision_threshold is None:\n decision_threshold = 0.5\n X_index = copy.deepcopy(X.index) if as_pandas else None\n y_pred_proba = self.predict_proba(\n X=X,\n model=model,\n as_pandas=False,\n as_multiclass=False,\n inverse_transform=False,\n transform_features=transform_features,\n )\n problem_type = self.label_cleaner.problem_type_transform or self.problem_type\n y_pred = get_pred_from_proba(\n y_pred_proba=y_pred_proba, problem_type=problem_type, decision_threshold=decision_threshold\n )\n y_pred = self._post_process_predict(\n y_pred=y_pred, as_pandas=as_pandas, index=X_index, inverse_transform=inverse_transform\n )\n return y_pred\n\n def _post_process_predict(\n self,\n y_pred: np.ndarray,\n as_pandas: bool = True,\n index=None,\n inverse_transform: bool = True,\n ):\n \"\"\"\n Given internal predictions, post-process them to vend to user.\n \"\"\"\n if self.problem_type != QUANTILE:\n if inverse_transform:\n y_pred = self.label_cleaner.inverse_transform(pd.Series(y_pred))\n else:\n y_pred = pd.Series(y_pred)\n if as_pandas:\n y_pred.index = index\n y_pred.name = self.label\n else:\n y_pred = y_pred.values\n else:\n if as_pandas:\n if len(y_pred) == 0:\n # avoid exception due to mismatched shape for empty predict\n y_pred = None\n y_pred = pd.DataFrame(data=y_pred, columns=self.quantile_levels, index=index)\n return y_pred\n\n def _post_process_predict_proba(\n self,\n y_pred_proba: np.ndarray,\n as_pandas: bool = True,\n index=None,\n as_multiclass: bool = True,\n inverse_transform: bool = True,\n ):\n \"\"\"\n Given internal prediction probabilities, post-process them to vend to user.\n \"\"\"\n if inverse_transform:\n y_pred_proba = self.label_cleaner.inverse_transform_proba(y_pred_proba)\n if as_multiclass and (self.problem_type == BINARY):\n y_pred_proba = LabelCleanerMulticlassToBinary.convert_binary_proba_to_multiclass_proba(y_pred_proba)\n if as_pandas:\n if self.problem_type == MULTICLASS or (as_multiclass and self.problem_type == BINARY):\n classes = self.class_labels if inverse_transform else self.class_labels_transformed\n y_pred_proba = pd.DataFrame(data=y_pred_proba, columns=classes, index=index)\n elif self.problem_type == QUANTILE:\n y_pred_proba = pd.DataFrame(data=y_pred_proba, columns=self.quantile_levels, index=index)\n else:\n y_pred_proba = pd.Series(data=y_pred_proba, name=self.label, index=index)\n return y_pred_proba\n\n def predict_proba_multi(\n self,\n X: DataFrame = None,\n models: List[str] = None,\n as_pandas: bool = True,\n as_multiclass: bool = True,\n transform_features: bool = True,\n inverse_transform: bool = True,\n use_refit_parent_oof: bool = True,\n ) -> dict:\n \"\"\"\n Returns a dictionary of prediction probabilities where the key is\n the model name and the value is the model's prediction probabilities on the data.\n\n Note that this will generally be much faster than calling `self.predict_proba` separately for each model\n because this method leverages the model dependency graph to avoid redundant computation.\n\n Parameters\n ----------\n X : DataFrame, default = None\n The data to predict on.\n If None:\n If self.trainer.has_val, the validation data is used.\n Else, the out-of-fold prediction probabilities are used.\n models : List[str], default = None\n The list of models to get predictions for.\n If None, all models that can infer are used.\n as_pandas : bool, default = True\n Whether to return the output of each model as a pandas object (True) or numpy array (False).\n Pandas object is a DataFrame if this is a multiclass problem or `as_multiclass=True`, otherwise it is a Series.\n If the output is a DataFrame, the column order will be equivalent to `predictor.class_labels`.\n as_multiclass : bool, default = True\n Whether to return binary classification probabilities as if they were for multiclass classification.\n Output will contain two columns, and if `as_pandas=True`, the column names will correspond to the binary class labels.\n The columns will be the same order as `predictor.class_labels`.\n If False, output will contain only 1 column for the positive class (get positive_class name via `predictor.positive_class`).\n Only impacts output for binary classification problems.\n transform_features : bool, default = True\n If True, preprocesses data before predicting with models.\n If False, skips global feature preprocessing.\n This is useful to save on inference time if you have already called `data = predictor.transform_features(data)`.\n inverse_transform : bool, default = True\n If True, will return prediction probabilities in the original format.\n If False (advanced), will return prediction probabilities in AutoGluon's internal format.\n use_refit_parent_oof: bool = True\n If True and data is None and returning OOF, will return the parent model's OOF for refit models instead of raising an exception.\n\n Returns\n -------\n Dictionary with model names as keys and model prediction probabilities as values.\n \"\"\"\n trainer = self.load_trainer()\n\n if models is None:\n models = trainer.get_model_names(can_infer=True)\n if X is not None:\n X_index = copy.deepcopy(X.index) if as_pandas else None\n if transform_features:\n X = self.transform_features(X)\n predict_proba_dict = trainer.get_model_pred_proba_dict(X=X, models=models)\n else:\n if trainer.has_val:\n # Return validation pred proba\n X = trainer.load_X_val()\n X_index = copy.deepcopy(X.index) if as_pandas else None\n predict_proba_dict = trainer.get_model_pred_proba_dict(X=X, models=models, use_val_cache=True)\n else:\n # Return out-of-fold pred proba\n X = trainer.load_X()\n X_index = copy.deepcopy(X.index) if as_pandas else None\n predict_proba_dict = dict()\n for m in models:\n predict_proba_dict[m] = trainer.get_model_oof(m, use_refit_parent=use_refit_parent_oof)\n\n # Inverse Transform labels\n for m, pred_proba in predict_proba_dict.items():\n predict_proba_dict[m] = self._post_process_predict_proba(\n y_pred_proba=pred_proba,\n as_pandas=as_pandas,\n as_multiclass=as_multiclass,\n index=X_index,\n inverse_transform=inverse_transform,\n )\n return predict_proba_dict\n\n def predict_multi(\n self,\n X: DataFrame = None,\n models: List[str] = None,\n as_pandas: bool = True,\n transform_features: bool = True,\n inverse_transform: bool = True,\n use_refit_parent_oof: bool = True,\n *,\n decision_threshold: float = None,\n ) -> dict:\n \"\"\"\n Identical to predict_proba_multi, except returns predictions instead of probabilities.\n \"\"\"\n predict_proba_dict = self.predict_proba_multi(\n X=X,\n models=models,\n as_pandas=as_pandas,\n transform_features=transform_features,\n inverse_transform=inverse_transform,\n use_refit_parent_oof=use_refit_parent_oof,\n )\n if self.problem_type in [REGRESSION, QUANTILE]:\n return predict_proba_dict\n predict_dict = {}\n for m in predict_proba_dict:\n predict_dict[m] = self.get_pred_from_proba(\n y_pred_proba=predict_proba_dict[m],\n decision_threshold=decision_threshold,\n inverse_transform=inverse_transform,\n )\n return predict_dict\n\n def get_pred_from_proba(\n self,\n y_pred_proba: np.ndarray | pd.DataFrame,\n decision_threshold: float | None = None,\n inverse_transform: bool = True,\n ) -> np.array | pd.Series:\n if isinstance(y_pred_proba, pd.DataFrame):\n y_pred = get_pred_from_proba_df(\n y_pred_proba, problem_type=self.problem_type, decision_threshold=decision_threshold\n )\n else:\n y_pred = get_pred_from_proba(\n y_pred_proba, problem_type=self.problem_type, decision_threshold=decision_threshold\n )\n y_pred = self._post_process_predict(\n y_pred=y_pred, as_pandas=False, index=None, inverse_transform=inverse_transform\n )\n return y_pred\n\n def _validate_fit_input(self, X: DataFrame, **kwargs):\n self.validate_label(X=X)\n X_val = kwargs.get(\"X_val\", None)\n self._validate_sample_weight(X, X_val)\n self._validate_groups(X, X_val)\n X_test = kwargs.get(\"X_test\", None)\n if X_test is not None:\n self._validate_sample_weight(X, X_test)\n self._validate_groups(X, X_test)\n\n def validate_label(self, X: DataFrame):\n \"\"\"\n Ensure that the label column is present in the training data\n \"\"\"\n if self.label not in X.columns:\n raise KeyError(\n f\"Label column '{self.label}' is missing from training data. Training data columns: {list(X.columns)}\"\n )\n\n def _validate_sample_weight(self, X, X_val):\n if self.sample_weight is not None:\n if self.sample_weight in [AUTO_WEIGHT, BALANCE_WEIGHT]:\n prefix = f\"Using predefined sample weighting strategy: {self.sample_weight}.\"\n if self.weight_evaluation:\n prefix += \" Warning: We do not recommend weight_evaluation=True with predefined sample weighting.\"\n else:\n if self.sample_weight not in X.columns:\n raise KeyError(\n f\"sample_weight column '{self.sample_weight}' is missing from training data. Training data columns: {list(X.columns)}\"\n )\n weight_vals = X[self.sample_weight]\n if weight_vals.isna().sum() > 0:\n raise ValueError(f\"Sample weights in column '{self.sample_weight}' cannot be nan\")\n if weight_vals.dtype.kind not in \"biuf\":\n raise ValueError(f\"Sample weights in column '{self.sample_weight}' must be numeric values\")\n if weight_vals.min() < 0:\n raise ValueError(f\"Sample weights in column '{self.sample_weight}' must be nonnegative\")\n if self.weight_evaluation and X_val is not None and self.sample_weight not in X_val.columns:\n raise KeyError(\n f\"sample_weight column '{self.sample_weight}' cannot be missing from validation data if weight_evaluation=True\"\n )\n prefix = (\n f\"Values in column '{self.sample_weight}' used as sample weights instead of predictive features.\"\n )\n if self.weight_evaluation:\n suffix = \" Evaluation will report weighted metrics, so ensure same column exists in test data.\"\n else:\n suffix = \" Evaluation metrics will ignore sample weights, specify weight_evaluation=True to instead report weighted metrics.\"\n logger.log(20, prefix + suffix)\n\n def _validate_groups(self, X, X_val):\n if self.groups is not None:\n if self.groups not in X.columns:\n raise KeyError(\n f\"groups column '{self.groups}' is missing from training data. Training data columns: {list(X.columns)}\"\n )\n groups_vals = X[self.groups]\n if len(groups_vals.unique()) < 2:\n raise ValueError(\n f\"Groups in column '{self.groups}' cannot have fewer than 2 unique values. Values: {list(groups_vals.unique())}\"\n )\n if X_val is not None and self.groups in X_val.columns:\n raise KeyError(\n f\"groups column '{self.groups}' cannot be in validation data. Validation data columns: {list(X_val.columns)}\"\n )\n logger.log(\n 20,\n f\"Values in column '{self.groups}' used as split folds instead of being automatically set. Bagged models will have {len(groups_vals.unique())} splits.\",\n )\n\n def get_inputs_to_stacker(self, dataset=None, model=None, base_models: list = None, use_orig_features=True):\n if model is not None or base_models is not None:\n if model is not None and base_models is not None:\n raise AssertionError(\"Only one of `model`, `base_models` is allowed to be set.\")\n\n trainer = self.load_trainer()\n if dataset is None:\n if trainer.bagged_mode:\n dataset_preprocessed = trainer.load_X()\n fit = True\n else:\n dataset_preprocessed = trainer.load_X_val()\n fit = False\n else:\n dataset_preprocessed = self.transform_features(dataset)\n fit = False\n dataset_preprocessed = trainer.get_inputs_to_stacker(\n X=dataset_preprocessed,\n model=model,\n base_models=base_models,\n fit=fit,\n use_orig_features=use_orig_features,\n )\n # Note: Below doesn't quite work here because weighted_ensemble has unique input format returned that isn't a DataFrame.\n # dataset_preprocessed = trainer.get_inputs_to_model(model=model_to_get_inputs_for, X=dataset_preprocessed, fit=fit)\n\n return dataset_preprocessed\n\n # Fits _FULL models and links them in the stack so _FULL models only use other _FULL models as input during stacking\n # If model is specified, will fit all _FULL models that are ancestors of the provided model, automatically linking them.\n # If no model is specified, all models are refit and linked appropriately.\n def refit_ensemble_full(self, model: str | List[str] = \"all\", **kwargs):\n return self.load_trainer().refit_ensemble_full(model=model, **kwargs)\n\n def fit_transform_features(self, X, y=None, **kwargs):\n if self.label in X:\n X = X.drop(columns=[self.label])\n if self.ignored_columns:\n logger.log(20, f\"Dropping user-specified ignored columns: {self.ignored_columns}\")\n X = X.drop(columns=self.ignored_columns, errors=\"ignore\")\n for feature_generator in self.feature_generators:\n X = feature_generator.fit_transform(X, y, **kwargs)\n return X\n\n def transform_features(self, X):\n for feature_generator in self.feature_generators:\n X = feature_generator.transform(X)\n return X\n\n def score(self, X: DataFrame, y=None, model: str = None, metric: Scorer = None, as_error: bool = False) -> float:\n if metric is None:\n metric = self.eval_metric\n if y is None:\n X, y = self.extract_label(X)\n self._validate_class_labels(y)\n weights = None\n if self.weight_evaluation:\n X, weights = extract_column(X, self.sample_weight)\n if self.eval_metric.needs_pred or self.eval_metric.needs_quantile:\n y_pred_proba = None\n y_pred = self.predict(X=X, model=model, as_pandas=False)\n if self.problem_type == BINARY:\n # Use 1 and 0, otherwise f1 can crash due to unknown pos_label.\n y_pred = self.label_cleaner.transform(y_pred)\n y = self.label_cleaner.transform(y)\n else:\n y_pred_proba = self.predict_proba(X=X, model=model, as_pandas=False, as_multiclass=False)\n y_pred = None\n y = self.label_cleaner.transform(y)\n\n return compute_metric(\n y=y,\n y_pred=y_pred,\n y_pred_proba=y_pred_proba,\n metric=metric,\n weights=weights,\n weight_evaluation=self.weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n # Scores both learner and all individual models, along with computing the optimal ensemble score + weights (oracle)\n def score_debug(\n self,\n X: DataFrame,\n y=None,\n extra_info=False,\n compute_oracle=False,\n extra_metrics=None,\n decision_threshold=None,\n skip_score=False,\n refit_full=None,\n set_refit_score_to_parent=False,\n display=False,\n ):\n leaderboard_df = self.leaderboard(\n extra_info=extra_info,\n refit_full=refit_full,\n set_refit_score_to_parent=set_refit_score_to_parent,\n display=display,\n )\n if extra_metrics is None:\n extra_metrics = []\n if y is None:\n error_if_missing = extra_metrics or not skip_score\n X, y = self.extract_label(X, error_if_missing=error_if_missing)\n w = None\n if self.weight_evaluation:\n X, w = extract_column(X, self.sample_weight)\n\n X = self.transform_features(X)\n if y is not None:\n self._validate_class_labels(y)\n y_internal = self.label_cleaner.transform(y)\n y_internal = y_internal.fillna(-1)\n else:\n y_internal = None\n\n trainer = self.load_trainer()\n scores = {}\n leaderboard_models = set(leaderboard_df[\"model\"].tolist())\n all_trained_models = trainer.get_model_names()\n all_trained_models = [m for m in all_trained_models if m in leaderboard_models]\n all_trained_models_can_infer = trainer.get_model_names(models=all_trained_models, can_infer=True)\n all_trained_models_original = all_trained_models.copy()\n model_pred_proba_dict, pred_time_test_marginal = trainer.get_model_pred_proba_dict(\n X=X, models=all_trained_models_can_infer, record_pred_time=True\n )\n\n if compute_oracle:\n pred_probas = list(model_pred_proba_dict.values())\n ensemble_selection = EnsembleSelection(\n ensemble_size=100,\n problem_type=trainer.problem_type,\n metric=self.eval_metric,\n quantile_levels=self.quantile_levels,\n )\n ensemble_selection.fit(\n predictions=pred_probas, labels=y_internal, identifiers=None, sample_weight=w\n ) # TODO: Only fit non-nan\n\n oracle_weights = ensemble_selection.weights_\n oracle_pred_time_start = time.time()\n oracle_pred_proba_norm = [pred * weight for pred, weight in zip(pred_probas, oracle_weights)]\n oracle_pred_proba_ensemble = np.sum(oracle_pred_proba_norm, axis=0)\n oracle_pred_time = time.time() - oracle_pred_time_start\n model_pred_proba_dict[\"OracleEnsemble\"] = oracle_pred_proba_ensemble\n pred_time_test_marginal[\"OracleEnsemble\"] = oracle_pred_time\n all_trained_models.append(\"OracleEnsemble\")\n\n scoring_args = dict(y=y, y_internal=y_internal, sample_weight=w)\n\n extra_scores = {}\n for model_name, y_pred_proba_internal in model_pred_proba_dict.items():\n if skip_score:\n scores[model_name] = np.nan\n else:\n scores[model_name] = self.score_with_pred_proba(\n y_pred_proba_internal=y_pred_proba_internal,\n metric=self.eval_metric,\n decision_threshold=decision_threshold,\n **scoring_args,\n )\n for metric in extra_metrics:\n metric = get_metric(metric, self.problem_type, \"leaderboard_metric\")\n if metric.name not in extra_scores:\n extra_scores[metric.name] = {}\n extra_scores[metric.name][model_name] = self.score_with_pred_proba(\n y_pred_proba_internal=y_pred_proba_internal,\n metric=metric,\n decision_threshold=decision_threshold,\n **scoring_args,\n )\n\n if extra_scores:\n series = []\n for metric in extra_scores:\n series.append(pd.Series(extra_scores[metric], name=metric))\n df_extra_scores = pd.concat(series, axis=1)\n extra_metrics_names = list(df_extra_scores.columns)\n df_extra_scores[\"model\"] = df_extra_scores.index\n df_extra_scores = df_extra_scores.reset_index(drop=True)\n else:\n df_extra_scores = None\n extra_metrics_names = None\n\n pred_time_test = {}\n # TODO: Add support for calculating pred_time_test_full for oracle_ensemble, need to copy graph from trainer and add oracle_ensemble to it with proper edges.\n for model in model_pred_proba_dict.keys():\n if model in all_trained_models_original:\n base_model_set = trainer.get_minimum_model_set(model)\n if len(base_model_set) == 1:\n pred_time_test[model] = pred_time_test_marginal[base_model_set[0]]\n else:\n pred_time_test_full_num = 0\n for base_model in base_model_set:\n pred_time_test_full_num += pred_time_test_marginal[base_model]\n pred_time_test[model] = pred_time_test_full_num\n else:\n pred_time_test[model] = None\n\n scored_models = list(scores.keys())\n for model in all_trained_models:\n if model not in scored_models:\n scores[model] = None\n pred_time_test[model] = None\n pred_time_test_marginal[model] = None\n\n model_names_final = list(scores.keys())\n df = pd.DataFrame(\n data={\n \"model\": model_names_final,\n \"score_test\": list(scores.values()),\n \"pred_time_test\": [pred_time_test.get(model, np.nan) for model in model_names_final],\n \"pred_time_test_marginal\": [pred_time_test_marginal.get(model, np.nan) for model in model_names_final],\n }\n )\n if df_extra_scores is not None:\n df = pd.merge(df, df_extra_scores, on=\"model\", how=\"left\")\n\n df_merged = pd.merge(df, leaderboard_df, on=\"model\", how=\"left\")\n df_merged = df_merged.sort_values(\n by=[\"score_test\", \"pred_time_test\", \"score_val\", \"pred_time_val\", \"model\"],\n ascending=[False, True, False, True, False],\n ).reset_index(drop=True)\n df_columns_lst = df_merged.columns.tolist()\n explicit_order = [\n \"model\",\n \"score_test\",\n ]\n if extra_metrics_names is not None:\n explicit_order += extra_metrics_names\n explicit_order += [\n \"score_val\",\n \"eval_metric\",\n \"pred_time_test\",\n \"pred_time_val\",\n \"fit_time\",\n \"pred_time_test_marginal\",\n \"pred_time_val_marginal\",\n \"fit_time_marginal\",\n \"stack_level\",\n \"can_infer\",\n \"fit_order\",\n ]\n df_columns_other = [column for column in df_columns_lst if column not in explicit_order]\n df_columns_new = explicit_order + df_columns_other\n df_merged = df_merged[df_columns_new]\n\n return df_merged\n\n def score_with_pred_proba(\n self,\n y,\n y_internal,\n y_pred_proba_internal: np.ndarray,\n metric: Scorer = None,\n sample_weight: np.ndarray = None,\n decision_threshold: float = None,\n weight_evaluation: bool = None,\n as_error: bool = False,\n ) -> float:\n if metric is None:\n metric = self.eval_metric\n metric = get_metric(metric, self.problem_type, \"leaderboard_metric\")\n if weight_evaluation is None:\n weight_evaluation = self.weight_evaluation\n if metric.needs_pred or metric.needs_quantile:\n if self.problem_type == BINARY:\n # Use 1 and 0, otherwise f1 can crash due to unknown pos_label.\n y_pred = self.get_pred_from_proba(\n y_pred_proba_internal, decision_threshold=decision_threshold, inverse_transform=False\n )\n y_pred_proba = None\n y_tmp = y_internal\n else:\n y_pred = self.label_cleaner.inverse_transform_proba(y_pred_proba_internal, as_pred=True)\n y_pred_proba = None\n y_tmp = y\n else:\n y_pred = None\n y_pred_proba = self.label_cleaner.inverse_transform_proba(y_pred_proba_internal, as_pred=False)\n if isinstance(self.label_cleaner, LabelCleanerMulticlass):\n # Ensures that logic works even when y contains previously dropped classes during fit.\n # If y contains never before seen classes, this will raise a ValueError in `self._validate_class_labels`.\n self._validate_class_labels(y=y, eval_metric=metric)\n y_tmp = self.label_cleaner.transform_pred_uncleaned(y)\n else:\n y_tmp = y_internal\n\n return compute_metric(\n y=y_tmp,\n y_pred=y_pred,\n y_pred_proba=y_pred_proba,\n metric=metric,\n weights=sample_weight,\n weight_evaluation=weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n def score_with_pred(\n self,\n y,\n y_internal,\n y_pred_internal,\n metric: Scorer = None,\n sample_weight: np.ndarray = None,\n weight_evaluation: bool = None,\n as_error: bool = False,\n ) -> float:\n if metric is None:\n metric = self.eval_metric\n metric = get_metric(metric, self.problem_type, \"leaderboard_metric\")\n if weight_evaluation is None:\n weight_evaluation = self.weight_evaluation\n if self.problem_type == BINARY:\n # Use 1 and 0, otherwise f1 can crash due to unknown pos_label.\n y_pred = y_pred_internal\n y_tmp = y_internal\n else:\n y_pred = self.label_cleaner.inverse_transform(y_pred_internal)\n y_tmp = y\n\n return compute_metric(\n y=y_tmp,\n y_pred=y_pred,\n y_pred_proba=None,\n metric=metric,\n weights=sample_weight,\n weight_evaluation=weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n def _validate_class_labels(self, y: Series, eval_metric: Scorer = None):\n null_count = y.isnull().sum()\n if null_count:\n raise ValueError(f\"Labels cannot contain missing (nan) values. Found {null_count} missing label values.\")\n if eval_metric is None:\n eval_metric = self.eval_metric\n if self.problem_type == MULTICLASS and not eval_metric.needs_pred:\n y_unique = np.unique(y)\n valid_class_set = set(self.class_labels)\n unknown_classes = []\n for cls in y_unique:\n if cls not in valid_class_set:\n unknown_classes.append(cls)\n if unknown_classes:\n # log_loss / pac_score\n raise ValueError(\n f\"Multiclass scoring with eval_metric='{eval_metric.name}' does not support unknown classes. \"\n f\"Please ensure the classes you wish to evaluate are present in the training data, otherwise they cannot be scored with this metric.\"\n f\"\\n\\tUnknown classes: {unknown_classes}\"\n f\"\\n\\t Known classes: {self.class_labels}\"\n )\n\n def evaluate_predictions(\n self,\n y_true,\n y_pred,\n sample_weight=None,\n decision_threshold=None,\n display=False,\n auxiliary_metrics=True,\n detailed_report=False,\n ):\n \"\"\"Evaluate predictions. Does not support sample weights since this method reports a variety of metrics.\n Args:\n display (bool): Should we print which metric is being used as well as performance.\n auxiliary_metrics (bool): Should we compute other (problem_type specific) metrics in addition to the default metric?\n detailed_report (bool): Should we computed more-detailed versions of the auxiliary_metrics? (requires auxiliary_metrics=True).\n\n Returns single performance-value if auxiliary_metrics=False.\n Otherwise returns dict where keys = metrics, values = performance along each metric.\n \"\"\"\n\n is_proba = False\n assert isinstance(y_true, (np.ndarray, pd.Series))\n assert isinstance(y_pred, (np.ndarray, pd.Series, pd.DataFrame))\n self._validate_class_labels(y_true)\n if isinstance(y_pred, np.ndarray):\n if self.problem_type == QUANTILE:\n y_pred = pd.DataFrame(data=y_pred, columns=self.quantile_levels)\n elif len(y_pred.shape) > 1:\n y_pred = pd.DataFrame(data=y_pred, columns=self.class_labels)\n\n if isinstance(y_pred, pd.DataFrame):\n is_proba = True\n elif not self.eval_metric.needs_pred:\n raise AssertionError(\n f\"`evaluate_predictions` requires y_pred_proba input \"\n f'when evaluating \"{self.eval_metric.name}\"... Please generate valid input via `predictor.predict_proba(data)`.\\n'\n f\"This may have occurred if you passed in predict input instead of predict_proba input, \"\n f\"or if you specified `as_multiclass=False` to `predictor.predict_proba(data, as_multiclass=False)`, \"\n f\"which is not supported by `evaluate_predictions`.\"\n )\n if is_proba:\n y_pred_proba = y_pred\n y_pred = self.get_pred_from_proba(y_pred_proba=y_pred_proba, decision_threshold=decision_threshold)\n if self.problem_type == BINARY:\n # roc_auc crashes if this isn't done\n y_pred_proba = y_pred_proba[self.positive_class]\n else:\n y_pred_proba = None\n y_pred = pd.Series(y_pred)\n if y_pred_proba is not None:\n y_pred_proba_internal = self.label_cleaner.transform_proba(y_pred_proba, as_pandas=True)\n else:\n y_pred_proba_internal = None\n y_true_internal = self.label_cleaner.transform(y_true) # Get labels in numeric order\n y_true_internal = y_true_internal.fillna(-1)\n y_pred_internal = self.label_cleaner.transform(y_pred) # Get labels in numeric order\n\n # Compute auxiliary metrics:\n auxiliary_metrics_lst = [self.eval_metric]\n performance_dict = {}\n\n if auxiliary_metrics:\n if self.problem_type == REGRESSION: # Adding regression metrics\n auxiliary_metrics_lst += [\n \"root_mean_squared_error\",\n \"mean_squared_error\",\n \"mean_absolute_error\",\n \"r2\",\n \"pearsonr\",\n \"median_absolute_error\",\n ]\n if self.problem_type in [BINARY, MULTICLASS]: # Adding classification metrics\n auxiliary_metrics_lst += [\n \"accuracy\",\n \"balanced_accuracy\",\n # 'log_loss', # Don't include as it probably adds more confusion to novice users (can be infinite)\n \"mcc\",\n ]\n if self.problem_type == BINARY: # binary-specific metrics\n auxiliary_metrics_lst += [\n \"roc_auc\",\n \"f1\",\n \"precision\",\n \"recall\",\n ]\n\n scoring_args = dict(\n y=y_true,\n y_internal=y_true_internal,\n weight_evaluation=False,\n )\n\n if sample_weight is not None:\n scoring_args[\"sample_weight\"] = sample_weight\n scoring_args[\"weight_evaluation\"] = True\n\n for aux_metric in auxiliary_metrics_lst:\n if isinstance(aux_metric, str):\n aux_metric = get_metric(metric=aux_metric, problem_type=self.problem_type, metric_type=\"aux_metric\")\n if not aux_metric.needs_pred and y_pred_proba_internal is None:\n logger.log(\n 15, f\"Skipping {aux_metric.name} because no prediction probabilities are available to score.\"\n )\n continue\n\n if aux_metric.name not in performance_dict:\n if y_pred_proba_internal is not None:\n score = self.score_with_pred_proba(\n y_pred_proba_internal=y_pred_proba_internal,\n metric=aux_metric,\n decision_threshold=decision_threshold,\n **scoring_args,\n )\n else:\n score = self.score_with_pred(y_pred_internal=y_pred_internal, metric=aux_metric, **scoring_args)\n performance_dict[aux_metric.name] = score\n\n if display:\n if self.eval_metric.name in performance_dict:\n score_eval = performance_dict[self.eval_metric.name]\n logger.log(20, f\"Evaluation: {self.eval_metric.name} on test data: {score_eval}\")\n if not self.eval_metric.greater_is_better_internal:\n logger.log(\n 20,\n f\"\\tNote: Scores are always higher_is_better. This metric score can be multiplied by -1 to get the metric value.\",\n )\n logger.log(20, \"Evaluations on test data:\")\n logger.log(20, json.dumps(performance_dict, indent=4))\n\n if detailed_report and (self.problem_type != REGRESSION):\n # Construct confusion matrix\n try:\n performance_dict[\"confusion_matrix\"] = confusion_matrix(\n y_true, y_pred, labels=self.label_cleaner.ordered_class_labels, output_format=\"pandas_dataframe\"\n )\n except ValueError:\n pass\n # One final set of metrics to report\n cl_metric = lambda y_true, y_pred: classification_report(y_true, y_pred, output_dict=True)\n metric_name = \"classification_report\"\n if metric_name not in performance_dict:\n try: # only compute auxiliary metrics which do not error (y_pred = class-probabilities may cause some metrics to error)\n performance_dict[metric_name] = cl_metric(y_true, y_pred)\n except ValueError:\n pass\n if display and metric_name in performance_dict:\n logger.log(20, \"Detailed (per-class) classification report:\")\n logger.log(20, json.dumps(performance_dict[metric_name], indent=4))\n return performance_dict\n\n def extract_label(self, X, error_if_missing=True):\n if self.label not in list(X.columns):\n if error_if_missing:\n raise ValueError(f\"Provided DataFrame does not contain label column: {self.label}\")\n else:\n return X, None\n y = X[self.label].copy()\n X = X.drop(self.label, axis=1)\n return X, y\n\n def leaderboard(\n self,\n X=None,\n y=None,\n extra_info=False,\n extra_metrics=None,\n decision_threshold=None,\n only_pareto_frontier=False,\n skip_score=False,\n score_format: str = \"score\",\n refit_full: bool = None,\n set_refit_score_to_parent: bool = False,\n display=False,\n ) -> pd.DataFrame:\n assert score_format in [\"score\", \"error\"]\n if X is not None:\n leaderboard = self.score_debug(\n X=X,\n y=y,\n extra_info=extra_info,\n extra_metrics=extra_metrics,\n decision_threshold=decision_threshold,\n skip_score=skip_score,\n refit_full=refit_full,\n set_refit_score_to_parent=set_refit_score_to_parent,\n display=False,\n )\n else:\n if extra_metrics:\n raise AssertionError(\"`extra_metrics` is only valid when data is specified.\")\n trainer = self.load_trainer()\n leaderboard = trainer.leaderboard(\n extra_info=extra_info, refit_full=refit_full, set_refit_score_to_parent=set_refit_score_to_parent\n )\n if only_pareto_frontier:\n if \"score_test\" in leaderboard.columns and \"pred_time_test\" in leaderboard.columns:\n score_col = \"score_test\"\n inference_time_col = \"pred_time_test\"\n else:\n score_col = \"score_val\"\n inference_time_col = \"pred_time_val\"\n leaderboard = get_leaderboard_pareto_frontier(\n leaderboard=leaderboard, score_col=score_col, inference_time_col=inference_time_col\n )\n if score_format == \"error\":\n leaderboard.rename(\n columns={\n \"score_test\": \"metric_error_test\",\n \"score_val\": \"metric_error_val\",\n },\n inplace=True,\n )\n if \"metric_error_test\" in leaderboard:\n leaderboard.loc[leaderboard[\"metric_error_test\"].notnull(), \"metric_error_test\"] = leaderboard.loc[\n leaderboard[\"metric_error_test\"].notnull(), \"metric_error_test\"\n ].apply(self.eval_metric.convert_score_to_error)\n leaderboard.loc[leaderboard[\"metric_error_val\"].notnull(), \"metric_error_val\"] = leaderboard.loc[\n leaderboard[\"metric_error_val\"].notnull(), \"metric_error_val\"\n ].apply(self.eval_metric.convert_score_to_error)\n if display:\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n print(leaderboard)\n return leaderboard\n\n # TODO: cache_data must be set to True to be able to pass X and y as None in this function, otherwise it will error.\n # Warning: This can take a very, very long time to compute if the data is large and the model is complex.\n # A value of 0.01 means that the objective metric error would be expected to increase by 0.01 if the feature were removed.\n # Negative values mean the feature is likely harmful.\n # model: model (str) to get feature importances for, if None will choose best model.\n # features: list of feature names that feature importances are calculated for and returned, specify None to get all feature importances.\n # feature_stage: Whether to compute feature importance on raw original features ('original'), transformed features ('transformed') or on the features used by the particular model ('transformed_model').\n def get_feature_importance(\n self,\n model=None,\n X=None,\n y=None,\n features: list = None,\n feature_stage=\"original\",\n subsample_size=5000,\n silent=False,\n **kwargs,\n ) -> DataFrame:\n valid_feature_stages = [\"original\", \"transformed\", \"transformed_model\"]\n if feature_stage not in valid_feature_stages:\n raise ValueError(f\"feature_stage must be one of: {valid_feature_stages}, but was {feature_stage}.\")\n trainer = self.load_trainer()\n if X is not None:\n if y is None:\n X, y = self.extract_label(X)\n y = self.label_cleaner.transform(y)\n X, y = self._remove_nan_label_rows(X, y)\n if self.ignored_columns:\n X = X.drop(columns=self.ignored_columns, errors=\"ignore\")\n unused_features = [f for f in list(X.columns) if f not in self.features]\n if len(unused_features) > 0:\n logger.log(\n 30,\n f\"These features in provided data are not utilized by the predictor and will be ignored: {unused_features}\",\n )\n X = X.drop(columns=unused_features)\n\n if feature_stage == \"original\":\n return trainer._get_feature_importance_raw(\n model=model,\n X=X,\n y=y,\n features=features,\n subsample_size=subsample_size,\n transform_func=self.transform_features,\n silent=silent,\n **kwargs,\n )\n X = self.transform_features(X)\n else:\n if feature_stage == \"original\":\n raise AssertionError(\n \"Feature importance `dataset` cannot be None if `feature_stage=='original'`. A test dataset must be specified.\"\n )\n y = None\n raw = feature_stage == \"transformed\"\n return trainer.get_feature_importance(\n X=X, y=y, model=model, features=features, raw=raw, subsample_size=subsample_size, silent=silent, **kwargs\n )\n\n @staticmethod\n def _remove_nan_label_rows(X, y):\n if y.isnull().any():\n y = y.dropna()\n X = X.loc[y.index]\n return X, y\n\n def infer_problem_type(self, y: Series, silent=False):\n problem_type = self._infer_problem_type(y, silent=silent)\n if problem_type == QUANTILE:\n if self.quantile_levels is None:\n raise AssertionError(\n f\"problem_type is inferred to be {QUANTILE}, yet quantile_levels is not specified.\"\n )\n elif self.quantile_levels is not None:\n if problem_type == REGRESSION:\n problem_type = QUANTILE\n else:\n raise AssertionError(\n f\"autogluon infers this to be classification problem ('{problem_type}'), yet quantile_levels is not None.\"\n \"If it is truly a quantile regression problem, \"\n f\"please specify problem_type='{QUANTILE}'.\"\n )\n return problem_type\n\n @staticmethod\n def _infer_problem_type(y: Series, silent=False):\n return infer_problem_type(y=y, silent=silent)\n\n # Loads models in memory so that they don't have to be loaded during predictions\n def persist_trainer(self, low_memory=False, models=\"all\", with_ancestors=False, max_memory=None) -> list:\n self.trainer = self.load_trainer()\n if not low_memory:\n return self.trainer.persist(models, with_ancestors=with_ancestors, max_memory=max_memory)\n # Warning: After calling this, it is not necessarily safe to save learner or trainer anymore\n # If neural network is persisted and then trainer or learner is saved, there will be an exception thrown\n else:\n return []\n\n def distill(\n self,\n X=None,\n y=None,\n X_val=None,\n y_val=None,\n time_limit=None,\n hyperparameters=None,\n holdout_frac=None,\n verbosity=None,\n models_name_suffix=None,\n teacher_preds=\"soft\",\n augmentation_data=None,\n augment_method=\"spunge\",\n augment_args={\"size_factor\": 5, \"max_size\": int(1e5)},\n ):\n \"\"\"See abstract_trainer.distill() for details.\"\"\"\n if X is not None:\n if (\n (self.eval_metric is not None)\n and (self.eval_metric.name == \"log_loss\")\n and (self.problem_type == MULTICLASS)\n ):\n X = augment_rare_classes(X, self.label, self.threshold)\n if y is None:\n X, y = self.extract_label(X)\n X = self.transform_features(X)\n y = self.label_cleaner.transform(y)\n if self.problem_type == MULTICLASS:\n y = y.fillna(-1)\n else:\n y = None\n\n if X_val is not None:\n if X is None:\n raise ValueError(\"Cannot specify X_val without specifying X\")\n if y_val is None:\n X_val, y_val = self.extract_label(X_val)\n X_val = self.transform_features(X_val)\n y_val = self.label_cleaner.transform(y_val)\n\n if augmentation_data is not None:\n augmentation_data = self.transform_features(augmentation_data)\n\n trainer = self.load_trainer()\n distilled_model_names = trainer.distill(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n time_limit=time_limit,\n hyperparameters=hyperparameters,\n holdout_frac=holdout_frac,\n verbosity=verbosity,\n teacher_preds=teacher_preds,\n models_name_suffix=models_name_suffix,\n augmentation_data=augmentation_data,\n augment_method=augment_method,\n augment_args=augment_args,\n )\n self.save_trainer(trainer=trainer)\n return distilled_model_names\n\n def transform_labels(self, y, inverse=False, proba=False):\n if inverse:\n if proba:\n y_transformed = self.label_cleaner.inverse_transform_proba(y=y, as_pandas=True)\n else:\n y_transformed = self.label_cleaner.inverse_transform(y=y)\n else:\n if proba:\n y_transformed = self.label_cleaner.transform_proba(y=y, as_pandas=True)\n else:\n y_transformed = self.label_cleaner.transform(y=y)\n return y_transformed\n\n def calibrate_decision_threshold(\n self,\n data: pd.DataFrame | None = None,\n metric: str | Scorer | None = None,\n model: str = \"best\",\n decision_thresholds: int | List[float] = 25,\n secondary_decision_thresholds: int | None = 19,\n verbose: bool = True,\n **kwargs,\n ) -> float:\n # TODO: docstring\n if metric is None:\n metric = self.eval_metric\n\n weights = None\n if data is None:\n X = None\n y = None\n else:\n if self.weight_evaluation:\n data, weights = extract_column(data, self.sample_weight)\n X = self.transform_features(X=data)\n y = self.transform_labels(y=data[self.label])\n\n return self.load_trainer().calibrate_decision_threshold(\n X=X,\n y=y,\n metric=metric,\n model=model,\n weights=weights,\n decision_thresholds=decision_thresholds,\n secondary_decision_thresholds=secondary_decision_thresholds,\n verbose=verbose,\n **kwargs,\n )\n\n def _verify_metric(self, eval_metric: Scorer, problem_type: str):\n \"\"\"\n Raises an exception if the eval_metric does not exist in the default metrics list for the problem type\n \"\"\"\n get_metric(metric=eval_metric.name, problem_type=problem_type, metric_type=\"eval_metric\")\n\n # TODO: Add data info gathering at beginning of .fit() that is used by all learners to add to get_info output\n # TODO: Add feature inference / feature engineering info to get_info output\n def get_info(self, **kwargs):\n learner_info = {\n \"path\": self.path,\n \"label\": self.label,\n \"random_state\": self.random_state,\n \"version\": self.version,\n \"features\": self.features,\n \"feature_metadata_in\": self.feature_metadata_in,\n }\n\n return learner_info\n"
},
{
"path": "tabular/src/autogluon/tabular/learner/default_learner.py",
"content": "from __future__ import annotations\n\nimport copy\nimport logging\nimport math\nimport time\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.utils.log_utils import convert_time_in_s_to_log_friendly\nfrom autogluon.core.constants import AUTO_WEIGHT, BALANCE_WEIGHT, BINARY, MULTICLASS, QUANTILE, REGRESSION\nfrom autogluon.core.data import LabelCleaner\nfrom autogluon.core.data.cleaner import Cleaner\nfrom autogluon.core.utils.time import sample_df_for_time_func, time_func\nfrom autogluon.core.utils.utils import augment_rare_classes, extract_column\n\nfrom ..trainer import AutoTrainer\nfrom .abstract_learner import AbstractTabularLearner\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add functionality for advanced feature generators such as gl_code_matrix_generator (inter-row dependencies, apply to train differently than test, etc., can only run after train/test split, rerun for each cv fold)\n# TODO: - Differentiate between advanced generators that require fit (stateful, gl_code_matrix) and those that do not (bucket label averaging in SCOT GC 2019)\n# TODO: - Those that do not could be added to preprocessing function of model, but would then have to be recomputed on each model.\n# TODO: Add cv / OOF generator option, so that AutoGluon can be used as a base model in an ensemble stacker\n# Learner encompasses full problem, loading initial data, feature generation, model training, model prediction\nclass DefaultLearner(AbstractTabularLearner):\n def __init__(self, trainer_type=AutoTrainer, **kwargs):\n super().__init__(**kwargs)\n self.trainer_type = trainer_type\n self.class_weights = None\n self._time_fit_total = None\n self._time_fit_preprocessing = None\n self._time_fit_training = None\n self._time_limit = None\n self.preprocess_1_time = None # Time required to preprocess 1 row of data\n self.preprocess_1_batch_size = None # Batch size used to calculate self.preprocess_1_time\n\n # TODO: v0.1 Document trainer_fit_kwargs\n def _fit(\n self,\n X: DataFrame,\n X_val: DataFrame | None = None,\n X_test: DataFrame | None = None,\n X_unlabeled: DataFrame | None = None,\n holdout_frac: float = 0.1,\n num_bag_folds: int = 0,\n num_bag_sets: int = 1,\n time_limit: float | None = None,\n infer_limit: float | None = None,\n infer_limit_batch_size: int | None = None,\n verbosity: int = 2,\n raise_on_model_failure: bool = False,\n **trainer_fit_kwargs,\n ):\n \"\"\"Arguments:\n X (DataFrame): training data\n X_val (DataFrame): data used for hyperparameter tuning. Note: final model may be trained using this data as well as training data\n X_test (DataFrame): data used for tracking model performance on test data during training. Note: this data is never used to train the model\n X_unlabeled (DataFrame): data used for pretraining a model. This is same data format as X, without label-column. This data is used for semi-supervised learning.\n holdout_frac (float): Fraction of data to hold out for evaluating validation performance (ignored if X_val != None, ignored if kfolds != 0)\n num_bag_folds (int): kfolds used for bagging of models, roughly increases model training time by a factor of k (0: disabled)\n num_bag_sets (int): number of repeats of kfold bagging to perform (values must be >= 1),\n total number of models trained during bagging = num_bag_folds * num_bag_sets\n \"\"\"\n # TODO: if provided, feature_types in X, X_val are ignored right now, need to pass to Learner/trainer and update this documentation.\n self._time_limit = time_limit\n if time_limit:\n logger.log(20, f\"Beginning AutoGluon training ... Time limit = {time_limit:.0f}s\")\n else:\n logger.log(20, \"Beginning AutoGluon training ...\")\n logger.log(20, f'AutoGluon will save models to \"{self.path}\"')\n logger.log(20, f\"Train Data Rows: {len(X)}\")\n logger.log(20, f\"Train Data Columns: {len([column for column in X.columns if column != self.label])}\")\n if X_val is not None:\n logger.log(20, f\"Tuning Data Rows: {len(X_val)}\")\n logger.log(20, f\"Tuning Data Columns: {len([column for column in X_val.columns if column != self.label])}\")\n logger.log(20, f\"Label Column: {self.label}\")\n time_preprocessing_start = time.time()\n self._pre_X_rows = len(X)\n if self.problem_type is None:\n self.problem_type = self.infer_problem_type(y=X[self.label])\n logger.log(20, f\"Problem Type: {self.problem_type}\")\n if self._eval_metric_was_str:\n # Ensure that the eval_metric is valid for the problem_type\n self._verify_metric(eval_metric=self.eval_metric, problem_type=self.problem_type)\n if self.groups is not None:\n num_bag_sets = 1\n num_bag_folds = len(X[self.groups].unique())\n X_og = None if infer_limit_batch_size is None else X\n logger.log(20, \"Preprocessing data ...\")\n X, y, X_val, y_val, X_test, y_test, X_unlabeled, holdout_frac, num_bag_folds, groups = (\n self.general_data_processing(\n X=X,\n X_val=X_val,\n X_test=X_test,\n X_unlabeled=X_unlabeled,\n holdout_frac=holdout_frac,\n num_bag_folds=num_bag_folds,\n )\n )\n if X_og is not None:\n infer_limit = self._update_infer_limit(\n X=X_og, infer_limit_batch_size=infer_limit_batch_size, infer_limit=infer_limit\n )\n\n self._post_X_rows = len(X)\n time_preprocessing_end = time.time()\n self._time_fit_preprocessing = time_preprocessing_end - time_preprocessing_start\n logger.log(\n 20, f\"Data preprocessing and feature engineering runtime = {round(self._time_fit_preprocessing, 2)}s ...\"\n )\n if time_limit:\n time_limit_trainer = time_limit - self._time_fit_preprocessing\n else:\n time_limit_trainer = None\n\n trainer = self.trainer_type(\n path=self.model_context,\n problem_type=self.label_cleaner.problem_type_transform,\n eval_metric=self.eval_metric,\n num_classes=self.label_cleaner.num_classes,\n quantile_levels=self.quantile_levels,\n feature_metadata=self.feature_generator.feature_metadata,\n low_memory=True,\n k_fold=num_bag_folds, # TODO: Consider moving to fit call\n n_repeats=num_bag_sets, # TODO: Consider moving to fit call\n sample_weight=self.sample_weight,\n weight_evaluation=self.weight_evaluation,\n save_data=self.cache_data,\n random_state=self.random_state,\n verbosity=verbosity,\n raise_on_model_failure=raise_on_model_failure,\n )\n\n self.trainer_path = trainer.path\n if self.eval_metric is None:\n self.eval_metric = trainer.eval_metric\n\n self.save()\n trainer.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n holdout_frac=holdout_frac,\n time_limit=time_limit_trainer,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n groups=groups,\n label_cleaner=copy.deepcopy(self.label_cleaner),\n **trainer_fit_kwargs,\n )\n self.save_trainer(trainer=trainer)\n time_end = time.time()\n self._time_fit_training = time_end - time_preprocessing_end\n self._time_fit_total = time_end - time_preprocessing_start\n log_throughput = \"\"\n if trainer.model_best is not None:\n predict_n_time_per_row = trainer.get_model_attribute_full(\n model=trainer.model_best, attribute=\"predict_n_time_per_row\"\n )\n predict_n_size = trainer.get_model_attribute_full(\n model=trainer.model_best, attribute=\"predict_n_size\", func=min\n )\n if predict_n_time_per_row is not None and predict_n_size is not None:\n log_throughput = f\" | Estimated inference throughput: {1 / (predict_n_time_per_row if predict_n_time_per_row else np.finfo(np.float16).eps):.1f} rows/s ({int(predict_n_size)} batch size)\"\n logger.log(\n 20,\n f\"AutoGluon training complete, total runtime = {round(self._time_fit_total, 2)}s ... Best model: {trainer.model_best}\"\n f\"{log_throughput}\",\n )\n\n def _update_infer_limit(self, X: DataFrame, *, infer_limit_batch_size: int, infer_limit: float = None):\n \"\"\"\n Calculates preprocessing time per row for a given unprocessed data X and infer_limit_batch_size\n Returns an updated infer_limit if not None with preprocessing time per row subtracted\n Raises an exception if preprocessing time is greater than or equal to the infer_limit\n \"\"\"\n X_batch = sample_df_for_time_func(df=X, sample_size=infer_limit_batch_size)\n infer_limit_batch_size_actual = len(X_batch)\n self.preprocess_1_time = time_func(f=self.transform_features, args=[X_batch]) / infer_limit_batch_size_actual\n self.preprocess_1_batch_size = infer_limit_batch_size\n preprocess_1_time_log, time_unit_preprocess_1_time = convert_time_in_s_to_log_friendly(self.preprocess_1_time)\n logger.log(\n 20,\n f\"\\t{round(preprocess_1_time_log, 3)}{time_unit_preprocess_1_time}\\t= Feature Preprocessing Time (1 row | {infer_limit_batch_size} batch size)\",\n )\n\n if infer_limit is not None:\n infer_limit_new = infer_limit - self.preprocess_1_time\n infer_limit_log, time_unit_infer_limit = convert_time_in_s_to_log_friendly(infer_limit)\n infer_limit_new_log, time_unit_infer_limit_new = convert_time_in_s_to_log_friendly(infer_limit_new)\n\n logger.log(\n 20,\n f\"\\t\\tFeature Preprocessing requires {round(self.preprocess_1_time / infer_limit * 100, 2)}% \"\n f\"of the overall inference constraint ({infer_limit_log}{time_unit_infer_limit})\\n\"\n f\"\\t\\t{round(infer_limit_new_log, 3)}{time_unit_infer_limit_new} inference time budget remaining for models...\",\n )\n if infer_limit_new <= 0:\n infer_limit_new = 0\n logger.log(\n 30,\n f\"WARNING: Impossible to satisfy inference constraint, budget is exceeded during data preprocessing!\\n\"\n f\"\\tAutoGluon will be unable to satisfy the constraint, but will return the fastest model it can.\\n\"\n f\"\\tConsider using fewer features, relaxing the inference constraint, or simplifying the feature generator.\",\n )\n infer_limit = infer_limit_new\n return infer_limit\n\n # TODO: Add default values to X_val, X_unlabeled, holdout_frac, and num_bag_folds\n def general_data_processing(\n self,\n X: DataFrame,\n X_val: DataFrame = None,\n X_test: DataFrame = None,\n X_unlabeled: DataFrame = None,\n holdout_frac: float = 1,\n num_bag_folds: int = 0,\n ):\n \"\"\"General data processing steps used for all models.\"\"\"\n X = self._check_for_non_finite_values(X, name=\"train\", is_train=True)\n if X_val is not None:\n X_val = self._check_for_non_finite_values(X_val, name=\"val\", is_train=False)\n if X_test is not None:\n X_test = self._check_for_non_finite_values(X_test, name=\"test\", is_train=False)\n\n holdout_frac_og = holdout_frac\n if X_val is not None and self.label in X_val.columns:\n holdout_frac = 1\n\n if self.eval_metric is not None and self.eval_metric.needs_proba and self.problem_type == MULTICLASS:\n # Metric requires all classes present in training to be able to compute a score\n if num_bag_folds > 0:\n self.threshold = 2\n if self.groups is None:\n X = augment_rare_classes(X, self.label, threshold=2)\n else:\n self.threshold = 1\n\n self.threshold, holdout_frac, num_bag_folds = self.adjust_threshold_if_necessary(\n X[self.label], threshold=self.threshold, holdout_frac=holdout_frac, num_bag_folds=num_bag_folds\n )\n\n # Gets labels prior to removal of infrequent classes\n y_uncleaned = X[self.label].copy()\n\n self.cleaner = Cleaner.construct(problem_type=self.problem_type, label=self.label, threshold=self.threshold)\n X = self.cleaner.fit_transform(X) # TODO: Consider merging cleaner into label_cleaner\n X, y = self.extract_label(X)\n self.label_cleaner = LabelCleaner.construct(\n problem_type=self.problem_type, y=y, y_uncleaned=y_uncleaned, positive_class=self._positive_class\n )\n y = self.label_cleaner.transform(y)\n X = self.set_predefined_weights(X, y)\n X, w = extract_column(X, self.sample_weight)\n X, groups = extract_column(X, self.groups)\n if self.label_cleaner.num_classes is not None and self.problem_type != BINARY:\n logger.log(20, f\"Train Data Class Count: {self.label_cleaner.num_classes}\")\n\n X_val, y_val, w_val, holdout_frac = self._apply_cleaner_transform(\n X=X_val,\n y_uncleaned=y_uncleaned,\n holdout_frac=holdout_frac,\n holdout_frac_og=holdout_frac_og,\n name=\"val\",\n is_test=False,\n )\n X_test, y_test, w_test, _ = self._apply_cleaner_transform(\n X=X_test,\n y_uncleaned=y_uncleaned,\n holdout_frac=holdout_frac,\n holdout_frac_og=holdout_frac_og,\n name=\"test\",\n is_test=True,\n )\n\n self._original_features = list(X.columns)\n # TODO: Move this up to top of data before removing data, this way our feature generator is better\n logger.log(20, f\"Using Feature Generators to preprocess the data ...\")\n\n if X_test is not None:\n logger.log(\n 15,\n \"Performing general data preprocessing with merged train & validation data, so validation/test performance may not accurately reflect performance on new test data\",\n )\n\n # TODO: extend this boolean flag into learner init parameter\n transform_with_test = False\n\n X_test_super = None\n y_test_super = None\n if transform_with_test:\n X_test_super = X_test\n y_test_super = y_test\n\n datasets = [X, X_val, X_test_super, X_unlabeled]\n X_super = pd.concat(datasets, ignore_index=True)\n\n if self.feature_generator.is_fit():\n logger.log(\n 20,\n f\"{self.feature_generator.__class__.__name__} is already fit, so the training data will be processed via .transform() instead of .fit_transform().\",\n )\n X_super = self.feature_generator.transform(X_super)\n if not transform_with_test and X_test is not None:\n X_test = self.feature_generator.transform(X_test)\n self.feature_generator.print_feature_metadata_info()\n else:\n y_unlabeled = pd.Series(np.nan, index=X_unlabeled.index) if X_unlabeled is not None else None\n y_list = [y, y_val, y_test_super, y_unlabeled]\n y_super = pd.concat(y_list, ignore_index=True)\n X_super = self.fit_transform_features(\n X_super, y_super, problem_type=self.label_cleaner.problem_type_transform, eval_metric=self.eval_metric\n )\n if not transform_with_test and X_test is not None:\n X_test = self.feature_generator.transform(X_test)\n\n idx = 0\n for i in range(len(datasets)):\n if datasets[i] is not None:\n length = len(datasets[i])\n datasets[i] = X_super.iloc[idx : idx + length].set_index(datasets[i].index)\n idx += length\n\n X, X_val, X_test_super, X_unlabeled = datasets\n del X_super\n\n if transform_with_test:\n X_test = X_test_super\n\n # TODO: consider not bundling sample-weights inside X, X_val\n X = self.bundle_weights(X, w, \"X\", is_train=True)\n X_val = self.bundle_weights(X_val, w_val, \"X_val\", is_train=False)\n X_test = self.bundle_weights(X_test, w_test, \"X_test\", is_train=False)\n return X, y, X_val, y_val, X_test, y_test, X_unlabeled, holdout_frac, num_bag_folds, groups\n\n def bundle_weights(self, X: DataFrame | None, w: Series | None, name: str, is_train=False) -> DataFrame:\n if is_train:\n if w is not None:\n X[self.sample_weight] = w\n elif X is not None:\n if w is not None:\n X[self.sample_weight] = w\n elif not self.weight_evaluation:\n nan_vals = np.empty((len(X),))\n nan_vals[:] = np.nan\n X[self.sample_weight] = nan_vals\n else:\n raise ValueError(\n f\"sample_weight column '{self.sample_weight}' \\\n cannot be missing from {name} dataset if weight_evaluation=True\"\n )\n\n return X\n\n def set_predefined_weights(self, X, y):\n if self.sample_weight not in [AUTO_WEIGHT, BALANCE_WEIGHT] or self.problem_type not in [BINARY, MULTICLASS]:\n return X\n if self.sample_weight in X.columns:\n raise ValueError(\n f\"Column name '{self.sample_weight}' cannot appear in your dataset with predefined weighting strategy. Please change it and try again.\"\n )\n if self.sample_weight == BALANCE_WEIGHT:\n if self.class_weights is None:\n class_counts = y.value_counts()\n n = len(y)\n k = len(class_counts)\n self.class_weights = {c: n / (class_counts[c] * k) for c in class_counts.index}\n logger.log(20, \"Assigning sample weights to balance differences in frequency of classes.\")\n logger.log(15, f\"Balancing classes via the following weights: {self.class_weights}\")\n w = y.map(self.class_weights)\n elif self.sample_weight == AUTO_WEIGHT: # TODO: support more sophisticated auto_weight strategy\n raise NotImplementedError(f\"{AUTO_WEIGHT} strategy not yet supported.\")\n X[self.sample_weight] = w # TODO: consider not bundling sample weights inside X\n return X\n\n def _check_for_non_finite_values(self, X: DataFrame, name: str = \"\", is_train: bool = False) -> DataFrame:\n if is_train or (X is not None and self.label in X.columns):\n X = copy.deepcopy(X)\n\n # treat None, NaN, INF, NINF as NA\n X[self.label] = X[self.label].replace([np.inf, -np.inf], np.nan)\n invalid_labels = X[self.label].isna()\n if invalid_labels.any():\n first_invalid_label_idx = invalid_labels.idxmax()\n raise ValueError(\n f\"{name} dataset label column cannot contain non-finite values (NaN, Inf, Ninf). First invalid label at data idx: {first_invalid_label_idx}\"\n )\n\n return X\n\n def _apply_cleaner_transform(\n self,\n X: DataFrame,\n y_uncleaned: Series,\n holdout_frac: float | int,\n holdout_frac_og: float | int,\n name: str,\n is_test: bool = False,\n ) -> tuple[DataFrame, Series, Series | None, float | int]:\n if X is not None and self.label in X.columns:\n y_og = X[self.label]\n len_og = len(X) if is_test else None\n X = self.cleaner.transform(X)\n if is_test and len(X) != len_og:\n # FIXME: Currently, there are ways in which this code can be reached (@innixma)\n raise AssertionError(\n f\"{name} cannot have low frequency classes! Please create a GitHub issue if you see this message, as it should never occur.\"\n )\n\n if len(X) == 0:\n logger.warning(\n \"############################################################################################################\\n\"\n f\"WARNING: All {name} data contained low frequency classes, ignoring {name} and generating from subset of X\\n\"\n \"\\tYour input validation data or training data labels might be corrupted, please manually inspect them for correctness!\"\n )\n if self.problem_type in [BINARY, MULTICLASS]:\n train_classes = sorted(list(y_uncleaned.unique()))\n val_classes = sorted(list(y_og.unique()))\n logger.warning(f\"\\ttrain Classes: {train_classes}\")\n logger.warning(f\"\\t{name} Classes: {val_classes}\")\n logger.warning(f\"\\ttrain Class Dtype: {y_uncleaned.dtype}\")\n logger.warning(f\"\\t{name} Class Dtype: {y_og.dtype}\")\n missing_classes = [c for c in val_classes if c not in train_classes]\n logger.warning(f\"\\tClasses missing from Training Data: {missing_classes}\")\n logger.warning(\n \"############################################################################################################\"\n )\n\n X = None\n y = None\n w = None\n holdout_frac = holdout_frac_og\n else:\n X, y = self.extract_label(X)\n y = self.label_cleaner.transform(y)\n X = self.set_predefined_weights(X, y)\n X, w = extract_column(X, self.sample_weight)\n else:\n y = None\n w = None\n\n return X, y, w, holdout_frac\n\n def adjust_threshold_if_necessary(self, y, threshold, holdout_frac, num_bag_folds):\n new_threshold, new_holdout_frac, new_num_bag_folds = self._adjust_threshold_if_necessary(\n y, threshold, holdout_frac, num_bag_folds\n )\n if new_threshold != threshold:\n if new_threshold < threshold:\n logger.warning(\n f\"Warning: Updated label_count_threshold from {threshold} to {new_threshold} to avoid cutting too many classes.\"\n )\n if new_holdout_frac != holdout_frac:\n if new_holdout_frac > holdout_frac:\n logger.warning(\n f\"Warning: Updated holdout_frac from {holdout_frac} to {new_holdout_frac} to avoid cutting too many classes.\"\n )\n if new_num_bag_folds != num_bag_folds:\n logger.warning(\n f\"Warning: Updated num_bag_folds from {num_bag_folds} to {new_num_bag_folds} to avoid cutting too many classes.\"\n )\n return new_threshold, new_holdout_frac, new_num_bag_folds\n\n def _adjust_threshold_if_necessary(self, y, threshold, holdout_frac, num_bag_folds):\n new_threshold = threshold\n num_rows = len(y)\n holdout_frac = max(holdout_frac, 1 / num_rows + 0.001)\n num_bag_folds = min(num_bag_folds, num_rows)\n\n if num_bag_folds < 2:\n minimum_safe_threshold = 1\n else:\n minimum_safe_threshold = 2\n\n if minimum_safe_threshold > new_threshold:\n new_threshold = minimum_safe_threshold\n\n if self.problem_type in [REGRESSION, QUANTILE]:\n return new_threshold, holdout_frac, num_bag_folds\n\n class_counts = y.value_counts()\n total_rows = class_counts.sum()\n minimum_percent_to_keep = 0.975\n minimum_rows_to_keep = math.ceil(total_rows * minimum_percent_to_keep)\n minimum_class_to_keep = 2\n\n num_classes = len(class_counts)\n class_counts_valid = class_counts[class_counts >= new_threshold]\n num_rows_valid = class_counts_valid.sum()\n num_classes_valid = len(class_counts_valid)\n\n if (num_rows_valid >= minimum_rows_to_keep) and (num_classes_valid >= minimum_class_to_keep):\n return new_threshold, holdout_frac, num_bag_folds\n\n num_classes_valid = 0\n num_rows_valid = 0\n new_threshold = None\n for i in range(num_classes):\n num_classes_valid += 1\n num_rows_valid += class_counts.iloc[i]\n new_threshold = class_counts.iloc[i]\n if (num_rows_valid >= minimum_rows_to_keep) and (num_classes_valid >= minimum_class_to_keep):\n break\n\n return new_threshold, holdout_frac, num_bag_folds\n\n def get_info(self, include_model_info=False, include_model_failures=False, **kwargs):\n learner_info = super().get_info(**kwargs)\n trainer = self.load_trainer()\n trainer_info = trainer.get_info(\n include_model_info=include_model_info, include_model_failures=include_model_failures\n )\n learner_info.update(\n {\n \"time_fit_preprocessing\": self._time_fit_preprocessing,\n \"time_fit_training\": self._time_fit_training,\n \"time_fit_total\": self._time_fit_total,\n \"time_limit\": self._time_limit,\n }\n )\n\n learner_info.update(trainer_info)\n return learner_info\n"
},
{
"path": "tabular/src/autogluon/tabular/models/__init__.py",
"content": "from autogluon.core.models.abstract.abstract_model import AbstractModel\n\nfrom .automm.automm_model import MultiModalPredictorModel\nfrom .automm.ft_transformer import FTTransformerModel\nfrom .catboost.catboost_model import CatBoostModel\nfrom .ebm.ebm_model import EBMModel\nfrom .fastainn.tabular_nn_fastai import NNFastAiTabularModel\nfrom .image_prediction.image_predictor import ImagePredictorModel\nfrom .imodels.imodels_models import (\n BoostedRulesModel,\n FigsModel,\n GreedyTreeModel,\n HSTreeModel,\n RuleFitModel,\n _IModelsModel,\n)\nfrom .knn.knn_model import KNNModel\nfrom .lgb.lgb_model import LGBModel\nfrom .lr.lr_model import LinearModel\nfrom .mitra.mitra_model import MitraModel\nfrom .realmlp.realmlp_model import RealMLPModel\nfrom .rf.rf_model import RFModel\nfrom .tabdpt.tabdpt_model import TabDPTModel\nfrom .tabicl.tabicl_model import TabICLModel\nfrom .tabm.tabm_model import TabMModel\nfrom .tabpfnmix.tabpfnmix_model import TabPFNMixModel\nfrom .tabpfnv2.tabpfnv2_5_model import RealTabPFNv2Model, RealTabPFNv25Model\nfrom .tabprep.prep_lgb_model import PrepLGBModel\nfrom .tabular_nn.torch.tabular_nn_torch import TabularNeuralNetTorchModel\nfrom .text_prediction.text_prediction_v1_model import TextPredictorModel\nfrom .xgboost.xgboost_model import XGBoostModel\nfrom .xt.xt_model import XTModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/_utils/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/_utils/rapids_utils.py",
"content": "from typing import Dict\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\n\nclass RapidsModelMixin:\n \"\"\"Mixin class for methods reused across RAPIDS models\"\"\"\n\n # FIXME: Efficient OOF doesn't work in RAPIDS\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\"use_child_oof\": False, \"fold_fitting_strategy\": \"sequential_local\"}\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n def _get_default_resources(self):\n num_cpus, _ = super()._get_default_resources()\n num_gpus = min(\n ResourceManager.get_gpu_count_torch(), 1\n ) # Use single gpu training by default. Consider revising it later.\n return num_cpus, num_gpus\n\n def get_minimum_resources(self, is_gpu_available=False) -> Dict[str, int]:\n return {\n \"num_cpus\": 1,\n \"num_gpus\": 1,\n }\n\n def _more_tags(self):\n return {\"valid_oof\": False}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/_utils/torch_utils.py",
"content": "import torch\n\n\nclass TorchThreadManager:\n \"\"\"\n Temporary updates torch num_threads to the specified value within the context, then reverts to the original num_threads upon exit.\n \"\"\"\n\n def __init__(self, num_threads):\n self.num_threads = num_threads\n self.num_threads_og = None\n\n def __enter__(self):\n self.num_threads_og = torch.get_num_threads()\n torch.set_num_threads(self.num_threads)\n\n def __exit__(self, exc_type, exc_value, exc_tb):\n torch.set_num_threads(self.num_threads_og)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/abstract/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/abstract/abstract_torch_model.py",
"content": "from __future__ import annotations\n\nimport logging\n\nfrom autogluon.core.models import AbstractModel\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add type hints once torch is a required dependency\nclass AbstractTorchModel(AbstractModel):\n \"\"\"\n .. versionadded:: 1.5.0\n \"\"\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.device = None\n self.device_train = None\n\n def suggest_device_infer(self, verbose: bool = False) -> str:\n import torch\n\n # Put the model on the same device it was trained on (GPU/MPS) if it is available; otherwise use CPU\n if self.device_train is None:\n original_device_type = None # skip update because no device is recorded\n elif isinstance(self.device_train, str):\n original_device_type = self.device_train\n else:\n original_device_type = self.device_train.type\n if original_device_type is None:\n # fallback to CPU\n device = torch.device(\"cpu\")\n elif \"cuda\" in original_device_type:\n # cuda: nvidia GPU\n device = torch.device(original_device_type if torch.cuda.is_available() else \"cpu\")\n elif \"mps\" in original_device_type:\n # mps: Apple Silicon\n device = torch.device(original_device_type if torch.backends.mps.is_available() else \"cpu\")\n else:\n device = torch.device(original_device_type)\n\n if verbose and (original_device_type != device.type):\n logger.log(\n 15,\n f\"Model is trained on {original_device_type}, but the device is not available - \"\n f\"loading on {device.type}...\",\n )\n\n return device.type\n\n @classmethod\n def to_torch_device(cls, device: str):\n import torch\n\n return torch.device(device)\n\n def get_device(self) -> str:\n \"\"\"\n Returns torch.device(...) of the fitted model\n\n Requires implementation by the inheriting model class.\n Refer to overriding methods in existing models for reference implementations.\n \"\"\"\n raise NotImplementedError\n\n def set_device(self, device: str):\n if not isinstance(device, str):\n device = device.type\n self.device = device\n self._set_device(device=device)\n\n def _set_device(self, device: str):\n \"\"\"\n Sets the device for the inner model object.\n\n Requires implementation by the inheriting model class.\n Refer to overriding methods in existing models for reference implementations.\n\n If your model does not need to edit inner model object details, you can simply make the logic `pass`.\n \"\"\"\n raise NotImplementedError\n\n def _post_fit(self, **kwargs):\n super()._post_fit(**kwargs)\n if self._get_class_tags().get(\"can_set_device\", False):\n self.device_train = self.get_device()\n self.device = self.device_train\n return self\n\n def save(self, path: str = None, verbose=True) -> str:\n \"\"\"\n Need to set device to CPU to be able to load on a non-GPU environment\n \"\"\"\n reset_device = False\n og_device = self.device\n\n # Save on CPU to ensure the model can be loaded without GPU\n if self.is_fit():\n device_save = self._get_class_tags().get(\"set_device_on_save_to\", None)\n if device_save is not None:\n self.set_device(device=device_save)\n reset_device = True\n path = super().save(path=path, verbose=verbose)\n # Put the model back to the device after the save\n if reset_device:\n self.set_device(device=og_device)\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True):\n \"\"\"\n Loads the model from disk to memory.\n The loaded model will be on the same device it was trained on (cuda/mps);\n if the device is not available (trained on GPU, deployed on CPU), then `cpu` will be used.\n\n Parameters\n ----------\n path : str\n Path to the saved model, minus the file name.\n This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).\n The model file is typically located in os.path.join(path, cls.model_file_name).\n reset_paths : bool, default True\n Whether to reset the self.path value of the loaded model to be equal to path.\n It is highly recommended to keep this value as True unless accessing the original self.path value is important.\n If False, the actual valid path and self.path may differ, leading to strange behaviour and potential exceptions if the model needs to load any other files at a later time.\n verbose : bool, default True\n Whether to log the location of the loaded file.\n\n Returns\n -------\n model : cls\n Loaded model object.\n \"\"\"\n model = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n\n # Put the model on the same device it was trained on (GPU/MPS) if it is available; otherwise use CPU\n if model.is_fit() and model._get_class_tags().get(\"set_device_on_load\", False):\n device = model.suggest_device_infer(verbose=verbose)\n model.set_device(device=device)\n\n return model\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_set_device\": True,\n \"set_device_on_save_to\": \"cpu\",\n \"set_device_on_load\": True,\n }\n"
},
{
"path": "tabular/src/autogluon/tabular/models/automm/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/automm/automm_model.py",
"content": "\"\"\"Wrapper of the MultiModalPredictor.\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nimport os\nfrom typing import Dict, Optional\n\nimport pandas as pd\n\nfrom autogluon.common.features.types import (\n R_CATEGORY,\n R_FLOAT,\n R_INT,\n R_OBJECT,\n S_IMAGE_PATH,\n S_TEXT,\n S_TEXT_AS_CATEGORY,\n S_TEXT_NGRAM,\n S_TEXT_SPECIAL,\n)\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_autogluon_multimodal\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.models import AbstractModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass MultiModalPredictorModel(AbstractModel):\n ag_key = \"AG_AUTOMM\"\n ag_name = \"MultiModalPredictor\"\n _NN_MODEL_NAME = \"automm_model\"\n\n def __init__(self, **kwargs):\n \"\"\"Wrapper of autogluon.multimodal.MultiModalPredictor.\n\n The features can be a mix of\n - image column\n - text column\n - categorical column\n - numerical column\n\n The labels can be categorical or numerical.\n\n Parameters\n ----------\n path\n The directory to store the modeling outputs.\n name\n Name of subdirectory inside path where model will be saved.\n problem_type\n Type of problem that this model will handle.\n Valid options: ['binary', 'multiclass', 'regression'].\n eval_metric\n The evaluation metric.\n num_classes\n The number of classes.\n stopping_metric\n The stopping metric.\n model\n The internal model object.\n hyperparameters\n The hyperparameters of the model\n features\n Names of the features.\n feature_metadata\n The feature metadata.\n\n .. versionadded:: 0.3.0\n \"\"\"\n super().__init__(**kwargs)\n self._label_column_name = None\n self._load_model = None # Whether to load inner model when loading.\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_INT, R_FLOAT, R_CATEGORY, R_OBJECT],\n ignored_type_group_special=[S_TEXT_NGRAM, S_TEXT_AS_CATEGORY, S_TEXT_SPECIAL],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args(cls) -> dict:\n default_ag_args = super()._get_default_ag_args()\n extra_ag_args = {\n \"valid_stacker\": False,\n }\n default_ag_args.update(extra_ag_args)\n return default_ag_args\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n # FIXME: Enable parallel bagging once AutoMM supports being run within Ray without hanging\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\"fold_fitting_strategy\": \"sequential_local\"}\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n def _set_default_params(self):\n super()._set_default_params()\n try_import_autogluon_multimodal()\n\n def preprocess_fit(self, X, y, X_val=None, y_val=None, **kwargs):\n \"\"\"\n Preprocessing training and validation data.\n This method is a placeholder for inheriting models to override with more complex functionality if needed.\n \"\"\"\n X = self.preprocess(X=X, y=y, **kwargs)\n if X_val is not None:\n X_val = self.preprocess(X=X_val, **kwargs)\n return X, y, X_val, y_val\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: Optional[pd.DataFrame] = None,\n y_val: Optional[pd.Series] = None,\n time_limit: Optional[int] = None,\n sample_weight=None,\n **kwargs,\n ):\n \"\"\"The internal fit function\n\n Parameters\n ----------\n X\n Features of the training dataset\n y\n Labels of the training dataset\n X_val\n Features of the validation dataset\n y_val\n Labels of the validation dataset\n time_limit\n The time limits for the fit function\n sample_weight\n The weights of the samples\n kwargs\n Other keyword arguments\n\n \"\"\"\n try_import_autogluon_multimodal()\n from autogluon.multimodal import MultiModalPredictor\n\n # Decide name of the label column\n if \"label\" in X.columns:\n label_col_id = 0\n while True:\n self._label_column_name = \"label{}\".format(label_col_id)\n if self._label_column_name not in X.columns:\n break\n label_col_id += 1\n else:\n self._label_column_name = \"label\"\n\n X, y, X_val, y_val = self.preprocess_fit(X=X, y=y, X_val=X_val, y_val=y_val)\n params = self._get_model_params()\n max_features = params.pop(\n \"_max_features\", None\n ) # FIXME: `_max_features` is a hack. Instead use ag_args_fit and make generic\n num_features = len(X.columns)\n if max_features is not None and num_features > max_features:\n raise AssertionError(\n f\"Feature count ({num_features}) is greater than max allowed features ({max_features}) for {self.name}. Skipping model... \"\n f\"To increase the max allowed features, specify the value via the `_max_features` parameter \"\n f\"(Fully ignore by specifying `None`. \"\n f\"`_max_features` is experimental and will likely change API without warning in future releases.\"\n )\n\n # Get arguments from kwargs\n verbosity = kwargs.get(\"verbosity\", 2)\n if verbosity <= 2:\n enable_progress_bar = False\n else:\n enable_progress_bar = True\n num_gpus = kwargs.get(\"num_gpus\", None)\n if sample_weight is not None: # TODO: support\n logger.log(\n 15,\n \"sample_weight not yet supported for MultiModalPredictorModel, \"\n \"this model will ignore them in training.\",\n )\n\n # Need to deep copy to avoid altering outer context\n X = X.copy()\n X.insert(len(X.columns), self._label_column_name, y)\n if X_val is not None:\n X_val = X_val.copy()\n X_val.insert(len(X_val.columns), self._label_column_name, y_val)\n\n column_types = self._construct_column_types()\n\n verbosity_text = max(0, verbosity - 1)\n root_logger = logging.getLogger(\"autogluon\")\n root_log_level = root_logger.level\n # in self.save(), the model is saved to automm_nn_path\n automm_nn_path = os.path.join(self.path, self._NN_MODEL_NAME)\n self.model = MultiModalPredictor(\n label=self._label_column_name,\n problem_type=self.problem_type,\n path=automm_nn_path,\n eval_metric=self.eval_metric,\n verbosity=verbosity_text,\n enable_progress_bar=enable_progress_bar,\n )\n\n if num_gpus is not None:\n params[\"env.num_gpus\"] = num_gpus\n presets = params.pop(\"presets\", None)\n seed = params.pop(\"seed\", 0)\n\n self.model.fit(\n train_data=X,\n tuning_data=X_val,\n time_limit=time_limit,\n presets=presets,\n hyperparameters=params,\n column_types=column_types,\n seed=seed,\n )\n\n self.model.set_verbosity(verbosity)\n root_logger.setLevel(root_log_level) # Reset log level\n\n def _predict_proba(self, X, **kwargs):\n X = self.preprocess(X, **kwargs)\n\n self.model._enable_progress_bar = False\n if self.problem_type == REGRESSION:\n return self.model.predict(X, as_pandas=False)\n\n y_pred_proba = self.model.predict_proba(X, as_pandas=False)\n return self._convert_proba_to_unified_form(y_pred_proba)\n\n def save(self, path: str = None, verbose=True) -> str:\n self._load_model = self.model is not None\n __model = self.model\n self.model = None\n # save this AbstractModel object without NN weights\n path = super().save(path=path, verbose=verbose)\n self.model = __model\n\n if self._load_model:\n automm_nn_path = os.path.join(path, self._NN_MODEL_NAME)\n self.model.save(automm_nn_path)\n logger.log(15, f\"\\tSaved AutoMM model weights and model hyperparameters to '{automm_nn_path}'.\")\n self._load_model = None\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True):\n model = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n if model._load_model:\n try_import_autogluon_multimodal()\n from autogluon.multimodal import MultiModalPredictor\n\n model.model = MultiModalPredictor.load(os.path.join(path, cls._NN_MODEL_NAME))\n model._load_model = None\n return model\n\n def _get_memory_size(self) -> int:\n \"\"\"Return the memory size by calculating the total number of parameters.\n\n Returns\n -------\n memory_size\n The total memory size in bytes.\n \"\"\"\n total_size = self.model.model_size * 1e6 # convert from megabytes to bytes\n\n return total_size\n\n def _get_default_resources(self):\n num_cpus = ResourceManager.get_cpu_count()\n num_gpus = min(\n ResourceManager.get_gpu_count_torch(), 1\n ) # Use single gpu training by default. Consider to revise it later.\n return num_cpus, num_gpus\n\n def get_minimum_resources(self, is_gpu_available=False) -> Dict[str, int]:\n return {\n \"num_cpus\": 1,\n \"num_gpus\": 1,\n }\n\n def _construct_column_types(self) -> dict:\n # Construct feature types input to MultimodalPredictor\n features_image_path = set(self._feature_metadata.get_features(required_special_types=[S_IMAGE_PATH]))\n features_text = set(self._feature_metadata.get_features(required_special_types=[S_TEXT]))\n features_categorical = set(self._feature_metadata.get_features(valid_raw_types=[R_CATEGORY]))\n features_numerical = set(self._feature_metadata.get_features(valid_raw_types=[R_INT, R_FLOAT]))\n\n key_map = {\n \"image_path\": features_image_path,\n \"text\": features_text,\n \"categorical\": features_categorical,\n \"numerical\": features_numerical,\n }\n\n features = self._feature_metadata.get_features()\n\n column_types = {}\n for feature in features:\n for key in [\"image_path\", \"text\", \"categorical\", \"numerical\"]:\n if feature in key_map[key]:\n column_types[feature] = key\n break\n return column_types\n\n def _more_tags(self):\n # `can_refit_full=False` because MultiModalPredictor does not communicate how to train until the best epoch in refit_full.\n return {\"can_refit_full\": False}\n\n @classmethod\n def _class_tags(cls):\n return {\"handles_text\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/automm/ft_transformer.py",
"content": "\"\"\"Wrapper of the MultiModalPredictor.\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nfrom typing import Dict\n\nfrom autogluon.common.features.types import R_CATEGORY, R_FLOAT, R_INT, S_TEXT_NGRAM, S_TEXT_SPECIAL\n\nfrom .automm_model import MultiModalPredictorModel\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add unit tests\nclass FTTransformerModel(MultiModalPredictorModel):\n ag_key = \"FT_TRANSFORMER\"\n ag_name = \"FTTransformer\"\n\n def __init__(self, **kwargs):\n \"\"\"\n FT-Transformer model.\n\n The features can be a mix of\n - categorical column\n - numerical column\n\n The labels can be categorical or numerical.\n\n Parameters\n ----------\n path\n The directory to store the modeling outputs.\n name\n Name of subdirectory inside path where model will be saved.\n problem_type\n Type of problem that this model will handle.\n Valid options: ['binary', 'multiclass', 'regression'].\n eval_metric\n The evaluation metric.\n num_classes\n The number of classes.\n stopping_metric\n The stopping metric.\n model\n The internal model object.\n hyperparameters\n The hyperparameters of the model\n features\n Names of the features.\n feature_metadata\n The feature metadata.\n\n .. versionadded:: 0.6.0\n \"\"\"\n super().__init__(**kwargs)\n\n def _fit(self, X, num_gpus=\"auto\", **kwargs):\n if not isinstance(num_gpus, str):\n if num_gpus == 0:\n logger.log(\n 30,\n f\"WARNING: Training {self.name} on CPU (no GPU specified). This could take a long time. Use GPU to speed up training.\",\n )\n super()._fit(X, num_gpus=num_gpus, **kwargs)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_INT, R_FLOAT, R_CATEGORY],\n ignored_type_group_special=[S_TEXT_NGRAM, S_TEXT_SPECIAL],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n def _set_default_params(self):\n default_params = {\n \"data.categorical.convert_to_text\": False,\n \"model.names\": [\"ft_transformer\"],\n \"model.ft_transformer.embedding_arch\": [\"linear\"],\n \"env.batch_size\": 128,\n \"env.per_gpu_batch_size\": 128,\n \"env.num_workers\": 0,\n \"env.num_workers_inference\": 0,\n \"optim.max_epochs\": 2000, # Specify a large value to train until convergence\n \"optim.weight_decay\": 1.0e-5,\n \"optim.lr_choice\": None,\n \"optim.lr_schedule\": \"polynomial_decay\",\n \"optim.warmup_steps\": 0.0,\n \"optim.patience\": 20,\n \"optim.top_k\": 3,\n \"_max_features\": 300, # FIXME: This is a hack, move to AG_ARGS_FIT for v0.7\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def get_minimum_resources(self, is_gpu_available=False) -> Dict[str, int]:\n return {\n \"num_cpus\": 1,\n \"num_gpus\": 0, # allow FT_Transformer to be trained on CPU only\n }\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\n \"fold_fitting_strategy\": \"auto\",\n \"fold_fitting_strategy_gpu\": \"sequential_local\", # Crashes when using GPU in parallel bagging\n }\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n @classmethod\n def _class_tags(cls):\n return {\"handles_text\": False}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/callbacks.py",
"content": "import logging\nimport time\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom .catboost_utils import CATBOOST_QUANTILE_PREFIX\n\nlogger = logging.getLogger(__name__)\n\n\nclass MemoryCheckCallback:\n \"\"\"\n Callback to ensure memory usage is safe, otherwise early stops the model to avoid OOM errors.\n\n This callback is CatBoost specific.\n\n Parameters\n ----------\n period : int, default = 10\n Number of iterations between checking memory status. Higher values are less precise but use less compute.\n verbose : bool, default = False\n Whether to log information on memory status even if memory usage is low.\n \"\"\"\n\n def __init__(self, period: int = 10, verbose=False):\n self.period = period\n self.init_mem_rss = ResourceManager.get_memory_rss()\n self.init_mem_avail = ResourceManager.get_available_virtual_mem()\n self.verbose = verbose\n\n self._cur_period = 1\n\n def after_iteration(self, info):\n iteration = info.iteration\n if iteration % self._cur_period == 0:\n not_enough_memory = self.memory_check(iteration)\n if not_enough_memory:\n logger.log(20, f\"\\tRan low on memory, early stopping on iteration {info.iteration}.\")\n return False\n return True\n\n def memory_check(self, iter: int) -> bool:\n \"\"\"\n Checks if memory usage is unsafe. If so, signals the model to stop training early.\n\n Parameters\n ----------\n iter: int\n The current training iteration.\n\n Returns\n -------\n bool: True if training should stop due to memory constraints, False otherwise.\n \"\"\"\n available_bytes = ResourceManager.get_available_virtual_mem()\n cur_rss = ResourceManager.get_memory_rss()\n\n # Update initial memory usage if current usage is lower\n if cur_rss < self.init_mem_rss:\n self.init_mem_rss = cur_rss\n\n # Convert memory values to MB\n estimated_model_size_mb = (cur_rss - self.init_mem_rss) / (1024**2)\n available_mb = available_bytes / (1024**2)\n\n model_size_memory_ratio = estimated_model_size_mb / available_mb\n early_stop = False\n\n if model_size_memory_ratio > 1.0:\n logger.warning(\n f\"Iteration {iter}: Model size exceeds available memory. \"\n f\"Available memory: {available_mb:.2f} MB, \"\n f\"Estimated model size: {estimated_model_size_mb:.2f} MB.\"\n )\n early_stop = True\n\n if available_mb < 512: # Less than 512 MB\n logger.warning(\n f\"Iteration {iter}: Low available memory (<512 MB). \"\n f\"Available memory: {available_mb:.2f} MB, \"\n f\"Estimated model size: {estimated_model_size_mb:.2f} MB.\"\n )\n early_stop = True\n\n if early_stop:\n logger.warning(\n \"Early stopping model prior to optimal result to avoid OOM error. \"\n \"Please increase available memory to avoid subpar model quality.\"\n )\n return True\n elif self.verbose or model_size_memory_ratio > 0.25:\n logger.debug(\n f\"Iteration {iter}: \"\n f\"Available memory: {available_mb:.2f} MB, \"\n f\"Estimated model size: {estimated_model_size_mb:.2f} MB.\"\n )\n\n # Adjust memory check frequency based on model size\n if model_size_memory_ratio > 0.5:\n self._cur_period = 1 # Increase frequency of memory checks\n elif iter > self.period:\n self._cur_period = self.period\n\n return False\n\n\nclass TimeCheckCallback:\n \"\"\"\n Callback to ensure time limit is respected, otherwise early stops the model to avoid going over time.\n\n This callback is CatBoost specific.\n\n Parameters\n ----------\n time_start : float\n The starting time (usually obtained via `time.time()`) of the training.\n time_limit : float\n The time in seconds before stopping training.\n \"\"\"\n\n def __init__(self, time_start, time_limit):\n self.time_end = time_start + time_limit\n self.time_start = time_start\n\n def after_iteration(self, info):\n time_cur = time.time()\n time_per_iter = (time_cur - self.time_start) / info.iteration\n if self.time_end < (time_cur + 2 * time_per_iter):\n logger.log(20, f\"\\tRan out of time, early stopping on iteration {info.iteration}.\")\n return False\n return True\n\n\nclass EarlyStoppingCallback:\n \"\"\"\n Early stopping callback.\n\n This callback is CatBoost specific.\n\n Parameters\n ----------\n stopping_rounds : int or tuple\n If int, The possible number of rounds without the trend occurrence.\n If tuple, contains early stopping class as first element and class init kwargs as second element.\n eval_metric : str\n The eval_metric to use for early stopping. Must also be specified in the CatBoost model params.\n compare_key : str, default = 'validation'\n The data to use for scoring. It is recommended to keep as default.\n \"\"\"\n\n def __init__(self, stopping_rounds, eval_metric, compare_key=\"validation\"):\n if isinstance(stopping_rounds, int):\n from autogluon.core.utils.early_stopping import SimpleES\n\n self.es = SimpleES(patience=stopping_rounds)\n else:\n self.es = stopping_rounds[0](**stopping_rounds[1])\n self.best_score = None\n self.compare_key = compare_key\n\n if isinstance(eval_metric, str):\n from catboost.core import is_maximizable_metric\n\n is_max_optimal = is_maximizable_metric(eval_metric)\n eval_metric_name = eval_metric\n else:\n is_max_optimal = eval_metric.is_max_optimal()\n # FIXME: Unsure if this works for custom metrics!\n eval_metric_name = eval_metric.__class__.__name__\n\n self.eval_metric_name = eval_metric_name\n self.is_max_optimal = is_max_optimal\n self.is_quantile = CATBOOST_QUANTILE_PREFIX in self.eval_metric_name\n\n def after_iteration(self, info):\n is_best_iter = False\n if self.is_quantile:\n # FIXME: CatBoost adds extra ',' in the metric name if quantile levels are not balanced\n # e.g., 'MultiQuantile:alpha=0.1,0.25,0.5,0.95' becomes 'MultiQuantile:alpha=0.1,,0.25,0.5,0.95'\n # `'Quantile:' in k` catches both multiquantile (MultiQuantile:) and single-quantile mode (Quantile:)\n eval_metric_name = [k for k in info.metrics[self.compare_key] if CATBOOST_QUANTILE_PREFIX in k][0]\n else:\n eval_metric_name = self.eval_metric_name\n cur_score = info.metrics[self.compare_key][eval_metric_name][-1]\n if not self.is_max_optimal:\n cur_score *= -1\n if self.best_score is None:\n self.best_score = cur_score\n elif cur_score > self.best_score:\n is_best_iter = True\n self.best_score = cur_score\n\n should_stop = self.es.update(cur_round=info.iteration, is_best=is_best_iter)\n return not should_stop\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/catboost_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport math\nimport os\nimport time\nfrom types import MappingProxyType\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_catboost\nfrom autogluon.core.constants import MULTICLASS, PROBLEM_TYPES_CLASSIFICATION, QUANTILE, REGRESSION, SOFTCLASS\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.models._utils import get_early_stopping_rounds\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\n\nfrom .callbacks import EarlyStoppingCallback, MemoryCheckCallback, TimeCheckCallback\nfrom .catboost_utils import CATBOOST_EVAL_METRIC_TO_LOSS_FUNCTION, get_catboost_metric_from_ag_metric\nfrom .hyperparameters.parameters import get_param_baseline\nfrom .hyperparameters.searchspaces import get_default_searchspace\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Consider having CatBoost variant that converts all categoricals to numerical as done in RFModel, was showing improved results in some problems.\nclass CatBoostModel(AbstractModel):\n \"\"\"\n CatBoost model: https://catboost.ai/\n\n Hyperparameter options: https://catboost.ai/en/docs/references/training-parameters\n \"\"\"\n\n ag_key = \"CAT\"\n ag_name = \"CatBoost\"\n ag_priority = 70\n ag_priority_by_problem_type = MappingProxyType({SOFTCLASS: 60})\n seed_name = \"random_seed\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._category_features = None\n\n def _set_default_params(self):\n default_params = get_param_baseline(problem_type=self.problem_type)\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n # Set 'allow_writing_files' to True in order to keep log files created by catboost during training (these will be saved in the directory where AutoGluon stores this model)\n self._set_default_param_value(\n \"allow_writing_files\", False\n ) # Disables creation of catboost logging files during training by default\n if self.problem_type != SOFTCLASS: # TODO: remove this after catboost 0.24\n default_eval_metric = get_catboost_metric_from_ag_metric(\n self.stopping_metric, self.problem_type, self.quantile_levels\n )\n self._set_default_param_value(\"eval_metric\", default_eval_metric)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(self.problem_type, num_classes=self.num_classes)\n\n def _preprocess(self, X, **kwargs):\n # Note: while this is nonadaptive preprocessing, we made it stateful because it\n # contains the logic for nan handling and nans can be created after\n # nonadaptive preprocessing by model-specific preprocessing.\n # Moreover, now CatBoost handles nan like most other models in `_preprocess`.\n X = super()._preprocess(X, **kwargs)\n if self._category_features is None:\n self._category_features = list(X.select_dtypes(include=\"category\").columns)\n if self._category_features:\n X = X.copy()\n for category in self._category_features:\n current_categories = X[category].cat.categories\n if \"__NaN__\" in current_categories:\n X[category] = X[category].fillna(\"__NaN__\")\n else:\n X[category] = X[category].cat.add_categories(\"__NaN__\").fillna(\"__NaN__\")\n return X\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n num_classes: int = 1,\n **kwargs,\n ) -> int:\n \"\"\"\n Returns the expected peak memory usage in bytes of the CatBoost model during fit.\n\n The memory usage of CatBoost is primarily made up of two sources:\n\n 1. The size of the data\n 2. The size of the histogram cache\n Scales roughly by 5080*num_features*2^depth bytes\n For 10000 features and 6 depth, the histogram would be 3.2 GB.\n \"\"\"\n if hyperparameters is None:\n hyperparameters = {}\n num_classes = (\n num_classes if num_classes else 1\n ) # self.num_classes could be None after initialization if it's a regression problem\n data_mem_usage = get_approximate_df_mem_usage(X).sum()\n data_mem_usage_bytes = (\n data_mem_usage * 5 + data_mem_usage / 4 * num_classes\n ) # TODO: Extremely crude approximation, can be vastly improved\n\n border_count = hyperparameters.get(\"border_count\", 254)\n depth = hyperparameters.get(\"depth\", 6)\n\n # if depth < 7, treat it as 1 step larger for histogram size estimate\n # this fixes cases where otherwise histogram size appears to be off by around a factor of 2 for depth=6\n histogram_effective_depth = max(min(depth + 1, 7), depth)\n\n # Formula based on manual testing, aligns with LightGBM histogram sizes\n histogram_mem_usage_bytes = 24 * math.pow(2, histogram_effective_depth) * len(X.columns) * border_count\n histogram_mem_usage_bytes *= 1.2 # Add a 20% buffer\n\n baseline_memory_bytes = 4e8 # 400 MB baseline memory\n\n approx_mem_size_req = data_mem_usage_bytes + histogram_mem_usage_bytes + baseline_memory_bytes\n return approx_mem_size_req\n\n # TODO: Use Pool in preprocess, optimize bagging to do Pool.split() to avoid re-computing pool for each fold! Requires stateful + y\n # Pool is much more memory efficient, avoids copying data twice in memory\n def _fit(\n self,\n X,\n y,\n X_val=None,\n y_val=None,\n time_limit=None,\n num_gpus=0,\n num_cpus=-1,\n sample_weight=None,\n sample_weight_val=None,\n **kwargs,\n ):\n time_start = time.time()\n try_import_catboost()\n from catboost import CatBoostClassifier, CatBoostRegressor, Pool\n\n ag_params = self._get_ag_params()\n params = self._get_model_params()\n\n params[\"thread_count\"] = num_cpus\n if self.problem_type == SOFTCLASS:\n # FIXME: This is extremely slow due to unoptimized metric / objective sent to CatBoost\n from .catboost_softclass_utils import SoftclassCustomMetric, SoftclassObjective\n\n params.setdefault(\"loss_function\", SoftclassObjective.SoftLogLossObjective())\n params[\"eval_metric\"] = SoftclassCustomMetric.SoftLogLossMetric()\n elif self.problem_type in [REGRESSION, QUANTILE]:\n # Choose appropriate loss_function that is as close as possible to the eval_metric\n params.setdefault(\n \"loss_function\",\n CATBOOST_EVAL_METRIC_TO_LOSS_FUNCTION.get(params[\"eval_metric\"], params[\"eval_metric\"]),\n )\n\n model_type = CatBoostClassifier if self.problem_type in PROBLEM_TYPES_CLASSIFICATION else CatBoostRegressor\n num_rows_train = len(X)\n num_cols_train = len(X.columns)\n num_classes = (\n self.num_classes if self.num_classes else 1\n ) # self.num_classes could be None after initialization if it's a regression problem\n\n X = self.preprocess(X, y=y, is_train=True)\n cat_features = list(X.select_dtypes(include=\"category\").columns)\n X = Pool(data=X, label=y, cat_features=cat_features, weight=sample_weight)\n\n if X_val is None:\n eval_set = None\n early_stopping_rounds = None\n else:\n X_val = self.preprocess(X_val)\n X_val = Pool(data=X_val, label=y_val, cat_features=cat_features, weight=sample_weight_val)\n eval_set = X_val\n early_stopping_rounds = ag_params.get(\"early_stop\", \"adaptive\")\n if isinstance(early_stopping_rounds, (str, tuple, list)):\n early_stopping_rounds = self._get_early_stopping_rounds(\n num_rows_train=num_rows_train, strategy=early_stopping_rounds\n )\n\n if params.get(\"allow_writing_files\", False):\n if \"train_dir\" not in params:\n try:\n # TODO: What if path is in S3?\n os.makedirs(os.path.dirname(self.path), exist_ok=True)\n except:\n pass\n else:\n params[\"train_dir\"] = os.path.join(self.path, \"catboost_info\")\n\n # TODO: Add more control over these params (specifically early_stopping_rounds)\n verbosity = kwargs.get(\"verbosity\", 2)\n if verbosity <= 1:\n verbose = False\n elif verbosity == 2:\n verbose = False\n elif verbosity == 3:\n verbose = 20\n else:\n verbose = True\n\n num_features = len(self._features)\n\n if num_gpus != 0:\n if \"task_type\" not in params:\n params[\"task_type\"] = \"GPU\"\n logger.log(\n 20,\n f\"\\tTraining {self.name} with GPU, note that this may negatively impact model quality compared to CPU training.\",\n )\n # TODO: Confirm if GPU is used in HPO (Probably not)\n # TODO: Adjust max_bins to 254?\n\n if params.get(\"task_type\", None) == \"GPU\":\n if \"colsample_bylevel\" in params:\n params.pop(\"colsample_bylevel\")\n logger.log(30, f\"\\t'colsample_bylevel' is not supported on GPU, using default value (Default = 1).\")\n if \"rsm\" in params:\n params.pop(\"rsm\")\n logger.log(30, f\"\\t'rsm' is not supported on GPU, using default value (Default = 1).\")\n\n if (\n self.problem_type == MULTICLASS\n and \"rsm\" not in params\n and \"colsample_bylevel\" not in params\n and num_features > 1000\n ):\n # Subsample columns to speed up training\n if params.get(\"task_type\", None) != \"GPU\": # RSM does not work on GPU\n params[\"colsample_bylevel\"] = max(min(1.0, 1000 / num_features), 0.05)\n logger.log(\n 30,\n f\"\\tMany features detected ({num_features}), dynamically setting 'colsample_bylevel' to {params['colsample_bylevel']} to speed up training (Default = 1).\",\n )\n logger.log(\n 30,\n f\"\\tTo disable this functionality, explicitly specify 'colsample_bylevel' in the model hyperparameters.\",\n )\n else:\n params[\"colsample_bylevel\"] = 1.0\n logger.log(30, f\"\\t'colsample_bylevel' is not supported on GPU, using default value (Default = 1).\")\n\n logger.log(15, f\"\\tCatboost model hyperparameters: {params}\")\n\n extra_fit_kwargs = dict()\n if params.get(\"task_type\", None) != \"GPU\":\n callbacks = []\n if early_stopping_rounds is not None:\n callbacks.append(\n EarlyStoppingCallback(stopping_rounds=early_stopping_rounds, eval_metric=params[\"eval_metric\"])\n )\n\n if num_rows_train * num_cols_train * num_classes > 5_000_000:\n # The data is large enough to potentially cause memory issues during training, so monitor memory usage via callback.\n callbacks.append(MemoryCheckCallback())\n if time_limit is not None:\n time_cur = time.time()\n time_left = time_limit - (time_cur - time_start)\n if (\n time_left <= time_limit * 0.4\n ): # if 60% of time was spent preprocessing, likely not enough time to train model\n raise TimeLimitExceeded\n callbacks.append(TimeCheckCallback(time_start=time_cur, time_limit=time_left))\n extra_fit_kwargs[\"callbacks\"] = callbacks\n else:\n logger.log(\n 30,\n f\"\\tWarning: CatBoost on GPU is experimental. If you encounter issues, use CPU for training CatBoost instead.\",\n )\n if time_limit is not None:\n params[\"iterations\"] = self._estimate_iter_in_time_gpu(\n X=X,\n eval_set=eval_set,\n time_limit=time_limit,\n verbose=verbose,\n params=params,\n num_rows_train=num_rows_train,\n time_start=time_start,\n model_type=model_type,\n )\n if early_stopping_rounds is not None:\n if isinstance(early_stopping_rounds, int):\n extra_fit_kwargs[\"early_stopping_rounds\"] = early_stopping_rounds\n elif isinstance(early_stopping_rounds, tuple):\n extra_fit_kwargs[\"early_stopping_rounds\"] = 50\n self.model = model_type(**params)\n\n # TODO: Custom metrics don't seem to work anymore\n # TODO: Custom metrics not supported in GPU mode\n # TODO: Callbacks not supported in GPU mode\n fit_final_kwargs = dict(\n eval_set=eval_set,\n verbose=verbose,\n **extra_fit_kwargs,\n )\n\n if eval_set is not None:\n fit_final_kwargs[\"use_best_model\"] = True\n\n self.model.fit(X, **fit_final_kwargs)\n\n self.params_trained[\"iterations\"] = self.model.tree_count_\n\n # FIXME: This logic is a hack made to maintain compatibility with GPU CatBoost.\n # GPU CatBoost does not support callbacks or custom metrics.\n # Since we use callbacks to check memory and training time in CPU mode, we need a way to estimate these things prior to training for GPU mode.\n # This method will train a model on a toy number of iterations to estimate memory and training time.\n # It will return an updated iterations to train on that will avoid running OOM and running over time limit.\n # Remove this logic once CatBoost fixes GPU support for callbacks and custom metrics.\n def _estimate_iter_in_time_gpu(\n self, *, X, eval_set, time_limit, verbose, params, num_rows_train, time_start, model_type\n ):\n import math\n import pickle\n import sys\n\n modifier = min(1.0, 10000 / num_rows_train)\n num_sample_iter_max = max(round(modifier * 50), 2)\n time_left_start = time_limit - (time.time() - time_start)\n if (\n time_left_start <= time_limit * 0.4\n ): # if 60% of time was spent preprocessing, likely not enough time to train model\n raise TimeLimitExceeded\n default_iters = params[\"iterations\"]\n params_init = params.copy()\n num_sample_iter = min(num_sample_iter_max, params_init[\"iterations\"])\n params_init[\"iterations\"] = num_sample_iter\n sample_model = model_type(\n **params_init,\n )\n sample_model.fit(\n X,\n eval_set=eval_set,\n use_best_model=True,\n verbose=verbose,\n )\n\n time_left_end = time_limit - (time.time() - time_start)\n time_taken_per_iter = (time_left_start - time_left_end) / num_sample_iter\n estimated_iters_in_time = round(time_left_end / time_taken_per_iter)\n\n available_mem = ResourceManager.get_available_virtual_mem()\n if self.problem_type == SOFTCLASS:\n model_size_bytes = 1 # skip memory check\n else:\n model_size_bytes = sys.getsizeof(pickle.dumps(sample_model))\n\n max_memory_proportion = 0.3\n mem_usage_per_iter = model_size_bytes / num_sample_iter\n max_memory_iters = math.floor(available_mem * max_memory_proportion / mem_usage_per_iter)\n\n final_iters = min(default_iters, min(max_memory_iters, estimated_iters_in_time))\n if final_iters < 1:\n raise TimeLimitExceeded\n return final_iters\n\n def _predict_proba(self, X, **kwargs):\n if self.problem_type != SOFTCLASS:\n return super()._predict_proba(X, **kwargs)\n # For SOFTCLASS problems, manually transform predictions into probabilities via softmax\n X = self.preprocess(X, **kwargs)\n y_pred_proba = self.model.predict(X, prediction_type=\"RawFormulaVal\")\n y_pred_proba = np.exp(y_pred_proba)\n y_pred_proba = np.multiply(y_pred_proba, 1 / np.sum(y_pred_proba, axis=1)[:, np.newaxis])\n if y_pred_proba.shape[1] == 2:\n y_pred_proba = y_pred_proba[:, 1]\n return y_pred_proba\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n def _get_early_stopping_rounds(self, num_rows_train, strategy=\"auto\"):\n return get_early_stopping_rounds(num_rows_train=num_rows_train, strategy=strategy)\n\n def _ag_params(self) -> set:\n return {\"early_stop\"}\n\n def _validate_fit_memory_usage(\n self,\n mem_error_threshold: float = 1,\n mem_warning_threshold: float = 0.75,\n mem_size_threshold: int = 1e9,\n **kwargs,\n ):\n return super()._validate_fit_memory_usage(\n mem_error_threshold=mem_error_threshold,\n mem_warning_threshold=mem_warning_threshold,\n mem_size_threshold=mem_size_threshold,\n **kwargs,\n )\n\n def get_minimum_resources(self, is_gpu_available=False):\n minimum_resources = {\n \"num_cpus\": 1,\n }\n if is_gpu_available:\n # Our custom implementation does not support partial GPU. No gpu usage according to nvidia-smi when the `num_gpus` passed to fit is fractional`\n minimum_resources[\"num_gpus\"] = 0.5\n return minimum_resources\n\n def _get_default_resources(self):\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\", \"softclass\"]\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n }\n\n def _more_tags(self):\n # `can_refit_full=True` because iterations is communicated at end of `_fit`\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/catboost_softclass_utils.py",
"content": "import math\n\nimport numpy as np\nfrom catboost import MultiRegressionCustomMetric, MultiRegressionCustomObjective\n\nfrom autogluon.core.metrics.softclass_metrics import EPS\n\nfrom .catboost_utils import CustomMetric\n\n\n# Ojectives for SOFTCLASS problem_type\nclass SoftclassCustomMetric(CustomMetric):\n def __init__(self, metric, is_higher_better, needs_pred_proba): # metric is ignored\n super().__init__(metric, is_higher_better, needs_pred_proba)\n self.softlogloss = self.SoftLogLossMetric() # the metric object to pass to CatBoostRegressor\n\n def evaluate(self, approxes, target, weight):\n return self.softlogloss.evaluate(approxes, target, weight)\n\n class SoftLogLossMetric(MultiRegressionCustomMetric):\n def get_final_error(self, error, weight):\n return error\n\n def is_max_optimal(self):\n return True\n\n def evaluate(self, approxes, target, weight):\n assert len(target) == len(approxes)\n assert len(target[0]) == len(approxes[0])\n weight_sum = len(target)\n # TODO: inefficient copy of approxes, targets to np.array from provided UniTuple (required for JIT to work)\n approxes2 = np.zeros((len(approxes[0]), len(approxes)))\n target2 = np.zeros((len(approxes[0]), len(approxes)))\n for i in range(len(approxes)):\n approxes2[:, i] = approxes[i]\n target2[:, i] = target[i]\n approxes = approxes2\n target = target2\n approxes = np.exp(approxes)\n approxes = (approxes.T / approxes.sum(axis=1)).T # softmax\n # Numpy implementation of soft logloss:\n approxes = np.clip(approxes, a_min=EPS, a_max=None) # clip 0s to avoid NaN\n approxes = (approxes.T / approxes.sum(axis=1)).T # renormalize\n losses = np.multiply(np.log(approxes), target).sum(axis=1)\n error_sum = np.mean(losses)\n return error_sum, weight_sum\n \"\"\" The above numpy evaluate() function is necessary for JIT to work and not print warnings, here is the original function (that works without JIT):\n def evaluate(self, approxes, target, weight):\n assert len(target) == len(approxes)\n assert len(target[0]) == len(approxes[0])\n weight_sum = len(target)\n approxes = np.array(approxes)\n approxes = np.exp(approxes)\n approxes = np.multiply(approxes, 1/np.sum(approxes, axis=1)[:, np.newaxis])\n error_sum = soft_log_loss(np.array(target), np.array(approxes))\n return error_sum, weight_sum\n \"\"\"\n\n\nclass SoftclassObjective(object):\n def __init__(self):\n self.softlogloss = self.SoftLogLossObjective() # the objective object to pass to CatBoostRegressor\n\n class SoftLogLossObjective(MultiRegressionCustomObjective):\n # TODO: Consider replacing with C++ implementation (but requires building catboost from source).\n # This pure Python is 3x faster than optimized Numpy implementation. Tested C++ implementation was 3x faster than this one.\n def calc_ders_multi(self, approxes, targets, weight):\n exp_approx = [math.exp(val) for val in approxes]\n # exp_sum = sum(exp_approx) # not yet supported in numba jit: https://stackoverflow.com/questions/64936311/numba-cannot-determine-numba-type-of-class-builtin-function-or-method, using for loop below instead:\n exp_sum = 0.0\n for x in exp_approx:\n exp_sum += x\n exp_approx = [val / exp_sum for val in exp_approx]\n grad = [(targets[j] - exp_approx[j]) * weight for j in range(len(targets))]\n hess = [\n [(exp_approx[j] * exp_approx[j2] - (j == j2) * exp_approx[j]) * weight for j in range(len(targets))]\n for j2 in range(len(targets))\n ]\n return (grad, hess)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/catboost_utils.py",
"content": "import logging\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\n\nlogger = logging.getLogger(__name__)\n\n\nCATBOOST_QUANTILE_PREFIX = \"Quantile:\"\n# Mapping from non-optimizable eval_metric to optimizable loss_function.\n# See https://catboost.ai/docs/en/concepts/loss-functions-regression#usage-information\nCATBOOST_EVAL_METRIC_TO_LOSS_FUNCTION = {\n \"MedianAbsoluteError\": \"MAE\",\n \"SMAPE\": \"MAPE\",\n \"R2\": \"RMSE\",\n}\n\n\n# TODO: Add weight support?\n# TODO: Can these be optimized? What computational cost do they have compared to the default catboost versions?\nclass CustomMetric:\n def __init__(self, metric, is_higher_better, needs_pred_proba):\n self.metric = metric\n self.is_higher_better = is_higher_better\n self.needs_pred_proba = needs_pred_proba\n\n @staticmethod\n def get_final_error(error, weight):\n return error\n\n def is_max_optimal(self):\n return self.is_higher_better\n\n def evaluate(self, approxes, target, weight):\n raise NotImplementedError\n\n\ndef get_catboost_metric_from_ag_metric(metric, problem_type, quantile_levels=None):\n if problem_type == SOFTCLASS:\n from .catboost_softclass_utils import SoftclassCustomMetric\n\n if metric.name != \"soft_log_loss\":\n logger.warning(\"Setting metric=soft_log_loss, the only metric supported for softclass problem_type\")\n return SoftclassCustomMetric(metric=None, is_higher_better=True, needs_pred_proba=True)\n elif problem_type == BINARY:\n metric_map = dict(\n log_loss=\"Logloss\",\n accuracy=\"Accuracy\",\n roc_auc=\"AUC\",\n f1=\"Logloss\", # f1 uses Logloss because f1 in CatBoost is not reliable (causes errors between versions)\n f1_macro=\"Logloss\",\n f1_micro=\"Logloss\",\n f1_weighted=\"Logloss\",\n balanced_accuracy=\"BalancedAccuracy\",\n recall=\"Recall\",\n recall_macro=\"Recall\",\n recall_micro=\"Recall\",\n recall_weighted=\"Recall\",\n precision=\"Precision\",\n precision_macro=\"Precision\",\n precision_micro=\"Precision\",\n precision_weighted=\"Precision\",\n )\n metric_class = metric_map.get(metric.name, \"Logloss\")\n elif problem_type == MULTICLASS:\n metric_map = dict(\n log_loss=\"MultiClass\",\n accuracy=\"Accuracy\",\n )\n metric_class = metric_map.get(metric.name, \"MultiClass\")\n elif problem_type == REGRESSION:\n metric_map = dict(\n mean_squared_error=\"RMSE\",\n root_mean_squared_error=\"RMSE\",\n mean_absolute_error=\"MAE\",\n mean_absolute_percentage_error=\"MAPE\",\n # Non-optimizable metrics, see CATBOOST_EVAL_METRIC_TO_LOSS_FUNCTION\n median_absolute_error=\"MedianAbsoluteError\",\n symmetric_mean_absolute_percentage_error=\"SMAPE\",\n r2=\"R2\",\n )\n metric_class = metric_map.get(metric.name, \"RMSE\")\n elif problem_type == QUANTILE:\n if quantile_levels is None:\n raise AssertionError(f\"quantile_levels must be provided for problem_type = {problem_type}\")\n if not all(0 < q < 1 for q in quantile_levels):\n raise AssertionError(\n f\"quantile_levels must fulfill 0 < q < 1, provided quantile_levels: {quantile_levels}\"\n )\n # Loss function MultiQuantile: can only be used if len(quantile_levels) >= 2, otherwise we must use Quantile:\n if len(quantile_levels) == 1:\n metric_class = f\"{CATBOOST_QUANTILE_PREFIX}alpha={quantile_levels[0]}\"\n else:\n quantile_string = \",\".join(str(q) for q in quantile_levels)\n metric_class = f\"Multi{CATBOOST_QUANTILE_PREFIX}alpha={quantile_string}\"\n else:\n raise AssertionError(f\"CatBoost does not support {problem_type} problem type.\")\n\n return metric_class\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/hyperparameters/parameters.py",
"content": "from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS\n\nDEFAULT_ITERATIONS = 10000\n\n\ndef get_param_baseline(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_param_binary_baseline()\n elif problem_type in [MULTICLASS, SOFTCLASS]:\n return get_param_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_param_regression_baseline()\n else:\n return get_param_binary_baseline()\n\n\ndef get_param_binary_baseline():\n params = {\n \"iterations\": DEFAULT_ITERATIONS,\n \"learning_rate\": 0.05,\n }\n return params\n\n\ndef get_param_multiclass_baseline(num_classes):\n params = {\n \"iterations\": DEFAULT_ITERATIONS,\n \"learning_rate\": 0.05,\n }\n return params\n\n\ndef get_param_regression_baseline():\n params = {\n \"iterations\": DEFAULT_ITERATIONS,\n \"learning_rate\": 0.05,\n }\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/catboost/hyperparameters/searchspaces.py",
"content": "\"\"\"Default hyperparameter search spaces used in CatBoost Boosting model\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\n\n\ndef get_default_searchspace(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_searchspace_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_searchspace_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_searchspace_regression_baseline()\n else:\n return get_searchspace_binary_baseline()\n\n\ndef get_searchspace_multiclass_baseline(num_classes):\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"depth\": space.Int(lower=5, upper=8, default=6),\n \"l2_leaf_reg\": space.Real(lower=1, upper=5, default=3),\n }\n return params\n\n\ndef get_searchspace_binary_baseline():\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"depth\": space.Int(lower=5, upper=8, default=6),\n \"l2_leaf_reg\": space.Real(lower=1, upper=5, default=3),\n }\n return params\n\n\ndef get_searchspace_regression_baseline():\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"depth\": space.Int(lower=5, upper=8, default=6),\n \"l2_leaf_reg\": space.Real(lower=1, upper=5, default=3),\n }\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/ebm/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/ebm/ebm_model.py",
"content": "from __future__ import annotations\n\nimport time\nimport warnings\nfrom typing import TYPE_CHECKING\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.models import AbstractModel\n\nfrom .hyperparameters.parameters import get_param_baseline\nfrom .hyperparameters.searchspaces import get_default_searchspace\n\nif TYPE_CHECKING:\n from autogluon.core.metrics import Scorer\n\n\nclass EbmCallback:\n \"\"\"Time limit callback for EBM.\"\"\"\n\n def __init__(self, seconds: float):\n self.seconds = seconds\n self.end_time: float | None = None\n\n def __call__(self, *args, **kwargs):\n if self.end_time is None:\n self.end_time = time.monotonic() + self.seconds\n return False\n return time.monotonic() > self.end_time\n\n\nclass EBMModel(AbstractModel):\n \"\"\"\n The Explainable Boosting Machine (EBM) is a glass-box generalized additive model\n with automatic interaction detection (https://interpret.ml/docs). EBMs are\n designed to be highly interpretable while achieving accuracy comparable to\n black-box models on a wide range of tabular datasets.\n\n Requires the 'interpret' or 'interpret-core' package. Install via:\n\n pip install interpret\n\n\n Paper: InterpretML: A Unified Framework for Machine Learning Interpretability\n\n Authors: H. Nori, S. Jenkins, P. Koch, and R. Caruana 2019\n\n Codebase: https://github.com/interpretml/interpret\n\n License: MIT\n\n .. versionadded:: 1.5.0\n \"\"\"\n\n ag_key = \"EBM\"\n ag_name = \"EBM\"\n ag_priority = 35\n seed_name = \"random_state\"\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame | None = None,\n y_val: pd.Series | None = None,\n time_limit: float | None = None,\n sample_weight: np.ndarray | None = None,\n sample_weight_val: np.ndarray | None = None,\n num_cpus: int | str = \"auto\",\n **kwargs,\n ):\n # Preprocess data.\n X = self.preprocess(X, y=y)\n if X_val is not None:\n X_val = self.preprocess(X_val)\n\n features = self._features\n if features is None:\n features = X.columns\n\n params = construct_ebm_params(\n self.problem_type,\n self._get_model_params(),\n features,\n self.stopping_metric,\n num_cpus,\n time_limit,\n )\n\n # Init Class\n model_cls = get_class_from_problem_type(self.problem_type)\n self.model = model_cls(**params)\n\n # Handle validation data format for EBM\n fit_X = X\n fit_y = y\n fit_sample_weight = sample_weight\n bags = None\n if X_val is not None:\n fit_X = pd.concat([X, X_val], ignore_index=True)\n fit_y = pd.concat([y, y_val], ignore_index=True)\n if sample_weight is not None:\n fit_sample_weight = np.hstack([sample_weight, sample_weight_val])\n bags = np.full((len(fit_X), 1), 1, np.int8)\n bags[len(X) :, 0] = -1\n\n with warnings.catch_warnings(): # try to filter joblib warnings\n warnings.filterwarnings(\n \"ignore\",\n category=UserWarning,\n message=\".*resource_tracker: process died.*\",\n )\n self.model.fit(fit_X, fit_y, sample_weight=fit_sample_weight, bags=bags)\n\n def _set_default_params(self):\n default_params = get_param_baseline(problem_type=self.problem_type, num_classes=self.num_classes)\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(problem_type=self.problem_type, num_classes=self.num_classes)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = {\n \"valid_raw_types\": [\"int\", \"float\", \"category\"],\n }\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n @classmethod\n def _class_tags(cls) -> dict:\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self) -> dict:\n \"\"\"EBMs support refit full.\"\"\"\n return {\"can_refit_full\": True}\n\n def _estimate_memory_usage(self, X: pd.DataFrame, y: pd.Series | None = None, **kwargs) -> int:\n return self.estimate_memory_usage_static(\n X=X,\n y=y,\n hyperparameters=self._get_model_params(),\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n features=self._features,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n y: pd.Series | None = None,\n hyperparameters: dict | None = None,\n problem_type: str = \"infer\",\n num_classes: int = 1,\n features=None,\n **kwargs,\n ) -> int:\n \"\"\"Returns the expected peak memory usage in bytes of the EBM model during fit.\"\"\"\n # TODO: we can improve the memory estimate slightly by using num_classes if y is None\n\n if features is None:\n features = X.columns\n\n model_cls = get_class_from_problem_type(problem_type)\n params = construct_ebm_params(problem_type, hyperparameters, features)\n baseline_memory_bytes = 400_000_000 # 400 MB baseline memory\n\n # assuming we call pd.concat([X, X_val], ignore_index=True), then X size will be doubled\n return baseline_memory_bytes + model_cls(**params).estimate_mem(X, y, data_multiplier=2.0)\n\n def _validate_fit_memory_usage(self, mem_error_threshold: float = 1, **kwargs):\n # Given the good mem estimates with overhead, we set the threshold to 1.\n return super()._validate_fit_memory_usage(mem_error_threshold=mem_error_threshold, **kwargs)\n\n\ndef construct_ebm_params(\n problem_type,\n hyperparameters=None,\n features=None,\n stopping_metric=None,\n num_cpus=-1,\n time_limit=None,\n):\n if hyperparameters is None:\n hyperparameters = {}\n\n hyperparameters = hyperparameters.copy() # we pop values below, so copy.\n\n # The user can specify nominal and continuous columns.\n continuous_columns = hyperparameters.pop(\"continuous_columns\", [])\n nominal_columns = hyperparameters.pop(\"nominal_columns\", [])\n\n feature_types = None\n if features is not None:\n feature_types = []\n for c in features:\n if c in continuous_columns:\n f_type = \"continuous\"\n elif c in nominal_columns:\n f_type = \"nominal\"\n else:\n f_type = \"auto\"\n feature_types.append(f_type)\n\n # Default parameters for EBM\n params = {\n \"outer_bags\": 1, # AutoGluon ensemble creates outer bags, no need for this overhead.\n \"n_jobs\": 1, # EBM only parallelizes across outer bags currently, so ignore num_cpus\n \"feature_names\": features,\n \"feature_types\": feature_types,\n }\n if stopping_metric is not None:\n params[\"objective\"] = get_metric_from_ag_metric(metric=stopping_metric, problem_type=problem_type)\n if time_limit is not None:\n params[\"callback\"] = EbmCallback(time_limit)\n\n params.update(hyperparameters)\n return params\n\n\ndef get_class_from_problem_type(problem_type: str):\n if problem_type in [BINARY, MULTICLASS]:\n from interpret.glassbox import ExplainableBoostingClassifier\n\n model_cls = ExplainableBoostingClassifier\n elif problem_type == REGRESSION:\n from interpret.glassbox import ExplainableBoostingRegressor\n\n model_cls = ExplainableBoostingRegressor\n else:\n raise ValueError(f\"Unsupported problem type: {problem_type}\")\n return model_cls\n\n\ndef get_metric_from_ag_metric(*, metric: Scorer, problem_type: str):\n \"\"\"Map AutoGluon metric to EBM metric for early stopping.\"\"\"\n if problem_type in [BINARY, MULTICLASS]:\n metric_class = \"log_loss\"\n elif problem_type == REGRESSION:\n metric_class = \"rmse\"\n else:\n raise AssertionError(f\"EBM does not support {problem_type} problem type.\")\n\n return metric_class\n"
},
{
"path": "tabular/src/autogluon/tabular/models/ebm/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/ebm/hyperparameters/parameters.py",
"content": "from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS\n\n\ndef get_param_baseline(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_param_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_param_multiclass_baseline(num_classes=num_classes)\n elif problem_type == SOFTCLASS:\n return get_param_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_param_regression_baseline()\n else:\n return get_param_binary_baseline()\n\n\ndef get_base_params():\n base_params = {}\n return base_params\n\n\ndef get_param_binary_baseline():\n params = get_base_params()\n baseline_params = {}\n params.update(baseline_params)\n return params\n\n\ndef get_param_multiclass_baseline(num_classes):\n params = get_base_params()\n baseline_params = {}\n params.update(baseline_params)\n return params\n\n\ndef get_param_regression_baseline():\n params = get_base_params()\n baseline_params = {}\n params.update(baseline_params)\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/ebm/hyperparameters/searchspaces.py",
"content": "\"\"\"Default hyperparameter search spaces used in EBM model\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\n\n\ndef get_default_searchspace(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_searchspace_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_searchspace_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_searchspace_regression_baseline()\n else:\n return get_searchspace_binary_baseline()\n\n\ndef get_base_searchspace():\n base_params = {\n \"max_leaves\": space.Int(2, 3, default=2),\n \"smoothing_rounds\": space.Int(0, 1000, default=200),\n \"learning_rate\": space.Real(0.0025, 0.2, default=0.02, log=True),\n \"interactions\": space.Categorical(\n 0,\n \"0.5x\",\n \"1x\",\n \"1.5x\",\n \"2x\",\n \"2.5x\",\n \"3x\",\n \"3.5x\",\n \"4x\",\n \"4.5x\",\n \"5x\",\n \"6x\",\n \"7x\",\n \"8x\",\n \"9x\",\n \"10x\",\n \"15x\",\n \"20x\",\n \"25x\",\n ),\n \"interaction_smoothing_rounds\": space.Int(0, 200, default=90),\n \"min_hessian\": space.Real(1e-10, 1e-2, default=1e-4, log=True),\n \"min_samples_leaf\": space.Int(2, 20, default=4),\n \"gain_scale\": space.Real(0.5, 5.0, default=5.0, log=True),\n \"min_cat_samples\": space.Int(5, 20, default=10),\n \"cat_smooth\": space.Real(5.0, 100.0, default=10.0, log=True),\n \"missing\": space.Categorical(\"separate\", \"low\", \"high\", \"gain\"),\n }\n return base_params\n\n\ndef get_searchspace_multiclass_baseline(num_classes):\n params = get_base_searchspace()\n baseline_params = {}\n params.update(baseline_params)\n return params\n\n\ndef get_searchspace_binary_baseline():\n params = get_base_searchspace()\n baseline_params = {}\n params.update(baseline_params)\n return params\n\n\ndef get_searchspace_regression_baseline():\n params = get_base_searchspace()\n baseline_params = {}\n params.update(baseline_params)\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/callbacks.py",
"content": "import logging\nimport time\n\nimport numpy as np\nfrom fastai.callback.core import Callback, CancelFitException\nfrom fastai.callback.tracker import TrackerCallback\nfrom fastcore.basics import store_attr\n\nlogger = logging.getLogger(__name__)\n\n\nclass BatchTimeTracker(Callback):\n \"\"\"\n Training callback which allows collecting batch training times. The primary use is epoch training time estimation in adaptive epoch number selection.\n \"\"\"\n\n def __init__(self, batches_to_measure):\n self.batches_to_measure = batches_to_measure\n self.batches_finished = 0\n self.batch_start_time = None\n self.batch_measured_time = None\n\n def after_batch(self):\n self.batches_finished += 1\n if self.batches_finished == 1:\n # skip first batch due to initialization overhead\n self.batch_start_time = self._time_now()\n if self.batches_finished > self.batches_to_measure:\n self.batch_measured_time = (self._time_now() - self.batch_start_time) / self.batches_to_measure\n raise CancelFitException()\n\n def _time_now(self):\n return time.time()\n\n\nclass EarlyStoppingCallbackWithTimeLimit(TrackerCallback):\n def __init__(\n self,\n monitor=\"valid_loss\",\n comp=None,\n min_delta=0.0,\n patience=1,\n reset_on_fit=True,\n time_limit=None,\n best_epoch_stop=None,\n ):\n super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)\n self.patience = patience\n self.time_limit = time_limit\n self.start_time = time.time()\n self.best_epoch_stop = best_epoch_stop\n self.wait = None\n\n def before_fit(self):\n self.wait = 0\n super().before_fit()\n\n def after_epoch(self):\n if self.best_epoch_stop is not None:\n if self.epoch >= self.best_epoch_stop:\n logger.log(20, f\"\\tStopping at the best epoch learned earlier - {self.epoch}.\")\n raise CancelFitException()\n\n super().after_epoch()\n\n if self.new_best:\n self.wait = 0\n else:\n loss_val = self.recorder.values[-1][self.idx]\n if np.isnan(loss_val):\n if self.epoch == 0:\n raise AssertionError(f\"WARNING: NaN loss encountered in epoch {self.epoch}!\")\n else:\n logger.log(30, f\"WARNING: NaN loss encountered in epoch {self.epoch}: early stopping\")\n raise CancelFitException()\n self.wait += 1\n if self.wait >= self.patience:\n logger.log(20, f\"No improvement since epoch {self.epoch - self.wait}: early stopping\")\n raise CancelFitException()\n\n if self.time_limit:\n time_elapsed = time.time() - self.start_time\n time_left = self.time_limit - time_elapsed\n time_per_epoch = time_elapsed / (self.epoch + 1)\n if time_left < time_per_epoch:\n logger.log(20, f\"\\tRan out of time, stopping training early. (Stopping on epoch {self.epoch})\")\n raise CancelFitException()\n\n\nclass AgSaveModelCallback(TrackerCallback):\n \"A `TrackerCallback` that saves the model's best during training and loads it at the end.\"\n\n _only_train_loop = True\n\n def __init__(\n self,\n monitor=\"valid_loss\",\n comp=None,\n min_delta=0.0,\n fname=\"model\",\n every_epoch=False,\n with_opt=False,\n reset_on_fit=True,\n best_epoch_stop=None,\n ):\n super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)\n # keep track of file path for loggers\n self.last_saved_path = None\n self.best_epoch_stop = best_epoch_stop\n store_attr(\"fname,every_epoch,with_opt\")\n\n def _save(self, name):\n self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)\n\n def after_epoch(self):\n \"Compare the value monitored to its best score and save if best.\"\n if self.best_epoch_stop is not None: # use epoch learned earlier\n if self.epoch >= self.best_epoch_stop:\n logger.log(15, f\"Saving model model at the best epoch learned earlier - {self.epoch}.\")\n self.best_epoch = self.epoch\n self.learn.save(f\"{self.fname}\")\n if self.every_epoch:\n self._save(f\"{self.fname}_{self.epoch}\")\n else: # every improvement\n super().after_epoch()\n if self.new_best:\n logger.log(15, f\"Better model found at epoch {self.epoch} with {self.monitor} value: {self.best}.\")\n self.best_epoch = self.epoch\n self._save(f\"{self.fname}\")\n\n def after_fit(self, **kwargs):\n if not self.every_epoch:\n self.learn.load(f\"{self.fname}\", with_opt=self.with_opt, weights_only=False) # nosec B614\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/fastai_helpers.py",
"content": "import pickle\nimport warnings\nfrom pathlib import Path\nfrom typing import Any\n\nimport torch\nfrom fastai.torch_core import flatten_check\n\n\ndef export(model, filename_or_stream=\"export.pkl\", pickle_module=pickle, pickle_protocol=2):\n import torch\n from fastai.torch_core import rank_distrib\n\n \"Export the content of `self` without the items and the optimizer state for inference\"\n if rank_distrib():\n return # don't export if child proc\n model._end_cleanup()\n old_dbunch = model.dls\n model.dls = model.dls.new_empty()\n state = model.opt.state_dict() if model.opt is not None else None\n model.opt = None\n target = open(model.path / filename_or_stream, \"wb\") if is_pathlike(filename_or_stream) else filename_or_stream\n with warnings.catch_warnings():\n # To avoid the warning that come from PyTorch about model not being checked\n warnings.simplefilter(\"ignore\")\n torch.save(model, target, pickle_module=pickle_module, pickle_protocol=pickle_protocol) # nosec B614\n model.create_opt()\n if state is not None:\n model.opt.load_state_dict(state)\n model.dls = old_dbunch\n\n\ndef is_pathlike(x: Any) -> bool:\n return isinstance(x, (str, Path))\n\n\ndef medae(inp, targ):\n \"Mean absolute error between `inp` and `targ`.\"\n inp, targ = flatten_check(inp, targ)\n e = torch.abs(inp - targ)\n return torch.median(e).item()\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/hyperparameters/parameters.py",
"content": "from autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\n\n\n# TODO this method is generalizable and potentially should be moved out into framework\ndef get_param_baseline(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_param_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_param_multiclass_baseline()\n elif problem_type == REGRESSION:\n return get_param_regression_baseline()\n elif problem_type == QUANTILE:\n return get_param_quantile_baseline()\n else:\n return get_param_binary_baseline()\n\n\ndef get_param_multiclass_baseline():\n # TODO: explore/add other hyperparameters like weight decay, use of batch-norm, activation-function choice, etc.\n params = {\n # See docs: https://docs.fast.ai/tabular.learner.html\n \"layers\": None, # layers configuration; None - use model's heuristics\n \"emb_drop\": 0.1, # embedding layers dropout\n \"ps\": 0.1, # linear layers dropout\n \"bs\": \"auto\", # batch size\n # maximum learning rate for one cycle policy https://docs.fast.ai/train.html#fit_one_cycle\n # One-cycle policy paper: https://arxiv.org/abs/1803.09820\n \"lr\": 1e-2,\n \"epochs\": \"auto\", # maximum number of epochs\n # Early stopping settings. See more details here: https://docs.fast.ai/callbacks.tracker.html#EarlyStoppingCallback\n \"early.stopping.min_delta\": 0.0001,\n \"early.stopping.patience\": 20,\n # If > 0, then use LabelSmoothingCrossEntropy loss function for binary/multi-class classification;\n # otherwise use default loss function for this type of problem\n \"smoothing\": 0.0,\n }\n return params\n\n\ndef get_param_binary_baseline():\n return get_param_multiclass_baseline()\n\n\ndef get_param_regression_baseline():\n return get_param_multiclass_baseline()\n\n\ndef get_param_quantile_baseline():\n params = get_param_regression_baseline()\n\n # residual threshold parameter in HuberPinballLoss\n params.update({\"alpha\": 0.01})\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/hyperparameters/searchspaces.py",
"content": "from autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\n\n\ndef get_default_searchspace(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_searchspace_binary().copy()\n elif problem_type == MULTICLASS:\n return get_searchspace_multiclass(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_searchspace_regression().copy()\n elif problem_type == QUANTILE:\n return get_searchspace_quantile().copy()\n else:\n return get_searchspace_binary().copy()\n\n\ndef get_searchspace_binary():\n spaces = {\n # See docs: https://docs.fast.ai/tabular.learner.html\n \"layers\": space.Categorical(\n None,\n [200, 100],\n [200],\n [500],\n [1000],\n [500, 200],\n [50, 25],\n [1000, 500],\n [200, 100, 50],\n [500, 200, 100],\n [1000, 500, 200],\n ),\n \"emb_drop\": space.Real(0.0, 0.5, default=0.1),\n \"ps\": space.Real(0.0, 0.5, default=0.1),\n \"bs\": space.Categorical(256, 64, 128, 512, 1024, 2048, 4096),\n \"lr\": space.Real(5e-5, 1e-1, default=1e-2, log=True),\n \"epochs\": space.Int(lower=5, upper=30, default=30),\n \"early.stopping.min_delta\": 0.0001,\n \"early.stopping.patience\": 20,\n }\n return spaces\n\n\ndef get_searchspace_multiclass(num_classes):\n return get_searchspace_binary()\n\n\ndef get_searchspace_regression():\n return get_searchspace_binary()\n\n\ndef get_searchspace_quantile():\n spaces = get_searchspace_regression()\n\n # residual threshold parameter in HuberPinballLoss\n spaces.update({\"alpha\": space.Categorical(0.001, 0.01, 0.1, 1.0)})\n return spaces\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/imports_helper.py",
"content": "from fastai.tabular.all import *\n\n# Helper to force fastai imports.\n# fastai is doing library setup during star imports. These are required for correct library functioning.\n# Local star imports are not possible in-place, so separate helper packages is created\n# see autogluon.common.utils.try_import.try_import_fastai() for usage\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/quantile_helpers.py",
"content": "\"Implements various metrics to measure training accuracy\"\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom sklearn.isotonic import IsotonicRegression\n\n\ndef isotonic(input_data, quantile_list):\n quantile_list = np.array(quantile_list).reshape(-1)\n batch_size = input_data.shape[0]\n new_output_data = []\n for i in range(batch_size):\n new_output_data.append(IsotonicRegression().fit_transform(quantile_list, input_data[i]))\n return np.stack(new_output_data, 0)\n\n\nclass HuberPinballLoss(nn.Module):\n __name__ = \"huber_pinball_loss\"\n\n def __init__(self, quantile_levels, alpha=0.01):\n super(HuberPinballLoss, self).__init__()\n if quantile_levels is not None:\n self.quantile_levels = torch.Tensor(quantile_levels).contiguous().reshape(1, -1)\n else:\n self.quantile_levels = None\n self.alpha = alpha\n\n def forward(self, predict_data, target_data):\n if self.quantile_levels is None:\n return None\n target_data = target_data.contiguous().reshape(-1, 1)\n batch_size = target_data.size()[0]\n predict_data = predict_data.contiguous().reshape(batch_size, -1)\n\n error_data = target_data - predict_data\n if self.alpha == 0.0:\n loss_data = torch.max(self.quantile_levels * error_data, (self.quantile_levels - 1) * error_data)\n else:\n loss_data = torch.where(\n torch.abs(error_data) < self.alpha,\n 0.5 * error_data * error_data,\n self.alpha * (torch.abs(error_data) - 0.5 * self.alpha),\n )\n loss_data = loss_data / self.alpha\n\n scale = torch.where(\n error_data >= 0,\n torch.ones_like(error_data) * self.quantile_levels,\n torch.ones_like(error_data) * (1 - self.quantile_levels),\n )\n loss_data *= scale\n return loss_data.mean()\n"
},
{
"path": "tabular/src/autogluon/tabular/models/fastainn/tabular_nn_fastai.py",
"content": "from __future__ import annotations\n\nimport copy\nimport logging\nimport os\nimport time\nimport warnings\nfrom builtins import classmethod\nfrom functools import partial\nfrom pathlib import Path\nfrom types import MappingProxyType\nfrom typing import Union\n\nimport numpy as np\nimport pandas as pd\nimport sklearn\n\nfrom autogluon.common.features.types import (\n R_BOOL,\n R_CATEGORY,\n R_DATETIME,\n R_FLOAT,\n R_INT,\n R_OBJECT,\n S_TEXT_AS_CATEGORY,\n S_TEXT_NGRAM,\n S_TEXT_SPECIAL,\n)\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_fastai\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\nfrom autogluon.core.hpo.constants import RAY_BACKEND\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\nfrom autogluon.core.utils.files import make_temp_directory\nfrom autogluon.core.utils.loaders import load_pkl\nfrom autogluon.core.utils.savers import save_pkl\nfrom autogluon.tabular.models.tabular_nn.utils.nn_architecture_utils import infer_y_range\n\nfrom .hyperparameters.parameters import get_param_baseline\nfrom .hyperparameters.searchspaces import get_default_searchspace\n\nwarnings.filterwarnings(\"ignore\", message=\"load_learner` uses Python's insecure pickle module\")\n\n# FIXME: Has a leak somewhere, training additional models in a single python script will slow down training for each additional model. Gets very slow after 20+ models (10x+ slowdown)\n# Slowdown does not appear to impact Mac OS\n# Reproduced with raw torch: https://github.com/pytorch/pytorch/issues/31867\n# https://forums.fast.ai/t/runtimeerror-received-0-items-of-ancdata/48935\n# https://github.com/pytorch/pytorch/issues/973\n# https://pytorch.org/docs/master/multiprocessing.html#file-system-file-system\n# Slowdown bug not experienced on Linux if 'torch.multiprocessing.set_sharing_strategy('file_system')' commented out\n# NOTE: If below line is commented out, Torch uses many file descriptors. If issues arise, increase ulimit through 'ulimit -n 2048' or larger. Default on Linux is 1024.\n# torch.multiprocessing.set_sharing_strategy('file_system')\n\n# MacOS issue: torchvision==0.7.0 + torch==1.6.0 can cause segfaults; use torch==1.2.0 torchvision==0.4.0\n\nLABEL = \"__label__\"\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Takes extremely long time prior to training start if many (10000) continuous features from ngrams, debug - explore TruncateSVD option to reduce input dimensionality\n# TODO: currently fastai automatically detect and use CUDA if available - add code to honor autogluon settings\nclass NNFastAiTabularModel(AbstractModel):\n \"\"\"Class for fastai v1 neural network models that operate on tabular data.\n\n Hyperparameters:\n 'y_scaler': on regression problems, the model can give unreasonable predictions on unseen data.\n To address this problem, AutoGluon scales y values by default for regression problems.\n This attribute allows to pass a custom scaler for y values. Please note that intermediate\n iteration metrics will be affected by this transform and as a result intermediate iteration scores will be\n different from the final ones (these will be correct).\n https://scikit-learn.org/stable/modules/classes.html#module-sklearn.preprocessing\n\n 'clipping': on regression problems, extreme outliers of y can hurt performance of the model during training and\n on unseen data. To address this problem, AutoGluon clips input y values and output predictions by default to a\n range inferred from the training data. Setting this attribute to False disables clipping.\n\n 'layers': list of hidden layers sizes; None - use model's heuristics; default is None\n\n 'emb_drop': embedding layers dropout; default is 0.1\n\n 'ps': linear layers dropout - list of values applied to every layer in `layers`; default is [0.1]\n\n 'bs': batch size; default is 256\n\n 'lr': maximum learning rate for one cycle policy; default is 1e-2;\n see also https://docs.fast.ai/callback.schedule.html#Learner.fit_one_cycle,\n One-cycle policy paper: https://arxiv.org/abs/1803.09820\n\n 'epochs': number of epochs; default is 30\n\n # Early stopping settings. See more details here: https://docs.fast.ai/callback.tracker.html#EarlyStoppingCallback\n 'early.stopping.min_delta': 0.0001,\n 'early.stopping.patience': 10,\n \"\"\"\n\n ag_key = \"FASTAI\"\n ag_name = \"NeuralNetFastAI\"\n ag_priority = 50\n # Increase priority for multiclass since neural networks\n # scale better than trees as a function of n_classes.\n ag_priority_by_problem_type = MappingProxyType(\n {\n MULTICLASS: 95,\n }\n )\n seed_name = \"random_seed\"\n\n model_internals_file_name = \"model-internals.pkl\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.cat_columns = None\n self.cont_columns = None\n self.columns_fills = None\n self._columns_fills_names = None\n self.procs = None\n self.y_scaler = None\n self._cont_normalization = None\n self._load_model = None # Whether to load inner model when loading.\n self._num_cpus_infer = None\n self.clipping = None\n\n def _preprocess_train(self, X, y, X_val, y_val):\n from fastai.data.block import CategoryBlock, RegressionBlock\n from fastai.data.transforms import IndexSplitter\n from fastai.tabular.core import TabularPandas\n from fastcore.basics import range_of\n\n X = self.preprocess(X, y=y, fit=True)\n if X_val is not None:\n X_val = self.preprocess(X_val)\n\n from fastai.tabular.core import Categorify\n\n self.procs = [Categorify]\n\n if self.problem_type in [REGRESSION, QUANTILE] and self.y_scaler is not None:\n y_norm = pd.Series(self.y_scaler.fit_transform(y.values.reshape(-1, 1)).reshape(-1))\n y_val_norm = (\n pd.Series(self.y_scaler.transform(y_val.values.reshape(-1, 1)).reshape(-1))\n if y_val is not None\n else None\n )\n logger.log(\n 0,\n f\"Training with scaled targets: {self.y_scaler} - !!! NN training metric will be different from the final results !!!\",\n )\n else:\n y_norm = y\n y_val_norm = y_val\n\n logger.log(15, f\"Using {len(self.cont_columns)} cont features\")\n df_train, train_idx, val_idx = self._generate_datasets(X, y_norm, X_val, y_val_norm)\n y_block = RegressionBlock() if self.problem_type in [REGRESSION, QUANTILE] else CategoryBlock()\n\n # Copy cat_columns and cont_columns because TabularList is mutating the list\n data = TabularPandas(\n df_train,\n cat_names=self.cat_columns.copy(),\n cont_names=self.cont_columns.copy(),\n procs=self.procs,\n y_block=y_block,\n y_names=LABEL,\n splits=IndexSplitter(val_idx)(range_of(df_train)),\n )\n return data\n\n def _preprocess(self, X: pd.DataFrame, fit=False, **kwargs):\n X = super()._preprocess(X=X, **kwargs)\n if fit:\n self.cont_columns = self._feature_metadata.get_features(valid_raw_types=[R_INT, R_FLOAT, R_DATETIME])\n self.cat_columns = self._feature_metadata.get_features(valid_raw_types=[R_OBJECT, R_CATEGORY, R_BOOL])\n if self.cont_columns:\n # Drop columns that have less than 2 unique values (ignoring NaNs)\n # If these columns are kept, it will raise an exception when trying to normalize.\n # TODO: Can instead treat them as boolean if 1 unique + NaN\n unique_vals = X[self.cont_columns].nunique()\n self.cont_columns = [c for c in self.cont_columns if unique_vals[c] > 1]\n if self.cont_columns:\n self._cont_normalization = (\n np.array(X[self.cont_columns].mean()),\n np.array(X[self.cont_columns].std()),\n )\n\n num_cat_cols_og = len(self.cat_columns)\n if self.cat_columns:\n try:\n X_stats = X[self.cat_columns].describe(include=\"all\").T.reset_index()\n cat_cols_to_drop = list(\n X_stats[\n (X_stats[\"unique\"] > self.params.get(\"max_unique_categorical_values\", 10000))\n | (X_stats[\"unique\"].isna())\n ][\"index\"].values\n )\n except:\n cat_cols_to_drop = []\n if len(cat_cols_to_drop) != 0:\n cat_cols_to_drop = set(cat_cols_to_drop)\n self.cat_columns = [col for col in self.cat_columns if (col not in cat_cols_to_drop)]\n num_cat_cols_use = len(self.cat_columns)\n logger.log(15, f\"Using {num_cat_cols_use}/{num_cat_cols_og} categorical features\")\n\n nullable_numeric_features = self._feature_metadata.get_features(\n valid_raw_types=[R_FLOAT, R_DATETIME], invalid_special_types=[S_TEXT_SPECIAL]\n )\n self.columns_fills = dict()\n self._columns_fills_names = nullable_numeric_features\n for c in self._columns_fills_names: # No need to do this for int features, int can't have null\n self.columns_fills[c] = X[c].mean()\n X = self._fill_missing(X)\n if self.cont_columns:\n cont_mean, cont_std = self._cont_normalization\n # Creating a new DataFrame is 10x+ faster than assigning results to X[self.cont_columns]\n X_cont = pd.DataFrame(\n (X[self.cont_columns].values - cont_mean) / cont_std,\n columns=self.cont_columns,\n index=X.index,\n )\n if self.cat_columns:\n # Creating a new DataFrame via concatenation is faster than editing values in-place\n X = pd.concat([X_cont, X[self.cat_columns]], axis=1)\n else:\n X = X_cont.copy()\n return X\n\n def _fill_missing(self, df: pd.DataFrame) -> pd.DataFrame:\n # FIXME: Consider representing categories as int\n if self.columns_fills:\n # Speed up preprocessing by only filling columns where NaNs are present\n is_null = df[self._columns_fills_names].isnull().values.max(axis=0)\n columns_to_fill = [self._columns_fills_names[i] for i in range(len(is_null)) if is_null[i]]\n column_fills = {k: self.columns_fills[k] for k in columns_to_fill}\n if column_fills:\n # TODO: pandas==1.5.3 fillna is 10x+ slower than pandas==1.3.5 with large column count\n # TODO: Remove warning later as a follow up to https://github.com/autogluon/autogluon/pull/3734\n with warnings.catch_warnings():\n warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n df = df.fillna(column_fills, inplace=False, downcast=False)\n else:\n df = df.copy()\n else:\n df = df.copy()\n return df\n\n def _fit(\n self, X, y, X_val=None, y_val=None, time_limit=None, num_cpus=None, num_gpus=0, sample_weight=None, **kwargs\n ):\n try_import_fastai()\n import torch\n from fastai import torch_core\n from fastai.tabular.learner import tabular_learner\n from fastai.tabular.model import tabular_config\n\n from .callbacks import AgSaveModelCallback, EarlyStoppingCallbackWithTimeLimit\n from .quantile_helpers import HuberPinballLoss\n\n torch.set_num_threads(num_cpus)\n start_time = time.time()\n if sample_weight is not None: # TODO: support\n logger.log(\n 15,\n \"sample_weight not yet supported for NNFastAiTabularModel, this model will ignore them in training.\",\n )\n\n params = self._get_model_params()\n self._num_cpus_infer = params.pop(\"_num_cpus_infer\", 1)\n\n self.y_scaler = params.get(\"y_scaler\", None)\n if self.y_scaler is None:\n if self.problem_type == REGRESSION:\n self.y_scaler = sklearn.preprocessing.StandardScaler()\n elif self.problem_type == QUANTILE:\n self.y_scaler = sklearn.preprocessing.MinMaxScaler()\n else:\n self.y_scaler = copy.deepcopy(self.y_scaler)\n\n self.clipping = params.pop(\"clipping\", True)\n if self.clipping and (self.problem_type == REGRESSION):\n # use code and concepts from TorchNN to infer y range. 0.05 follows default from TorchNN.\n y_min, y_max = infer_y_range(y, y_range_extend=0.05)\n clip_func = partial(np.clip, a_min=y_min, a_max=y_max)\n\n steps = [\n # needs both func and inverse func, as the FastAI model calls inverse_transform.\n (\"clipper\", sklearn.preprocessing.FunctionTransformer(func=clip_func, inverse_func=clip_func)),\n ]\n\n # Support the case where no scaler is defined.\n if self.y_scaler is not None:\n steps.append((\"scaler\", self.y_scaler))\n\n self.y_scaler = sklearn.pipeline.Pipeline(steps=steps)\n\n if num_gpus is not None:\n # TODO: Control CPU vs GPU usage during inference\n if num_gpus == 0:\n torch_core.default_device(False)\n else:\n # TODO: respect CUDA_VISIBLE_DEVICES to select proper GPU\n torch_core.default_device(True)\n\n logger.log(15, f\"Fitting Neural Network with parameters {params}...\")\n data = self._preprocess_train(X, y, X_val, y_val)\n\n nn_metric, objective_func_name = self.__get_objective_func_name(self.stopping_metric)\n objective_func_name_to_monitor = self.__get_objective_func_to_monitor(objective_func_name)\n objective_optim_mode = (\n np.less\n if objective_func_name\n in [\n \"log_loss\",\n \"root_mean_squared_error\",\n \"mean_squared_error\",\n \"mean_absolute_error\",\n \"median_absolute_error\", # Regression objectives\n \"pinball_loss\", # Quantile objective\n ]\n else np.greater\n )\n\n # TODO: calculate max emb concat layer size and use 1st layer as that value and 2nd in between number of classes and the value\n if params.get(\"layers\", None) is not None:\n layers = params[\"layers\"]\n if isinstance(layers, tuple):\n layers = list(layers)\n elif self.problem_type in [REGRESSION, BINARY]:\n layers = [200, 100]\n elif self.problem_type == QUANTILE:\n base_size = max(len(self.quantile_levels) * 4, 128)\n layers = [base_size, base_size, base_size]\n else:\n base_size = max(data.c * 2, 100)\n layers = [base_size * 2, base_size]\n\n loss_func = None\n if self.problem_type == QUANTILE:\n loss_func = HuberPinballLoss(self.quantile_levels, alpha=self.params[\"alpha\"])\n\n best_epoch_stop = params.get(\"best_epoch\", None) # Use best epoch for refit_full.\n batch_size = self._get_batch_size(X)\n dls = data.dataloaders(bs=batch_size)\n\n # Make deterministic\n from fastai.torch_core import set_seed\n\n random_seed = params.pop(self.seed_name, self.default_random_seed)\n set_seed(random_seed, True)\n dls.rng.seed(random_seed)\n\n if self.problem_type == QUANTILE:\n dls.c = len(self.quantile_levels)\n\n self.model = tabular_learner(\n dls,\n layers=layers,\n metrics=nn_metric,\n config=tabular_config(ps=params[\"ps\"], embed_p=params[\"emb_drop\"]),\n loss_func=loss_func,\n )\n logger.log(15, self.model.model)\n\n fname = \"model\"\n save_callback = AgSaveModelCallback(\n monitor=objective_func_name_to_monitor,\n comp=objective_optim_mode,\n fname=fname,\n best_epoch_stop=best_epoch_stop,\n with_opt=True,\n )\n\n if time_limit is not None:\n time_elapsed = time.time() - start_time\n time_left = time_limit - time_elapsed\n if (\n time_left <= time_limit * 0.7\n ): # if 30% of time was spent preprocessing, likely not enough time to train model\n raise TimeLimitExceeded\n else:\n time_left = None\n\n early_stopping = EarlyStoppingCallbackWithTimeLimit(\n monitor=objective_func_name_to_monitor,\n comp=objective_optim_mode,\n min_delta=params[\"early.stopping.min_delta\"],\n patience=params[\"early.stopping.patience\"],\n time_limit=time_left,\n best_epoch_stop=best_epoch_stop,\n )\n\n callbacks = [save_callback, early_stopping]\n\n # TODO: Optimize by using io.BytesIO() instead of temp_dir for checkpointing?\n with make_temp_directory() as temp_dir:\n with self.model.no_bar():\n with self.model.no_logging():\n original_path = self.model.path\n self.model.path = Path(temp_dir)\n\n len_val = len(X_val) if X_val is not None else 0\n epochs = self._get_epochs_number(\n samples_num=len(X) + len_val,\n epochs=params[\"epochs\"],\n batch_size=batch_size,\n time_left=time_left,\n )\n if epochs == 0:\n # Stop early if there is not enough time to train a full epoch\n raise TimeLimitExceeded\n\n self.model.fit_one_cycle(epochs, params[\"lr\"], cbs=callbacks)\n\n # Load the best one and export it\n self.model = self.model.load(fname, weights_only=False) # nosec B614\n\n if objective_func_name == \"log_loss\":\n eval_result = self.model.validate(dl=dls.valid)[0]\n else:\n eval_result = self.model.validate(dl=dls.valid)[1]\n\n logger.log(15, f\"Model validation metrics: {eval_result}\")\n self.model.path = original_path\n\n self.params_trained[\"epochs\"] = epochs\n self.params_trained[\"best_epoch\"] = save_callback.best_epoch\n\n def _get_batch_size(self, X, default_batch_size_for_small_inputs=32):\n bs = self.params[\"bs\"]\n if bs == \"auto\":\n bs = 512 if len(X) >= 200000 else 256\n bs = bs if len(X) > bs else default_batch_size_for_small_inputs\n\n if self.params[\"bs\"] == \"auto\":\n logger.log(15, f\"Automated batch size selection: {bs}\")\n\n return bs\n\n def _get_epochs_number(\n self,\n samples_num: int,\n epochs: int | str,\n batch_size: int,\n time_left: float | None = None,\n min_batches_count: int = 30,\n default_epochs: int = 30,\n ) -> int:\n \"\"\"Get the number of epochs to train during fit\"\"\"\n if epochs == \"auto\":\n batches_count = int(samples_num / batch_size) + 1\n if not time_left:\n return default_epochs\n elif batches_count < min_batches_count:\n return default_epochs\n else:\n est_batch_time = self._measure_batch_times(min_batches_count)\n est_epoch_time = batches_count * est_batch_time * 1.1\n est_max_epochs = int(time_left / est_epoch_time)\n epochs = min(default_epochs, est_max_epochs)\n epochs = max(epochs, 0)\n logger.log(\n 15,\n f\"Automated epochs selection: training for {epochs} epoch(s). Estimated time budget use {epochs * est_epoch_time:.2f} / {time_left:.2f} sec\",\n )\n return epochs\n\n def _measure_batch_times(self, min_batches_count: int) -> float:\n \"\"\"Returns the time in seconds taken to fit a single batch\"\"\"\n from fastai.callback.core import CancelFitException\n\n from .callbacks import BatchTimeTracker\n\n batch_time_tracker_callback = BatchTimeTracker(batches_to_measure=min_batches_count)\n try:\n with self.model.no_bar():\n with self.model.no_logging():\n self.model.fit(1, lr=0, cbs=[batch_time_tracker_callback])\n except CancelFitException:\n pass # expected early exit\n batch_time = batch_time_tracker_callback.batch_measured_time\n if batch_time is None or batch_time < 0.00001:\n # Fixes rare issue where batch_time = None if the operation occurs too quickly\n batch_time = 0.00001\n return batch_time\n\n def _generate_datasets(self, X, y, X_val, y_val):\n df_train = pd.concat([X, X_val], ignore_index=True)\n df_train[LABEL] = pd.concat([y, y_val], ignore_index=True)\n train_idx = np.arange(len(X))\n if X_val is None:\n # use validation set for refit_full case - it's not going to be used for early stopping\n val_idx = np.array([0, 1]) + len(train_idx)\n df_train = pd.concat([df_train, df_train[:2]], ignore_index=True)\n else:\n val_idx = np.arange(len(X_val)) + len(X)\n return df_train, train_idx, val_idx\n\n def __get_objective_func_name(self, stopping_metric):\n metrics_map = self.__get_metrics_map()\n\n # Unsupported metrics will be replaced by defaults for a given problem type\n objective_func_name = stopping_metric.name\n if objective_func_name not in metrics_map.keys():\n if self.problem_type == REGRESSION:\n objective_func_name = \"mean_squared_error\"\n elif self.problem_type == QUANTILE:\n objective_func_name = \"pinball_loss\"\n else:\n objective_func_name = \"log_loss\"\n logger.warning(\n f\"Metric {stopping_metric.name} is not supported by this model - using {objective_func_name} instead\"\n )\n\n nn_metric = metrics_map.get(objective_func_name, None)\n\n return nn_metric, objective_func_name\n\n def __get_objective_func_to_monitor(self, objective_func_name):\n monitor_obj_func = {\n **{\n k: m.name if hasattr(m, \"name\") else m.__name__\n for k, m in self.__get_metrics_map().items()\n if m is not None\n },\n \"log_loss\": \"valid_loss\",\n }\n objective_func_name_to_monitor = objective_func_name\n if objective_func_name in monitor_obj_func:\n objective_func_name_to_monitor = monitor_obj_func[objective_func_name]\n return objective_func_name_to_monitor\n\n def _predict_proba(self, X, **kwargs):\n X = self.preprocess(X, **kwargs)\n\n single_row = len(X) == 1\n # fastai has issues predicting on a single row, duplicating the row as a workaround\n if single_row:\n X = pd.concat([X, X]).reset_index(drop=True)\n # Copy cat_columns and cont_columns because TabularList is mutating the list\n # TODO: This call has very high fixed cost with many features (0.7s for a single row with 3k features)\n # Primarily due to normalization performed on the inputs\n test_dl = self.model.dls.test_dl(X, inplace=True)\n with self.model.no_bar():\n with self.model.no_logging():\n preds, _ = self.model.get_preds(dl=test_dl)\n if single_row:\n preds = preds[:1, :]\n if self.problem_type == REGRESSION:\n if self.y_scaler is not None:\n return self.y_scaler.inverse_transform(preds.numpy()).reshape(-1)\n else:\n return preds.numpy().reshape(-1)\n elif self.problem_type == QUANTILE:\n from .quantile_helpers import isotonic\n\n if self.y_scaler is not None:\n preds = self.y_scaler.inverse_transform(preds.numpy()).reshape(-1, len(self.quantile_levels))\n else:\n preds = preds.numpy().reshape(-1, len(self.quantile_levels))\n return isotonic(preds, self.quantile_levels)\n elif self.problem_type == BINARY:\n return preds[:, 1].numpy()\n else:\n return preds.numpy()\n\n def save(self, path: str = None, verbose=True) -> str:\n from .fastai_helpers import export\n\n self._load_model = self.model is not None\n __model = self.model\n self.model = None\n path = super().save(path=path, verbose=verbose)\n self.model = __model\n # Export model\n if self._load_model:\n save_pkl.save_with_fn(\n self._model_internals_path, self.model, pickle_fn=lambda m, buffer: export(m, buffer), verbose=verbose\n )\n self._load_model = None\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True):\n from fastai.learner import load_learner\n\n model = super().load(path, reset_paths=reset_paths, verbose=verbose)\n if model._load_model:\n # Need the following logic to allow cross os loading of fastai model\n # https://github.com/fastai/fastai/issues/1482\n import pathlib\n import platform\n\n plt = platform.system()\n og_windows_path = None\n if plt != \"Windows\":\n og_windows_path = pathlib.WindowsPath\n pathlib.WindowsPath = pathlib.PosixPath\n model_internals_path = os.path.join(path, model.model_internals_file_name)\n model.model = load_pkl.load_with_fn(model_internals_path, lambda p: load_learner(p), verbose=verbose)\n if og_windows_path is not None:\n pathlib.WindowsPath = og_windows_path\n model._load_model = None\n return model\n\n @property\n def _model_internals_path(self) -> str:\n \"\"\"Path to model-internals.pkl\"\"\"\n return os.path.join(self.path, self.model_internals_file_name)\n\n def _set_default_params(self):\n \"\"\"Specifies hyperparameter values to use by default\"\"\"\n default_params = get_param_baseline(self.problem_type)\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(self.problem_type, num_classes=None)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n ignored_type_group_special=[S_TEXT_NGRAM, S_TEXT_AS_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n def _get_default_resources(self):\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n def __get_metrics_map(self):\n from fastai.metrics import FBeta, Precision, R2Score, Recall, RocAucBinary, accuracy, mae, mse, rmse\n\n from .fastai_helpers import medae\n from .quantile_helpers import HuberPinballLoss\n\n metrics_map = {\n # Regression\n \"root_mean_squared_error\": rmse,\n \"mean_squared_error\": mse,\n \"mean_absolute_error\": mae,\n \"r2\": R2Score(),\n \"median_absolute_error\": medae,\n # Classification\n \"accuracy\": accuracy,\n \"f1\": FBeta(beta=1),\n \"f1_macro\": FBeta(beta=1, average=\"macro\"),\n \"f1_micro\": FBeta(beta=1, average=\"micro\"),\n \"f1_weighted\": FBeta(beta=1, average=\"weighted\"), # this one has some issues\n \"roc_auc\": RocAucBinary(),\n \"precision\": Precision(),\n \"precision_macro\": Precision(average=\"macro\"),\n \"precision_micro\": Precision(average=\"micro\"),\n \"precision_weighted\": Precision(average=\"weighted\"),\n \"recall\": Recall(),\n \"recall_macro\": Recall(average=\"macro\"),\n \"recall_micro\": Recall(average=\"micro\"),\n \"recall_weighted\": Recall(average=\"weighted\"),\n \"log_loss\": None,\n \"pinball_loss\": HuberPinballLoss(quantile_levels=self.quantile_levels),\n # Not supported: pac_score\n }\n return metrics_map\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n return 10 * get_approximate_df_mem_usage(X).sum()\n\n def _get_hpo_backend(self):\n \"\"\"Choose which backend(Ray or Custom) to use for hpo\"\"\"\n return RAY_BACKEND\n\n def _get_maximum_resources(self) -> dict[str, Union[int, float]]:\n # fastai model trains slower when utilizing virtual cores and this issue scale up when the number of cpu cores increases\n return {\"num_cpus\": ResourceManager.get_cpu_count(only_physical_cores=True)}\n\n def get_minimum_resources(self, is_gpu_available=False):\n minimum_resources = {\n \"num_cpus\": 1,\n }\n if is_gpu_available:\n minimum_resources[\"num_gpus\"] = 0.5\n return minimum_resources\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\"]\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"reset_torch_threads\": True,\n \"reset_torch_cudnn_deterministic\": True,\n }\n\n def _more_tags(self):\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/image_prediction/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/image_prediction/image_predictor.py",
"content": "from __future__ import annotations\n\nimport logging\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_OBJECT, S_IMAGE_PATH\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\n\nfrom ..automm.automm_model import MultiModalPredictorModel\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Handle multiple image columns?\n# TODO: Handle multiple images in a single image column?\n# TODO: Consider fully replacing with MultiModalPredictorModel\n# first check if the null handling in this class provides value\nclass ImagePredictorModel(MultiModalPredictorModel):\n \"\"\"\n MultimodalPredictor that only uses image features.\n Currently only supports 1 image column, with 1 image per sample.\n Additionally has special null image handling to improve performance in the presence of null images (aka image path of '')\n Note: null handling has not been compared to the built-in null handling of MultimodalPredictor yet.\n \"\"\"\n\n ag_key = \"AG_IMAGE_NN\"\n ag_name = \"ImagePredictor\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._dummy_pred_proba = None # Dummy value to predict if image is NaN\n self._image_col_name = None\n\n @property\n def _has_predict_proba(self):\n return self.problem_type in [BINARY, MULTICLASS, SOFTCLASS]\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n get_features_kwargs=dict(\n valid_raw_types=[R_OBJECT],\n required_special_types=[S_IMAGE_PATH],\n ),\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args(cls) -> dict:\n default_ag_args = super()._get_default_ag_args()\n extra_ag_args = {\n \"valid_stacker\": False,\n \"problem_types\": [BINARY, MULTICLASS, REGRESSION],\n }\n default_ag_args.update(extra_ag_args)\n return default_ag_args\n\n def preprocess_fit(self, X, y, X_val=None, y_val=None, **kwargs):\n X, y, X_val, y_val = super().preprocess_fit(X=X, y=y, X_val=X_val, y_val=y_val, **kwargs)\n X_features = list(X.columns)\n if len(X_features) != 1:\n raise AssertionError(\n f\"ImagePredictorModel only supports one image feature, but {len(X_features)} were given: {X_features}\"\n )\n self._image_col_name = X_features[0]\n null_indices = X[self._image_col_name] == \"\"\n\n # TODO: Consider some kind of weighting of the two options so there isn't a harsh cutoff at 50\n # FIXME: What if all rows in a class are null? Will probably crash.\n if null_indices.sum() > 50:\n self._dummy_pred_proba = self._compute_dummy_pred_proba(\n y[null_indices]\n ) # FIXME: Do this one for better results\n else:\n # Not enough null to get a confident estimate of null label average, instead use all data average\n self._dummy_pred_proba = self._compute_dummy_pred_proba(y)\n\n if null_indices.sum() > 0:\n X = X[~null_indices]\n y = y[~null_indices]\n\n if X_val is not None:\n null_indices_val = X_val[self._image_col_name] == \"\"\n if null_indices_val.sum() > 0:\n X_val = X_val[~null_indices_val]\n y_val = y_val[~null_indices_val]\n\n return X, y, X_val, y_val\n\n def _predict_proba(self, X, **kwargs):\n X = self.preprocess(X, **kwargs)\n pred_method = self.model.predict_proba if self._has_predict_proba else self.model.predict\n # TODO: Add option to crash if null is present for faster predict_proba\n null_indices = X[self._image_col_name] == \"\"\n if null_indices.sum() > 0:\n if self.num_classes is None:\n y_pred_proba = np.zeros(len(X))\n else:\n y_pred_proba = np.zeros((len(X), self.num_classes))\n X = X.reset_index(drop=True)\n null_indices = X[self._image_col_name] == \"\"\n X = X[~null_indices]\n null_indices_rows = list(null_indices[null_indices].index)\n non_null_indices_rows = list(null_indices[~null_indices].index)\n y_pred_proba[null_indices_rows] = self._dummy_pred_proba\n y_pred_proba[non_null_indices_rows] = pred_method(X, as_pandas=False)\n else:\n y_pred_proba = pred_method(X, as_pandas=False)\n return self._convert_proba_to_unified_form(y_pred_proba)\n\n # TODO: Consider moving to AbstractModel or as a separate function\n # TODO: Test softclass\n def _compute_dummy_pred_proba(self, y):\n num_classes = self.num_classes\n if self.problem_type in [BINARY, MULTICLASS]:\n dummies = pd.get_dummies(y)\n dummy_columns = set(list(dummies.columns))\n if len(dummies.columns) < num_classes:\n for c in range(num_classes):\n if c not in dummy_columns:\n dummies[c] = 0\n dummies = dummies[list(range(num_classes))]\n pred_proba_mean = dummies.mean().values\n\n elif self.problem_type in [REGRESSION, QUANTILE, SOFTCLASS]:\n pred_proba_mean = y.mean()\n else:\n raise NotImplementedError(f\"Computing dummy pred_proba is not implemented for {self.problem_type}.\")\n return pred_proba_mean\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n"
},
{
"path": "tabular/src/autogluon/tabular/models/imodels/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/imodels/imodels_models.py",
"content": "from __future__ import annotations\n\nfrom abc import abstractmethod\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom autogluon.common.utils.try_import import try_import_imodels\nfrom autogluon.core.models import AbstractModel\n\n\nclass _IModelsModel(AbstractModel):\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._feature_generator = None\n self._ohe = None\n self._ohe_columns = None\n self._categorical_featnames = None\n self._other_featnames = None\n\n @abstractmethod\n def get_model(self):\n return NotImplemented\n\n def get_info(self):\n info = super().get_info()\n info[\"complexity\"] = self.model.complexity_\n return info\n\n def _preprocess(self, X: pd.DataFrame, is_train=False, **kwargs) -> pd.DataFrame:\n X = super()._preprocess(X, **kwargs)\n if is_train:\n self._categorical_featnames = self._feature_metadata.get_features(valid_raw_types=[\"category\"])\n self._other_featnames = self._feature_metadata.get_features(invalid_raw_types=[\"category\"])\n if self._categorical_featnames:\n self._ohe = OneHotEncoder(dtype=np.uint8, handle_unknown=\"ignore\")\n self._ohe.fit(X=X[self._categorical_featnames])\n self._ohe_columns = self._ohe.get_feature_names_out()\n else:\n self._ohe = None # otherwise no model is fitted when there are not self._categorical_featnames\n\n if self._ohe is not None:\n X_index = X.index\n X_ohe = self._ohe.transform(X[self._categorical_featnames])\n X_ohe = pd.DataFrame.sparse.from_spmatrix(X_ohe, columns=self._ohe_columns, index=X_index)\n if self._other_featnames:\n X = pd.concat([X[self._other_featnames], X_ohe], axis=1)\n else:\n X = X_ohe\n\n return X.fillna(0)\n\n def _fit(self, X: pd.DataFrame, y: pd.Series, **kwargs): # training data # training labels\n model_cls = self.get_model()\n X = self.preprocess(X, y=y, is_train=True)\n params = self._get_model_params()\n self.model = model_cls(**params)\n self.model.fit(X, y, feature_names=X.columns.values.tolist())\n\n def _set_default_params(self):\n default_params = {\n \"random_state\": 0,\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n get_features_kwargs=dict(\n valid_raw_types=[\"int\", \"float\", \"category\"],\n ),\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n\nclass RuleFitModel(_IModelsModel):\n ag_key = \"IM_RULEFIT\"\n ag_name = \"RuleFit\"\n\n def get_model(self):\n try_import_imodels()\n from imodels import RuleFitClassifier, RuleFitRegressor\n\n if self.problem_type in [\"regression\", \"softclass\"]:\n return RuleFitRegressor\n else:\n return RuleFitClassifier\n\n\nclass GreedyTreeModel(_IModelsModel):\n ag_key = \"IM_GREEDYTREE\"\n ag_name = \"GreedyTree\"\n\n def get_model(self):\n try_import_imodels()\n from imodels import GreedyTreeClassifier\n from sklearn.tree import DecisionTreeRegressor\n\n if self.problem_type in [\"regression\", \"softclass\"]:\n return DecisionTreeRegressor\n else:\n return GreedyTreeClassifier\n\n\nclass BoostedRulesModel(_IModelsModel):\n ag_key = \"IM_BOOSTEDRULES\"\n ag_name = \"BoostedRules\"\n\n def get_model(self):\n try_import_imodels()\n from imodels import BoostedRulesClassifier\n\n if self.problem_type in [\"binary\"]:\n return BoostedRulesClassifier\n else:\n raise Exception(\"Boosted Rule Set only supports binary classification!\")\n\n\nclass HSTreeModel(_IModelsModel):\n ag_key = \"IM_HSTREE\"\n ag_name = \"HierarchicalShrinkageTree\"\n\n def get_model(self):\n try_import_imodels()\n from imodels import HSTreeClassifierCV, HSTreeRegressorCV\n\n if self.problem_type in [\"regression\", \"softclass\"]:\n return HSTreeRegressorCV\n else:\n return HSTreeClassifierCV\n\n\nclass FigsModel(_IModelsModel):\n ag_key = \"IM_FIGS\"\n ag_name = \"Figs\"\n\n def get_model(self):\n try_import_imodels()\n from imodels import FIGSClassifier, FIGSRegressor\n\n if self.problem_type in [\"regression\", \"softclass\"]:\n return FIGSRegressor\n else:\n return FIGSClassifier\n"
},
{
"path": "tabular/src/autogluon/tabular/models/knn/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/knn/_knn_loo_variants.py",
"content": "import logging\n\nlogger = logging.getLogger(__name__)\n\nimport numpy as np\nfrom scipy import stats\nfrom sklearn.neighbors._base import _get_weights\nfrom sklearn.utils.extmath import weighted_mode\n\n# These are variants of sklearn KNN models which have the ability to calculate unbiased predictions on the training data via leave-one-out (LOO) calculation.\n# TODO: Consider contributing to sklearn officially\n# TODO: This uses private methods in sklearn, could potentially break without warning in future sklearn releases\n# TODO: Code is largely identical to `predict` and `predict_proba` methods, but due to how those methods are coded, we can't call them directly.\n# This means if code within those methods changes, the LOO equivalents may start to become outdated.\n\n__all__ = [\"KNeighborsClassifierLOOMixin\", \"KNeighborsRegressorLOOMixin\"]\n\n\nclass KNeighborsClassifierLOOMixin:\n @staticmethod\n def predict_loo(self):\n \"\"\"Predict the class labels for the training data via leave-one-out.\n\n Returns\n -------\n y : ndarray of shape (n_queries,) or (n_queries, n_outputs)\n Class labels for each training data sample.\n \"\"\"\n\n neigh_dist, neigh_ind = self.kneighbors()\n classes_ = self.classes_\n _y = self._y\n if not self.outputs_2d_:\n _y = self._y.reshape((-1, 1))\n classes_ = [self.classes_]\n\n n_outputs = len(classes_)\n n_queries = len(neigh_dist)\n weights = _get_weights(neigh_dist, self.weights)\n\n y_pred = np.empty((n_queries, n_outputs), dtype=classes_[0].dtype)\n for k, classes_k in enumerate(classes_):\n if weights is None:\n mode, _ = stats.mode(_y[neigh_ind, k], axis=1)\n else:\n mode, _ = weighted_mode(_y[neigh_ind, k], weights, axis=1)\n\n mode = np.asarray(mode.ravel(), dtype=np.intp)\n y_pred[:, k] = classes_k.take(mode)\n\n if not self.outputs_2d_:\n y_pred = y_pred.ravel()\n\n return y_pred\n\n @staticmethod\n def predict_proba_loo(self):\n \"\"\"Return probability estimates for the training data via leave-one-out.\n\n Returns\n -------\n p : ndarray of shape (n_queries, n_classes), or a list of n_outputs\n of such arrays if n_outputs > 1.\n The class probabilities of the training data samples. Classes are ordered\n by lexicographic order.\n \"\"\"\n\n neigh_dist, neigh_ind = self.kneighbors()\n\n classes_ = self.classes_\n _y = self._y\n if not self.outputs_2d_:\n _y = self._y.reshape((-1, 1))\n classes_ = [self.classes_]\n\n n_queries = len(neigh_dist)\n\n weights = _get_weights(neigh_dist, self.weights)\n if weights is None:\n weights = np.ones_like(neigh_ind)\n\n all_rows = np.arange(n_queries)\n probabilities = []\n for k, classes_k in enumerate(classes_):\n pred_labels = _y[:, k][neigh_ind]\n proba_k = np.zeros((n_queries, classes_k.size))\n\n # a simple ':' index doesn't work right\n for i, idx in enumerate(pred_labels.T): # loop is O(n_neighbors)\n proba_k[all_rows, idx] += weights[:, i]\n\n # normalize 'votes' into real [0,1] probabilities\n normalizer = proba_k.sum(axis=1)[:, np.newaxis]\n normalizer[normalizer == 0.0] = 1.0\n proba_k /= normalizer\n\n probabilities.append(proba_k)\n\n if not self.outputs_2d_:\n probabilities = probabilities[0]\n\n return probabilities\n\n\nclass KNeighborsRegressorLOOMixin:\n @staticmethod\n def predict_loo(self):\n \"\"\"Predict the target for the training data via leave-one-out.\n\n Returns\n -------\n y : ndarray of shape (n_queries,) or (n_queries, n_outputs), dtype=int\n Target values.\n \"\"\"\n neigh_dist, neigh_ind = self.kneighbors()\n\n weights = _get_weights(neigh_dist, self.weights)\n\n _y = self._y\n if _y.ndim == 1:\n _y = _y.reshape((-1, 1))\n\n if weights is None:\n y_pred = np.mean(_y[neigh_ind], axis=1)\n else:\n y_pred = np.empty((len(neigh_dist), _y.shape[1]), dtype=np.float64)\n denom = np.sum(weights, axis=1)\n\n for j in range(_y.shape[1]):\n num = np.sum(_y[neigh_ind, j] * weights, axis=1)\n y_pred[:, j] = num / denom\n\n if self._y.ndim == 1:\n y_pred = y_pred.ravel()\n\n return y_pred\n"
},
{
"path": "tabular/src/autogluon/tabular/models/knn/knn_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport math\nimport time\nfrom typing import Dict, Union\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_FLOAT, R_INT, S_BOOL\nfrom autogluon.common.utils.log_utils import fix_sklearnex_logging_if_kaggle\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils.exceptions import NotEnoughMemoryError\nfrom autogluon.core.utils.utils import normalize_pred_probas\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Normalize data!\nclass KNNModel(AbstractModel):\n \"\"\"\n KNearestNeighbors model (scikit-learn): https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html\n \"\"\"\n\n ag_key = \"KNN\"\n ag_name = \"KNeighbors\"\n ag_priority = 100\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._X_unused_index = None # Keeps track of unused training data indices, necessary for LOO OOF generation\n\n def _get_model_type(self):\n if self.params_aux.get(\"use_daal\", True):\n try:\n from sklearnex.neighbors import KNeighborsClassifier, KNeighborsRegressor\n\n fix_sklearnex_logging_if_kaggle() # Fix logging verbosity if in Kaggle notebook environment\n\n # sklearnex backend for KNN seems to be 20-40x+ faster than native sklearn with no downsides.\n logger.log(15, \"\\tUsing sklearnex KNN backend...\")\n except:\n from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor\n else:\n from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor\n if self.problem_type == REGRESSION:\n return KNeighborsRegressor\n else:\n return KNeighborsClassifier\n\n def _preprocess(self, X, **kwargs):\n X = super()._preprocess(X, **kwargs)\n X = X.fillna(0).to_numpy(dtype=np.float32)\n return X\n\n def _set_default_params(self):\n default_params = {\n \"weights\": \"uniform\",\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_INT, R_FLOAT], # TODO: Eventually use category features\n ignored_type_group_special=[S_BOOL],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args(cls) -> dict:\n default_ag_args = super()._get_default_ag_args()\n extra_ag_args = {\n \"valid_stacker\": False,\n }\n default_ag_args.update(extra_ag_args)\n return default_ag_args\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\"use_child_oof\": True}\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n # TODO: Enable HPO for KNN\n def _get_default_searchspace(self):\n spaces = {}\n return spaces\n\n def _fit(self, X, y, num_cpus=-1, time_limit=None, sample_weight=None, **kwargs):\n time_start = time.time()\n X = self.preprocess(X, y=y)\n params = self._get_model_params()\n if \"n_jobs\" not in params:\n params[\"n_jobs\"] = num_cpus\n if sample_weight is not None: # TODO: support\n logger.log(15, \"sample_weight not yet supported for KNNModel, this model will ignore them in training.\")\n\n num_rows_max = len(X)\n # FIXME: v0.1 Must store final num rows for refit_full or else will use everything! Worst case refit_full could train far longer than the original model.\n if time_limit is None or num_rows_max <= 10000:\n self.model = self._get_model_type()(**params).fit(X, y)\n else:\n self.model = self._fit_with_samples(\n X=X, y=y, model_params=params, time_limit=time_limit - (time.time() - time_start)\n )\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n model_size_bytes = 4 * X.shape[0] * X.shape[1] # Assuming float32 types\n expected_final_model_size_bytes = int(\n model_size_bytes * 3.6\n ) # Roughly what can be expected of the final KNN model in memory size\n return expected_final_model_size_bytes\n\n def _validate_fit_memory_usage(\n self,\n mem_error_threshold: float = 0.2,\n mem_warning_threshold: float = 0.15,\n mem_size_threshold: int = 1e7,\n **kwargs,\n ):\n return super()._validate_fit_memory_usage(\n mem_error_threshold=mem_error_threshold,\n mem_warning_threshold=mem_warning_threshold,\n mem_size_threshold=mem_size_threshold,\n **kwargs,\n )\n\n # TODO: Won't work for RAPIDS without modification\n # TODO: Technically isn't OOF, but can be used inplace of OOF. Perhaps rename to something more accurate?\n def predict_proba_oof(self, X, normalize=None, **kwargs):\n \"\"\"X should be the same X passed to `.fit`\"\"\"\n y_oof_pred_proba = self._predict_proba_oof(X=X, **kwargs)\n if normalize is None:\n normalize = self.normalize_pred_probas\n if normalize:\n y_oof_pred_proba = normalize_pred_probas(y_oof_pred_proba, self.problem_type)\n y_oof_pred_proba = y_oof_pred_proba.astype(np.float32)\n return y_oof_pred_proba\n\n def _predict_proba_oof(self, X, **kwargs):\n from ._knn_loo_variants import KNeighborsClassifierLOOMixin, KNeighborsRegressorLOOMixin\n\n if self.problem_type in [BINARY, MULTICLASS]:\n y_oof_pred_proba = KNeighborsClassifierLOOMixin.predict_proba_loo(self.model)\n else:\n y_oof_pred_proba = KNeighborsRegressorLOOMixin.predict_loo(self.model)\n y_oof_pred_proba = self._convert_proba_to_unified_form(y_oof_pred_proba)\n if X is not None and self._X_unused_index:\n X_unused = X.iloc[self._X_unused_index]\n y_pred_proba_new = self.predict_proba(X_unused)\n X_unused_index = set(self._X_unused_index)\n num_rows = len(X)\n X_used_index = [i for i in range(num_rows) if i not in X_unused_index]\n oof_pred_shape = y_oof_pred_proba.shape\n if len(oof_pred_shape) == 1:\n y_oof_tmp = np.zeros(num_rows, dtype=np.float32)\n y_oof_tmp[X_used_index] = y_oof_pred_proba\n y_oof_tmp[self._X_unused_index] = y_pred_proba_new\n else:\n y_oof_tmp = np.zeros((num_rows, oof_pred_shape[1]), dtype=np.float32)\n y_oof_tmp[X_used_index, :] = y_oof_pred_proba\n y_oof_tmp[self._X_unused_index, :] = y_pred_proba_new\n y_oof_pred_proba = y_oof_tmp\n return y_oof_pred_proba\n\n # TODO: Consider making this fully generic and available to all models\n def _fit_with_samples(\n self,\n X,\n y,\n model_params,\n time_limit,\n start_samples=10000,\n max_samples=None,\n sample_growth_factor=2,\n sample_time_growth_factor=8,\n ):\n \"\"\"\n Fit model with samples of the data repeatedly, gradually increasing the amount of data until time_limit is reached or all data is used.\n\n X and y must already be preprocessed.\n\n Parameters\n ----------\n X : np.ndarray\n The training data features (preprocessed).\n y : Series\n The training data ground truth labels.\n time_limit : float, default = None\n Time limit in seconds to adhere to when fitting model.\n start_samples : int, default = 10000\n Number of samples to start with. This will be multiplied by sample_growth_factor after each model fit to determine the next number of samples.\n For example, if start_samples=10000, sample_growth_factor=2, then the number of samples per model fit would be [10000, 20000, 40000, 80000, ...]\n max_samples : int, default = None\n The maximum number of samples to use.\n If None or greater than the number of rows in X, then it is set equal to the number of rows in X.\n sample_growth_factor : float, default = 2\n The rate of growth in sample size between each model fit. If 2, then the sample size doubles after each fit.\n sample_time_growth_factor : float, default = 8\n The multiplier to the expected fit time of the next model. If `sample_time_growth_factor=8` and a model took 10 seconds to train, the next model fit will be expected to take 80 seconds.\n If an expected time is greater than the remaining time in `time_limit`, the model will not be trained and the method will return early.\n \"\"\"\n time_start = time.time()\n\n num_rows_samples = []\n if max_samples is None:\n num_rows_max = len(X)\n else:\n num_rows_max = min(len(X), max_samples)\n num_rows_cur = start_samples\n while True:\n num_rows_cur = min(num_rows_cur, num_rows_max)\n num_rows_samples.append(num_rows_cur)\n if num_rows_cur == num_rows_max:\n break\n num_rows_cur *= sample_growth_factor\n num_rows_cur = math.ceil(num_rows_cur)\n if num_rows_cur * 1.5 >= num_rows_max:\n num_rows_cur = num_rows_max\n\n def sample_func(chunk, frac):\n # Guarantee at least 1 sample (otherwise log_loss would crash or model would return different column counts in pred_proba)\n n = max(math.ceil(len(chunk) * frac), 1)\n return chunk.sample(n=n, replace=False, random_state=self.random_seed)\n\n if self.problem_type != REGRESSION:\n y_df = y.to_frame(name=\"label\").reset_index(drop=True)\n else:\n y_df = None\n\n time_start_sample_loop = time.time()\n time_limit_left = time_limit - (time_start_sample_loop - time_start)\n model_type = self._get_model_type()\n idx = None\n for i, samples in enumerate(num_rows_samples):\n if samples != num_rows_max:\n if self.problem_type == REGRESSION:\n idx = np.random.choice(num_rows_max, size=samples, replace=False)\n else:\n idx = y_df.groupby(\"label\", group_keys=False).apply(sample_func, frac=samples / num_rows_max).index\n X_samp = X[idx, :]\n y_samp = y.iloc[idx]\n else:\n X_samp = X\n y_samp = y\n idx = None\n self.model = model_type(**model_params).fit(X_samp, y_samp)\n time_limit_left_prior = time_limit_left\n time_fit_end_sample = time.time()\n time_limit_left = time_limit - (time_fit_end_sample - time_start)\n time_fit_sample = time_limit_left_prior - time_limit_left\n time_required_for_next = time_fit_sample * sample_time_growth_factor\n logger.log(\n 15,\n f\"\\t{round(time_fit_sample, 2)}s \\t= Train Time (Using {samples}/{num_rows_max} rows) ({round(time_limit_left, 2)}s remaining time)\",\n )\n if time_required_for_next > time_limit_left and i != len(num_rows_samples) - 1:\n logger.log(\n 20,\n f\"\\tNot enough time to train KNN model on all training rows. Fit {samples}/{num_rows_max} rows. (Training KNN model on {num_rows_samples[i + 1]} rows is expected to take {round(time_required_for_next, 2)}s)\",\n )\n break\n if idx is not None:\n idx = set(idx)\n self._X_unused_index = [i for i in range(num_rows_max) if i not in idx]\n return self.model\n\n def _get_maximum_resources(self) -> dict[str, int | float]:\n # use at most 32 cpus to avoid OpenBLAS error: https://github.com/autogluon/autogluon/issues/1020\n # no GPU support\n return {\n \"num_cpus\": 32,\n \"num_gpus\": 0,\n }\n\n def _get_default_resources(self):\n # use at most 32 cpus to avoid OpenBLAS error: https://github.com/autogluon/autogluon/issues/1020\n num_cpus = ResourceManager.get_cpu_count()\n num_gpus = 0\n return num_cpus, num_gpus\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n }\n\n def _more_tags(self):\n return {\n \"valid_oof\": True,\n \"can_refit_full\": True,\n }\n\n\nclass FAISSModel(KNNModel):\n def _get_model_type(self):\n from .knn_utils import FAISSNeighborsClassifier, FAISSNeighborsRegressor\n\n if self.problem_type == REGRESSION:\n return FAISSNeighborsRegressor\n else:\n return FAISSNeighborsClassifier\n\n def _set_default_params(self):\n default_params = {\n \"index_factory_string\": \"Flat\",\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n super()._set_default_params()\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\"use_child_oof\": False}\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n def _more_tags(self):\n return {\"valid_oof\": False}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/knn/knn_rapids_model.py",
"content": "import logging\n\nfrom autogluon.common.utils.try_import import try_import_rapids_cuml\nfrom autogluon.core.constants import REGRESSION\n\nfrom .._utils.rapids_utils import RapidsModelMixin\nfrom .knn_model import KNNModel\n\nlogger = logging.getLogger(__name__)\n\n\n# FIXME: Benchmarks show that CPU KNN can be trained in ~3 seconds with 0.2 second validation time for CoverType on automlbenchmark (m5.2xlarge)\n# This is over 100 seconds validation time on CPU with rapids installed, investigate how it was so fast on CPU.\n# \"2021_02_26/autogluon_hpo_auto.openml_s_271.1h8c.aws.20210228T000327/aws.openml_s_271.1h8c.covertype.0.autogluon_hpo_auto/\"\n# Noticed: different input data types, investigate locally with openml dataset version and dtypes.\n# TODO: Given this is so fast, consider doing rapid feature pruning\nclass KNNRapidsModel(RapidsModelMixin, KNNModel):\n \"\"\"\n RAPIDS KNearestNeighbors model : https://rapids.ai/start.html\n\n NOTE: This code is experimental, it is recommend to not use this unless you are a developer.\n This was tested on rapids-21.06 via:\n\n conda create -n rapids-21.06 -c rapidsai -c nvidia -c conda-forge rapids=21.06 python=3.8 cudatoolkit=11.2\n conda activate rapids-21.06\n pip install --pre autogluon.tabular[all]\n \"\"\"\n\n def _get_model_type(self):\n try_import_rapids_cuml()\n from cuml.neighbors import KNeighborsClassifier, KNeighborsRegressor\n\n if self.problem_type == REGRESSION:\n return KNeighborsRegressor\n else:\n return KNeighborsClassifier\n"
},
{
"path": "tabular/src/autogluon/tabular/models/knn/knn_utils.py",
"content": "import logging\n\nimport numpy as np\nfrom pandas import DataFrame\nfrom scipy.stats import mode\nfrom sklearn.utils.extmath import weighted_mode\n\nfrom autogluon.common.utils.try_import import try_import_faiss\n\nlogger = logging.getLogger(__name__)\n\n\n# Rather than try to import non-public sklearn internals, we implement our own weighting functions here\n# These support the same operations as the sklearn functions - at least as far as possible with FAISS\ndef _check_weights(weights):\n \"\"\"Check to make sure weights are valid\"\"\"\n if weights in (None, \"uniform\", \"distance\"):\n return weights\n elif callable(weights):\n return weights\n else:\n raise ValueError(\"weights not recognized: should be 'uniform', 'distance', or a callable function\")\n\n\ndef _get_weights(dist, weights):\n \"\"\"Get the weights from an array of distances and a parameter weights\"\"\"\n if weights in (None, \"uniform\"):\n return None\n elif weights == \"distance\":\n # if user attempts to classify a point that was zero distance from one\n # or more training points, those training points are weighted as 1.0\n # and the other points as 0.0\n with np.errstate(divide=\"ignore\"):\n dist = 1.0 / dist\n inf_mask = np.isinf(dist)\n inf_row = np.any(inf_mask, axis=1)\n dist[inf_row] = inf_mask[inf_row]\n return dist\n elif callable(weights):\n return weights(dist)\n else:\n raise ValueError(\"weights not recognized: should be 'uniform', 'distance', or a callable function\")\n\n\nclass FAISSNeighborsRegressor:\n def __init__(self, n_neighbors=5, weights=\"uniform\", n_jobs=-1, index_factory_string=\"Flat\"):\n \"\"\"\n Creates a KNN regressor model based on FAISS. FAISS allows you to compose different\n near-neighbor search algorithms from several different preprocessing / search algorithms\n This composition is specified by the string that is passed to the FAISS index_factory.\n Here are good guidelines for choosing the index string:\n https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index\n\n The model itself is a clone of the sklearn one\n \"\"\"\n try_import_faiss()\n import faiss\n\n self.faiss = faiss\n self.index_factory_string = index_factory_string\n self.n_neighbors = n_neighbors\n self.weights = weights\n self.n_jobs = n_jobs\n if n_jobs > 0:\n # global config, affects all faiss indexes\n faiss.omp_set_num_threads(n_jobs)\n\n def fit(self, X, y):\n if isinstance(X, DataFrame):\n X = X.to_numpy(dtype=np.float32)\n else:\n X = X.astype(np.float32)\n if not X.flags[\"C_CONTIGUOUS\"]:\n X = np.ascontiguousarray(X)\n d = X.shape[1]\n self.index = self.faiss.index_factory(d, self.index_factory_string)\n self.y = np.array(y)\n self.index.train(X)\n self.index.add(X)\n return self\n\n def predict(self, X):\n X = X.astype(np.float32)\n X = np.ascontiguousarray(X)\n if X.ndim == 1:\n X = X[np.newaxis]\n D, I = self.index.search(X, self.n_neighbors)\n outputs = np.squeeze(self.y[I])\n\n weights = _get_weights(D, self.weights)\n\n if weights is None:\n y_pred = np.mean(outputs, axis=1)\n else:\n denom = np.sum(weights, axis=1)\n if outputs.ndim == 1:\n y_pred = np.sum(weights * outputs, axis=1)\n y_pred /= denom\n else:\n y_pred = np.sum(weights * outputs, axis=1)\n y_pred /= denom\n\n return y_pred\n\n def __getstate__(self):\n state = {}\n for k, v in self.__dict__.items():\n if (v is not self.index) and (v is not self.faiss):\n state[k] = v\n else:\n state[k] = self.faiss.serialize_index(self.index)\n return state\n\n def __setstate__(self, state):\n try_import_faiss()\n import faiss\n\n self.__dict__.update(state)\n self.faiss = faiss\n self.index = self.faiss.deserialize_index(self.index)\n\n\nclass FAISSNeighborsClassifier:\n def __init__(self, n_neighbors=5, weights=\"uniform\", n_jobs=-1, index_factory_string=\"Flat\"):\n \"\"\"\n Creates a KNN classifier model based on FAISS. FAISS allows you to compose different\n near-neighbor search algorithms from several different preprocessing / search algorithms\n This composition is specified by the string that is passed to the FAISS index_factory.\n Here are good guidelines for choosing the index string:\n https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index\n\n The model itself is a clone of the sklearn one\n \"\"\"\n try_import_faiss()\n import faiss\n\n self.faiss = faiss\n self.index_factory_string = index_factory_string\n self.n_neighbors = n_neighbors\n self.weights = weights\n self.classes = []\n self.n_jobs = n_jobs\n if n_jobs > 0:\n # global config, affects all faiss indexes\n faiss.omp_set_num_threads(n_jobs)\n\n def fit(self, X, y):\n if isinstance(X, DataFrame):\n X = X.to_numpy(dtype=np.float32)\n else:\n X = X.astype(np.float32)\n if not X.flags[\"C_CONTIGUOUS\"]:\n X = np.ascontiguousarray(X)\n d = X.shape[1]\n self.index = self.faiss.index_factory(d, self.index_factory_string)\n self.labels = np.array(y)\n self.index.train(X)\n self.index.add(X)\n self.classes = np.unique(y)\n return self\n\n def predict(self, X):\n X = X.astype(np.float32)\n X = np.ascontiguousarray(X)\n if X.ndim == 1:\n X = X[np.newaxis]\n D, I = self.index.search(X, self.n_neighbors)\n outputs = np.squeeze(self.labels[I])\n weights = _get_weights(D, self.weights)\n if weights is None:\n y_pred, _ = mode(outputs, axis=1)\n else:\n y_pred, _ = weighted_mode(outputs, weights, axis=1)\n return y_pred\n\n def predict_proba(self, X):\n X = X.astype(np.float32)\n X = np.ascontiguousarray(X)\n if X.ndim == 1:\n X = X[np.newaxis]\n D, I = self.index.search(X, self.n_neighbors)\n outputs = np.squeeze(self.labels[I])\n weights = _get_weights(D, self.weights)\n if weights is None:\n weights = np.ones_like(I)\n\n probabilities = np.empty((X.shape[0], len(self.classes)), dtype=np.float64)\n for k, class_k in enumerate(self.classes):\n proba_k = np.sum(np.multiply(outputs == class_k, weights), axis=1)\n probabilities[:, k] = proba_k\n\n normalizer = np.sum(probabilities, axis=1)\n normalizer[normalizer == 0.0] = 1.0\n probabilities /= normalizer[:, np.newaxis]\n return probabilities\n\n def __getstate__(self):\n state = {}\n for k, v in self.__dict__.items():\n if (v is not self.index) and (v is not self.faiss):\n state[k] = v\n else:\n state[k] = self.faiss.serialize_index(self.index)\n return state\n\n def __setstate__(self, state):\n try_import_faiss()\n import faiss\n\n self.__dict__.update(state)\n self.faiss = faiss\n self.index = self.faiss.deserialize_index(self.index)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/callbacks.py",
"content": "import copy\nimport logging\nimport os\nimport time\nimport warnings\nfrom operator import gt, lt\n\nfrom lightgbm.callback import EarlyStopException, _format_eval_result\n\nfrom autogluon.common.utils.lite import disable_if_lite_mode\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.utils.early_stopping import SimpleES\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add option to stop if current run's metric value is X% lower, such as min 30%, current 40% -> Stop\ndef early_stopping_custom(\n stopping_rounds,\n first_metric_only=False,\n metrics_to_use=None,\n start_time=None,\n time_limit=None,\n verbose=True,\n max_diff=None,\n ignore_dart_warning=False,\n manual_stop_file=None,\n train_loss_name=None,\n reporter=None,\n):\n \"\"\"Create a callback that activates early stopping.\n\n Note\n ----\n Activates early stopping.\n The model will train until the validation score stops improving.\n Validation score needs to improve at least every ``early_stopping_rounds`` round(s)\n to continue training.\n Requires at least one validation data and one metric.\n If there's more than one, will check all of them. But the training data is ignored anyway.\n To check only the first metric set ``first_metric_only`` to True.\n\n Parameters\n ----------\n stopping_rounds : int or tuple\n If int, The possible number of rounds without the trend occurrence.\n If tuple, contains early stopping class as first element and class init kwargs as second element.\n first_metric_only : bool, optional (default=False)\n Whether to use only the first metric for early stopping.\n verbose : bool, optional (default=True)\n Whether to print message with early stopping information.\n train_loss_name : str, optional (default=None):\n Name of metric that contains training loss value.\n reporter : optional (default=None):\n reporter object from AutoGluon scheduler.\n\n Returns\n -------\n callback : function\n The callback that activates early stopping.\n \"\"\"\n best_score = []\n best_iter = []\n best_score_list = []\n best_trainloss = [] # stores training losses at corresponding best_iter\n cmp_op = []\n enabled = [True]\n indices_to_check = []\n init_mem_rss = []\n init_mem_avail = []\n es = []\n\n mem_status = ResourceManager.get_process()\n\n def _init(env):\n if not ignore_dart_warning:\n enabled[0] = not any(\n (boost_alias in env.params and env.params[boost_alias] == \"dart\")\n for boost_alias in (\"boosting\", \"boosting_type\", \"boost\")\n )\n if not enabled[0]:\n warnings.warn(\"Early stopping is not available in dart mode\")\n return\n if not env.evaluation_result_list:\n raise ValueError(\"For early stopping, at least one dataset and eval metric is required for evaluation\")\n\n if verbose:\n msg = \"Training until validation scores don't improve for {} rounds.\"\n logger.debug(msg.format(stopping_rounds))\n if manual_stop_file:\n logger.debug(\"Manually stop training by creating file at location: \", manual_stop_file)\n\n if isinstance(stopping_rounds, int):\n es_template = SimpleES(patience=stopping_rounds)\n else:\n es_template = stopping_rounds[0](**stopping_rounds[1])\n\n for eval_ret in env.evaluation_result_list:\n best_iter.append(0)\n best_score_list.append(None)\n best_trainloss.append(None)\n es.append(copy.deepcopy(es_template))\n if eval_ret[3]:\n best_score.append(float(\"-inf\"))\n cmp_op.append(gt)\n else:\n best_score.append(float(\"inf\"))\n cmp_op.append(lt)\n\n if metrics_to_use is None:\n for i in range(len(env.evaluation_result_list)):\n indices_to_check.append(i)\n if first_metric_only:\n break\n else:\n for i, eval in enumerate(env.evaluation_result_list):\n if (eval[0], eval[1]) in metrics_to_use:\n indices_to_check.append(i)\n if first_metric_only:\n break\n\n @disable_if_lite_mode()\n def _init_mem():\n init_mem_rss.append(mem_status.memory_info().rss)\n init_mem_avail.append(ResourceManager.get_available_virtual_mem())\n\n _init_mem()\n\n @disable_if_lite_mode()\n def _mem_early_stop():\n available = ResourceManager.get_available_virtual_mem()\n cur_rss = mem_status.memory_info().rss\n\n if cur_rss < init_mem_rss[0]:\n init_mem_rss[0] = cur_rss\n estimated_model_size_mb = (cur_rss - init_mem_rss[0]) >> 20\n available_mb = available >> 20\n\n if available_mb != 0:\n model_size_memory_ratio = estimated_model_size_mb / available_mb\n else:\n model_size_memory_ratio = 100\n\n if verbose or (model_size_memory_ratio > 0.25):\n logger.debug(\"Available Memory: \" + str(available_mb) + \" MB\")\n logger.debug(\"Estimated Model Size: \" + str(estimated_model_size_mb) + \" MB\")\n\n early_stop = False\n # FIXME: during parallel fits, model only knows its own memory usage and the overall system memory.\n # Because memory usage can spike during saving, OOM can occur if many models finish at the same time and spike in memory at the same time during save.\n # To fix this, we need to provide the per-model memory limit as a constraint passed to this method,\n # so that we can ensure a given model isn't exceeding its portion of the memory budget.\n # Ditto for XGBoost and CatBoost (ex: \"kropt\" dataset with 8-fold bagging and 32 GB memory. Fits 10k iterations on all 8 folds, then goes OOM)\n # We also need to estimate the peak memory usage given the estimated_model_size_mb if we were to save. Otherwise we will go OOM during save anyways.\n if model_size_memory_ratio > 0.66:\n logger.warning(\"Warning: Large GBM model size may cause OOM error if training continues\")\n logger.warning(\"Available Memory: \" + str(available_mb) + \" MB\")\n logger.warning(\"Estimated GBM model size: \" + str(estimated_model_size_mb) + \" MB\")\n early_stop = True\n\n # TODO: We will want to track size of model as well, even if we early stop before OOM, we will still crash when saving if the model is large enough\n if available_mb < 512: # Less than 500 MB\n logger.warning(\"Warning: Low available memory may cause OOM error if training continues\")\n logger.warning(\"Available Memory: \" + str(available_mb) + \" MB\")\n logger.warning(\"Estimated GBM model size: \" + str(estimated_model_size_mb) + \" MB\")\n early_stop = True\n\n if early_stop:\n logger.warning(\n \"Warning: Early stopped GBM model prior to optimal result to avoid OOM error. Please increase available memory to avoid subpar model quality.\"\n )\n logger.log(\n 15,\n \"Early stopping, best iteration is:\\n[%d]\\t%s\"\n % (best_iter[0] + 1, \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[0]])),\n )\n raise EarlyStopException(best_iter[0], best_score_list[0])\n\n def _callback(env):\n if not cmp_op:\n _init(env)\n if not enabled[0]:\n return\n if train_loss_name is not None:\n train_loss_evals = [\n eval for eval in env.evaluation_result_list if eval[0] == \"train_set\" and eval[1] == train_loss_name\n ]\n train_loss_val = train_loss_evals[0][2]\n else:\n train_loss_val = 0.0\n for i in indices_to_check:\n is_best_iter = False\n eval_result = env.evaluation_result_list[i]\n _, eval_metric, score, greater_is_better = eval_result\n if best_score_list[i] is None or cmp_op[i](score, best_score[i]):\n is_best_iter = True\n best_score[i] = score\n best_iter[i] = env.iteration\n best_score_list[i] = env.evaluation_result_list\n best_trainloss[i] = train_loss_val\n if reporter is not None: # Report current best scores for iteration, used in HPO\n if (\n i == indices_to_check[0]\n ): # TODO: documentation needs to note that we assume 0th index is the 'official' validation performance metric.\n if cmp_op[i] == gt:\n validation_perf = score\n else:\n validation_perf = -score\n reporter(\n epoch=env.iteration + 1,\n validation_performance=validation_perf,\n train_loss=best_trainloss[i],\n best_iter_sofar=best_iter[i] + 1,\n best_valperf_sofar=best_score[i],\n eval_metric=eval_metric, # eval_metric here is the stopping_metric from LGBModel\n greater_is_better=greater_is_better,\n )\n early_stop = es[i].update(cur_round=env.iteration, is_best=is_best_iter)\n if early_stop:\n if verbose:\n logger.log(\n 15,\n \"Early stopping, best iteration is:\\n[%d]\\t%s\"\n % (\n best_iter[i] + 1,\n \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[i]]),\n ),\n )\n raise EarlyStopException(best_iter[i], best_score_list[i])\n elif (max_diff is not None) and (abs(score - best_score[i]) > max_diff):\n if verbose:\n logger.debug(\"max_diff breached!\")\n logger.debug(abs(score - best_score[i]))\n logger.log(\n 15,\n \"Early stopping, best iteration is:\\n[%d]\\t%s\"\n % (\n best_iter[i] + 1,\n \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[i]]),\n ),\n )\n raise EarlyStopException(best_iter[i], best_score_list[i])\n if env.iteration == env.end_iteration - 1:\n if verbose:\n logger.log(\n 15,\n \"Did not meet early stopping criterion. Best iteration is:\\n[%d]\\t%s\"\n % (\n best_iter[i] + 1,\n \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[i]]),\n ),\n )\n raise EarlyStopException(best_iter[i], best_score_list[i])\n if verbose:\n logger.debug((env.iteration - best_iter[i], eval_result))\n if manual_stop_file:\n if os.path.exists(manual_stop_file):\n i = indices_to_check[0]\n logger.log(\n 20,\n \"Found manual stop file, early stopping. Best iteration is:\\n[%d]\\t%s\"\n % (\n best_iter[i] + 1,\n \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[i]]),\n ),\n )\n raise EarlyStopException(best_iter[i], best_score_list[i])\n if time_limit:\n time_elapsed = time.time() - start_time\n time_left = time_limit - time_elapsed\n if time_left <= 0:\n i = indices_to_check[0]\n logger.log(\n 20,\n \"\\tRan out of time, early stopping on iteration \"\n + str(env.iteration + 1)\n + \". Best iteration is:\\n\\t[%d]\\t%s\"\n % (\n best_iter[i] + 1,\n \"\\t\".join([_format_eval_result(x, show_stdv=False) for x in best_score_list[i]]),\n ),\n )\n raise EarlyStopException(best_iter[i], best_score_list[i])\n\n # TODO: Add toggle parameter to early_stopping to disable this\n # TODO: Identify optimal threshold values for early_stopping based on lack of memory\n if env.iteration % 10 == 0:\n _mem_early_stop()\n\n _callback.order = 30\n return _callback\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/hyperparameters/parameters.py",
"content": "\"\"\"Default (fixed) hyperparameter values used in Gradient Boosting model.\"\"\"\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\n\nDEFAULT_NUM_BOOST_ROUND = 10000 # default for single training run\n\n\ndef get_lgb_objective(problem_type):\n return {\n BINARY: \"binary\",\n MULTICLASS: \"multiclass\",\n QUANTILE: \"quantile\",\n REGRESSION: \"regression\",\n SOFTCLASS: \"multiclass\",\n }[problem_type]\n\n\ndef get_param_baseline(problem_type):\n if problem_type == BINARY:\n return get_param_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_param_multiclass_baseline()\n elif problem_type == REGRESSION:\n return get_param_regression_baseline()\n elif problem_type == SOFTCLASS:\n return get_param_softclass_baseline()\n else:\n return get_param_binary_baseline()\n\n\ndef get_param_binary_baseline():\n params = {\n \"learning_rate\": 0.05,\n }\n return params\n\n\ndef get_param_multiclass_baseline():\n params = {\n \"learning_rate\": 0.05,\n }\n return params\n\n\ndef get_param_regression_baseline():\n params = {\n \"learning_rate\": 0.05,\n }\n return params\n\n\ndef get_param_softclass_baseline():\n params = get_param_multiclass_baseline()\n params.pop(\"metric\", None)\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/hyperparameters/searchspaces.py",
"content": "\"\"\"Default hyperparameter search spaces used in Gradient Boosting model\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\n\n\ndef get_default_searchspace(problem_type):\n if problem_type == BINARY:\n return get_searchspace_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_searchspace_multiclass_baseline()\n elif problem_type == REGRESSION:\n return get_searchspace_regression_baseline()\n else:\n return get_searchspace_binary_baseline()\n\n\ndef get_searchspace_multiclass_baseline():\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"feature_fraction\": space.Real(lower=0.75, upper=1.0, default=1.0),\n \"min_data_in_leaf\": space.Int(\n lower=2, upper=60, default=20\n ), # TODO: Use size of dataset to set upper, if row count is small upper should be small\n \"num_leaves\": space.Int(\n lower=16, upper=96, default=31\n ), # TODO: Use row count and feature count to set this, the higher feature count the higher num_leaves upper\n # TODO: Bin size max increase\n }\n return params\n\n\ndef get_searchspace_binary_baseline():\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"feature_fraction\": space.Real(lower=0.75, upper=1.0, default=1.0),\n \"min_data_in_leaf\": space.Int(lower=2, upper=60, default=20),\n \"num_leaves\": space.Int(lower=16, upper=96, default=31),\n }\n return params\n\n\ndef get_searchspace_regression_baseline():\n params = {\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.05, log=True),\n \"feature_fraction\": space.Real(lower=0.75, upper=1.0, default=1.0),\n \"min_data_in_leaf\": space.Int(lower=2, upper=60, default=20),\n \"num_leaves\": space.Int(lower=16, upper=96, default=31),\n }\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/lgb_model.py",
"content": "from __future__ import annotations\n\nimport gc\nimport logging\nimport os\nimport re\nimport time\nimport warnings\nfrom types import MappingProxyType\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_lightgbm\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.models._utils import get_early_stopping_rounds\n\nfrom . import lgb_utils\nfrom .hyperparameters.parameters import DEFAULT_NUM_BOOST_ROUND, get_lgb_objective, get_param_baseline\nfrom .hyperparameters.searchspaces import get_default_searchspace\nfrom .lgb_utils import construct_dataset, train_lgb_model\n\nwarnings.filterwarnings(\n \"ignore\", category=UserWarning, message=\"Starting from version\"\n) # lightGBM brew libomp warning\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, message=\"Dask dataframe query\") # lightGBM dask-expr warning\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Save dataset to binary and reload for HPO. This will avoid the memory spike overhead when training each model and instead it will only occur once upon saving the dataset.\nclass LGBModel(AbstractModel):\n \"\"\"\n LightGBM model: https://lightgbm.readthedocs.io/en/latest/\n\n Hyperparameter options: https://lightgbm.readthedocs.io/en/latest/Parameters.html\n\n Extra hyperparameter options:\n ag.early_stop : int, specifies the early stopping rounds. Defaults to an adaptive strategy. Recommended to keep default.\n \"\"\"\n\n ag_key = \"GBM\"\n ag_name = \"LightGBM\"\n ag_priority = 90\n ag_priority_by_problem_type = MappingProxyType({SOFTCLASS: 100})\n seed_name = \"seed\"\n seed_name_alt = [\"seed_value\", \"random_seed\", \"random_state\"]\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n self._features_internal_map = None\n self._requires_remap = None\n self._features_internal_lgbm = None\n\n def _set_default_params(self):\n default_params = get_param_baseline(problem_type=self.problem_type)\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(problem_type=self.problem_type)\n\n # Use specialized LightGBM metric if available (fast), otherwise use custom func generator\n def _get_stopping_metric_internal(self):\n stopping_metric = lgb_utils.convert_ag_metric_to_lgbm(\n ag_metric_name=self.stopping_metric.name, problem_type=self.problem_type\n )\n if stopping_metric is None:\n stopping_metric = lgb_utils.func_generator(\n metric=self.stopping_metric,\n is_higher_better=True,\n needs_pred_proba=not self.stopping_metric.needs_pred,\n problem_type=self.problem_type,\n )\n stopping_metric_name = self.stopping_metric.name\n else:\n stopping_metric_name = stopping_metric\n return stopping_metric, stopping_metric_name\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n # FIXME: Don't use `hyperparameters.get(\"max_bins\", 255)`, instead get the defaults all at once!\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: DataFrame,\n hyperparameters: dict = None,\n num_classes: int = 1,\n **kwargs,\n ) -> int:\n \"\"\"\n Returns the expected peak memory usage in bytes of the LightGBM model during fit.\n\n The memory usage of LightGBM is primarily made up of three sources:\n\n 1. The size of the data\n 2. The size of the histogram cache\n Scales roughly by 5100*num_features*num_leaves bytes\n For 10000 features and 128 num_leaves, the histogram would be 6.5 GB.\n 3. The size of the model\n Scales linearly with the number of estimators, number of classes, and number of leaves.\n Memory usage peaks during model saving, with the peak consuming approximately 2-4x the size of the model in memory.\n \"\"\"\n data_mem_usage = get_approximate_df_mem_usage(X).sum()\n return cls._estimate_memory_usage_common(\n num_features=X.shape[1],\n data_mem_usage=data_mem_usage,\n hyperparameters=hyperparameters,\n num_classes=num_classes,\n )\n\n @classmethod\n def _estimate_memory_usage_static_lite(\n cls,\n num_samples: int,\n num_features: int,\n num_bytes_per_cell: float = 4,\n hyperparameters: dict = None,\n num_classes: int = 1,\n **kwargs,\n ) -> int:\n data_mem_usage = num_samples * num_features * num_bytes_per_cell\n return cls._estimate_memory_usage_common(\n num_features=num_features,\n data_mem_usage=data_mem_usage,\n hyperparameters=hyperparameters,\n num_classes=num_classes,\n )\n\n @classmethod\n def _estimate_memory_usage_common(\n cls,\n num_features: int,\n data_mem_usage: int | float,\n hyperparameters: dict | None = None,\n num_classes: int = 1,\n ) -> int:\n \"\"\"\n Utility method to avoid code duplication\n \"\"\"\n if hyperparameters is None:\n hyperparameters = {}\n num_classes = (\n num_classes if num_classes else 1\n ) # num_classes could be None after initialization if it's a regression problem\n data_mem_usage_bytes = (\n data_mem_usage * 5 + data_mem_usage / 4 * num_classes\n ) # TODO: Extremely crude approximation, can be vastly improved\n\n n_trees_per_estimator = num_classes if num_classes > 2 else 1\n\n max_bins = hyperparameters.get(\"max_bins\", 255)\n num_leaves = hyperparameters.get(\"num_leaves\", 31)\n # Memory usage of histogram based on https://github.com/microsoft/LightGBM/issues/562#issuecomment-304524592\n histogram_mem_usage_bytes = 20 * max_bins * num_features * num_leaves\n histogram_mem_usage_bytes_max = hyperparameters.get(\"histogram_pool_size\", None)\n if histogram_mem_usage_bytes_max is not None:\n histogram_mem_usage_bytes_max *= 1e6 # Convert megabytes to bytes, `histogram_pool_size` is in MB.\n if histogram_mem_usage_bytes > histogram_mem_usage_bytes_max:\n histogram_mem_usage_bytes = histogram_mem_usage_bytes_max\n histogram_mem_usage_bytes *= 1.2 # Add a 20% buffer\n\n mem_size_per_estimator = n_trees_per_estimator * num_leaves * 100 # very rough estimate\n n_estimators = hyperparameters.get(\"num_boost_round\", DEFAULT_NUM_BOOST_ROUND)\n n_estimators_min = min(n_estimators, 5000)\n mem_size_estimators = (\n n_estimators_min * mem_size_per_estimator\n ) # memory estimate after fitting up to 5000 estimators\n\n approx_mem_size_req = data_mem_usage_bytes + histogram_mem_usage_bytes + mem_size_estimators\n return int(approx_mem_size_req)\n\n def _fit(\n self,\n X,\n y,\n X_val=None,\n y_val=None,\n time_limit=None,\n num_gpus=0,\n num_cpus=0,\n sample_weight=None,\n sample_weight_val=None,\n verbosity=2,\n **kwargs,\n ):\n try_import_lightgbm() # raise helpful error message if LightGBM isn't installed\n start_time = time.time()\n ag_params = self._get_ag_params()\n params = self._get_model_params()\n generate_curves = ag_params.get(\"generate_curves\", False)\n\n if generate_curves:\n X_test = kwargs.get(\"X_test\", None)\n y_test = kwargs.get(\"y_test\", None)\n else:\n X_test = None\n y_test = None\n\n if verbosity <= 1:\n log_period = False\n elif verbosity == 2:\n log_period = 1000\n elif verbosity == 3:\n log_period = 50\n else:\n log_period = 1\n\n stopping_metric, stopping_metric_name = self._get_stopping_metric_internal()\n\n num_boost_round = params.pop(\"num_boost_round\", DEFAULT_NUM_BOOST_ROUND)\n dart_retrain = params.pop(\n \"dart_retrain\", False\n ) # Whether to retrain the model to get optimal iteration if model is trained in 'dart' mode.\n if num_gpus != 0:\n if \"device\" not in params:\n # TODO: lightgbm must have a special install to support GPU: https://github.com/Microsoft/LightGBM/tree/master/python-package#build-gpu-version\n # Before enabling GPU, we should add code to detect that GPU-enabled version is installed and that a valid GPU exists.\n # GPU training heavily alters accuracy, often in a negative manner. We will have to be careful about when to use GPU.\n params[\"device\"] = \"gpu\"\n logger.log(\n 20,\n f\"\\tWarning: Training LightGBM with GPU. This may negatively impact model quality compared to CPU training.\",\n )\n logger.log(15, f\"\\tFitting {num_boost_round} rounds... Hyperparameters: {params}\")\n\n if \"num_threads\" not in params:\n params[\"num_threads\"] = num_cpus\n if \"objective\" not in params:\n params[\"objective\"] = get_lgb_objective(problem_type=self.problem_type)\n if self.problem_type in [MULTICLASS, SOFTCLASS] and \"num_classes\" not in params:\n params[\"num_classes\"] = self.num_classes\n if \"verbose\" not in params:\n params[\"verbose\"] = -1\n\n num_rows_train = len(X)\n dataset_train, dataset_val, dataset_test = self.generate_datasets(\n X=X,\n y=y,\n params=params,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n sample_weight=sample_weight,\n sample_weight_val=sample_weight_val,\n )\n gc.collect()\n\n callbacks = []\n valid_names = []\n valid_sets = []\n if dataset_val is not None:\n from .callbacks import early_stopping_custom\n\n # TODO: Better solution: Track trend to early stop when score is far worse than best score, or score is trending worse over time\n early_stopping_rounds = ag_params.get(\"early_stop\", \"adaptive\")\n if isinstance(early_stopping_rounds, (str, tuple, list)):\n early_stopping_rounds = self._get_early_stopping_rounds(\n num_rows_train=num_rows_train, strategy=early_stopping_rounds\n )\n if early_stopping_rounds is None:\n early_stopping_rounds = 999999\n reporter = kwargs.get(\"reporter\", None)\n train_loss_name = self._get_train_loss_name() if reporter is not None else None\n if train_loss_name is not None:\n if \"metric\" not in params or params[\"metric\"] == \"\":\n params[\"metric\"] = train_loss_name\n elif train_loss_name not in params[\"metric\"]:\n params[\"metric\"] = f\"{params['metric']},{train_loss_name}\"\n # early stopping callback will be added later by QuantileBooster if problem_type==QUANTILE\n early_stopping_callback_kwargs = dict(\n stopping_rounds=early_stopping_rounds,\n metrics_to_use=[(\"valid_set\", stopping_metric_name)],\n max_diff=None,\n start_time=start_time,\n time_limit=time_limit,\n ignore_dart_warning=True,\n verbose=False,\n manual_stop_file=False,\n reporter=reporter,\n train_loss_name=train_loss_name,\n )\n callbacks += [\n # Note: Don't use self.params_aux['max_memory_usage_ratio'] here as LightGBM handles memory per iteration optimally. # TODO: Consider using when ratio < 1.\n early_stopping_custom(**early_stopping_callback_kwargs)\n ]\n valid_names = [\"valid_set\"] + valid_names\n valid_sets = [dataset_val] + valid_sets\n else:\n early_stopping_callback_kwargs = None\n\n from lightgbm.callback import log_evaluation, record_evaluation\n\n if log_period is not None:\n callbacks.append(log_evaluation(period=log_period))\n\n train_params = {\n \"params\": params,\n \"train_set\": dataset_train,\n \"num_boost_round\": num_boost_round,\n \"valid_names\": valid_names,\n \"valid_sets\": valid_sets,\n \"callbacks\": callbacks,\n \"keep_training_booster\": generate_curves,\n }\n\n if generate_curves:\n scorers = ag_params.get(\"curve_metrics\", [self.eval_metric])\n use_curve_metric_error = ag_params.get(\"use_error_for_curve_metrics\", False)\n metric_names = [scorer.name for scorer in scorers]\n\n if stopping_metric_name in metric_names:\n idx = metric_names.index(stopping_metric_name)\n scorers[idx].name = f\"_{stopping_metric_name}\"\n metric_names[idx] = scorers[idx].name\n\n custom_metrics = [\n lgb_utils.func_generator(\n metric=scorer,\n is_higher_better=scorer.greater_is_better_internal,\n needs_pred_proba=not scorer.needs_pred,\n problem_type=self.problem_type,\n error=use_curve_metric_error,\n )\n for scorer in scorers\n ]\n\n eval_results = {}\n train_params[\"callbacks\"].append(record_evaluation(eval_results))\n train_params[\"feval\"] = custom_metrics\n\n if dataset_test is not None:\n train_params[\"valid_names\"] = [\"train_set\", \"test_set\"] + train_params[\"valid_names\"]\n train_params[\"valid_sets\"] = [dataset_train, dataset_test] + train_params[\"valid_sets\"]\n else:\n train_params[\"valid_names\"] = [\"train_set\"] + train_params[\"valid_names\"]\n train_params[\"valid_sets\"] = [dataset_train] + train_params[\"valid_sets\"]\n\n # NOTE: lgb stops based on first metric if more than one\n if not isinstance(stopping_metric, str):\n if generate_curves:\n train_params[\"feval\"].insert(0, stopping_metric)\n else:\n train_params[\"feval\"] = stopping_metric\n elif isinstance(stopping_metric, str):\n if \"metric\" not in train_params[\"params\"] or train_params[\"params\"][\"metric\"] == \"\":\n train_params[\"params\"][\"metric\"] = stopping_metric\n elif stopping_metric not in train_params[\"params\"][\"metric\"]:\n train_params[\"params\"][\"metric\"] = f\"{stopping_metric},{train_params['params']['metric']}\"\n\n if self.problem_type == SOFTCLASS:\n train_params[\"params\"][\"objective\"] = lgb_utils.softclass_lgbobj\n train_params[\"params\"][\"num_classes\"] = self.num_classes\n elif self.problem_type == QUANTILE:\n train_params[\"params\"][\"quantile_levels\"] = self.quantile_levels\n\n # Train LightGBM model:\n # Note that self.model contains a not a LightBGMClassifier or LightGBMRegressor object\n from lightgbm.basic import LightGBMError\n\n with warnings.catch_warnings():\n # Filter harmless warnings introduced in lightgbm 3.0, future versions plan to remove: https://github.com/microsoft/LightGBM/issues/3379\n warnings.filterwarnings(\"ignore\", message=\"Overriding the parameters from Reference Dataset.\")\n warnings.filterwarnings(\"ignore\", message=\"categorical_column in param dict is overridden.\")\n try:\n self.model = train_lgb_model(\n early_stopping_callback_kwargs=early_stopping_callback_kwargs, **train_params\n )\n except LightGBMError:\n if train_params[\"params\"].get(\"device\", \"cpu\") not in [\"gpu\", \"cuda\"]:\n raise\n else:\n if train_params[\"params\"][\"device\"] == \"gpu\":\n logger.warning(\n \"Warning: GPU mode might not be installed for LightGBM, \"\n \"GPU training raised an exception. Falling back to CPU training...\"\n \"Refer to LightGBM GPU documentation: \"\n \"https://github.com/Microsoft/LightGBM/tree/master/python-package#build-gpu-version\"\n \"One possible method is:\"\n \"\\tpip uninstall lightgbm -y\"\n \"\\tpip install lightgbm --install-option=--gpu\"\n )\n elif train_params[\"params\"][\"device\"] == \"cuda\":\n # Current blocker for using CUDA over GPU: https://github.com/microsoft/LightGBM/issues/6828\n # Note that device=\"cuda\" works if AutoGluon (and therefore LightGBM) is installed via conda.\n logger.warning(\n \"Warning: CUDA mode might not be installed for LightGBM, \"\n \"CUDA training raised an exception. Falling back to CPU training...\"\n \"Refer to LightGBM CUDA documentation: \"\n \"https://github.com/Microsoft/LightGBM/tree/master/python-package#build-cuda-version\"\n )\n train_params[\"params\"][\"device\"] = \"cpu\"\n self.model = train_lgb_model(\n early_stopping_callback_kwargs=early_stopping_callback_kwargs, **train_params\n )\n retrain = False\n if train_params[\"params\"].get(\"boosting_type\", \"\") == \"dart\":\n if dataset_val is not None and dart_retrain and (self.model.best_iteration != num_boost_round):\n retrain = True\n if time_limit is not None:\n time_left = time_limit + start_time - time.time()\n if time_left < 0.5 * time_limit:\n retrain = False\n if retrain:\n logger.log(15, f\"Retraining LGB model to optimal iterations ('dart' mode).\")\n train_params.pop(\"callbacks\", None)\n train_params.pop(\"valid_sets\", None)\n train_params.pop(\"valid_names\", None)\n train_params[\"num_boost_round\"] = self.model.best_iteration\n self.model = train_lgb_model(**train_params)\n else:\n logger.log(15, f\"Not enough time to retrain LGB model ('dart' mode)...\")\n\n if generate_curves:\n\n def og_name(key):\n if key == f\"_{stopping_metric_name}\":\n return stopping_metric_name\n return key\n\n def filter(d, keys):\n return {og_name(key): d[key] for key in keys if key in d}\n\n curves = {\"train\": filter(eval_results[\"train_set\"], metric_names)}\n if X_val is not None:\n curves[\"val\"] = filter(eval_results[\"valid_set\"], metric_names)\n if X_test is not None:\n curves[\"test\"] = filter(eval_results[\"test_set\"], metric_names)\n\n if f\"_{stopping_metric_name}\" in metric_names:\n idx = metric_names.index(f\"_{stopping_metric_name}\")\n metric_names[idx] = stopping_metric_name\n\n self.save_learning_curves(metrics=metric_names, curves=curves)\n\n if dataset_val is not None and not retrain:\n self.params_trained[\"num_boost_round\"] = self.model.best_iteration\n else:\n self.params_trained[\"num_boost_round\"] = self.model.current_iteration()\n\n def _predict_proba(self, X, num_cpus=0, **kwargs) -> np.ndarray:\n X = self.preprocess(X, **kwargs)\n\n y_pred_proba = self.model.predict(X, num_threads=num_cpus)\n return self._post_process_predictions(y_pred_proba=y_pred_proba)\n\n def _post_process_predictions(self, y_pred_proba) -> np.ndarray:\n if self.problem_type == QUANTILE:\n # y_pred_proba is a pd.DataFrame, need to convert\n y_pred_proba = y_pred_proba.to_numpy()\n if self.problem_type in [REGRESSION, QUANTILE, MULTICLASS]:\n return y_pred_proba\n elif self.problem_type == BINARY:\n if len(y_pred_proba.shape) == 1:\n return y_pred_proba\n elif y_pred_proba.shape[1] > 1:\n return y_pred_proba[:, 1]\n else:\n return y_pred_proba\n elif self.problem_type == SOFTCLASS: # apply softmax\n y_pred_proba = np.exp(y_pred_proba)\n y_pred_proba = np.multiply(y_pred_proba, 1 / np.sum(y_pred_proba, axis=1)[:, np.newaxis])\n return y_pred_proba\n else:\n if len(y_pred_proba.shape) == 1:\n return y_pred_proba\n elif y_pred_proba.shape[1] > 2: # Should this ever happen?\n return y_pred_proba\n else: # Should this ever happen?\n return y_pred_proba[:, 1]\n\n @staticmethod\n def _clean_column_name_for_lgb(column_name):\n \"\"\"Clean column names while keeping most semantic meaning.\"\"\"\n if not isinstance(column_name, str):\n return column_name\n for symbol in ['\"', \",\", \":\", \"{\", \"}\", \"[\", \"]\"]:\n column_name = column_name.replace(symbol, \"_\")\n return column_name\n\n @classmethod\n def _rename_columns(cls, features: list) -> dict:\n \"\"\"\n Generate a deterministic, one-to-one mapping from original feature names to\n LightGBM-safe, unique column names.\n\n This method:\n - Cleans feature names using `_clean_column_name_for_lgb`\n - Resolves naming collisions by appending numeric suffixes (`_2`, `_3`, ...)\n - Guarantees that all output column names are unique\n - Guarantees a strict 1-to-1 mapping between input features and output names\n\n The mapping is deterministic with respect to input order. If two or more\n features clean to the same base name, the first occurrence keeps the base\n name and subsequent occurrences receive incrementing suffixes.\n\n Parameters\n ----------\n features : list\n List of feature names. All entries must be unique under Python equality\n semantics (e.g., `\"a\"` and `\"a\"` or `1` and `True` are considered duplicates).\n\n Returns\n -------\n dict\n Mapping from original feature name to a unique, cleaned column name\n suitable for use in LightGBM.\n\n Raises\n ------\n ValueError\n If `features` contains duplicate entries, since a dictionary cannot\n represent a one-to-one mapping in that case.\n\n \"\"\"\n if len(features) != len(set(features)):\n raise ValueError(\"features contains duplicates; cannot create 1-to-1 mapping with a dict.\")\n\n unique_features = set()\n features_map = {}\n for feature in features:\n cleaned_feature = cls._clean_column_name_for_lgb(feature)\n\n unique_feature = cleaned_feature\n if unique_feature in unique_features:\n is_unique = False\n count = 2\n while not is_unique:\n unique_feature = f\"{cleaned_feature}_{count}\"\n if unique_feature not in unique_features:\n is_unique = True\n else:\n count += 1\n unique_features.add(unique_feature)\n features_map[feature] = unique_feature\n return features_map\n\n def _preprocess_nonadaptive(self, X: pd.DataFrame, is_train: bool = False, **kwargs):\n X = super()._preprocess_nonadaptive(X=X, **kwargs)\n\n if is_train:\n self._requires_remap = False\n for column in X.columns:\n if isinstance(column, str):\n new_column = re.sub(r'[\",:{}[\\]]', \"\", column)\n if new_column != column:\n self._requires_remap = True\n break\n if self._requires_remap:\n self._features_internal_map = self._rename_columns(features=list(X.columns))\n self._features_internal_lgbm = [self._features_internal_map[feature] for feature in list(X.columns)]\n\n if not self._requires_remap:\n return X\n\n X_new = X.copy(deep=False)\n X_new.columns = self._features_internal_lgbm\n\n # Update feature metadata\n if is_train:\n new_feature_metadata = self._feature_metadata.rename_features(self._features_internal_map)\n self._preprocess_set_features_internal(X=X_new, feature_metadata=new_feature_metadata)\n\n return X_new\n\n def generate_datasets(\n self,\n X: DataFrame,\n y: Series,\n params: dict,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n sample_weight=None,\n sample_weight_val=None,\n sample_weight_test=None,\n save=False,\n init_train=None,\n init_val=None,\n init_test=None,\n ):\n lgb_dataset_params_keys = [\"two_round\"] # Keys that are specific to lightGBM Dataset object construction.\n data_params = {key: params[key] for key in lgb_dataset_params_keys if key in params}.copy()\n\n X = self.preprocess(X, y=y, is_train=True)\n if X_val is not None:\n X_val = self.preprocess(X_val)\n if X_test is not None:\n X_test = self.preprocess(X_test)\n # TODO: Try creating multiple Datasets for subsets of features, then combining with Dataset.add_features_from(), this might avoid memory spike\n\n y_og = None\n y_val_og = None\n y_test_og = None\n if self.problem_type == SOFTCLASS:\n y_og = np.array(y)\n y = None\n if X_val is not None:\n y_val_og = np.array(y_val)\n y_val = None\n if X_test is not None:\n y_test_og = np.array(y_test)\n y_test = None\n\n # X, W_train = self.convert_to_weight(X=X)\n dataset_train = construct_dataset(\n x=X,\n y=y,\n location=os.path.join(\"self.path\", \"datasets\", \"train\"),\n params=data_params,\n save=save,\n weight=sample_weight,\n init_score=init_train,\n )\n # dataset_train = construct_dataset_lowest_memory(X=X, y=y, location=self.path + 'datasets/train', params=data_params)\n if X_val is not None:\n # X_val, W_val = self.convert_to_weight(X=X_val)\n dataset_val = construct_dataset(\n x=X_val,\n y=y_val,\n location=os.path.join(self.path, \"datasets\", \"val\"),\n reference=dataset_train,\n params=data_params,\n save=save,\n weight=sample_weight_val,\n init_score=init_val,\n )\n # dataset_val = construct_dataset_lowest_memory(X=X_val, y=y_val, location=self.path + 'datasets/val', reference=dataset_train, params=data_params)\n else:\n dataset_val = None\n\n if X_test is not None:\n dataset_test = construct_dataset(\n x=X_test,\n y=y_test,\n location=os.path.join(self.path, \"datasets\", \"test\"),\n reference=dataset_train,\n params=data_params,\n save=save,\n weight=sample_weight_test,\n init_score=init_test,\n )\n else:\n dataset_test = None\n\n if self.problem_type == SOFTCLASS:\n if y_og is not None:\n dataset_train.softlabels = y_og\n if y_val_og is not None:\n dataset_val.softlabels = y_val_og\n if y_test_og is not None:\n dataset_test.softlabels = y_test_og\n return dataset_train, dataset_val, dataset_test\n\n def _get_train_loss_name(self):\n if self.problem_type == BINARY:\n train_loss_name = \"binary_logloss\"\n elif self.problem_type == MULTICLASS:\n train_loss_name = \"multi_logloss\"\n elif self.problem_type == REGRESSION:\n train_loss_name = \"l2\"\n else:\n raise ValueError(f\"unknown problem_type for LGBModel: {self.problem_type}\")\n return train_loss_name\n\n def _get_early_stopping_rounds(self, num_rows_train, strategy=\"auto\"):\n return get_early_stopping_rounds(num_rows_train=num_rows_train, strategy=strategy)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @staticmethod\n def _is_gpu_lgbm_installed():\n # Taken from https://github.com/microsoft/LightGBM/issues/3939\n try_import_lightgbm()\n import lightgbm\n\n rng = np.random.RandomState(42)\n data = rng.rand(25, 2)\n label = rng.randint(2, size=25)\n\n try:\n train_data = lightgbm.Dataset(data, label=label)\n params = {\n \"device\": \"gpu\",\n \"verbose\": -1,\n }\n gbm = lightgbm.train(params, num_boost_round=10, train_set=train_data)\n return True\n except Exception as e:\n return False\n\n @staticmethod\n def _is_cuda_lgbm_installed():\n # Taken from https://github.com/microsoft/LightGBM/issues/3939\n try_import_lightgbm()\n import lightgbm\n\n rng = np.random.RandomState(42)\n data = rng.rand(25, 2)\n label = rng.randint(2, size=25)\n\n try:\n train_data = lightgbm.Dataset(data, label=label)\n params = {\n \"device\": \"cuda\",\n \"verbose\": -1,\n }\n gbm = lightgbm.train(params, num_boost_round=10, train_set=train_data)\n return True\n except Exception as e:\n return False\n\n def get_minimum_resources(self, is_gpu_available=False):\n minimum_resources = {\n \"num_cpus\": 1,\n }\n if is_gpu_available:\n minimum_resources[\"num_gpus\"] = 0.5\n return minimum_resources\n\n def _get_default_resources(self):\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\", \"softclass\"]\n\n def _ag_params(self) -> set:\n return {\"early_stop\", \"generate_curves\", \"curve_metrics\", \"use_error_for_curve_metrics\"}\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"can_estimate_memory_usage_static_lite\": True,\n \"supports_learning_curves\": True,\n }\n\n def _more_tags(self):\n # `can_refit_full=True` because num_boost_round is communicated at end of `_fit`\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lgb/lgb_utils.py",
"content": "import copy\nimport os\nimport time\nfrom typing import Optional\n\nimport numpy as np\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.utils.try_import import try_import_lightgbm\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\n\n# Mapping to specialized LightGBM metrics that are much faster than the standard metric computation\n_ag_to_lgbm_metric_dict = {\n BINARY: dict(\n accuracy=\"binary_error\",\n log_loss=\"binary_logloss\",\n roc_auc=\"auc\",\n ),\n MULTICLASS: dict(\n accuracy=\"multi_error\",\n log_loss=\"multi_logloss\",\n ),\n QUANTILE: dict(\n pinball_loss=\"quantile\",\n ),\n REGRESSION: dict(\n mean_absolute_error=\"l1\",\n mean_squared_error=\"l2\",\n root_mean_squared_error=\"rmse\",\n ),\n}\n\n\ndef convert_ag_metric_to_lgbm(ag_metric_name, problem_type):\n return _ag_to_lgbm_metric_dict.get(problem_type, dict()).get(ag_metric_name, None)\n\n\ndef func_generator(metric, is_higher_better, needs_pred_proba, problem_type, error=False):\n if error:\n is_higher_better = False\n\n compute = metric.error if error else metric\n if problem_type in [REGRESSION, QUANTILE]:\n # TODO: Might not work for custom quantile metrics\n def function_template(y_hat, data):\n y_true = data.get_label()\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n elif needs_pred_proba:\n if problem_type == MULTICLASS:\n\n def function_template(y_hat, data):\n y_true = data.get_label()\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n elif problem_type == SOFTCLASS: # metric must take in soft labels array, like soft_log_loss\n\n def function_template(y_hat, data):\n y_true = data.softlabels\n y_hat = y_hat.reshape(y_true.shape[1], -1).T\n y_hat = np.exp(y_hat)\n y_hat = np.multiply(y_hat, 1 / np.sum(y_hat, axis=1)[:, np.newaxis])\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n else:\n\n def function_template(y_hat, data):\n y_true = data.get_label()\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n else:\n if problem_type == MULTICLASS:\n\n def function_template(y_hat, data):\n y_true = data.get_label()\n y_hat = y_hat.argmax(axis=1)\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n else:\n\n def function_template(y_hat, data):\n y_true = data.get_label()\n y_hat = np.round(y_hat)\n return metric.name, compute(y_true, y_hat), is_higher_better\n\n # allows lgb library to output autogluon metric name in the evaluation logs\n function_template.__name__ = metric.name\n\n return function_template\n\n\ndef softclass_lgbobj(preds, train_data):\n \"\"\"Custom LightGBM loss function for soft (probabilistic, vector-valued) class-labels only,\n which have been appended to lgb.Dataset (train_data) as additional \".softlabels\" attribute (2D numpy array).\n \"\"\"\n softlabels = train_data.softlabels\n num_classes = softlabels.shape[1]\n preds = np.reshape(preds, (len(softlabels), num_classes), order=\"F\")\n preds = np.exp(preds)\n preds = np.multiply(preds, 1 / np.sum(preds, axis=1)[:, np.newaxis])\n grad = preds - softlabels\n hess = 2.0 * preds * (1.0 - preds)\n return grad.flatten(\"F\"), hess.flatten(\"F\")\n\n\ndef construct_dataset(\n x: DataFrame, y: Series, location=None, reference=None, params=None, save=False, weight=None, init_score=None\n):\n try_import_lightgbm()\n import lightgbm as lgb\n\n dataset = lgb.Dataset(\n data=x, label=y, reference=reference, free_raw_data=True, params=params, weight=weight, init_score=init_score\n )\n\n if save:\n assert location is not None\n saving_path = f\"{location}.bin\"\n if os.path.exists(saving_path):\n os.remove(saving_path)\n\n os.makedirs(os.path.dirname(saving_path), exist_ok=True)\n dataset.save_binary(saving_path)\n # dataset_binary = lgb.Dataset(location + '.bin', reference=reference, free_raw_data=False)# .construct()\n\n return dataset\n\n\ndef train_lgb_model(early_stopping_callback_kwargs=None, **train_params):\n import lightgbm as lgb\n\n if train_params[\"params\"][\"objective\"] == \"quantile\":\n quantile_levels = train_params[\"params\"].pop(\"quantile_levels\")\n booster = QuantileBooster(\n quantile_levels=quantile_levels, early_stopping_callback_kwargs=early_stopping_callback_kwargs\n )\n return booster.fit(**train_params)\n else:\n return lgb.train(**train_params)\n\n\nclass QuantileBooster:\n \"\"\"Wrapper that trains a separate LGBM Booster for each quantile level.\"\"\"\n\n def __init__(self, quantile_levels: list[float], early_stopping_callback_kwargs: Optional[dict] = None):\n if quantile_levels is None:\n raise AssertionError\n if not all(0 < q < 1 for q in quantile_levels):\n raise AssertionError(\n f\"quantile_levels must fulfill 0 < q < 1, provided quantile_levels: {quantile_levels}\"\n )\n\n self.quantile_levels = quantile_levels\n\n self.early_stopping_callback_kwargs = None\n self.time_limit_global = None\n\n if early_stopping_callback_kwargs is not None:\n self.early_stopping_callback_kwargs = early_stopping_callback_kwargs\n self.time_limit_global = early_stopping_callback_kwargs.pop(\"time_limit\")\n self.model_dict = {}\n\n def fit(self, **train_params_base):\n import lightgbm as lgb\n\n from .callbacks import early_stopping_custom\n\n start_time_global = time.time()\n\n for q in self.quantile_levels:\n train_params = copy.deepcopy(train_params_base)\n train_params[\"params\"][\"alpha\"] = q\n if self.early_stopping_callback_kwargs is not None:\n es_kwargs = copy.deepcopy(self.early_stopping_callback_kwargs)\n if self.time_limit_global is not None:\n es_kwargs[\"start_time\"] = time.time()\n es_kwargs[\"time_limit\"] = self.time_limit_global / len(self.quantile_levels)\n # Don't add a logging callback to avoid printing logs for each base booster\n train_params[\"callbacks\"] = [early_stopping_custom(**es_kwargs)]\n else:\n train_params[\"callbacks\"] = []\n\n self.model_dict[q] = lgb.train(**train_params)\n if self.time_limit_global is not None:\n time_left = self.time_limit_global - (time.time() - start_time_global)\n if time_left <= 0 and len(self.model_dict) != len(self.quantile_levels):\n raise TimeLimitExceeded\n return self\n\n def predict(self, X, num_threads=0):\n predictions = {}\n for q in self.quantile_levels:\n predictions[q] = self.model_dict[q].predict(X, num_threads=num_threads)\n return DataFrame(predictions)\n\n @property\n def best_iteration(self):\n return int(np.ceil(np.mean([model.best_iteration for model in self.model_dict.values()])))\n\n def current_iteration(self):\n return int(np.ceil(np.mean([model.current_iteration() for model in self.model_dict.values()])))\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lr/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/lr/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/lr/hyperparameters/parameters.py",
"content": "import logging\n\nfrom autogluon.core.constants import BINARY\n\nIGNORE = \"ignore\"\nONLY = \"only\"\nINCLUDE = \"include\"\n\nlogger = logging.getLogger(__name__)\n\npreprocess_params_set = {\n \"vectorizer_dict_size\",\n \"proc.ngram_range\",\n \"proc.skew_threshold\",\n \"proc.impute_strategy\",\n \"penalty\",\n \"handle_text\",\n}\n\n\ndef get_param_baseline():\n default_params = {\n \"C\": 1,\n \"vectorizer_dict_size\": 75000, # size of TFIDF vectorizer dictionary; used only in text model\n \"proc.ngram_range\": (1, 5), # range of n-grams for TFIDF vectorizer dictionary; used only in text model\n \"proc.skew_threshold\": 0.99, # numerical features whose absolute skewness is greater than this receive special power-transform preprocessing. Choose big value to avoid using power-transforms\n \"proc.impute_strategy\": \"median\", # strategy argument of sklearn.SimpleImputer() used to impute missing numeric values\n \"penalty\": \"L2\", # regularization to use with regression models\n \"handle_text\": IGNORE, # how text should be handled: `ignore` - don't use NLP features; `only` - only use NLP features; `include` - use both regular and NLP features\n }\n return default_params\n\n\ndef _get_solver(problem_type):\n if problem_type == BINARY:\n # TODO explore using liblinear for smaller datasets\n solver = \"lbfgs\"\n else:\n solver = \"lbfgs\"\n return solver\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lr/hyperparameters/searchspaces.py",
"content": "from autogluon.common.space import Categorical, Real\n\n\ndef get_default_searchspace(problem_type, num_classes=None):\n spaces = {\n \"C\": Real(lower=0.1, upper=1e3, default=1),\n \"proc.skew_threshold\": Categorical(0.99, None),\n \"penalty\": Categorical(\"L2\", \"L1\"),\n }\n return spaces\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lr/lr_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport re\nimport time\nimport warnings\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import QuantileTransformer, StandardScaler\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT, R_OBJECT, S_BOOL, S_TEXT_AS_CATEGORY\nfrom autogluon.common.utils.log_utils import fix_sklearnex_logging_if_kaggle\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.core.constants import BINARY, REGRESSION\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\n\nfrom .hyperparameters.parameters import IGNORE, INCLUDE, ONLY, _get_solver, get_param_baseline, preprocess_params_set\nfrom .hyperparameters.searchspaces import get_default_searchspace\nfrom .lr_preprocessing_utils import NlpDataPreprocessor, OheFeaturesGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Can Bagged LinearModels be combined during inference to 1 model by averaging their weights?\n# What about just always using refit_full model? Should we even bag at all? Do we care that its slightly overfit?\nclass LinearModel(AbstractModel):\n \"\"\"\n Linear model (scikit-learn): https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\n\n Model backend differs depending on problem_type:\n\n 'binary' & 'multiclass': https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\n\n 'regression': https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge\n \"\"\"\n\n ag_key = \"LR\"\n ag_name = \"LinearModel\"\n ag_priority = 30\n seed_name = \"random_state\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._pipeline = None\n\n # noinspection PyUnresolvedReferences\n def _get_model_type(self):\n penalty = self.params.get(\"penalty\", \"L2\")\n # FIXME: False by default because AdultIncome dataset shows worse results with use_daal=True.\n # Version: scikit-learn-intelex-2024.4.0\n # model score_test score_val eval_metric\n # 0 LinearModel 0.902293 0.904318 roc_auc\n # 1 LinearModel_SKLEARNEX 0.863535 0.873544 roc_auc\n if self.params_aux.get(\"use_daal\", False):\n # Appears to give 20x training speedup when enabled\n try:\n from sklearnex.linear_model import Lasso, LogisticRegression, Ridge\n\n fix_sklearnex_logging_if_kaggle() # Fix logging verbosity if in Kaggle notebook environment\n\n logger.log(15, \"\\tUsing sklearnex LR backend...\")\n except Exception:\n from sklearn.linear_model import Lasso, LogisticRegression, Ridge\n else:\n from sklearn.linear_model import Lasso, LogisticRegression, Ridge\n if self.problem_type == REGRESSION:\n if penalty == \"L2\":\n model_type = Ridge\n elif penalty == \"L1\":\n model_type = Lasso\n else:\n raise AssertionError(f'Unknown value for penalty \"{penalty}\" - supported types are [\"L1\", \"L2\"]')\n else:\n model_type = LogisticRegression\n return model_type\n\n def _tokenize(self, s):\n return re.split(\"[ ]+\", s)\n\n def _get_types_of_features(self, df):\n \"\"\"Returns dict with keys: : 'continuous', 'skewed', 'onehot', 'embed', 'language', values = ordered list of feature-names falling into each category.\n Each value is a list of feature-names corresponding to columns in original dataframe.\n \"\"\"\n continuous_featnames = self._feature_metadata.get_features(\n valid_raw_types=[R_INT, R_FLOAT], invalid_special_types=[S_BOOL]\n )\n categorical_featnames = self._feature_metadata.get_features(valid_raw_types=[R_CATEGORY, R_OBJECT])\n bool_featnames = self._feature_metadata.get_features(required_special_types=[S_BOOL])\n language_featnames = [] # TODO: Disabled currently, have to pass raw text data features here to function properly\n return self._select_features(\n df=df,\n categorical_featnames=categorical_featnames,\n language_featnames=language_featnames,\n continuous_featnames=continuous_featnames,\n bool_featnames=bool_featnames,\n )\n\n def _select_features(self, df, **kwargs):\n features_selector = {\n INCLUDE: self._select_features_handle_text_include,\n ONLY: self._select_features_handle_text_only,\n IGNORE: self._select_features_handle_text_ignore,\n }.get(self.params.get(\"handle_text\", IGNORE), self._select_features_handle_text_ignore)\n return features_selector(df=df, **kwargs)\n\n # TODO: handle collinear features - they will impact results quality\n def _preprocess(self, X, is_train=False, **kwargs):\n if is_train:\n feature_types = self._get_types_of_features(X)\n X = self._preprocess_train(X, feature_types, self.params[\"vectorizer_dict_size\"])\n else:\n X = self._pipeline.transform(X)\n return X\n\n def _preprocess_train(self, X, feature_types, vect_max_features):\n transformer_list = []\n if feature_types.get(\"language\", None):\n pipeline = Pipeline(\n steps=[\n (\"preparator\", NlpDataPreprocessor(nlp_cols=feature_types[\"language\"])),\n (\n \"vectorizer\",\n TfidfVectorizer(\n ngram_range=self.params[\"proc.ngram_range\"],\n sublinear_tf=True,\n max_features=vect_max_features,\n tokenizer=self._tokenize,\n ),\n ),\n ]\n )\n transformer_list.append((\"vect\", pipeline, feature_types[\"language\"]))\n if feature_types.get(\"onehot\", None):\n pipeline = Pipeline(\n steps=[\n (\"generator\", OheFeaturesGenerator()),\n ]\n )\n transformer_list.append((\"cats\", pipeline, feature_types[\"onehot\"]))\n if feature_types.get(\"continuous\", None):\n pipeline = Pipeline(\n steps=[\n (\"imputer\", SimpleImputer(strategy=self.params[\"proc.impute_strategy\"])),\n (\"scaler\", StandardScaler()),\n ]\n )\n transformer_list.append((\"cont\", pipeline, feature_types[\"continuous\"]))\n if feature_types.get(\"bool\", None):\n pipeline = Pipeline(steps=[(\"scaler\", StandardScaler())])\n transformer_list.append((\"bool\", pipeline, feature_types[\"bool\"]))\n if feature_types.get(\"skewed\", None):\n pipeline = Pipeline(\n steps=[\n (\"imputer\", SimpleImputer(strategy=self.params[\"proc.impute_strategy\"])),\n (\n \"quantile\",\n QuantileTransformer(output_distribution=\"normal\"),\n ), # Or output_distribution = 'uniform'\n ]\n )\n transformer_list.append((\"skew\", pipeline, feature_types[\"skewed\"]))\n self._pipeline = ColumnTransformer(transformers=transformer_list)\n return self._pipeline.fit_transform(X)\n\n def _set_default_params(self):\n default_params = {\"fit_intercept\": True}\n if self.problem_type != REGRESSION:\n default_params.update({\"solver\": _get_solver(self.problem_type)})\n default_params.update(get_param_baseline())\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(self.problem_type)\n\n def _fit(self, X, y, time_limit=None, num_cpus=-1, sample_weight=None, **kwargs):\n time_fit_start = time.time()\n X = self.preprocess(X, y=y, is_train=True)\n if self.problem_type == BINARY:\n y = y.astype(int).values\n\n params = {k: v for k, v in self.params.items() if k not in preprocess_params_set}\n if \"n_jobs\" not in params:\n if self.problem_type != REGRESSION:\n params[\"n_jobs\"] = num_cpus\n\n # Ridge/Lasso are using alpha instead of C, which is C^-1\n # https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge\n if self.problem_type == REGRESSION and \"alpha\" not in params:\n # For numerical reasons, using alpha = 0 with the Lasso object is not advised, so we add epsilon\n params[\"alpha\"] = 1 / (params[\"C\"] if params[\"C\"] != 0 else 1e-8)\n params.pop(\"C\", None)\n\n logger.log(15, f\"Training Model with the following hyperparameter settings:\")\n logger.log(15, params)\n\n max_iter = params.pop(\"max_iter\", 10000)\n\n # TODO: copy_X=True currently set during regression problem type, could potentially set to False to avoid unnecessary data copy.\n model_cls = self._get_model_type()\n\n time_fit_model_start = time.time()\n if time_limit is not None:\n time_left = time_limit - (time_fit_model_start - time_fit_start)\n time_left = time_left - 0.2 # Account for 0.2s of overhead\n if time_left <= 0:\n raise TimeLimitExceeded\n else:\n time_left = None\n\n if time_left is not None and max_iter >= 200 and self.problem_type != REGRESSION:\n max_iter_list = [100, max_iter - 100]\n else:\n max_iter_list = [max_iter]\n\n fit_args = dict(X=X, y=y)\n if sample_weight is not None:\n fit_args[\"sample_weight\"] = sample_weight\n\n if len(max_iter_list) > 1:\n params[\"warm_start\"] = True # Force True\n\n total_iter = 0\n total_iter_used = 0\n total_max_iter = sum(max_iter_list)\n model = model_cls(max_iter=max_iter_list[0], **params)\n early_stop = False\n for i, cur_max_iter in enumerate(max_iter_list):\n if time_left is not None and (i > 0):\n time_spent = time.time() - time_fit_model_start\n time_left_train = time_left - time_spent\n time_per_iter = time_spent / total_iter\n time_to_train_cur_max_iter = time_per_iter * cur_max_iter\n if time_to_train_cur_max_iter > time_left_train:\n cur_max_iter = min(int(time_left_train / time_per_iter) - 1, cur_max_iter)\n if cur_max_iter <= 0:\n logger.warning(\n f\"\\tEarly stopping due to lack of time remaining. Fit {total_iter}/{total_max_iter} iters...\"\n )\n break\n early_stop = True\n\n model.max_iter = cur_max_iter\n with warnings.catch_warnings():\n # Filter the not-converged warning since we are purposefully training in increments.\n # FIXME: Annoyingly, this doesn't filter the warning on Mac due to how multiprocessing works when n_cpus>1. Unsure how to fix.\n warnings.simplefilter(action=\"ignore\", category=UserWarning)\n model = model.fit(**fit_args)\n total_iter += model.max_iter\n if model.n_iter_ is not None:\n if isinstance(model.n_iter_, int):\n total_iter_used += model.n_iter_\n else:\n try:\n # FIXME: For some reason this crashes on regression with some versions of scikit-learn.\n total_iter_used += model.n_iter_[0]\n except Exception:\n pass\n else:\n total_iter_used += model.max_iter\n if early_stop:\n if total_iter_used == total_iter: # Not yet converged\n logger.warning(\n f\"\\tEarly stopping due to lack of time remaining. Fit {total_iter}/{total_max_iter} iters...\"\n )\n break\n\n self.model = model\n self.params_trained[\"max_iter\"] = total_iter\n\n def _select_features_handle_text_include(\n self, df, categorical_featnames, language_featnames, continuous_featnames, bool_featnames\n ):\n types_of_features = dict()\n types_of_features.update(self._select_continuous(df, continuous_featnames))\n types_of_features.update(self._select_bool(df, bool_featnames))\n types_of_features.update(self._select_categorical(df, categorical_featnames))\n types_of_features.update(self._select_text(df, language_featnames))\n return types_of_features\n\n def _select_features_handle_text_only(\n self, df, categorical_featnames, language_featnames, continuous_featnames, bool_featnames\n ):\n types_of_features = dict()\n types_of_features.update(self._select_text(df, language_featnames))\n return types_of_features\n\n def _select_features_handle_text_ignore(\n self, df, categorical_featnames, language_featnames, continuous_featnames, bool_featnames\n ):\n types_of_features = dict()\n types_of_features.update(self._select_continuous(df, continuous_featnames))\n types_of_features.update(self._select_bool(df, bool_featnames))\n types_of_features.update(self._select_categorical(df, categorical_featnames))\n return types_of_features\n\n def _select_categorical(self, df, features):\n return dict(onehot=features)\n\n def _select_continuous(self, df, features):\n # continuous = numeric features to rescale\n # skewed = features to which we will apply power (ie. log / box-cox) transform before normalization\n types_of_features = defaultdict(list)\n skew_threshold = self.params[\"proc.skew_threshold\"]\n for feature in features:\n if skew_threshold is not None and (np.abs(df[feature].skew()) > self.params[\"proc.skew_threshold\"]):\n types_of_features[\"skewed\"].append(feature)\n else:\n types_of_features[\"continuous\"].append(feature)\n return types_of_features\n\n def _select_text(self, df, features):\n return dict(language=features)\n\n def _select_bool(self, df, features):\n return dict(bool=features)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n ignored_type_group_special=[S_TEXT_AS_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n return 4 * get_approximate_df_mem_usage(X).sum()\n\n def _get_maximum_resources(self) -> dict[str, int | float]:\n # no GPU support\n return {\"num_gpus\": 0}\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n @classmethod\n def _class_tags(cls):\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self):\n # `can_refit_full=True` because validation data isn't used during fit.\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lr/lr_preprocessing_utils.py",
"content": "from sklearn.base import BaseEstimator, TransformerMixin\n\nfrom autogluon.features.generators import OneHotEncoderFeatureGenerator\n\n\nclass OheFeaturesGenerator(BaseEstimator, TransformerMixin):\n def __init__(self):\n pass\n\n def fit(self, X, y=None):\n self.encoder_ = OneHotEncoderFeatureGenerator(max_levels=10000, verbosity=0)\n self.encoder_.fit(X)\n self.feature_names_ = self.encoder_.features_out\n return self\n\n def transform(self, X, y=None):\n return self.encoder_.transform_ohe(X)\n\n def get_feature_names(self):\n return self.feature_names_\n\n\nclass NlpDataPreprocessor(BaseEstimator, TransformerMixin):\n def __init__(self, nlp_cols):\n self.nlp_cols = nlp_cols\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n X = X[self.nlp_cols].copy()\n for c in self.nlp_cols:\n X[c] = X[c].astype(str).fillna(\" \")\n X = X.apply(\" \".join, axis=1).str.replace(\"[ ]+\", \" \", regex=True)\n return X.values.tolist()\n"
},
{
"path": "tabular/src/autogluon/tabular/models/lr/lr_rapids_model.py",
"content": "import logging\n\nfrom autogluon.common.utils.try_import import try_import_rapids_cuml\nfrom autogluon.core.constants import REGRESSION\n\nfrom .._utils.rapids_utils import RapidsModelMixin\nfrom .hyperparameters.parameters import get_param_baseline\nfrom .lr_model import LinearModel\n\nlogger = logging.getLogger(__name__)\n\n\n# FIXME: If rapids is installed, normal CPU LinearModel crashes.\nclass LinearRapidsModel(RapidsModelMixin, LinearModel):\n \"\"\"\n RAPIDS Linear model : https://rapids.ai/start.html\n\n NOTE: This code is experimental, it is recommend to not use this unless you are a developer.\n This was tested on rapids-21.06 via:\n\n conda create -n rapids-21.06 -c rapidsai -c nvidia -c conda-forge rapids=21.06 python=3.8 cudatoolkit=11.2\n conda activate rapids-21.06\n pip install --pre autogluon.tabular[all]\n \"\"\"\n\n def _get_model_type(self):\n penalty = self.params.get(\"penalty\", \"L2\")\n try_import_rapids_cuml()\n from cuml.linear_model import Lasso, LogisticRegression, Ridge\n\n if self.problem_type == REGRESSION:\n if penalty == \"L2\":\n model_type = Ridge\n elif penalty == \"L1\":\n model_type = Lasso\n else:\n raise AssertionError(f'Unknown value for penalty \"{penalty}\" - supported types are [\"L1\", \"L2\"]')\n else:\n model_type = LogisticRegression\n return model_type\n\n def _set_default_params(self):\n default_params = {\"fit_intercept\": True, \"max_iter\": 10000}\n if self.problem_type != REGRESSION:\n default_params.update({\"solver\": \"qn\"})\n default_params.update(get_param_baseline())\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _preprocess(self, X, **kwargs):\n X = super()._preprocess(X=X, **kwargs)\n if hasattr(X, \"toarray\"): # Check if it's a sparse matrix\n X = X.toarray()\n return X\n\n def _fit(self, X, y, **kwargs):\n \"\"\"\n Custom fit method for RAPIDS cuML models that handles parameter compatibility\n and bypasses sklearn-specific incremental training approach.\n \"\"\"\n # Preprocess data\n X = self.preprocess(X, y=y, is_train=True)\n if self.problem_type == \"binary\":\n y = y.astype(int).values\n\n # Create cuML model with filtered parameters\n model_cls = self._get_model_type()\n\n # Comprehensive parameter filtering for cuML compatibility\n cuml_incompatible_params = {\n # AutoGluon-specific preprocessing parameters\n \"vectorizer_dict_size\",\n \"proc.ngram_range\",\n \"proc.skew_threshold\",\n \"proc.impute_strategy\",\n \"handle_text\",\n # sklearn-specific parameters not supported by cuML\n \"n_jobs\",\n \"warm_start\",\n \"multi_class\",\n \"dual\",\n \"intercept_scaling\",\n \"class_weight\",\n \"random_state\",\n \"verbose\",\n # Parameters that need conversion or special handling\n \"penalty\",\n \"C\",\n }\n\n # Filter out incompatible parameters\n filtered_params = {k: v for k, v in self.params.items() if k not in cuml_incompatible_params}\n\n # Handle parameter conversions for cuML\n if self.problem_type == REGRESSION:\n # Convert sklearn's C parameter to cuML's alpha\n if \"C\" in self.params:\n filtered_params[\"alpha\"] = 1.0 / self.params[\"C\"]\n else:\n # For classification, keep C parameter\n if \"C\" in self.params:\n filtered_params[\"C\"] = self.params[\"C\"]\n\n # Create and fit cuML model - let cuML handle its own error messages\n self.model = model_cls(**filtered_params)\n self.model.fit(X, y)\n\n # Add missing sklearn-compatible attributes for AutoGluon compatibility\n self.model.n_iter_ = None # cuML doesn't track iterations like sklearn\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/__init__.py",
"content": "# Internal modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/config/__init__.py",
"content": "# Configuration modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/config/config_pretrain.py",
"content": "from __future__ import annotations\n\nimport os\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom typing import Optional\n\nimport torch\nimport yaml\nfrom omegaconf import DictConfig, OmegaConf\n\nfrom ..._internal.config.enums import GeneratorName, LossName, ModelName, Task\n\n\n@dataclass\nclass ConfigData:\n generator: GeneratorName\n min_samples_support: int\n max_samples_support: int\n n_samples_query: int\n min_features: int\n max_features: int\n max_classes: int\n sample_multinomial_categorical: bool\n sample_multinomial_label: bool\n generator_hyperparams: dict\n task: Task\n\n def __post_init__(self):\n assert self.min_samples_support <= self.max_samples_support\n assert self.min_features <= self.max_features\n\n\n@dataclass\nclass ConfigModel:\n name: ModelName\n hyperparams: dict\n\n\n@dataclass\nclass ConfigPreprocessing:\n use_quantile_transformer: bool\n use_feature_count_scaling: bool\n\n\n@dataclass\nclass ConfigGradScaler:\n enabled: bool\n scale_init: float\n scale_min: float\n growth_interval: int\n\n def __post_init__(self):\n assert self.scale_init >= self.scale_min, \"Scale init must be greater than scale min\"\n assert self.scale_min >= 1, \"Scale min lower than 1 makes no sense for mixed precision training\"\n assert type(self.scale_init) == float, \"Scale init must be a float, otherwise gradscaler will return an error\"\n assert type(self.scale_min) == float, \"Scale min must be a float, otherwise gradscaler will return an error\"\n\n\n@dataclass\nclass ConfigOptim:\n steps: int\n log_every_n_steps: int\n eval_every_n_steps: int\n batch_size: int\n gradient_accumulation_steps: int\n lr: float\n weight_decay: float\n beta1: float\n beta2: float\n warmup_steps: int\n cosine_scheduler: bool\n max_grad_norm: float\n label_smoothing: float\n regression_loss: LossName\n use_pretrained_weights: bool\n path_to_weights: str\n resume_states: bool\n path_to_states: str\n precision: str\n grad_scaler: ConfigGradScaler\n\n @classmethod\n def from_hydra(cls, cfg_hydra: DictConfig) -> Self:\n grad_scaler = ConfigGradScaler(**cfg_hydra.grad_scaler)\n cfg_dict: dict = OmegaConf.to_container(cfg_hydra) # type: ignore\n del cfg_dict[\"grad_scaler\"]\n\n regression_loss = LossName[cfg_dict[\"regression_loss\"]]\n del cfg_dict[\"regression_loss\"]\n\n return cls(grad_scaler=grad_scaler, regression_loss=regression_loss, **cfg_dict)\n\n def __post_init__(self):\n assert hasattr(torch, self.precision), f\"Precision {self.precision} not supported by torch\"\n\n\nclass ConfigSaveLoadMixin(yaml.YAMLObject):\n def save(self, path: Path) -> None:\n path.parent.mkdir(parents=True, exist_ok=True)\n\n with open(path, \"w\") as f:\n yaml.dump(self, f, default_flow_style=False)\n\n @classmethod\n def load(cls, path: Path) -> Self:\n with open(path, \"r\") as f:\n # It's unsafe, but not unsafer than the pickle module\n config = yaml.unsafe_load(f)\n\n return config\n\n\n@dataclass\nclass ConfigPretrain(ConfigSaveLoadMixin):\n run_name: str\n output_dir: Path\n seed: int\n devices: list[torch.device]\n device: torch.device\n max_cpus_per_device: Optional[int]\n use_ddp: bool\n workers_per_gpu: int\n model: ConfigModel\n data: ConfigData\n optim: ConfigOptim\n preprocessing: ConfigPreprocessing\n load_from_file: bool\n load_path_x: str\n load_path_y: str\n save_file: bool\n save_file_only: bool\n save_path_x: str\n save_path_y: str\n number_of_runs: int\n\n @classmethod\n def from_hydra(cls, cfg_hydra: DictConfig):\n assert not os.path.exists(cfg_hydra.output_dir), (\n f\"Output directory {cfg_hydra.output_dir} already exists! Please change to a new folder.\"\n )\n\n output_dir = Path(cfg_hydra.output_dir)\n\n devices = [torch.device(device) for device in cfg_hydra.devices]\n\n # Initialize device to cpu, DDP will overwrite this\n device = torch.device(\"cpu\")\n\n return cls(\n run_name=cfg_hydra.run_name,\n output_dir=output_dir,\n devices=devices,\n device=device,\n max_cpus_per_device=cfg_hydra.max_cpus_per_device,\n use_ddp=len(devices) > 1,\n seed=cfg_hydra.seed,\n workers_per_gpu=cfg_hydra.workers_per_gpu,\n model=ConfigModel(\n name=ModelName[cfg_hydra.model.name],\n hyperparams=OmegaConf.to_container(cfg_hydra.model.hyperparams),\n ),\n data=ConfigData(\n generator=GeneratorName(cfg_hydra.data.generator),\n min_samples_support=cfg_hydra.data.min_samples_support,\n max_samples_support=cfg_hydra.data.max_samples_support,\n n_samples_query=cfg_hydra.data.n_samples_query,\n min_features=cfg_hydra.data.min_features,\n max_features=cfg_hydra.data.max_features,\n max_classes=cfg_hydra.data.max_classes,\n task=Task[cfg_hydra.data.task],\n sample_multinomial_categorical=cfg_hydra.data.sample_multinomial_categorical,\n sample_multinomial_label=cfg_hydra.data.sample_multinomial_label,\n generator_hyperparams=OmegaConf.to_container(cfg_hydra.data.generator_hyperparams), # type: ignore\n ),\n optim=ConfigOptim.from_hydra(cfg_hydra.optim),\n preprocessing=ConfigPreprocessing(**cfg_hydra.preprocessing),\n load_from_file=cfg_hydra.load_from_file,\n load_path_x=cfg_hydra.load_path_x,\n load_path_y=cfg_hydra.load_path_y,\n save_file=cfg_hydra.save_file,\n save_file_only=cfg_hydra.save_file_only,\n save_path_x=cfg_hydra.save_path_x,\n save_path_y=cfg_hydra.save_path_y,\n number_of_runs=cfg_hydra.number_of_runs,\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/config/config_run.py",
"content": "from __future__ import annotations\n\nfrom dataclasses import dataclass\n\nimport torch\n\nfrom ..._internal.config.config_pretrain import ConfigSaveLoadMixin\nfrom ..._internal.config.enums import ModelName\n\n\n@dataclass\nclass ConfigRun(ConfigSaveLoadMixin):\n device: torch.device\n seed: int\n model_name: ModelName\n hyperparams: dict\n\n @classmethod\n def create(cls, device: torch.device, seed: int, model_name: ModelName, hyperparams: dict) -> \"ConfigRun\":\n return cls(device=device, seed=seed, model_name=model_name, hyperparams=hyperparams)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/config/enums.py",
"content": "from enum import IntEnum\n\ntry:\n from enum import StrEnum\nexcept ImportError:\n # StrEnum is not available in Python < 3.11, so we create a compatible version\n from enum import Enum\n\n class StrEnum(str, Enum):\n \"\"\"\n Enum where members are also (and must be) strings\n \"\"\"\n\n def __new__(cls, value):\n if not isinstance(value, str):\n raise TypeError(f\"{value!r} is not a string\")\n return super().__new__(cls, value)\n\n def __str__(self):\n return self.value\n\n\nclass Task(StrEnum):\n CLASSIFICATION = \"classification\"\n REGRESSION = \"regression\"\n\n\nclass FeatureType(StrEnum):\n NUMERICAL = \"numerical\"\n CATEGORICAL = \"categorical\"\n MIXED = \"mixed\"\n\n\nclass SearchType(StrEnum):\n DEFAULT = \"default\"\n RANDOM = \"random\"\n\n\nclass DatasetSize(IntEnum):\n SMALL = 1000\n MEDIUM = 10000\n LARGE = 50000\n\n\nclass DataSplit(StrEnum):\n TRAIN = \"train\"\n VALID = \"valid\"\n TEST = \"test\"\n\n\nclass Phase(StrEnum):\n TRAINING = \"training\"\n VALIDATION = \"validation\"\n TESTING = \"testing\"\n\n\nclass ModelName(StrEnum):\n PLACEHOLDER = \"_placeholder_\" # This is a placeholder for the current running model\n FT_TRANSFORMER = \"FT-Transformer\"\n TABPFN = \"TabPFN\"\n FOUNDATION = \"Foundation\"\n FOUNDATION_FLASH = \"FoundationFlash\"\n TAB2D = \"Tab2D\"\n TAB2D_COL_ROW = \"Tab2D_COL_ROW\"\n TAB2D_SDPA = \"Tab2D_SDPA\"\n SAINT = \"SAINT\"\n MLP = \"MLP\"\n MLP_RTDL = \"MLP-rtdl\"\n RESNET = \"Resnet\"\n RANDOM_FOREST = \"RandomForest\"\n XGBOOST = \"XGBoost\"\n CATBOOST = \"CatBoost\"\n LIGHTGBM = \"LightGBM\"\n GRADIENT_BOOSTING_TREE = \"GradientBoostingTree\"\n HIST_GRADIENT_BOOSTING_TREE = \"HistGradientBoostingTree\"\n LOGISTIC_REGRESSION = \"LogisticRegression\"\n LINEAR_REGRESSION = \"LinearRegression\"\n DECISION_TREE = \"DecisionTree\"\n KNN = \"KNN\"\n STG = \"STG\"\n SVM = \"SVM\"\n TABNET = \"TabNet\"\n TABTRANSFORMER = \"TabTransformer\"\n DEEPFM = \"DeepFM\"\n VIME = \"VIME\"\n DANET = \"DANet\"\n NODE = \"NODE\"\n AUTOGLUON = \"AutoGluon\"\n\n\nclass ModelClass(StrEnum):\n BASE = \"base\"\n GBDT = \"GBDT\"\n NN = \"NN\"\n ICLT = \"ICLT\"\n\n\nclass DownstreamTask(StrEnum):\n ZEROSHOT = \"zeroshot\"\n FINETUNE = \"finetune\"\n\n\nclass BenchmarkName(StrEnum):\n DEBUG_CLASSIFICATION = \"debug_classification\"\n DEBUG_REGRESSION = \"debug_regression\"\n DEBUG_TABZILLA = \"debug_tabzilla\"\n\n CATEGORICAL_CLASSIFICATION = \"categorical_classification\"\n NUMERICAL_CLASSIFICATION = \"numerical_classification\"\n CATEGORICAL_REGRESSION = \"categorical_regression\"\n NUMERICAL_REGRESSION = \"numerical_regression\"\n CATEGORICAL_CLASSIFICATION_LARGE = \"categorical_classification_large\"\n NUMERICAL_CLASSIFICATION_LARGE = \"numerical_classification_large\"\n CATEGORICAL_REGRESSION_LARGE = \"categorical_regression_large\"\n NUMERICAL_REGRESSION_LARGE = \"numerical_regression_large\"\n\n TABZILLA_HARD = \"tabzilla_hard\"\n TABZILLA_HARD_MAX_TEN_CLASSES = \"tabzilla_hard_max_ten_classes\"\n TABZILLA_HAS_COMPLETED_RUNS = \"tabzilla_has_completed_runs\"\n\n\nclass BenchmarkOrigin(StrEnum):\n TABZILLA = \"tabzilla\"\n WHYTREES = \"whytrees\"\n\n\nclass GeneratorName(StrEnum):\n TABPFN = \"tabpfn\"\n TREE = \"tree\"\n RANDOMFOREST = \"randomforest\"\n NEIGHBOR = \"neighbor\"\n MIX = \"mix\"\n PERLIN = \"perlin\"\n MIX_7 = \"mix_7\"\n MIX_6 = \"mix_6\"\n MIX_5 = \"mix_5\"\n MIX_5_GP = \"mix_5_gp\"\n MIX_4 = \"mix_4\"\n MIX_4_AG = \"mix_4_ag\"\n LR = \"lr\"\n POLY = \"poly\"\n SAMPLE_RF = \"sample_rf\"\n SAMPLE_GP = \"sample_gp\"\n TABREPO = \"tabrepo\"\n MIX_4_TABREPO = \"mix_4_tabrepo\"\n MIX_4_TABPFNV2 = \"mix_4_tabpfnv2\"\n\n\nclass MetricName(StrEnum):\n ACCURACY = \"accuracy\"\n F1 = \"f1\"\n AUC = \"auc\"\n MSE = \"mse\"\n MAE = \"mae\"\n R2 = \"r2\"\n LOG_LOSS = \"log_loss\"\n RMSE = \"rmse\"\n\n\nclass LossName(StrEnum):\n CROSS_ENTROPY = \"cross_entropy\"\n MSE = \"mse\"\n MAE = \"mae\"\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/config/tab2d.yaml",
"content": "device:\n _target_: torch.device\n _args_: [cpu]\n\nhyperparams_finetuning:\n dim_embedding: null\n early_stopping_data_split: VALID\n early_stopping_max_samples: 2048\n early_stopping_patience: 40\n grad_scaler_enabled: false\n grad_scaler_growth_interval: 1000\n grad_scaler_scale_init: 65536.0\n grad_scaler_scale_min: 65536.0\n label_smoothing: 0.0\n lr_scheduler:\n default: false\n values:\n - true\n - false\n lr_scheduler_patience: 25\n max_epochs: 300\n max_samples_query: 1024\n max_samples_support: 8192\n optimizer: adamw\n lr: 0.0001\n weight_decay: 0.1 \n warmup_steps: 1000\n path_to_weights: outputs/runs/2024-08-08/23-58-03/weights/model_step_12000.pt\n precision: bfloat16\n random_mirror_regression: true\n random_mirror_x: true\n shuffle_classes: true\n shuffle_features: false\n use_feature_count_scaling: false\n use_pretrained_weights: false\n use_quantile_transformer: false\n\nmodel_name: Tab2D\nseed: 0\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/__init__.py",
"content": "# Core modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/callbacks.py",
"content": "import numpy as np\nimport torch\n\n\nclass EarlyStopping:\n def __init__(self, patience=10, delta=0.0001, metric=\"log_loss\"):\n self.patience = patience\n self.counter = 0\n self.best_score = None\n self.early_stop = False\n self.delta = delta\n self.metric = metric\n\n def __call__(self, val_loss):\n # smaller is better for these metrics\n if self.metric in [\"log_loss\", \"mse\", \"mae\", \"rmse\"]:\n score = -val_loss\n # larger is better for these metrics\n elif self.metric in [\"accuracy\", \"roc_auc\", \"r2\"]:\n score = val_loss\n else:\n raise ValueError(\n f\"Unsupported metric: {self.metric}. Supported metrics are: log_loss, mse, mae, rmse, accuracy, roc_auc, r2.\"\n )\n\n if self.best_score is None:\n self.best_score = score\n elif score < self.best_score + self.delta:\n self.counter += 1\n if self.counter >= self.patience:\n self.early_stop = True\n else:\n self.best_score = score\n self.counter = 0\n\n def we_should_stop(self):\n return self.early_stop\n\n\nclass Checkpoint:\n def __init__(self):\n self.curr_best_loss = np.inf\n self.best_model: dict\n\n def reset(self, net: torch.nn.Module):\n self.curr_best_loss = np.inf\n self.best_model = net.state_dict()\n for key in self.best_model:\n self.best_model[key] = self.best_model[key].to(\"cpu\")\n\n def __call__(self, net: torch.nn.Module, loss: float):\n if loss < self.curr_best_loss:\n self.curr_best_loss = loss\n self.best_model = net.state_dict()\n for key in self.best_model:\n self.best_model[key] = self.best_model[key].to(\"cpu\")\n\n def set_to_best(self, net):\n net.load_state_dict(self.best_model)\n\n\nclass EpochStatistics:\n def __init__(self) -> None:\n self.n = 0\n self.loss = 0\n self.score = 0\n\n def update(self, loss, score, n):\n self.n += n\n self.loss += loss * n\n self.score += score * n\n\n def get(self):\n return self.loss / self.n, self.score / self.n\n\n\nclass TrackOutput:\n def __init__(self) -> None:\n self.y_true: list[np.ndarray] = []\n self.y_pred: list[np.ndarray] = []\n\n def update(self, y_true: np.ndarray, y_pred: np.ndarray):\n self.y_true.append(y_true)\n self.y_pred.append(y_pred)\n\n def get(self):\n return np.concatenate(self.y_true, axis=0), np.concatenate(self.y_pred, axis=0)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/get_loss.py",
"content": "import einops\nimport torch\n\nfrom ..._internal.config.config_pretrain import ConfigPretrain\nfrom ..._internal.config.config_run import ConfigRun\nfrom ..._internal.config.enums import LossName, Task\n\n\nclass CrossEntropyLossExtraBatch(torch.nn.Module):\n def __init__(self, label_smoothing: float):\n super().__init__()\n\n self.loss = torch.nn.CrossEntropyLoss(label_smoothing=label_smoothing)\n\n def forward(self, input, target):\n \"\"\"\n Input has shape (batch_size, num_samples, num_classes)\n Target has shape (batch_size, num_samples)\n\n Compared to the original CrossEntropyLoss, accepts (batch_size, num_samples) as batch\n \"\"\"\n\n input = einops.rearrange(input, \"b s c -> (b s) c\")\n target = einops.rearrange(target, \"b s -> (b s)\")\n\n return self.loss(input, target)\n\n\ndef get_loss(cfg: ConfigRun):\n if cfg.task == Task.REGRESSION and cfg.hyperparams[\"regression_loss\"] == LossName.MSE:\n return torch.nn.MSELoss()\n elif cfg.task == Task.REGRESSION and cfg.hyperparams[\"regression_loss\"] == LossName.MAE:\n return torch.nn.L1Loss()\n elif cfg.task == Task.REGRESSION and cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY:\n return CrossEntropyLossExtraBatch(cfg.hyperparams[\"label_smoothing\"])\n elif cfg.task == Task.CLASSIFICATION:\n return CrossEntropyLossExtraBatch(cfg.hyperparams[\"label_smoothing\"])\n else:\n raise ValueError(f\"Unsupported task {cfg.task} and (regression) loss {cfg.hyperparams['regression_loss']}\")\n\n\ndef get_loss_pretrain(cfg: ConfigPretrain):\n if cfg.data.task == Task.REGRESSION and cfg.optim.regression_loss == LossName.MSE:\n return torch.nn.MSELoss()\n elif cfg.data.task == Task.REGRESSION and cfg.optim.regression_loss == LossName.MAE:\n return torch.nn.L1Loss()\n elif cfg.data.task == Task.REGRESSION and cfg.optim.regression_loss == LossName.CROSS_ENTROPY:\n return CrossEntropyLossExtraBatch(cfg.optim.label_smoothing)\n elif cfg.data.task == Task.CLASSIFICATION:\n return CrossEntropyLossExtraBatch(cfg.optim.label_smoothing)\n else:\n raise ValueError(f\"Unsupported task {cfg.data.task} and (regression) loss {cfg.optim.regression_loss}\")\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/get_optimizer.py",
"content": "import torch\nfrom torch.optim import SGD, Adam, AdamW\n\nfrom ..._internal.config.config_pretrain import ConfigPretrain\n\n\ndef get_optimizer(hyperparams: dict, model: torch.nn.Module) -> torch.optim.Optimizer:\n optimizer: torch.optim.Optimizer\n\n if hyperparams[\"optimizer\"] == \"adam\":\n optimizer = Adam(\n model.parameters(), lr=hyperparams[\"lr\"], betas=(0.9, 0.999), weight_decay=hyperparams[\"weight_decay\"]\n )\n elif hyperparams[\"optimizer\"] == \"adamw\":\n optimizer = AdamW(\n model.parameters(), lr=hyperparams[\"lr\"], betas=(0.9, 0.999), weight_decay=hyperparams[\"weight_decay\"]\n )\n elif hyperparams[\"optimizer\"] == \"sgd\":\n optimizer = SGD(model.parameters(), lr=hyperparams[\"lr\"], weight_decay=hyperparams[\"weight_decay\"])\n else:\n raise ValueError(\"Optimizer not recognized\")\n\n return optimizer\n\n\ndef get_optimizer_pretrain(cfg: ConfigPretrain, model: torch.nn.Module) -> torch.optim.Optimizer:\n parameters = [(name, param) for name, param in model.named_parameters()]\n\n parameters_with_weight_decay = []\n parameters_without_weight_decay = []\n\n for name, param in parameters:\n if name.endswith(\"bias\") or \"norm\" in name or \"embedding\" in name:\n parameters_without_weight_decay.append(param)\n else:\n parameters_with_weight_decay.append(param)\n\n optimizer_parameters = [\n {\"params\": parameters_with_weight_decay, \"weight_decay\": cfg.optim.weight_decay},\n {\"params\": parameters_without_weight_decay, \"weight_decay\": 0.0},\n ]\n\n optimizer = torch.optim.AdamW(\n optimizer_parameters,\n lr=cfg.optim.lr,\n betas=(cfg.optim.beta1, cfg.optim.beta2),\n weight_decay=cfg.optim.weight_decay,\n )\n\n return optimizer\n\n\nclass GradScaler(torch.amp.GradScaler):\n def __init__(\n self,\n enabled: bool = True,\n scale_init: float = 2.0**16,\n scale_min: float = 1.0,\n growth_interval: int = 2000,\n device: str = \"cuda\",\n ):\n super().__init__(enabled=enabled, device=\"cpu\", init_scale=scale_init, growth_interval=growth_interval) # type: ignore\n self._enabled = enabled\n self.scale_min = scale_min\n self.device = device\n\n if not self._enabled:\n # We write scale=1 to log if the scaler is disabled\n self._scale = torch.tensor((1,), dtype=torch.float32, device=self.device)\n\n def update(self):\n if not self._enabled:\n return\n\n super().update()\n\n if self._scale < self.scale_min:\n super().update(self.scale_min)\n\n\ndef move_optimizer_to(optim, device):\n for param in optim.state.values():\n # Not sure there are any global tensors in the state dict\n if isinstance(param, torch.Tensor):\n param.data = param.data.to(device)\n if param._grad is not None:\n param._grad.data = param._grad.data.to(device)\n elif isinstance(param, dict):\n for subparam in param.values():\n if isinstance(subparam, torch.Tensor):\n subparam.data = subparam.data.to(device)\n if subparam._grad is not None:\n subparam._grad.data = subparam._grad.data.to(device)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/get_scheduler.py",
"content": "import torch\nfrom torch.optim.lr_scheduler import LinearLR, ReduceLROnPlateau\nfrom transformers import get_constant_schedule_with_warmup\nfrom transformers.optimization import get_cosine_with_min_lr_schedule_with_warmup\n\nfrom ..._internal.config.config_pretrain import ConfigPretrain\n\n\ndef get_scheduler(\n hyperparams: dict, optimizer: torch.optim.Optimizer\n) -> tuple[torch.optim.lr_scheduler.LambdaLR, ReduceLROnPlateau]:\n warmup_steps = hyperparams[\"warmup_steps\"]\n\n # if warmup_steps > 0:\n # scheduler_warmup = torch.optim.lr_scheduler.LambdaLR(\n # optimizer, lambda step: min((step + 1) / warmup_steps, 1.0)\n # )\n # else:\n # scheduler_warmup = torch.optim.lr_scheduler.LambdaLR(\n # optimizer, lambda step: 1.0\n # )\n\n if warmup_steps > 0:\n scheduler_warmup = LinearLR(\n optimizer,\n start_factor=1.0 / warmup_steps,\n end_factor=1.0,\n total_iters=warmup_steps,\n )\n else:\n scheduler_warmup = torch.optim.lr_scheduler.ConstantLR(optimizer, factor=1.0, total_iters=1)\n\n if hyperparams[\"lr_scheduler\"]:\n scheduler_reduce_on_plateau = ReduceLROnPlateau(\n optimizer, patience=hyperparams[\"lr_scheduler_patience\"], min_lr=0, factor=0.2\n )\n else:\n # With ReduceLROnPlateau, the scheduler accepts a metric to monitor, so our dummy metric must also be a ReduceLRonPlateau scheduler\n scheduler_reduce_on_plateau = ReduceLROnPlateau(optimizer, patience=1000000000, min_lr=0, factor=0.2)\n\n return scheduler_warmup, scheduler_reduce_on_plateau\n\n\ndef get_scheduler_pretrain(cfg: ConfigPretrain, optimizer: torch.optim.Optimizer):\n if cfg.optim.cosine_scheduler:\n schedule = get_cosine_with_min_lr_schedule_with_warmup(\n optimizer, num_warmup_steps=cfg.optim.warmup_steps, num_training_steps=cfg.optim.steps, min_lr_rate=0.1\n )\n else:\n schedule = get_constant_schedule_with_warmup(optimizer, num_warmup_steps=cfg.optim.warmup_steps)\n\n return schedule\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/prediction_metrics.py",
"content": "from dataclasses import dataclass\n\nimport numpy as np\nimport scipy.special\nimport torch\nfrom loguru import logger\nfrom sklearn.metrics import f1_score, mean_squared_error, r2_score, roc_auc_score, root_mean_squared_error\n\nfrom ..._internal.config.enums import MetricName, Task\nfrom ..._internal.data.preprocessor import Preprocessor\n\n\n@dataclass\nclass PredictionMetrics:\n task: Task\n loss: float\n score: float\n metrics: dict[MetricName, float]\n\n @classmethod\n def from_prediction(cls, y_pred: np.ndarray, y_true: np.ndarray, task: Task) -> \"PredictionMetrics\":\n loss, score, metrics = compute_metrics(y_pred, y_true, task)\n\n return cls(task=task, loss=loss, score=score, metrics=metrics)\n\n\ndef compute_metrics(y_pred: np.ndarray, y_true: np.ndarray, task: Task) -> tuple[float, float, dict]:\n if task == Task.CLASSIFICATION:\n return compute_classification_metrics(y_pred, y_true)\n elif task == Task.REGRESSION:\n return compute_regression_metrics(y_pred, y_true)\n\n\ndef compute_classification_metrics(y_pred: np.ndarray, y_true: np.ndarray) -> tuple[float, float, dict]:\n # predictions are assumed to be log-probabilities\n\n y_pred_class = np.argmax(y_pred, axis=1)\n y_pred_proba = scipy.special.softmax(y_pred, axis=1)\n y_pred_proba = y_pred_proba / y_pred_proba.sum(\n axis=1, keepdims=True\n ) # softmax not completely numerically stable, so a small correction is needed\n labels = np.arange(y_pred_proba.shape[1])\n\n metrics = {\n MetricName.ACCURACY: (y_true == y_pred_class).mean(),\n MetricName.F1: f1_score(y_true, y_pred_class, average=\"weighted\"),\n MetricName.AUC: roc_auc_score_multiclass(\n y_true, y_pred_proba, multi_class=\"ovo\", average=\"macro\", labels=labels\n ),\n MetricName.LOG_LOSS: torch.nn.functional.cross_entropy(\n torch.from_numpy(y_pred), torch.from_numpy(y_true)\n ).item(),\n }\n\n loss = metrics[MetricName.LOG_LOSS]\n score = metrics[MetricName.ACCURACY]\n\n return loss, score, metrics\n\n\ndef roc_auc_score_multiclass(y_true, y_pred_proba, multi_class=\"ovo\", average=\"macro\", labels=None) -> float:\n \"\"\"\n The roc_auc_score multi_class is not supported for binary classification\n \"\"\"\n\n if np.unique(y_true).shape[0] == 1:\n # AUC is not defined if there is only one class\n return float(\"nan\")\n\n try:\n if y_pred_proba.shape[1] == 2:\n return roc_auc_score(y_true, y_pred_proba[:, 1])\n else:\n return roc_auc_score(y_true, y_pred_proba, multi_class=multi_class, average=average, labels=labels)\n except ValueError as e:\n logger.error(f\"Error computing roc_auc_score: {e}\")\n logger.error(f\"Returning {-1}\")\n return -1\n\n\ndef compute_regression_metrics(y_pred: np.ndarray, y_true: np.ndarray) -> tuple[float, float, dict]:\n metrics = {\n MetricName.RMSE: root_mean_squared_error(y_true, y_pred),\n MetricName.MSE: mean_squared_error(y_true, y_pred),\n MetricName.MAE: np.abs(y_true - y_pred).mean(),\n MetricName.R2: r2_score(y_true, y_pred),\n }\n\n loss = metrics[MetricName.MSE]\n score = metrics[MetricName.R2]\n\n return loss, score, metrics\n\n\nclass PredictionMetricsTracker:\n \"\"\"\n Prediction metrics tracker that accumulates predictions and true values to compute metrics at the end.\n Uses torch.Tensor for predictions and true values.\n \"\"\"\n\n def __init__(self, task: Task, preprocessor: Preprocessor) -> None:\n self.task = task\n self.preprocessor = preprocessor\n self.reset()\n\n def reset(self) -> None:\n self.ys_pred: list[np.ndarray] = []\n self.ys_true: list[np.ndarray] = []\n\n def update(self, y_pred: torch.Tensor, y_true: torch.Tensor, train: bool) -> None:\n y_pred_np = y_pred.detach().cpu().numpy()[0]\n y_pred_ori = self.preprocessor.inverse_transform_y(y_pred_np)\n\n y_true_np = y_true.detach().cpu().numpy()[0]\n if train:\n y_true_ori = self.preprocessor.inverse_transform_y(y_true_np)\n else:\n y_true_ori = y_true_np\n\n self.ys_pred.append(y_pred_ori)\n self.ys_true.append(y_true_ori)\n\n def get_metrics(self) -> PredictionMetrics:\n y_pred = np.concatenate(self.ys_pred, axis=0)\n y_true = np.concatenate(self.ys_true, axis=0)\n\n return PredictionMetrics.from_prediction(y_pred, y_true, self.task)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/core/trainer_finetune.py",
"content": "import time\n\nimport numpy as np\nimport torch\nfrom loguru import logger\nfrom sklearn.base import BaseEstimator\n\nfrom ..._internal.config.config_run import ConfigRun\nfrom ..._internal.config.enums import LossName, MetricName, ModelName, Task\nfrom ..._internal.core.callbacks import Checkpoint, EarlyStopping\nfrom ..._internal.core.get_loss import get_loss\nfrom ..._internal.core.get_optimizer import GradScaler, get_optimizer\nfrom ..._internal.core.get_scheduler import get_scheduler\nfrom ..._internal.core.prediction_metrics import PredictionMetrics, PredictionMetricsTracker\nfrom ..._internal.data.collator import CollatorWithPadding\nfrom ..._internal.data.dataset_finetune import DatasetFinetune, DatasetFinetuneGenerator\nfrom ..._internal.data.preprocessor import Preprocessor\n\n\nclass TrainerFinetune(BaseEstimator):\n def __init__(\n self,\n cfg: ConfigRun,\n model: torch.nn.Module,\n n_classes: int,\n device: str,\n rng: np.random.RandomState = None,\n verbose: bool = True,\n ):\n self.cfg = cfg\n if rng is None:\n rng = np.random.RandomState(self.cfg.seed)\n self.rng = rng\n self.verbose = verbose\n self.device = device\n self.model = model.to(self.device, non_blocking=True)\n self.n_classes = n_classes\n\n self.loss = get_loss(self.cfg)\n self.optimizer = get_optimizer(self.cfg.hyperparams, self.model)\n self.scheduler_warmup, self.scheduler_reduce_on_plateau = get_scheduler(self.cfg.hyperparams, self.optimizer)\n self.scaler = GradScaler(\n enabled=self.cfg.hyperparams[\"grad_scaler_enabled\"],\n scale_init=self.cfg.hyperparams[\"grad_scaler_scale_init\"],\n scale_min=self.cfg.hyperparams[\"grad_scaler_scale_min\"],\n growth_interval=self.cfg.hyperparams[\"grad_scaler_growth_interval\"],\n device=self.device,\n )\n\n self.early_stopping = EarlyStopping(patience=self.cfg.hyperparams[\"early_stopping_patience\"])\n self.checkpoint = Checkpoint()\n self.preprocessor = Preprocessor(\n dim_embedding=self.cfg.hyperparams[\"dim_embedding\"],\n n_classes=self.n_classes,\n dim_output=self.cfg.hyperparams[\"dim_output\"],\n use_quantile_transformer=self.cfg.hyperparams[\"use_quantile_transformer\"],\n use_feature_count_scaling=self.cfg.hyperparams[\"use_feature_count_scaling\"],\n use_random_transforms=self.cfg.hyperparams[\"use_random_transforms\"],\n shuffle_classes=self.cfg.hyperparams[\"shuffle_classes\"],\n shuffle_features=self.cfg.hyperparams[\"shuffle_features\"],\n random_mirror_x=self.cfg.hyperparams[\"random_mirror_x\"],\n random_mirror_regression=self.cfg.hyperparams[\"random_mirror_regression\"],\n task=self.cfg.task,\n )\n\n self.checkpoint.reset(self.model)\n\n if self.cfg.task == Task.REGRESSION and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY:\n self.bins = torch.linspace(-0.5, 1.5, self.cfg.hyperparams[\"dim_output\"] + 1, device=cfg.device)\n self.bin_width = self.bins[1] - self.bins[0]\n\n self.metric = self.cfg.hyperparams[\"metric\"]\n\n def set_device(self, device: str):\n self.device = device\n self.model = self.model.to(device=device, non_blocking=True)\n\n def post_fit_optimize(self):\n # Minimize memory usage post-fit\n self.checkpoint = None\n self.optimizer = None\n self.scaler = None\n self.scheduler_warmup = None\n self.scheduler_reduce_on_plateau = None\n self.loss = None\n self.early_stopping = None\n self.metric = None\n\n def train(self, x_train: np.ndarray, y_train: np.ndarray, x_val: np.ndarray, y_val: np.ndarray):\n self.preprocessor.fit(x_train, y_train)\n\n x_train_transformed = self.preprocessor.transform_X(x_train)\n y_train_transformed = self.preprocessor.transform_y(y_train)\n\n dataset_train_generator = DatasetFinetuneGenerator(\n self.cfg,\n x=x_train_transformed,\n y=y_train_transformed,\n task=self.cfg.task,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n rng=self.rng,\n )\n\n self.checkpoint.reset(self.model)\n\n metrics_valid = self.evaluate(x_train, y_train, x_val, y_val)\n if self.verbose:\n self.log_start_metrics(metrics_valid)\n self.checkpoint(self.model, metrics_valid.loss)\n\n start_time = time.time()\n\n for epoch in range(1, self.cfg.hyperparams[\"max_epochs\"] + 1):\n dataset_train = next(dataset_train_generator)\n loader_train = self.make_loader(dataset_train, training=True)\n self.model.train()\n\n prediction_metrics_tracker = PredictionMetricsTracker(task=self.cfg.task, preprocessor=self.preprocessor)\n\n for batch in loader_train:\n with torch.autocast(device_type=self.device, dtype=getattr(torch, self.cfg.hyperparams[\"precision\"])):\n x_support = batch[\"x_support\"].to(self.device, non_blocking=True)\n y_support = batch[\"y_support\"].to(self.device, non_blocking=True)\n x_query = batch[\"x_query\"].to(self.device, non_blocking=True)\n y_query = batch[\"y_query\"].to(self.device, non_blocking=True)\n padding_features = batch[\"padding_features\"].to(self.device, non_blocking=True)\n padding_obs_support = batch[\"padding_obs_support\"].to(self.device, non_blocking=True)\n padding_obs_query = batch[\"padding_obs_query\"].to(self.device, non_blocking=True)\n\n # Convert numerical y_support to bin ids\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_support = torch.bucketize(y_support, self.bins) - 1\n y_support = torch.clamp(y_support, 0, self.cfg.hyperparams[\"dim_output\"] - 1).to(torch.int64)\n y_query_bin_ids = torch.bucketize(y_query, self.bins) - 1\n y_query_bin_ids = torch.clamp(y_query_bin_ids, 0, self.cfg.hyperparams[\"dim_output\"] - 1).to(\n torch.int64\n )\n\n if self.cfg.model_name == ModelName.TABPFN:\n y_hat = self.model(x_support, y_support, x_query, task=self.cfg.task).squeeze(-1)\n elif self.cfg.model_name in [ModelName.TAB2D, ModelName.TAB2D_COL_ROW, ModelName.TAB2D_SDPA]:\n y_hat = self.model(\n x_support, y_support, x_query, padding_features, padding_obs_support, padding_obs_query\n )\n\n # Convert numerical y_query to bin ids\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n loss = self.loss(y_hat, y_query_bin_ids)\n elif self.cfg.task == Task.CLASSIFICATION:\n # for b in range(y_support.shape[0]):\n # unique_classes = len(torch.unique(torch.cat((y_support[b], y_query[b]))))\n # y_hat[b, :, unique_classes:] = 0\n loss = self.loss(y_hat, y_query)\n else:\n loss = self.loss(y_hat, y_query)\n\n self.optimizer.zero_grad()\n self.scaler.scale(loss).backward()\n self.scaler.step(self.optimizer)\n self.scaler.update()\n\n # Convert bin id predictions to numerical values\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_hat = torch.argmax(y_hat, dim=-1)\n y_hat = self.bins[y_hat] + self.bin_width / 2\n\n y_hat = y_hat.float()\n if self.cfg.task == Task.REGRESSION:\n prediction_metrics_tracker.update(y_hat, y_query, train=True)\n else:\n prediction_metrics_tracker.update(y_hat, y_query, train=False)\n\n metrics_train = prediction_metrics_tracker.get_metrics()\n metrics_valid = self.evaluate(x_train, y_train, x_val, y_val)\n\n if self.verbose:\n self.log_metrics(epoch, metrics_train, metrics_valid)\n\n self.checkpoint(self.model, metrics_valid.loss)\n\n self.early_stopping(metrics_valid.metrics[self.metric])\n if self.early_stopping.we_should_stop():\n if self.verbose:\n logger.info(\"Early stopping\")\n break\n\n if (\n self.cfg.hyperparams[\"budget\"] is not None\n and self.cfg.hyperparams[\"budget\"] > 0\n and time.time() - start_time > self.cfg.hyperparams[\"budget\"]\n ):\n logger.info(\"Time limit reached\")\n break\n\n if epoch < self.cfg.hyperparams[\"warmup_steps\"]:\n self.scheduler_warmup.step()\n else:\n self.scheduler_reduce_on_plateau.step(metrics_valid.loss)\n\n self.checkpoint.set_to_best(self.model)\n\n def evaluate(\n self, x_support: np.ndarray, y_support: np.ndarray, x_query: np.ndarray, y_query: np.ndarray\n ) -> PredictionMetrics:\n self.model.eval()\n\n x_support_transformed = self.preprocessor.transform_X(x_support)\n x_query_transformed = self.preprocessor.transform_X(x_query)\n y_support_transformed = self.preprocessor.transform_y(y_support)\n # y_query_transformed = self.preprocessor.transform_y(y_query)\n\n dataset = DatasetFinetune(\n self.cfg,\n x_support=x_support_transformed,\n y_support=y_support_transformed,\n x_query=x_query_transformed,\n y_query=y_query,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n rng=self.rng,\n )\n\n loader = self.make_loader(dataset, training=False)\n prediction_metrics_tracker = PredictionMetricsTracker(task=self.cfg.task, preprocessor=self.preprocessor)\n\n with torch.no_grad():\n for batch in loader:\n with torch.autocast(device_type=self.device, dtype=getattr(torch, self.cfg.hyperparams[\"precision\"])):\n x_s = batch[\"x_support\"].to(self.device, non_blocking=True)\n y_s = batch[\"y_support\"].to(self.device, non_blocking=True)\n x_q = batch[\"x_query\"].to(self.device, non_blocking=True)\n y_q = batch[\"y_query\"].to(self.device, non_blocking=True)\n padding_features = batch[\"padding_features\"].to(self.device, non_blocking=True)\n padding_obs_support = batch[\"padding_obs_support\"].to(self.device, non_blocking=True)\n padding_obs_query = batch[\"padding_obs_query\"].to(self.device, non_blocking=True)\n\n # Convert numerical y_support to bin ids\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_s = torch.bucketize(y_s, self.bins) - 1\n y_s = torch.clamp(y_s, 0, self.cfg.hyperparams[\"dim_output\"] - 1).to(torch.int64)\n\n if self.cfg.model_name == ModelName.TABPFN:\n y_hat = self.model(x_s, y_s, x_q, task=self.cfg.task).squeeze(-1)\n elif self.cfg.model_name in [ModelName.TAB2D, ModelName.TAB2D_COL_ROW, ModelName.TAB2D_SDPA]:\n y_hat = self.model(x_s, y_s, x_q, padding_features, padding_obs_support, padding_obs_query)\n\n # Convert bin id predictions to numerical values\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_hat = torch.argmax(y_hat, dim=-1)\n y_hat = self.bins[y_hat] + self.bin_width / 2\n\n y_hat = y_hat.float()\n prediction_metrics_tracker.update(y_hat, y_q, train=False)\n\n metrics_eval = prediction_metrics_tracker.get_metrics()\n return metrics_eval\n\n def predict(self, x_support: np.ndarray, y_support: np.ndarray, x_query: np.ndarray) -> np.ndarray:\n x_support_transformed = self.preprocessor.transform_X(x_support)\n x_query_transformed = self.preprocessor.transform_X(x_query)\n y_support_transformed = self.preprocessor.transform_y(y_support)\n\n dataset = DatasetFinetune(\n self.cfg,\n x_support=x_support_transformed,\n y_support=y_support_transformed,\n x_query=x_query_transformed,\n y_query=None,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n rng=self.rng,\n )\n\n loader = self.make_loader(dataset, training=False)\n self.model.eval()\n\n y_pred_list = []\n\n with torch.no_grad():\n for batch in loader:\n with torch.autocast(device_type=self.device, dtype=getattr(torch, self.cfg.hyperparams[\"precision\"])):\n x_s = batch[\"x_support\"].to(self.device, non_blocking=True)\n y_s = batch[\"y_support\"].to(self.device, non_blocking=True)\n x_q = batch[\"x_query\"].to(self.device, non_blocking=True)\n padding_features = batch[\"padding_features\"].to(self.device, non_blocking=True)\n padding_obs_support = batch[\"padding_obs_support\"].to(self.device, non_blocking=True)\n padding_obs_query = batch[\"padding_obs_query\"].to(self.device, non_blocking=True)\n\n # Convert numerical y_support to bin ids\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_s = torch.bucketize(y_s, self.bins) - 1\n y_s = torch.clamp(y_s, 0, self.cfg.hyperparams[\"dim_output\"] - 1).to(torch.int64)\n\n if self.cfg.model_name == ModelName.TABPFN:\n y_hat = self.model(x_s, y_s, x_q, task=self.cfg.task).squeeze(-1)\n elif self.cfg.model_name in [ModelName.TAB2D, ModelName.TAB2D_COL_ROW, ModelName.TAB2D_SDPA]:\n y_hat = self.model(x_s, y_s, x_q, padding_features, padding_obs_support, padding_obs_query)\n\n y_hat = y_hat[0].float().cpu().numpy()\n\n # Convert bin id predictions to numerical values\n if (\n self.cfg.task == Task.REGRESSION\n and self.cfg.hyperparams[\"regression_loss\"] == LossName.CROSS_ENTROPY\n ):\n y_hat = np.argmax(y_hat, axis=-1)\n y_hat = (self.bins[y_hat] + self.bin_width / 2).cpu().numpy()\n\n y_hat = self.preprocessor.inverse_transform_y(y_hat)\n y_pred_list.append(y_hat)\n\n y_pred = np.concatenate(y_pred_list, axis=0)\n\n return y_pred\n\n def load_params(self, path):\n self.model.load_state_dict(torch.load(path))\n\n def make_loader(self, dataset: torch.utils.data.Dataset, training: bool) -> torch.utils.data.DataLoader:\n if self.cfg.model_name == ModelName.TABPFN:\n pad_to_max_features = True\n elif self.cfg.model_name in [ModelName.TAB2D, ModelName.TAB2D_COL_ROW, ModelName.TAB2D_SDPA]:\n pad_to_max_features = False\n else:\n raise NotImplementedError(f\"Model {self.cfg.model_name} not implemented\")\n\n return torch.utils.data.DataLoader(\n dataset,\n batch_size=1,\n shuffle=training,\n pin_memory=True,\n num_workers=0,\n drop_last=False,\n collate_fn=CollatorWithPadding(\n max_features=self.cfg.hyperparams[\"dim_embedding\"], pad_to_max_features=pad_to_max_features\n ),\n )\n\n def log_start_metrics(self, metrics_valid: PredictionMetrics):\n if self.cfg.task == Task.REGRESSION:\n logger.info(\n (\n f\"Epoch 000 \"\n f\"| Train MSE: -.---- \"\n f\"| Train MAE: -.---- \"\n f\"| Train r2: -.---- \"\n f\"| Val MSE: {metrics_valid.metrics[MetricName.MSE]:.4f} \"\n f\"| Val MAE: {metrics_valid.metrics[MetricName.MAE]:.4f} \"\n f\"| Val r2: {metrics_valid.metrics[MetricName.R2]:.4f}\"\n )\n )\n\n elif self.cfg.task == Task.CLASSIFICATION:\n logger.info(\n (\n f\"Epoch 000 \"\n f\"| Train CE: -.---- \"\n f\"| Train acc: -.---- \"\n f\"| Val CE: {metrics_valid.metrics[MetricName.LOG_LOSS]:.4f} \"\n f\"| Val acc: {metrics_valid.metrics[MetricName.ACCURACY]:.4f}\"\n )\n )\n\n def log_metrics(self, epoch: int, metrics_train: PredictionMetrics, metrics_valid: PredictionMetrics):\n if self.cfg.task == Task.REGRESSION:\n logger.info(\n (\n f\"Epoch {epoch:03d} \"\n f\"| Train MSE: {metrics_train.metrics[MetricName.MSE]:.4f} \"\n f\"| Train MAE: {metrics_train.metrics[MetricName.MAE]:.4f} \"\n f\"| Train r2: {metrics_train.metrics[MetricName.R2]:.4f} \"\n f\"| Val MSE: {metrics_valid.metrics[MetricName.MSE]:.4f} \"\n f\"| Val MAE: {metrics_valid.metrics[MetricName.MAE]:.4f} \"\n f\"| Val r2: {metrics_valid.metrics[MetricName.R2]:.4f}\"\n )\n )\n elif self.cfg.task == Task.CLASSIFICATION:\n logger.info(\n (\n f\"Epoch {epoch:03d} \"\n f\"| Train CE: {metrics_train.metrics[MetricName.LOG_LOSS]:.4f} \"\n f\"| Train acc: {metrics_train.metrics[MetricName.ACCURACY]:.4f} \"\n f\"| Val CE: {metrics_valid.metrics[MetricName.LOG_LOSS]:.4f} \"\n f\"| Val acc: {metrics_valid.metrics[MetricName.ACCURACY]:.4f}\"\n )\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/data/__init__.py",
"content": "# Data processing modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/data/collator.py",
"content": "import torch\n\n\nclass CollatorWithPadding:\n def __init__(\n self,\n max_features: int,\n pad_to_max_features: bool,\n ) -> None:\n self.max_features = max_features\n self.pad_to_max_features = pad_to_max_features\n\n def __call__(self, batch: list[dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]:\n max_support_samples = max(dataset[\"x_support\"].shape[0] for dataset in batch)\n max_query_samples = max(dataset[\"x_query\"].shape[0] for dataset in batch)\n max_features = max(dataset[\"x_support\"].shape[1] for dataset in batch)\n\n if self.pad_to_max_features:\n max_features = self.max_features\n\n batch_size = len(batch)\n\n tensor_dict = {\n \"x_support\": torch.zeros(\n (batch_size, max_support_samples, max_features), dtype=batch[0][\"x_support\"].dtype\n ),\n \"y_support\": torch.full(\n (batch_size, max_support_samples), fill_value=-100, dtype=batch[0][\"y_support\"].dtype\n ),\n \"x_query\": torch.zeros((batch_size, max_query_samples, max_features), dtype=batch[0][\"x_query\"].dtype),\n \"y_query\": torch.full((batch_size, max_query_samples), fill_value=-100, dtype=batch[0][\"y_query\"].dtype),\n \"padding_features\": torch.ones((batch_size, max_features), dtype=torch.bool),\n \"padding_obs_support\": torch.ones((batch_size, max_support_samples), dtype=torch.bool),\n \"padding_obs_query\": torch.ones((batch_size, max_query_samples), dtype=torch.bool),\n }\n\n for i, dataset in enumerate(batch):\n tensor_dict[\"x_support\"][i, : dataset[\"x_support\"].shape[0], : dataset[\"x_support\"].shape[1]] = dataset[\n \"x_support\"\n ]\n tensor_dict[\"y_support\"][i, : dataset[\"y_support\"].shape[0]] = dataset[\"y_support\"]\n tensor_dict[\"x_query\"][i, : dataset[\"x_query\"].shape[0], : dataset[\"x_support\"].shape[1]] = dataset[\n \"x_query\"\n ]\n tensor_dict[\"y_query\"][i, : dataset[\"y_query\"].shape[0]] = dataset[\"y_query\"]\n tensor_dict[\"padding_features\"][i, : dataset[\"x_support\"].shape[1]] = False\n tensor_dict[\"padding_obs_support\"][i, : dataset[\"x_support\"].shape[0]] = False\n tensor_dict[\"padding_obs_query\"][i, : dataset[\"x_query\"].shape[0]] = False\n\n return tensor_dict\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/data/dataset_finetune.py",
"content": "from typing import Optional\n\nimport numpy as np\nimport torch\n\nfrom ..._internal.config.config_run import ConfigRun\nfrom ..._internal.config.enums import Task\nfrom ..._internal.data.dataset_split import make_dataset_split\n\n\nclass DatasetFinetune(torch.utils.data.Dataset):\n \"\"\"\n The main goal of this class is to generate a dataset for fine-tuning.\n The input data are the full (x_support, y_support, x_query, y_query)\n But these arrays are too large to be pushed through the model at once.\n So here we split query the data into chunks if the query data is too large.\n If the support data is too large, we randomly sample from it.\n Furthermore, we transition from numpy to tensors.\n \"\"\"\n\n def __init__(\n self,\n cfg: ConfigRun,\n x_support: np.ndarray,\n y_support: np.ndarray,\n x_query: np.ndarray,\n y_query: Optional[np.ndarray],\n max_samples_support: int,\n max_samples_query: int,\n rng: np.random.RandomState,\n ):\n \"\"\"\n :param: max_features: number of features the tab pfn model has been trained on\n \"\"\"\n\n self.cfg = cfg\n self.rng = rng\n\n self.x_support = x_support\n self.y_support = y_support\n self.x_query = x_query\n self.y_query = y_query\n\n if self.y_query is None:\n self.y_query = np.zeros((self.x_query.shape[0],)) - 1\n\n self.max_samples_support = max_samples_support\n self.max_samples_query = max_samples_query\n\n self.x_queries = self.split_in_chunks(self.x_query, max_samples_query)\n self.y_queries = self.split_in_chunks(self.y_query, max_samples_query)\n\n self.n_samples_support = self.x_support.shape[0]\n\n # We push the whole training data through the model, unless it is too large\n self.support_size = min(self.max_samples_support, self.n_samples_support)\n\n def __len__(self):\n return len(self.x_queries)\n\n def __getitem__(self, idx):\n support_indices = self.rng.choice(self.n_samples_support, size=self.support_size, replace=False)\n\n x_support = self.x_support[support_indices]\n y_support = self.y_support[support_indices]\n\n x_support_tensor = torch.as_tensor(x_support)\n y_support_tensor = torch.as_tensor(y_support)\n x_query_tensor = torch.as_tensor(self.x_queries[idx])\n y_query_tensor = torch.as_tensor(self.y_queries[idx])\n\n return {\n \"x_support\": x_support_tensor,\n \"y_support\": y_support_tensor,\n \"x_query\": x_query_tensor,\n \"y_query\": y_query_tensor,\n }\n\n def split_in_chunks(self, x: np.ndarray, batch_size: int) -> list[np.ndarray]:\n \"\"\"\n Splits the data into chunks of size batch_size\n \"\"\"\n\n n_chunks = int(np.ceil(x.shape[0] / batch_size))\n x_chunks = []\n\n for i in range(n_chunks):\n x_chunks.append(x[i * batch_size : (i + 1) * batch_size])\n\n return x_chunks\n\n\ndef DatasetFinetuneGenerator(\n cfg: ConfigRun,\n x: np.ndarray,\n y: np.ndarray,\n task: Task,\n max_samples_support: int,\n max_samples_query: int,\n rng: np.random.RandomState,\n):\n \"\"\"\n The dataset fine-tune generator is a generator that yields a dataset for fine-tuning.\n The idea is to split the training dataset into a support and query set.\n Every single iteration, the generator yields a different support and query set split.\n The dataset made always has exactly one batch.\n \"\"\"\n\n while True:\n x_support, x_query, y_support, y_query = make_dataset_split(x=x, y=y, task=task, seed=rng)\n n_samples_support = x_support.shape[0]\n n_samples_query = x_query.shape[0]\n\n support_size = min(max_samples_support, n_samples_support)\n query_size = min(max_samples_query, n_samples_query)\n\n dataset_finetune = DatasetFinetune(\n cfg=cfg,\n x_support=x_support[:support_size],\n y_support=y_support[:support_size],\n x_query=x_query[:query_size],\n y_query=y_query[:query_size],\n max_samples_support=max_samples_support,\n max_samples_query=max_samples_query,\n rng=rng,\n )\n\n yield dataset_finetune\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/data/dataset_split.py",
"content": "from __future__ import annotations\n\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\nfrom ..._internal.config.enums import Task\n\n\ndef make_dataset_split(x: np.ndarray, y: np.ndarray, task: Task, seed: int) -> tuple[np.ndarray, ...]:\n # Splits the dataset into train and validation sets with ratio 80/20\n\n if task == Task.REGRESSION:\n return make_standard_dataset_split(x, y, seed=seed)\n\n size_of_smallest_class = np.min(np.bincount(y))\n\n if size_of_smallest_class >= 5:\n # stratification needs have at least 5 samples in each class if split is 80/20\n return make_stratified_dataset_split(x, y, seed=seed)\n else:\n return make_standard_dataset_split(x, y, seed=seed)\n\n\ndef make_stratified_dataset_split(x, y, n_splits=5, seed=0):\n if isinstance(seed, int):\n seed = np.random.RandomState(seed)\n\n # Stratify doesn't shuffle the data, so we shuffle it first\n permutation = seed.permutation(len(y))\n x, y = x[permutation], y[permutation]\n\n min_samples_per_class = np.min(np.bincount(y))\n\n # Adjust n_splits based on both total samples and minimum samples per class\n n_samples = len(y)\n max_possible_splits = min(n_samples - 1, min_samples_per_class)\n n_splits = min(n_splits, max_possible_splits)\n\n # Ensure we have at least 2 splits if possible\n if n_samples >= 2 and min_samples_per_class >= 2:\n n_splits = max(2, n_splits)\n else:\n # If we can't do stratified splitting, fall back to standard split\n return make_standard_dataset_split(x, y, seed)\n\n skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n indices = next(skf.split(x, y))\n x_t_train, x_t_valid = x[indices[0]], x[indices[1]] # 80%, 20%\n y_t_train, y_t_valid = y[indices[0]], y[indices[1]]\n\n return x_t_train, x_t_valid, y_t_train, y_t_valid\n\n\ndef make_standard_dataset_split(x, y, seed):\n return train_test_split(\n x,\n y,\n test_size=0.2,\n random_state=seed,\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/data/preprocessor.py",
"content": "import random\nfrom typing import Optional\n\nimport numpy as np\nfrom loguru import logger\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.pipeline import FeatureUnion, Pipeline\nfrom sklearn.preprocessing import OrdinalEncoder, QuantileTransformer, StandardScaler\n\nfrom ..._internal.config.enums import Task\n\n\nclass NoneTransformer(BaseEstimator, TransformerMixin):\n def fit(self, X, y=None):\n return self\n\n def transform(self, X):\n return X\n\n\nclass Preprocessor:\n \"\"\"\n This class is used to preprocess the data before it is pushed through the model.\n The preprocessor assures that the data has the right shape and is normalized,\n This way the model always gets the same input distribution,\n no matter whether the input data is synthetic or real.\n\n \"\"\"\n\n def __init__(\n self,\n dim_embedding: Optional[\n int\n ], # Size of the feature embedding. For some models this is None, which means the embedding does not depend on the number of features\n n_classes: int, # Actual number of classes in the dataset, assumed to be numbered 0, ..., n_classes - 1\n dim_output: int, # Maximum number of classes the model has been trained on -> size of the output\n use_quantile_transformer: bool,\n use_feature_count_scaling: bool,\n use_random_transforms: bool,\n shuffle_classes: bool,\n shuffle_features: bool,\n random_mirror_regression: bool,\n random_mirror_x: bool,\n task: Task,\n ):\n self.dim_embedding = dim_embedding\n self.n_classes = n_classes\n self.dim_output = dim_output\n self.use_quantile_transformer = use_quantile_transformer\n self.use_feature_count_scaling = use_feature_count_scaling\n self.use_random_transforms = use_random_transforms\n self.shuffle_classes = shuffle_classes\n self.shuffle_features = shuffle_features\n self.random_mirror_regression = random_mirror_regression\n self.random_mirror_x = random_mirror_x\n self.task = task\n\n def fit(self, X: np.ndarray, y: np.ndarray) -> \"Preprocessor\":\n \"\"\"\n X: np.ndarray [n_samples, n_features]\n y: np.ndarray [n_samples]\n \"\"\"\n\n if self.task == Task.CLASSIFICATION:\n # We assume that y properly presents classes [0, 1, 2, ...] before passing to the preprocessor\n # If the test set has a class that is not in the training set, we will throw an error\n\n assert np.all(y < self.n_classes), \"y contains class values that are not in the range of n_classes\"\n\n self.compute_pre_nan_mean(X)\n X = self.impute_nan_features_with_mean(X)\n\n self.determine_which_features_are_singular(X)\n X = self.cutoff_singular_features(X, self.singular_features)\n\n self.determine_which_features_to_select(X, y)\n X = self.select_features(X)\n\n if self.use_quantile_transformer:\n # If use quantile transform is off, it means that the preprocessing will happen on the GPU.\n X = self.fit_transform_quantile_transformer(X)\n\n self.mean, self.std = self.calc_mean_std(X)\n X = self.normalize_by_mean_std(X, self.mean, self.std)\n\n if self.use_random_transforms:\n X = self.transform_tabpfn(X)\n\n if self.task == Task.CLASSIFICATION and self.shuffle_classes:\n self.determine_shuffle_class_order()\n\n if self.shuffle_features:\n self.determine_feature_order(X)\n\n if self.task == Task.REGRESSION:\n self.determine_mix_max_scale(y)\n\n if self.task == Task.REGRESSION and self.random_mirror_regression:\n self.determine_regression_mirror()\n\n if self.random_mirror_x:\n self.determine_mirror(X)\n\n X[np.isnan(X)] = 0\n X[np.isinf(X)] = 0\n\n return self\n\n def transform_X(self, X: np.ndarray):\n X = self.impute_nan_features_with_mean(X)\n X = self.cutoff_singular_features(X, self.singular_features)\n X = self.select_features(X)\n\n if self.use_quantile_transformer:\n # If use quantile transform is off, it means that the preprocessing will happen on the GPU.\n\n X = self.quantile_transformer.transform(X)\n\n X = self.normalize_by_mean_std(X, self.mean, self.std)\n\n if self.use_feature_count_scaling:\n X = self.normalize_by_feature_count(X)\n\n if self.use_random_transforms:\n X = self.random_transforms.transform(X)\n\n if self.shuffle_features:\n X = self.randomize_feature_order(X)\n\n if self.random_mirror_x:\n X = self.apply_random_mirror_x(X)\n\n X = X.astype(np.float32)\n\n X[np.isnan(X)] = 0\n X[np.isinf(X)] = 0\n\n return X\n\n def transform_tabpfn(self, X: np.ndarray):\n n_samples = X.shape[0]\n n_features = X.shape[1]\n\n use_config1 = random.random() < 0.5\n random_state = random.randint(0, 1000000)\n\n if use_config1:\n self.random_transforms = Pipeline(\n [\n (\n \"quantile\",\n QuantileTransformer(\n output_distribution=\"normal\",\n n_quantiles=max(n_samples // 10, 2),\n random_state=random_state,\n ),\n ),\n (\n \"svd\",\n FeatureUnion(\n [\n (\"passthrough\", NoneTransformer()),\n (\n \"svd\",\n Pipeline(\n [\n (\"standard\", StandardScaler(with_mean=False)),\n (\n \"svd\",\n TruncatedSVD(\n algorithm=\"arpack\",\n n_components=max(1, min(n_samples // 10 + 1, n_features // 2)),\n random_state=random_state,\n ),\n ),\n ]\n ),\n ),\n ]\n ),\n ),\n ]\n )\n else:\n self.random_transforms = ColumnTransformer(\n [(\"ordinal\", OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=np.nan), [])],\n remainder=\"passthrough\",\n )\n\n return self.random_transforms.fit_transform(X)\n\n def transform_y(self, y: np.ndarray):\n if self.task == Task.CLASSIFICATION:\n # We assume that y properly presents classes [0, 1, 2, ...] before passing to the preprocessor\n # If the test set has a class that is not in the training set, we will throw an error\n assert np.all(y < self.n_classes), \"y contains class values that are not in the range of n_classes\"\n\n if self.task == Task.CLASSIFICATION and self.shuffle_classes:\n y = self.randomize_class_order(y)\n\n if self.task == Task.REGRESSION:\n y = self.normalize_y(y)\n\n if self.task == Task.REGRESSION and self.random_mirror_regression:\n y = self.apply_random_mirror_regression(y)\n\n if self.task == Task.CLASSIFICATION:\n y = y.astype(np.int64)\n elif self.task == Task.REGRESSION:\n y = y.astype(np.float32)\n\n return y\n\n def inverse_transform_y(self, y: np.ndarray):\n # Function used during the prediction to transform the model output back to the original space\n # For classification, y is assumed to be logits of shape [n_samples, n_classes]\n\n if self.task == Task.CLASSIFICATION:\n y = self.extract_correct_classes(y)\n\n if self.shuffle_classes:\n y = self.undo_randomize_class_order(y)\n\n elif self.task == Task.REGRESSION:\n if self.random_mirror_regression:\n y = self.apply_random_mirror_regression(y)\n\n y = self.undo_normalize_y(y)\n\n return y\n\n def fit_transform_quantile_transformer(self, X: np.ndarray) -> np.ndarray:\n n_obs, n_features = X.shape\n n_quantiles = min(n_obs, 1000)\n self.quantile_transformer = QuantileTransformer(n_quantiles=n_quantiles, output_distribution=\"normal\")\n X = self.quantile_transformer.fit_transform(X)\n\n return X\n\n def determine_which_features_are_singular(self, x: np.ndarray) -> None:\n self.singular_features = np.array([len(np.unique(x_col)) for x_col in x.T]) == 1\n\n def determine_which_features_to_select(self, x: np.ndarray, y: np.ndarray) -> None:\n if self.dim_embedding is None:\n # All features are selected\n return\n\n if x.shape[1] > self.dim_embedding:\n logger.info(\n f\"Number of features is capped at {self.dim_embedding}, but the dataset has {x.shape[1]} features. A subset of {self.dim_embedding} are selected using SelectKBest\"\n )\n\n self.select_k_best = SelectKBest(k=self.dim_embedding)\n self.select_k_best.fit(x, y)\n\n def compute_pre_nan_mean(self, x: np.ndarray) -> None:\n \"\"\"\n Computes the mean of the data before the NaNs are imputed\n \"\"\"\n self.pre_nan_mean = np.nanmean(x, axis=0)\n\n def impute_nan_features_with_mean(self, x: np.ndarray) -> np.ndarray:\n inds = np.where(np.isnan(x))\n x[inds] = np.take(self.pre_nan_mean, inds[1])\n return x\n\n def select_features(self, x: np.ndarray) -> np.ndarray:\n if self.dim_embedding is None:\n # All features are selected\n return x\n\n if x.shape[1] > self.dim_embedding:\n x = self.select_k_best.transform(x)\n\n return x\n\n def cutoff_singular_features(self, x: np.ndarray, singular_features: np.ndarray) -> np.ndarray:\n if singular_features.any():\n x = x[:, ~singular_features]\n\n return x\n\n def calc_mean_std(self, x: np.ndarray) -> tuple[np.ndarray, np.ndarray]:\n \"\"\"\n Calculates the mean and std of the training data\n \"\"\"\n mean = x.mean(axis=0)\n std = x.std(axis=0) + 1e-6\n return mean, std\n\n def normalize_by_mean_std(self, x: np.ndarray, mean: np.ndarray, std: np.ndarray) -> np.ndarray:\n \"\"\"\n Normalizes the data by the mean and std\n \"\"\"\n\n x = (x - mean) / std\n return x\n\n def normalize_by_feature_count(self, x: np.ndarray) -> np.ndarray:\n \"\"\"\n An interesting way of normalization by the tabPFN paper\n \"\"\"\n\n assert self.dim_embedding is not None, \"dim_embedding must be set to use this feature count scaling\"\n\n x = x * self.dim_embedding / x.shape[1]\n\n return x\n\n def extend_feature_dim_to_dim_embedding(self, x: np.ndarray, dim_embedding) -> np.ndarray:\n \"\"\"\n Increases the number of features to the number of features the model has been trained on\n \"\"\"\n\n assert self.dim_embedding is not None, \"dim_embedding must be set to extend the feature dimension\"\n\n added_zeros = np.zeros((x.shape[0], dim_embedding - x.shape[1]), dtype=np.float32)\n x = np.concatenate([x, added_zeros], axis=1)\n return x\n\n def determine_mix_max_scale(self, y: np.ndarray) -> None:\n self.y_min = y.min()\n self.y_max = y.max()\n assert self.y_min != self.y_max, \"y_min and y_max are the same, cannot normalize, regression makes no sense\"\n\n def normalize_y(self, y: np.ndarray) -> np.ndarray:\n y = (y - self.y_min) / (self.y_max - self.y_min)\n return y\n\n def undo_normalize_y(self, y: np.ndarray) -> np.ndarray:\n y = y * (self.y_max - self.y_min) + self.y_min\n return y\n\n def determine_regression_mirror(self) -> None:\n self.regression_mirror = np.random.choice([True, False], size=(1,)).item()\n\n def apply_random_mirror_regression(self, y: np.ndarray) -> np.ndarray:\n if self.regression_mirror:\n y = 1 - y\n return y\n\n def determine_mirror(self, x: np.ndarray) -> None:\n n_features = x.shape[1]\n self.mirror = np.random.choice([1, -1], size=(1, n_features))\n\n def apply_random_mirror_x(self, x: np.ndarray) -> np.ndarray:\n x = x * self.mirror\n return x\n\n def determine_shuffle_class_order(self) -> None:\n if self.shuffle_classes:\n self.new_shuffle_classes = np.random.permutation(self.n_classes)\n else:\n self.new_shuffle_classes = np.arange(self.n_classes)\n\n def randomize_class_order(self, y: np.ndarray) -> np.ndarray:\n mapping = {i: self.new_shuffle_classes[i] for i in range(self.n_classes)}\n y = np.array([mapping[i.item()] for i in y], dtype=np.int64)\n\n return y\n\n def undo_randomize_class_order(self, y_logits: np.ndarray) -> np.ndarray:\n \"\"\"\n We assume y_logits has shape [n_samples, n_classes]\n \"\"\"\n\n # mapping = {self.new_shuffle_classes[i]: i for i in range(self.n_classes)}\n mapping = {i: self.new_shuffle_classes[i] for i in range(self.n_classes)}\n y = np.concatenate([y_logits[:, mapping[i] : mapping[i] + 1] for i in range(self.n_classes)], axis=1)\n\n return y\n\n def extract_correct_classes(self, y_logits: np.ndarray) -> np.ndarray:\n # Even though our network might be able to support 10 classes,\n # If the problem only has three classes, we should give three classes as output.\n # We assume y_logits has shape [n_samples, n_classes]\n y_logits = y_logits[:, : self.n_classes]\n return y_logits\n\n def determine_feature_order(self, x: np.ndarray) -> None:\n n_features = x.shape[1]\n self.new_feature_order = np.random.permutation(n_features)\n\n def randomize_feature_order(self, x: np.ndarray) -> np.ndarray:\n x = x[:, self.new_feature_order]\n\n return x\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/models/__init__.py",
"content": "# Model architecture modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/models/base.py",
"content": "from abc import ABC, abstractmethod\n\nimport torch\nimport torch.nn as nn\n\n\nclass BaseModel(nn.Module, ABC):\n def __init__(self):\n super().__init__()\n\n def init_weights(self):\n \"\"\"Initialize model weights.\"\"\"\n pass\n\n @abstractmethod\n def forward(self, x_support: torch.Tensor, y_support: torch.Tensor, x_query: torch.Tensor, **kwargs):\n \"\"\"Forward pass for the model.\"\"\"\n pass\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/models/embedding.py",
"content": "import einops\nimport einx\nimport torch\nimport torch.nn as nn\n\n\nclass Tab2DEmbeddingX(torch.nn.Module):\n def __init__(self, dim: int) -> None:\n super().__init__()\n\n self.dim = dim\n self.x_embedding = nn.Linear(1, dim)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n x = einx.rearrange(\"b s f -> b s f 1\", x)\n x = self.x_embedding(x)\n\n return x\n\n\nclass Tab2DQuantileEmbeddingX(torch.nn.Module):\n def __init__(\n self,\n dim: int,\n ) -> None:\n super().__init__()\n\n self.dim = dim\n\n def forward(\n self,\n x_support: torch.Tensor,\n x_query__: torch.Tensor,\n padding_mask: torch.Tensor,\n feature_mask: torch.Tensor,\n ) -> tuple[torch.Tensor, torch.Tensor]:\n \"\"\"\n Syntax:\n b = batch size\n s = number of observations\n f = number of features\n q = number of quantiles\n \"\"\"\n\n batch_size = padding_mask.shape[0]\n seq_len = einx.sum(\"b [s]\", ~padding_mask)\n feature_count = einx.sum(\"b [f]\", ~feature_mask)\n\n # By setting the padded tokens to 9999 we ensure they don't participate in the quantile calculation\n x_support[padding_mask] = 9999\n\n q = torch.arange(1, 1000, dtype=torch.float, device=x_support.device) / 1000\n quantiles = torch.quantile(x_support, q=q, dim=1)\n quantiles = einx.rearrange(\"q b f -> (b f) q\", quantiles)\n x_support = einx.rearrange(\"b s f -> (b f) s\", x_support).contiguous()\n x_query__ = einx.rearrange(\"b s f -> (b f) s\", x_query__).contiguous()\n\n bucketize = torch.vmap(torch.bucketize, in_dims=(0, 0), out_dims=0)\n x_support = bucketize(x_support, quantiles).float()\n x_query__ = bucketize(x_query__, quantiles).float()\n x_support = einx.rearrange(\"(b f) s -> b s f\", x_support, b=batch_size).contiguous()\n x_query__ = einx.rearrange(\"(b f) s -> b s f\", x_query__, b=batch_size).contiguous()\n\n # If 30% is padded, the minimum will have quantile 0.0 and the maximum will have quantile 0.7 times max_length.\n # Here we correct the quantiles so that the minimum has quantile 0.0 and the maximum has quantile 1.0.\n x_support = x_support / seq_len[:, None, None]\n x_query__ = x_query__ / seq_len[:, None, None]\n\n # Make sure that the padding is not used in the calculation of the mean\n x_support[padding_mask] = 0\n x_support_mean = einx.sum(\"b [s] f\", x_support, keepdims=True) / seq_len[:, None, None]\n\n x_support = x_support - x_support_mean\n x_query__ = x_query__ - x_support_mean\n\n # Make sure that the padding is not used in the calculation of the variance\n x_support[padding_mask] = 0\n x_support_var = einx.sum(\"b [s] f\", x_support**2, keepdims=True) / seq_len[:, None, None]\n\n x_support = x_support / x_support_var.sqrt()\n x_query__ = x_query__ / x_support_var.sqrt()\n\n # In case an x_support feature column contains one unique feature, set the feature to zero\n x_support = torch.where(x_support_var == 0, 0, x_support)\n x_query__ = torch.where(x_support_var == 0, 0, x_query__)\n\n return x_support, x_query__\n\n\nclass Tab2DEmbeddingY(torch.nn.Module):\n def __init__(self, dim: int, n_classes: int) -> None:\n super().__init__()\n\n self.dim = dim\n self.n_classes = n_classes\n self.y_embedding_support = nn.Linear(1, dim)\n self.y_embedding_query = nn.Embedding(1, dim)\n\n def forward(\n self, y_support: torch.Tensor, padding_obs_support: torch.Tensor, n_obs_query: int\n ) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = y_support.shape[0]\n\n y_support = y_support.type(torch.float32)\n y_support = y_support / self.n_classes - 0.5\n y_support = einops.rearrange(y_support, \"b n -> b n 1\")\n\n y_support = self.y_embedding_support(y_support)\n y_support[padding_obs_support] = 0\n\n y_query = torch.zeros((batch_size, n_obs_query, 1), device=y_support.device, dtype=torch.int64)\n y_query = self.y_embedding_query(y_query)\n\n return y_support, y_query\n\n\nclass Tab2DEmbeddingYClasses(torch.nn.Module):\n def __init__(\n self,\n dim: int,\n n_classes: int,\n ) -> None:\n super().__init__()\n\n self.n_classes = n_classes\n self.dim = dim\n\n self.y_embedding = nn.Embedding(\n n_classes,\n dim,\n )\n self.y_mask = nn.Embedding(1, dim) # masking is also modeled as a separate class\n\n def forward(\n self, y_support: torch.Tensor, padding_obs_support: torch.Tensor, n_obs_query: int\n ) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = y_support.shape[0]\n n_obs_support = y_support.shape[1]\n\n y_support = y_support.type(torch.int64)\n y_support = einops.rearrange(y_support, \"b n -> b n 1\")\n y_support[padding_obs_support] = 0 # padded tokens are -100 -> set it to zero so nn.Embedding can handle it\n y_support = self.y_embedding(y_support)\n y_support[padding_obs_support] = 0 # just to make sure, set it to zero again\n\n y_query = torch.zeros((batch_size, n_obs_query, 1), device=y_support.device, dtype=torch.int64)\n y_query = self.y_mask(y_query)\n\n return y_support, y_query\n\n\nclass Tab2DEmbeddingYRegression(torch.nn.Module):\n def __init__(self, dim: int) -> None:\n super().__init__()\n\n self.dim = dim\n self.y_embedding = nn.Linear(1, dim)\n self.y_mask = nn.Embedding(1, dim)\n\n def forward(\n self, y_support: torch.Tensor, padding_obs_support: torch.Tensor, n_obs_query: int\n ) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = y_support.shape[0]\n y_support = y_support.type(torch.float32)\n y_support = einops.rearrange(y_support, \"b n -> b n 1\")\n y_support = self.y_embedding(y_support)\n y_support[padding_obs_support] = 0\n\n y_query = torch.zeros((batch_size, n_obs_query, 1), device=y_support.device, dtype=torch.int64)\n y_query = self.y_mask(y_query)\n\n return y_support, y_query\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/models/tab2d.py",
"content": "import json\nimport logging\nimport os\nfrom typing import Optional, Union\n\nimport einops\nimport einx\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom huggingface_hub import hf_hub_download\nfrom safetensors.torch import load_file, save_file\n\n# Try to import flash attention, but make it optional\ntry:\n from flash_attn.bert_padding import pad_input, unpad_input\n from flash_attn.flash_attn_interface import flash_attn_varlen_func\n\n FLASH_ATTN_AVAILABLE = True\nexcept ImportError:\n FLASH_ATTN_AVAILABLE = False\n\nfrom torch.utils.checkpoint import checkpoint\n\nfrom ..._internal.config.enums import Task\nfrom ..._internal.models.base import BaseModel\nfrom ..._internal.models.embedding import (\n Tab2DEmbeddingX,\n Tab2DEmbeddingYClasses,\n Tab2DEmbeddingYRegression,\n Tab2DQuantileEmbeddingX,\n)\n\nlogger = logging.getLogger(__name__)\n\n\nclass Tab2D(BaseModel):\n def __init__(\n self,\n dim: int,\n dim_output: int,\n n_layers: int,\n n_heads: int,\n task: Union[str, Task],\n use_pretrained_weights: bool,\n path_to_weights: str,\n device: str = \"cuda\", # Add device parameter\n ) -> None:\n super().__init__()\n\n self.dim = dim\n self.dim_output = dim_output\n self.n_layers = n_layers\n self.n_heads = n_heads\n self.task = task\n self.device_type = device\n\n # Determine if we can use flash attention\n self.use_flash_attn = FLASH_ATTN_AVAILABLE and device.startswith(\"cuda\")\n\n if isinstance(self.task, str):\n self.task = Task[self.task]\n\n self.x_quantile = Tab2DQuantileEmbeddingX(dim)\n self.x_embedding = Tab2DEmbeddingX(dim)\n\n if self.task == Task.CLASSIFICATION:\n self.y_embedding = Tab2DEmbeddingYClasses(dim, dim_output) # type: nn.Module\n elif self.task == Task.REGRESSION:\n if self.dim_output == 1:\n self.y_embedding = Tab2DEmbeddingYRegression(dim)\n else:\n self.y_embedding = Tab2DEmbeddingYClasses(dim, dim_output)\n else:\n raise ValueError(f\"Task {task} not supported\")\n\n self.layers = nn.ModuleList()\n\n for _ in range(n_layers):\n self.layers.append(Layer(dim, n_heads, self.use_flash_attn))\n\n self.final_layer_norm = nn.LayerNorm(dim)\n\n self.final_layer = nn.Linear(dim, dim_output, bias=True)\n\n if use_pretrained_weights:\n if device == \"cpu\":\n # For CPU, use weights_only=False since CUDA checkpoints are incompatible with weights_only=True\n self.load_state_dict(torch.load(path_to_weights, weights_only=False, map_location=torch.device(\"cpu\")))\n else:\n # For GPU, use weights_only=True for security\n self.load_state_dict(torch.load(path_to_weights, weights_only=True, map_location=device))\n else:\n self.init_weights()\n\n def forward(\n self,\n x_support: torch.Tensor, # (b, n_s, f)\n y_support: torch.Tensor, # (b, n_s)\n x_query: torch.Tensor, # (b, n_q, f)\n padding_features: torch.Tensor, # (b, f), \"1\" represents padding, \"0\" represents valid values\n padding_obs_support: torch.Tensor, # (b, n_s)\n padding_obs_query__: torch.Tensor, # (b, n_q)\n ):\n \"\"\"\n x_support is (batch_size, n_observations_support, n_features)\n y_support is (batch_size, n_observations_support)\n\n x_query is (batch_size, n_observations_query, n_features)\n\n returns:\n\n y_query is (batch_size, n_observations_query, n_classes)\n\n syntax:\n b = batch size\n s = number of observations\n f = number of features\n d = dimension of embedding\n c = number of classes\n \"\"\"\n\n x_query__ = x_query\n\n batch_size = x_support.shape[0]\n n_obs_support = x_support.shape[1]\n n_obs_query__ = x_query__.shape[1]\n\n x_support, x_query__ = self.x_quantile(x_support, x_query__, padding_obs_support, padding_features)\n x_support = self.x_embedding(x_support) # (b, n_s, f, d)\n x_query__ = self.x_embedding(x_query__) # (b, n_q, f, d)\n y_support, y_query__ = self.y_embedding(\n y_support, padding_obs_support, n_obs_query__\n ) # (b, n_s, 1, d), (b, n_q, 1, d)\n\n support, pack_support = einops.pack((y_support, x_support), \"b s * d\") # (b, n_s, f+1, d)\n query__, pack_query__ = einops.pack((y_query__, x_query__), \"b s * d\") # (b, n_q, f+1, d)\n\n padding_features_y = torch.zeros((batch_size, 1), device=padding_features.device, dtype=torch.bool) # (b, 1)\n padding_features, _ = einops.pack((padding_features_y, padding_features), \"b *\") # (b, f+1)\n\n if self.use_flash_attn:\n padder_support = Padder(support, padding_obs_support, padding_features)\n padder_query__ = Padder(query__, padding_obs_query__, padding_features)\n\n support = padder_support.base_to_obs(support) # (n_valid_s, d)\n query__ = padder_query__.base_to_obs(query__) # (n_valid_q, d)\n\n for layer in self.layers:\n support, query__ = checkpoint(\n layer, support, query__, padder_support, padder_query__, use_reentrant=False\n ) # (n_valid_s, d), (n_valid_q, d)\n\n query__ = self.final_layer_norm(query__)\n query__ = self.final_layer(query__) # (n_valid_q, d)\n\n query__ = padder_query__.obs_to_base(query__) # (b, n_q, f+1, c)\n else:\n # For CPU/non-flash attention, work with standard tensor format\n for layer in self.layers:\n support, query__ = checkpoint(\n layer,\n support,\n query__,\n None,\n None,\n batch_size,\n padding_obs_support,\n padding_obs_query__,\n padding_features,\n use_reentrant=False,\n )\n\n query__ = self.final_layer_norm(query__)\n query__ = self.final_layer(query__) # (b, n_q, f+1, c)\n\n y_query__, x_query__ = einops.unpack(query__, pack_query__, \"b s * c\") # (b, n_q, 1, c), (b, n_q, f, c)\n\n if self.task == Task.REGRESSION:\n # output has shape (batch_size, n_observations_query, n_features, n_classes)\n # we want to remove the n_features dimension, and for regression, the n_classes dimension\n if self.dim_output == 1:\n y_query__ = y_query__[:, :, 0, 0]\n else:\n y_query__ = y_query__[:, :, 0, :]\n elif self.task == Task.CLASSIFICATION:\n y_query__ = y_query__[:, :, 0, :]\n else:\n raise ValueError(f\"Task {self.task} not supported\")\n\n return y_query__\n\n def init_weights(self) -> None:\n nn.init.normal_(self.x_embedding.x_embedding.weight, mean=0.0, std=1.0)\n nn.init.normal_(self.x_embedding.x_embedding.bias, mean=0.0, std=1.0)\n nn.init.normal_(self.y_embedding.y_embedding.weight, mean=0.0, std=1.0)\n nn.init.normal_(self.y_embedding.y_mask.weight, mean=0.0, std=1.0)\n\n # default PyTorch initialization for everything else\n\n def save_pretrained(self, save_directory: str):\n os.makedirs(save_directory, exist_ok=True)\n\n save_file(self.state_dict(), os.path.join(save_directory, \"model.safetensors\"))\n\n config = {\n \"dim\": self.dim,\n \"dim_output\": self.dim_output,\n \"n_layers\": self.n_layers,\n \"n_heads\": self.n_heads,\n \"task\": str(self.task).upper(),\n }\n with open(os.path.join(save_directory, \"config.json\"), \"w\") as f:\n json.dump(config, f)\n\n @classmethod\n def from_pretrained(cls, path_or_repo_id: str, device: str = \"cuda\") -> \"Tab2D\":\n config_path = hf_hub_download(repo_id=path_or_repo_id, filename=\"config.json\")\n with open(config_path, \"r\") as f:\n config = json.load(f)\n\n model = cls(\n dim=config[\"dim\"],\n dim_output=config[\"dim_output\"],\n n_layers=config[\"n_layers\"],\n n_heads=config[\"n_heads\"],\n task=config[\"task\"],\n use_pretrained_weights=False,\n path_to_weights=\"\",\n device=device,\n )\n\n weights_path = hf_hub_download(repo_id=path_or_repo_id, filename=\"model.safetensors\")\n state_dict = load_file(weights_path, device=device)\n model.load_state_dict(state_dict)\n\n return model\n\n\nclass Padder(torch.nn.Module):\n def __init__(self, x: torch.Tensor, padding_mask: torch.Tensor, feature_mask: torch.Tensor) -> None:\n super().__init__()\n\n self.padding_mask = padding_mask\n self.feature_mask = feature_mask\n\n n_obs, n_feat = x.shape[1], x.shape[2]\n self.batch_size = x.shape[0]\n\n if not FLASH_ATTN_AVAILABLE:\n # CPU fallback: implement simplified padding logic without flash attention\n self._init_cpu_fallback(x, n_obs, n_feat)\n return\n\n # GPU path with flash attention\n self._init_flash_attn(x, n_obs, n_feat)\n\n def _init_cpu_fallback(self, x: torch.Tensor, n_obs: int, n_feat: int):\n \"\"\"Initialize CPU-compatible version without flash attention dependencies.\"\"\"\n # For CPU, we don't need the complex unpadding/padding logic\n # We'll implement pass-through methods that preserve tensor shapes\n self.cpu_mode = True\n\n # Store original shapes for reference\n self.original_shape = x.shape\n self.n_obs = n_obs\n self.n_feat = n_feat\n\n # These attributes won't be used in CPU mode but need to exist for compatibility\n self.cu_seqlens_o = None\n self.cu_seqlens_f = None\n self.cu_seqlens_fo = None\n self.cu_seqlens_of = None\n self.max_seqlen_in_batch_o = None\n self.max_seqlen_in_batch_f = None\n self.max_seqlen_in_batch_fo = None\n self.max_seqlen_in_batch_of = None\n\n def _init_flash_attn(self, x: torch.Tensor, n_obs: int, n_feat: int):\n \"\"\"Initialize GPU version with flash attention.\"\"\"\n self.cpu_mode = False\n\n # Original flash attention initialization logic\n x_o, self.indices_o, self.cu_seqlens_o, self.max_seqlen_in_batch_o, *_ = unpad_input(x, ~self.padding_mask)\n\n self.feature_mask_big = einops.repeat(self.feature_mask, \"b f -> b s f\", s=n_obs)\n self.feature_mask_big, _, _, _, *_ = unpad_input(self.feature_mask_big, ~self.padding_mask)\n x_of, self.indices_of, self.cu_seqlens_of, self.max_seqlen_in_batch_of, *_ = unpad_input(\n x_o, ~self.feature_mask_big\n )\n\n x_rearranged = einx.rearrange(\"b s f d -> b f s d\", x)\n x_f, self.indices_f, self.cu_seqlens_f, self.max_seqlen_in_batch_f, *_ = unpad_input(\n x_rearranged, ~self.feature_mask\n )\n\n self.padding_mask_big = einops.repeat(self.padding_mask, \"b s -> b f s\", f=n_feat)\n self.padding_mask_big, _, _, _, *_ = unpad_input(self.padding_mask_big, ~self.feature_mask)\n x_fo, self.indices_fo, self.cu_seqlens_fo, self.max_seqlen_in_batch_fo, *_ = unpad_input(\n x_f, ~self.padding_mask_big\n )\n\n self.batch_size_f = x_f.shape[0]\n self.batch_size_o = x_o.shape[0]\n\n t = torch.arange(self.indices_fo.shape[0]).unsqueeze(1).to(x.device)\n self.obs_to_feat_indices = self.base_to_feat(self.obs_to_base(t)).squeeze(1)\n self.feat_to_obs_indices = self.base_to_obs(self.feat_to_base(t)).squeeze(1)\n\n def base_to_obs(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: reshape for standard attention\n # Convert from (b, s, f, d) to (b*s, f*d) or similar flattened format\n b, s, f, d = x.shape\n return x.view(b * s, f * d)\n\n # GPU path with flash attention\n x = einx.rearrange(\"b s f d -> b f s d\", x)\n x, _, _, _, *_ = unpad_input(x, ~self.feature_mask)\n x, _, _, _, *_ = unpad_input(x, ~self.padding_mask_big)\n return x\n\n def base_to_feat(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: reshape for standard attention\n # Convert from (b, s, f, d) to (b*f, s*d) or similar flattened format\n b, s, f, d = x.shape\n return x.view(b * f, s * d)\n\n # GPU path with flash attention\n x, _, _, _, *_ = unpad_input(x, ~self.padding_mask)\n x, _, _, _, *_ = unpad_input(x, ~self.feature_mask_big)\n return x\n\n def obs_to_base(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: reshape back to base format\n # This is the inverse of base_to_obs\n total_elements = x.numel()\n expected_d = self.original_shape[-1] # last dimension\n b, s, f = self.batch_size, self.n_obs, self.n_feat\n return x.view(b, s, f, expected_d)\n\n # GPU path with flash attention\n x = pad_input(x, self.indices_fo, self.batch_size_f, self.max_seqlen_in_batch_fo)\n x = pad_input(x, self.indices_f, self.batch_size, self.max_seqlen_in_batch_f)\n x = einx.rearrange(\"b f s d -> b s f d\", x)\n return x\n\n def feat_to_base(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: reshape back to base format\n # This is the inverse of base_to_feat\n total_elements = x.numel()\n expected_d = self.original_shape[-1] # last dimension\n b, s, f = self.batch_size, self.n_obs, self.n_feat\n return x.view(b, s, f, expected_d)\n\n # GPU path with flash attention\n x = pad_input(x, self.indices_of, self.batch_size_o, self.max_seqlen_in_batch_of)\n x = pad_input(x, self.indices_o, self.batch_size, self.max_seqlen_in_batch_o)\n return x\n\n def obs_to_feat(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: simple pass-through or basic reshaping\n return x\n\n # GPU path with flash attention\n x = x[self.obs_to_feat_indices]\n return x\n\n def feat_to_obs(self, x: torch.Tensor) -> torch.Tensor:\n if hasattr(self, \"cpu_mode\") and self.cpu_mode:\n # CPU fallback: simple pass-through or basic reshaping\n return x\n\n # GPU path with flash attention\n x = x[self.feat_to_obs_indices]\n return x\n\n\nclass Layer(torch.nn.Module):\n def __init__(self, dim: int, n_heads: int, use_flash_attn: bool) -> None:\n super().__init__()\n\n self.layer_norm1 = nn.LayerNorm(dim)\n self.attention1 = MultiheadAttention(dim, n_heads, use_flash_attn)\n self.layer_norm2 = nn.LayerNorm(dim)\n self.linear1 = nn.Linear(dim, dim * 4, bias=True)\n self.linear2 = nn.Linear(dim * 4, dim, bias=True)\n\n self.layer_norm3 = nn.LayerNorm(dim)\n self.attention2 = MultiheadAttention(dim, n_heads, use_flash_attn)\n self.layer_norm4 = nn.LayerNorm(dim)\n self.linear3 = nn.Linear(dim, dim * 4, bias=True)\n self.linear4 = nn.Linear(dim * 4, dim, bias=True)\n\n self.use_flash_attn = use_flash_attn\n\n def forward(\n self,\n support: torch.Tensor,\n query__: torch.Tensor,\n padder_support: Optional[Padder],\n padder_query__: Optional[Padder],\n batch_size: Optional[int] = None,\n padding_obs_support: Optional[torch.Tensor] = None,\n padding_obs_query__: Optional[torch.Tensor] = None,\n padding_features: Optional[torch.Tensor] = None,\n ) -> tuple[torch.Tensor, torch.Tensor]:\n \"\"\"\n Input:\n support in 'obs' format\n query__ in 'obs' format\n\n Output:\n support in 'obs' format\n query__ in 'obs' format\n \"\"\"\n\n if self.use_flash_attn and padder_support is not None and padder_query__ is not None:\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm1(support)\n query__ = self.layer_norm1(query__)\n\n # attention across rows\n support_att = self.attention1(\n support,\n support,\n support,\n cu_seqlens_q=padder_support.cu_seqlens_fo,\n max_seqlen_q=padder_support.max_seqlen_in_batch_fo,\n cu_seqlens_k=padder_support.cu_seqlens_fo,\n max_seqlen_k=padder_support.max_seqlen_in_batch_fo,\n )\n query___att = self.attention1(\n query__,\n support,\n support,\n cu_seqlens_q=padder_query__.cu_seqlens_fo,\n max_seqlen_q=padder_query__.max_seqlen_in_batch_fo,\n cu_seqlens_k=padder_support.cu_seqlens_fo,\n max_seqlen_k=padder_support.max_seqlen_in_batch_fo,\n )\n\n support = support_residual + support_att\n query__ = query___residual + query___att\n\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm2(support)\n query__ = self.layer_norm2(query__)\n\n support = self.linear1(support)\n query__ = self.linear1(query__)\n\n support = torch.nn.functional.gelu(support)\n query__ = torch.nn.functional.gelu(query__)\n\n support = self.linear2(support)\n query__ = self.linear2(query__)\n\n support = support_residual + support\n query__ = query___residual + query__\n\n support = padder_support.obs_to_feat(support)\n query__ = padder_query__.obs_to_feat(query__)\n\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm3(support)\n query__ = self.layer_norm3(query__)\n\n # attention across features\n support = self.attention2(\n support,\n support,\n support,\n cu_seqlens_q=padder_support.cu_seqlens_of,\n max_seqlen_q=padder_support.max_seqlen_in_batch_of,\n cu_seqlens_k=padder_support.cu_seqlens_of,\n max_seqlen_k=padder_support.max_seqlen_in_batch_of,\n )\n query__ = self.attention2(\n query__,\n query__,\n query__,\n cu_seqlens_q=padder_query__.cu_seqlens_of,\n max_seqlen_q=padder_query__.max_seqlen_in_batch_of,\n cu_seqlens_k=padder_query__.cu_seqlens_of,\n max_seqlen_k=padder_query__.max_seqlen_in_batch_of,\n )\n\n support = support_residual + support\n query__ = query___residual + query__\n\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm4(support)\n query__ = self.layer_norm4(query__)\n\n support = self.linear3(support)\n query__ = self.linear3(query__)\n\n support = torch.nn.functional.gelu(support)\n query__ = torch.nn.functional.gelu(query__)\n\n support = self.linear4(support)\n query__ = self.linear4(query__)\n\n support = support_residual + support\n query__ = query___residual + query__\n\n support = padder_support.feat_to_obs(support)\n query__ = padder_query__.feat_to_obs(query__)\n\n return support, query__\n else:\n # CPU/Standard attention path - ensure it matches the GPU logic exactly\n # Input format: (b, s, f+1, d) where f+1 includes the y embedding\n batch_size_actual, n_obs_support, n_feat_plus_one, dim = support.shape\n _, n_obs_query, _, _ = query__.shape\n\n if batch_size is None:\n batch_size = batch_size_actual\n\n # First attention block - attention across observations (rows)\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm1(support)\n query__ = self.layer_norm1(query__)\n\n # Reshape for row attention: (b, s, f+1, d) -> (b*(f+1), s, d)\n support_flat = einops.rearrange(support, \"b s f d -> (b f) s d\")\n query___flat = einops.rearrange(query__, \"b s f d -> (b f) s d\")\n\n # attention across observations\n support_att_flat = self.attention1(support_flat, support_flat, support_flat)\n query___att_flat = self.attention1(query___flat, support_flat, support_flat)\n\n # Reshape back: (b*(f+1), s, d) -> (b, s, f+1, d)\n support_att = einops.rearrange(support_att_flat, \"(b f) s d -> b s f d\", b=batch_size)\n query___att = einops.rearrange(query___att_flat, \"(b f) s d -> b s f d\", b=batch_size)\n\n support = support_residual + support_att\n query__ = query___residual + query___att\n\n # First MLP block\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm2(support)\n query__ = self.layer_norm2(query__)\n\n support = self.linear1(support)\n query__ = self.linear1(query__)\n\n support = torch.nn.functional.gelu(support)\n query__ = torch.nn.functional.gelu(query__)\n\n support = self.linear2(support)\n query__ = self.linear2(query__)\n\n support = support_residual + support\n query__ = query___residual + query__\n\n # Second attention block - attention across features\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm3(support)\n query__ = self.layer_norm3(query__)\n\n # Reshape for feature attention: (b, s, f+1, d) -> (b*s, f+1, d)\n support_feat = einops.rearrange(support, \"b s f d -> (b s) f d\")\n query___feat = einops.rearrange(query__, \"b s f d -> (b s) f d\")\n\n # attention across features\n support_feat_att = self.attention2(support_feat, support_feat, support_feat)\n query___feat_att = self.attention2(query___feat, query___feat, query___feat)\n\n # Reshape back: (b*s, f+1, d) -> (b, s, f+1, d)\n support_feat_att = einops.rearrange(support_feat_att, \"(b s) f d -> b s f d\", b=batch_size)\n query___feat_att = einops.rearrange(query___feat_att, \"(b s) f d -> b s f d\", b=batch_size)\n\n support = support_residual + support_feat_att\n query__ = query___residual + query___feat_att\n\n # Second MLP block\n support_residual = support\n query___residual = query__\n\n support = self.layer_norm4(support)\n query__ = self.layer_norm4(query__)\n\n support = self.linear3(support)\n query__ = self.linear3(query__)\n\n support = torch.nn.functional.gelu(support)\n query__ = torch.nn.functional.gelu(query__)\n\n support = self.linear4(support)\n query__ = self.linear4(query__)\n\n support = support_residual + support\n query__ = query___residual + query__\n\n return support, query__\n\n\nclass MultiheadAttention(torch.nn.Module):\n def __init__(self, dim: int, n_heads: int, use_flash_attn: bool) -> None:\n super().__init__()\n\n self.use_flash_attn = use_flash_attn\n self.dim = dim\n self.n_heads = n_heads\n\n self.q = nn.Linear(dim, dim, bias=True)\n self.k = nn.Linear(dim, dim, bias=True)\n self.v = nn.Linear(dim, dim, bias=True)\n self.o = nn.Linear(dim, dim, bias=True)\n\n def forward(\n self,\n query: torch.Tensor,\n key: torch.Tensor,\n value: torch.Tensor,\n cu_seqlens_q: Optional[torch.Tensor] = None,\n cu_seqlens_k: Optional[torch.Tensor] = None,\n max_seqlen_q: Optional[int] = None,\n max_seqlen_k: Optional[int] = None,\n ) -> torch.Tensor:\n \"\"\"\n b = batch size\n s = number of observations\n f = number of features\n t = flashattention-compressed sequences of (batch, observations, features)\n h = heads\n d = dimension of embedding\n\n input: (bsf, d) for flash attention or (b, s, d) for standard attention\n output: (bsf, d) for flash attention or (b, s, d) for standard attention\n \"\"\"\n\n q = self.q(query)\n k = self.k(key)\n v = self.v(value)\n\n if self.use_flash_attn and cu_seqlens_q is not None:\n q = einops.rearrange(\n q, \"t (h d) -> t h d\", h=self.n_heads\n ) # (tokens, heads, dim), tokens is b*n*f w/o pad\n k = einops.rearrange(k, \"t (h d) -> t h d\", h=self.n_heads)\n v = einops.rearrange(v, \"t (h d) -> t h d\", h=self.n_heads)\n\n output = flash_attn_varlen_func(\n q=q,\n k=k,\n v=v,\n cu_seqlens_q=cu_seqlens_q, # num_seq+1, either b*n (w/o pad)+1, or b*f (w/o pad)+1\n cu_seqlens_k=cu_seqlens_k,\n max_seqlen_q=max_seqlen_q, # max sequence length, either n or f\n max_seqlen_k=max_seqlen_k,\n deterministic=True,\n )\n\n output = einops.rearrange(output, \"t h d -> t (h d)\")\n else:\n # Standard scaled dot-product attention for CPU\n q = einops.rearrange(q, \"b t (h d) -> b h t d\", h=self.n_heads)\n k = einops.rearrange(k, \"b t (h d) -> b h t d\", h=self.n_heads)\n v = einops.rearrange(v, \"b t (h d) -> b h t d\", h=self.n_heads)\n\n output = F.scaled_dot_product_attention(q, k, v)\n output = einops.rearrange(output, \"b h t d -> b t (h d)\")\n\n output = self.o(output)\n\n return output\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/utils/__init__.py",
"content": "# Utility modules for MitraModel\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/_internal/utils/set_seed.py",
"content": "import random\n\nimport numpy as np\nimport torch\n\n\ndef set_seed(seed: int) -> None:\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n\n\ndef seed_worker(worker_id: int) -> None:\n worker_seed = torch.initial_seed() % 2**32\n set_seed(worker_seed)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/mitra_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport os\nfrom pathlib import Path\nfrom typing import List, Optional\n\nimport pandas as pd\nfrom typing_extensions import Self\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\nfrom autogluon.tabular import __version__\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass MitraModel(AbstractTorchModel):\n \"\"\"\n Mitra is a tabular foundation model pre-trained purely on synthetic data with the goal\n of optimizing fine-tuning performance over in-context learning performance.\n Mitra was developed by the AutoGluon team @ AWS AI.\n\n Mitra's default hyperparameters outperforms all methods for small datasets on TabArena-v0.1 (excluding ensembling): https://tabarena.ai\n\n Authors: Xiyuan Zhang, Danielle C. Maddix, Junming Yin, Nick Erickson, Abdul Fatir Ansari, Boran Han, Shuai Zhang, Leman Akoglu, Christos Faloutsos, Michael W. Mahoney, Cuixiong Hu, Huzefa Rangwala, George Karypis, Bernie Wang\n Blog Post: https://www.amazon.science/blog/mitra-mixed-synthetic-priors-for-enhancing-tabular-foundation-models\n License: Apache-2.0\n\n .. versionadded:: 1.4.0\n \"\"\"\n\n ag_key = \"MITRA\"\n ag_name = \"Mitra\"\n weights_file_name = \"model.pt\"\n ag_priority = 55\n seed_name = \"seed\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._weights_saved = False\n self._feature_generator = None\n\n @staticmethod\n def _get_default_device():\n \"\"\"Get the best available device for the current system.\"\"\"\n if ResourceManager.get_gpu_count_torch(cuda_only=True) > 0:\n logger.log(15, \"Using CUDA GPU\")\n return \"cuda\"\n else:\n return \"cpu\"\n\n def get_model_cls(self):\n if self.problem_type in [\"binary\", \"multiclass\"]:\n from .sklearn_interface import MitraClassifier\n\n model_cls = MitraClassifier\n elif self.problem_type == \"regression\":\n from .sklearn_interface import MitraRegressor\n\n model_cls = MitraRegressor\n else:\n raise AssertionError(f\"Unsupported problem_type: {self.problem_type}\")\n return model_cls\n\n def _preprocess(self, X: pd.DataFrame, is_train: bool = False, **kwargs) -> pd.DataFrame:\n X = super()._preprocess(X, **kwargs)\n\n if is_train:\n # X will be the training data.\n self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0)\n self._feature_generator.fit(X=X)\n\n # This converts categorical features to numeric via stateful label encoding.\n if self._feature_generator.features_in:\n X = X.copy()\n X[self._feature_generator.features_in] = self._feature_generator.transform(X=X)\n\n return X\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame = None,\n y_val: pd.Series = None,\n time_limit: float = None,\n num_cpus: int = 1,\n num_gpus: float = 0,\n verbosity: int = 2,\n **kwargs,\n ):\n # TODO: Reset the number of threads based on the specified num_cpus\n need_to_reset_torch_threads = False\n torch_threads_og = None\n\n try:\n model_cls = self.get_model_cls()\n import torch\n except ImportError as err:\n logger.log(\n 40,\n f\"\\tFailed to import Mitra! To use the Mitra model, \"\n f\"do: `pip install autogluon.tabular[mitra]=={__version__}`.\",\n )\n raise err\n\n if num_cpus is not None and isinstance(num_cpus, (int, float)):\n torch_threads_og = torch.get_num_threads()\n if torch_threads_og != num_cpus:\n # reset torch threads back to original value after fit\n torch.set_num_threads(num_cpus)\n need_to_reset_torch_threads = True\n\n hyp = self._get_model_params()\n\n hf_cls_model = hyp.pop(\"hf_cls_model\", None)\n hf_reg_model = hyp.pop(\"hf_reg_model\", None)\n if self.problem_type in [\"binary\", \"multiclass\"]:\n hf_model = hf_cls_model\n elif self.problem_type == \"regression\":\n hf_model = hf_reg_model\n else:\n raise AssertionError(f\"Unsupported problem_type: {self.problem_type}\")\n if hf_model is None:\n hf_model = hyp.pop(\"hf_general_model\", None)\n if hf_model is None:\n hf_model = hyp.pop(\"hf_model\", None)\n if hf_model is not None:\n logger.log(30, f\"\\tCustom hf_model specified: {hf_model}\")\n hyp[\"hf_model\"] = hf_model\n\n if hyp.get(\"device\", None) is None:\n if num_gpus == 0:\n hyp[\"device\"] = \"cpu\"\n else:\n hyp[\"device\"] = self._get_default_device()\n\n if hyp[\"device\"] == \"cpu\" and hyp.get(\"fine_tune\", True):\n logger.log(\n 30,\n f\"\\tWarning: Attempting to fine-tune Mitra on CPU. This will be very slow. \"\n f\"We strongly recommend using a GPU instance to fine-tune Mitra.\",\n )\n\n if \"state_dict_classification\" in hyp:\n state_dict_classification = hyp.pop(\"state_dict_classification\")\n if self.problem_type in [\"binary\", \"multiclass\"]:\n hyp[\"state_dict\"] = state_dict_classification\n if \"state_dict_regression\" in hyp:\n state_dict_regression = hyp.pop(\"state_dict_regression\")\n if self.problem_type in [\"regression\"]:\n hyp[\"state_dict\"] = state_dict_regression\n\n if \"verbose\" not in hyp:\n hyp[\"verbose\"] = verbosity >= 3\n\n self.model = model_cls(**hyp)\n\n X = self.preprocess(X, y=y, is_train=True)\n if X_val is not None:\n X_val = self.preprocess(X_val)\n\n model = self.model.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n time_limit=time_limit,\n )\n\n for i in range(len(model.trainers)):\n model.trainers[i].post_fit_optimize()\n\n self.model = model\n\n if need_to_reset_torch_threads:\n torch.set_num_threads(torch_threads_og)\n\n def _set_default_params(self):\n default_params = {\n \"n_estimators\": 1,\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_rows\": 10000,\n \"max_features\": 500,\n \"max_classes\": 10,\n }\n )\n return default_auxiliary_params\n\n def weights_path(self, path: str | None = None) -> str:\n if path is None:\n path = self.path\n return str(Path(path) / self.weights_file_name)\n\n def save(self, path: str = None, verbose=True) -> str:\n _model_weights_list = None\n if self.model is not None:\n self._save_model_artifact(path=path)\n _model_weights_list = []\n for i in range(len(self.model.trainers)):\n _model_weights_list.append(self.model.trainers[i].model)\n self.model.trainers[i].model = None\n\n path = super().save(path=path, verbose=verbose)\n if _model_weights_list is not None:\n for i in range(len(self.model.trainers)):\n self.model.trainers[i].model = _model_weights_list[i]\n return path\n\n def _save_model_artifact(self, path: str | None):\n if path is None:\n path = self.path\n import torch\n\n device_og = self.device\n self.set_device(\"cpu\")\n\n _model_weights_list = []\n for i in range(len(self.model.trainers)):\n _model_weights_list.append(self.model.trainers[i].model)\n\n os.makedirs(path, exist_ok=True)\n torch.save(_model_weights_list, self.weights_path(path=path))\n self.set_device(device_og)\n self._weights_saved = True\n\n def _load_model_artifact(self):\n import torch\n\n device = self.suggest_device_infer()\n model_weights_list = torch.load(self.weights_path(), weights_only=False) # nosec B614\n for i in range(len(self.model.trainers)):\n self.model.trainers[i].model = model_weights_list[i]\n self.set_device(device)\n\n def _set_device(self, device: str):\n for i in range(len(self.model.trainers)):\n self.model.trainers[i].set_device(device)\n\n def get_device(self) -> str:\n return self.model.trainers[0].device\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True) -> Self:\n model: MitraModel = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n\n if model._weights_saved:\n model._load_model_artifact()\n model._weights_saved = False\n return model\n\n @classmethod\n def download_weights(cls, repo_id: str):\n \"\"\"\n Download weights for Mitra from HuggingFace from `repo_id`.\n Requires an internet connection.\n \"\"\"\n from huggingface_hub import hf_hub_download\n\n hf_hub_download(repo_id=repo_id, filename=\"config.json\")\n hf_hub_download(repo_id=repo_id, filename=\"model.safetensors\")\n\n @classmethod\n def download_default_weights(cls):\n \"\"\"\n Download default weights for Mitra from HuggingFace.\n Includes both classifier and regressor weights.\n\n This is useful to call when building a docker image to avoid having to download Mitra weights for each instance.\n This is also useful for benchmarking as a first sanity check\n to avoid HuggingFace potentially blocking the download.\n\n Requires an internet connection.\n \"\"\"\n cls.download_weights(repo_id=\"autogluon/mitra-classifier\")\n cls.download_weights(repo_id=\"autogluon/mitra-regressor\")\n\n @classmethod\n def supported_problem_types(cls) -> Optional[List[str]]:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n # FIXME: Test if it works with parallel, need to enable n_cpus support\n extra_ag_args_ensemble = {\n \"fold_fitting_strategy\": \"sequential_local\", # FIXME: Comment out after debugging for large speedup\n }\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n\n return num_cpus, num_gpus\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n return self.estimate_memory_usage_static(\n X=X, problem_type=self.problem_type, num_classes=self.num_classes, **kwargs\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n # Multiply by 0.9 as currently this is overly safe\n return int(\n 0.9\n * max(\n cls._estimate_memory_usage_static_cpu_icl(X=X, **kwargs),\n cls._estimate_memory_usage_static_cpu_ft_icl(X=X, **kwargs),\n cls._estimate_memory_usage_static_gpu_cpu(X=X, **kwargs),\n cls._estimate_memory_usage_static_gpu_gpu(X=X, **kwargs),\n )\n )\n\n @classmethod\n def _estimate_memory_usage_static_cpu_icl(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n rows, features = X.shape[0], X.shape[1]\n\n # For very small datasets, use a more conservative estimate\n if rows * features < 100: # Small dataset threshold\n # Use a simpler linear formula for small datasets\n cpu_memory_kb = 1.3 * (100 * rows * features + 1000000) # 1GB base + linear scaling\n else:\n # Original formula for larger datasets\n cpu_memory_kb = 1.3 * (\n 0.001748 * (rows**2) * features + 0.001206 * rows * (features**2) + 10.3482 * rows * features + 6409698\n )\n return int(cpu_memory_kb * 1e3)\n\n @classmethod\n def _estimate_memory_usage_static_cpu_ft_icl(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n rows, features = X.shape[0], X.shape[1]\n\n # For very small datasets, use a more conservative estimate\n if rows * features < 100: # Small dataset threshold\n # Use a simpler linear formula for small datasets\n cpu_memory_kb = 1.3 * (200 * rows * features + 2000000) # 2GB base + linear scaling\n else:\n # Original formula for larger datasets\n cpu_memory_kb = 1.3 * (\n 0.001 * (rows**2) * features + 0.004541 * rows * (features**2) + 46.2974 * rows * features + 5605681\n )\n return int(cpu_memory_kb * 1e3)\n\n @classmethod\n def _estimate_memory_usage_static_gpu_cpu(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n rows, features = X.shape[0], X.shape[1]\n\n # For very small datasets, use a more conservative estimate\n if rows * features < 100: # Small dataset threshold\n return int(2.5 * 1e9) # 2.5GB for small datasets\n else:\n return int(5 * 1e9) # 5GB for larger datasets\n\n @classmethod\n def _estimate_memory_usage_static_gpu_gpu(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n rows, features = X.shape[0], X.shape[1]\n\n # For very small datasets, use a more conservative estimate\n if rows * features < 100: # Small dataset threshold\n # Use a simpler linear formula for small datasets\n gpu_memory_mb = 1.3 * (10 * rows * features + 2000) # 2GB base + linear scaling\n else:\n # Original formula for larger datasets\n gpu_memory_mb = 1.3 * (0.05676 * rows * features + 3901)\n return int(gpu_memory_mb * 1e6)\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"can_set_device\": True,\n \"set_device_on_save_to\": None,\n \"set_device_on_load\": False,\n }\n\n def _more_tags(self) -> dict:\n tags = {\"can_refit_full\": True}\n return tags\n"
},
{
"path": "tabular/src/autogluon/tabular/models/mitra/sklearn_interface.py",
"content": "from __future__ import annotations\n\nimport contextlib\nimport os\nimport time\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin\n\nfrom ._internal.config.config_run import ConfigRun\nfrom ._internal.config.enums import ModelName\nfrom ._internal.core.trainer_finetune import TrainerFinetune\nfrom ._internal.data.dataset_split import make_stratified_dataset_split\nfrom ._internal.models.tab2d import Tab2D\nfrom ._internal.utils.set_seed import set_seed\n\n# Hyperparameter search space\nDEFAULT_FINE_TUNE = True # [True, False]\nDEFAULT_FINE_TUNE_STEPS = 50 # [50, 60, 70, 80, 90, 100]\nDEFAULT_CLS_METRIC = \"log_loss\" # ['log_loss', 'accuracy', 'auc']\nDEFAULT_REG_METRIC = \"mse\" # ['mse', 'mae', 'rmse', 'r2']\nSHUFFLE_CLASSES = False # [True, False]\nSHUFFLE_FEATURES = False # [True, False]\nUSE_RANDOM_TRANSFORMS = False # [True, False]\nRANDOM_MIRROR_REGRESSION = True # [True, False]\nRANDOM_MIRROR_X = True # [True, False]\nLR = 0.0001 # [0.00001, 0.000025, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005, 0.00075, 0.001]\nPATIENCE = 40 # [30, 35, 40, 45, 50]\nWARMUP_STEPS = 1000 # [500, 750, 1000, 1250, 1500]\nDEFAULT_CLS_MODEL = \"autogluon/mitra-classifier\"\nDEFAULT_REG_MODEL = \"autogluon/mitra-regressor\"\n\n# Constants\nSEED = 0\nDEFAULT_MODEL_TYPE = \"Tab2D\"\n\n\ndef _get_default_device():\n \"\"\"Get the best available device for the current system.\"\"\"\n if torch.cuda.is_available():\n return \"cuda\"\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n return \"mps\" # Apple silicon\n else:\n return \"cpu\"\n\n\nDEFAULT_DEVICE = _get_default_device()\nDEFAULT_ENSEMBLE = 1\nDEFAULT_DIM = 512\nDEFAULT_LAYERS = 12\nDEFAULT_HEADS = 4\nDEFAULT_CLASSES = 10\nDEFAULT_VALIDATION_SPLIT = 0.2\nUSE_HF = True # Use Hugging Face pretrained models if available\n\n\nclass MitraBase(BaseEstimator):\n \"\"\"Base class for Mitra models with common functionality.\"\"\"\n\n def __init__(\n self,\n model_type=DEFAULT_MODEL_TYPE,\n n_estimators=DEFAULT_ENSEMBLE,\n device=DEFAULT_DEVICE,\n fine_tune=DEFAULT_FINE_TUNE,\n fine_tune_steps=DEFAULT_FINE_TUNE_STEPS,\n metric=DEFAULT_CLS_METRIC,\n state_dict=None,\n hf_model=None,\n patience=PATIENCE,\n lr=LR,\n warmup_steps=WARMUP_STEPS,\n shuffle_classes=SHUFFLE_CLASSES,\n shuffle_features=SHUFFLE_FEATURES,\n use_random_transforms=USE_RANDOM_TRANSFORMS,\n random_mirror_regression=RANDOM_MIRROR_REGRESSION,\n random_mirror_x=RANDOM_MIRROR_X,\n seed=SEED,\n verbose=True,\n ):\n \"\"\"\n Initialize the base Mitra model.\n\n Parameters\n ----------\n model_type : str, default=\"Tab2D\"\n The type of model to use. Options: \"Tab2D\", \"Tab2D_COL_ROW\"\n n_estimators : int, default=1\n Number of models in the ensemble\n device : str, default=\"cuda\"\n Device to run the model on\n fine_tune_steps: int, default=0\n Number of epochs to train for\n state_dict : str, optional\n Path to the pretrained weights\n \"\"\"\n self.model_type = model_type\n self.n_estimators = n_estimators\n self.device = device\n self.fine_tune = fine_tune\n self.fine_tune_steps = fine_tune_steps\n self.metric = metric\n self.state_dict = state_dict\n self.hf_model = hf_model\n self.patience = patience\n self.lr = lr\n self.warmup_steps = warmup_steps\n self.shuffle_classes = shuffle_classes\n self.shuffle_features = shuffle_features\n self.use_random_transforms = use_random_transforms\n self.random_mirror_regression = random_mirror_regression\n self.random_mirror_x = random_mirror_x\n self.trainers = []\n self.train_time = 0\n self.seed = seed\n self.verbose = verbose\n\n # FIXME: set_seed was removed in v1.4 as quality and speed reduction was observed when setting seed.\n # This should be investigated and fixed for v1.5\n # set_seed(self.seed)\n\n def _create_config(self, task, dim_output, time_limit=None):\n cfg = ConfigRun(\n device=self.device,\n model_name=ModelName.TAB2D,\n seed=self.seed,\n hyperparams={\n \"dim_embedding\": None,\n \"early_stopping_data_split\": \"VALID\",\n \"early_stopping_max_samples\": 2048,\n \"early_stopping_patience\": self.patience,\n \"grad_scaler_enabled\": False,\n \"grad_scaler_growth_interval\": 1000,\n \"grad_scaler_scale_init\": 65536.0,\n \"grad_scaler_scale_min\": 65536.0,\n \"label_smoothing\": 0.0,\n \"lr_scheduler\": False,\n \"lr_scheduler_patience\": 25,\n \"max_epochs\": self.fine_tune_steps if self.fine_tune else 0,\n \"max_samples_query\": 1024,\n \"max_samples_support\": 8192,\n \"optimizer\": \"adamw\",\n \"lr\": self.lr,\n \"weight_decay\": 0.1,\n \"warmup_steps\": self.warmup_steps,\n \"path_to_weights\": self.state_dict,\n \"precision\": \"bfloat16\",\n \"random_mirror_regression\": self.random_mirror_regression,\n \"random_mirror_x\": self.random_mirror_x,\n \"shuffle_classes\": self.shuffle_classes,\n \"shuffle_features\": self.shuffle_features,\n \"use_random_transforms\": self.use_random_transforms,\n \"use_feature_count_scaling\": False,\n \"use_pretrained_weights\": False,\n \"use_quantile_transformer\": False,\n \"budget\": time_limit,\n \"metric\": self.metric,\n },\n )\n\n cfg.task = task\n cfg.hyperparams.update(\n {\n \"n_ensembles\": self.n_estimators,\n \"dim\": DEFAULT_DIM,\n \"dim_output\": dim_output,\n \"n_layers\": DEFAULT_LAYERS,\n \"n_heads\": DEFAULT_HEADS,\n \"regression_loss\": \"mse\",\n }\n )\n\n return cfg, Tab2D\n\n def _split_data(self, X, y):\n \"\"\"Split data into training and validation sets.\"\"\"\n if hasattr(self, \"task\") and self.task == \"classification\":\n return make_stratified_dataset_split(X, y, seed=self.seed)\n else:\n # For regression, use random split\n val_indices = np.random.choice(\n range(len(X)), int(DEFAULT_VALIDATION_SPLIT * len(X)), replace=False\n ).tolist()\n train_indices = [i for i in range(len(X)) if i not in val_indices]\n return X[train_indices], X[val_indices], y[train_indices], y[val_indices]\n\n def _train_ensemble(self, X_train, y_train, X_valid, y_valid, task, dim_output, n_classes=0, time_limit=None):\n \"\"\"Train the ensemble of models.\"\"\"\n\n cfg, Tab2D = self._create_config(task, dim_output, time_limit)\n rng = np.random.RandomState(cfg.seed)\n\n success = False\n while not (\n success and cfg.hyperparams[\"max_samples_support\"] > 0 and cfg.hyperparams[\"max_samples_query\"] > 0\n ):\n try:\n self.trainers.clear()\n\n self.train_time = 0\n for _ in range(self.n_estimators):\n if USE_HF:\n assert self.hf_model is not None, f\"hf_model must not be None.\"\n model = Tab2D.from_pretrained(self.hf_model, device=self.device)\n else:\n model = Tab2D(\n dim=cfg.hyperparams[\"dim\"],\n dim_output=dim_output,\n n_layers=cfg.hyperparams[\"n_layers\"],\n n_heads=cfg.hyperparams[\"n_heads\"],\n task=task.upper(),\n use_pretrained_weights=True,\n path_to_weights=Path(self.state_dict),\n device=self.device,\n )\n trainer = TrainerFinetune(\n cfg, model, n_classes=n_classes, device=self.device, rng=rng, verbose=self.verbose\n )\n\n start_time = time.time()\n trainer.train(X_train, y_train, X_valid, y_valid)\n end_time = time.time()\n\n self.trainers.append(trainer)\n self.train_time += end_time - start_time\n\n success = True\n\n except torch.cuda.OutOfMemoryError:\n if cfg.hyperparams[\"max_samples_support\"] >= 2048:\n cfg.hyperparams[\"max_samples_support\"] = int(cfg.hyperparams[\"max_samples_support\"] // 2)\n print(\n f\"Reducing max_samples_support from {cfg.hyperparams['max_samples_support'] * 2}\"\n f\"to {cfg.hyperparams['max_samples_support']} due to OOM error.\"\n )\n else:\n cfg.hyperparams[\"max_samples_support\"] = int(cfg.hyperparams[\"max_samples_support\"] // 2)\n print(\n f\"Reducing max_samples_support from {cfg.hyperparams['max_samples_support'] * 2}\"\n f\"to {cfg.hyperparams['max_samples_support']} due to OOM error.\"\n )\n cfg.hyperparams[\"max_samples_query\"] = int(cfg.hyperparams[\"max_samples_query\"] // 2)\n print(\n f\"Reducing max_samples_query from {cfg.hyperparams['max_samples_query'] * 2}\"\n f\"to {cfg.hyperparams['max_samples_query']} due to OOM error.\"\n )\n\n if not success:\n raise RuntimeError(\"Failed to train Mitra model after multiple attempts due to out of memory error.\")\n\n return self\n\n\nclass MitraClassifier(MitraBase, ClassifierMixin):\n \"\"\"Classifier implementation of Mitra model.\"\"\"\n\n def __init__(\n self,\n model_type=DEFAULT_MODEL_TYPE,\n n_estimators=DEFAULT_ENSEMBLE,\n device=DEFAULT_DEVICE,\n fine_tune=DEFAULT_FINE_TUNE,\n fine_tune_steps=DEFAULT_FINE_TUNE_STEPS,\n metric=DEFAULT_CLS_METRIC,\n state_dict=None,\n hf_model=DEFAULT_CLS_MODEL,\n patience=PATIENCE,\n lr=LR,\n warmup_steps=WARMUP_STEPS,\n shuffle_classes=SHUFFLE_CLASSES,\n shuffle_features=SHUFFLE_FEATURES,\n use_random_transforms=USE_RANDOM_TRANSFORMS,\n random_mirror_regression=RANDOM_MIRROR_REGRESSION,\n random_mirror_x=RANDOM_MIRROR_X,\n seed=SEED,\n verbose=True,\n ):\n \"\"\"Initialize the classifier.\"\"\"\n super().__init__(\n model_type,\n n_estimators,\n device,\n fine_tune,\n fine_tune_steps,\n metric,\n state_dict,\n hf_model=hf_model,\n patience=patience,\n lr=lr,\n warmup_steps=warmup_steps,\n shuffle_classes=shuffle_classes,\n shuffle_features=shuffle_features,\n use_random_transforms=use_random_transforms,\n random_mirror_regression=random_mirror_regression,\n random_mirror_x=random_mirror_x,\n seed=seed,\n verbose=verbose,\n )\n self.task = \"classification\"\n\n def fit(self, X, y, X_val=None, y_val=None, time_limit=None):\n \"\"\"\n Fit the ensemble of models.\n\n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n Training data\n y : array-like of shape (n_samples,)\n Target values\n\n Returns\n -------\n self : object\n Returns self\n \"\"\"\n\n with mitra_deterministic_context():\n if isinstance(X, pd.DataFrame):\n X = X.values\n if isinstance(y, pd.Series):\n y = y.values\n\n self.X, self.y = X, y\n\n if X_val is not None and y_val is not None:\n if isinstance(X_val, pd.DataFrame):\n X_val = X_val.values\n if isinstance(y_val, pd.Series):\n y_val = y_val.values\n X_train, X_valid, y_train, y_valid = X, X_val, y, y_val\n else:\n X_train, X_valid, y_train, y_valid = self._split_data(X, y)\n\n return self._train_ensemble(\n X_train,\n y_train,\n X_valid,\n y_valid,\n self.task,\n DEFAULT_CLASSES,\n n_classes=DEFAULT_CLASSES,\n time_limit=time_limit,\n )\n\n def predict(self, X):\n \"\"\"\n Predict class labels for samples in X.\n\n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n The input samples\n\n Returns\n -------\n y : ndarray of shape (n_samples,)\n The predicted classes\n \"\"\"\n\n if isinstance(X, pd.DataFrame):\n X = X.values\n\n return self.predict_proba(X).argmax(axis=1)\n\n def predict_proba(self, X):\n \"\"\"\n Predict class probabilities for samples in X.\n\n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n The input samples\n\n Returns\n -------\n p : ndarray of shape (n_samples, n_classes)\n The class probabilities of the input samples\n \"\"\"\n\n with mitra_deterministic_context():\n if isinstance(X, pd.DataFrame):\n X = X.values\n\n preds = []\n for trainer in self.trainers:\n logits = trainer.predict(self.X, self.y, X)[..., : len(np.unique(self.y))] # Remove extra classes\n preds.append(np.exp(logits) / np.exp(logits).sum(axis=1, keepdims=True)) # Softmax\n preds = sum(preds) / len(preds) # Averaging ensemble predictions\n\n return preds\n\n\nclass MitraRegressor(MitraBase, RegressorMixin):\n \"\"\"Regressor implementation of Mitra model.\"\"\"\n\n def __init__(\n self,\n model_type=DEFAULT_MODEL_TYPE,\n n_estimators=DEFAULT_ENSEMBLE,\n device=DEFAULT_DEVICE,\n fine_tune=DEFAULT_FINE_TUNE,\n fine_tune_steps=DEFAULT_FINE_TUNE_STEPS,\n metric=DEFAULT_REG_METRIC,\n state_dict=None,\n hf_model=DEFAULT_REG_MODEL,\n patience=PATIENCE,\n lr=LR,\n warmup_steps=WARMUP_STEPS,\n shuffle_classes=SHUFFLE_CLASSES,\n shuffle_features=SHUFFLE_FEATURES,\n use_random_transforms=USE_RANDOM_TRANSFORMS,\n random_mirror_regression=RANDOM_MIRROR_REGRESSION,\n random_mirror_x=RANDOM_MIRROR_X,\n seed=SEED,\n verbose=True,\n ):\n \"\"\"Initialize the regressor.\"\"\"\n super().__init__(\n model_type,\n n_estimators,\n device,\n fine_tune,\n fine_tune_steps,\n metric,\n state_dict,\n hf_model=hf_model,\n patience=patience,\n lr=lr,\n warmup_steps=warmup_steps,\n shuffle_classes=shuffle_classes,\n shuffle_features=shuffle_features,\n use_random_transforms=use_random_transforms,\n random_mirror_regression=random_mirror_regression,\n random_mirror_x=random_mirror_x,\n seed=seed,\n verbose=verbose,\n )\n self.task = \"regression\"\n\n def fit(self, X, y, X_val=None, y_val=None, time_limit=None):\n \"\"\"\n Fit the ensemble of models.\n\n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n Training data\n y : array-like of shape (n_samples,)\n Target values\n\n Returns\n -------\n self : object\n Returns self\n \"\"\"\n\n with mitra_deterministic_context():\n if isinstance(X, pd.DataFrame):\n X = X.values\n if isinstance(y, pd.Series):\n y = y.values\n\n self.X, self.y = X, y\n\n if X_val is not None and y_val is not None:\n if isinstance(X_val, pd.DataFrame):\n X_val = X_val.values\n if isinstance(y_val, pd.Series):\n y_val = y_val.values\n X_train, X_valid, y_train, y_valid = X, X_val, y, y_val\n else:\n X_train, X_valid, y_train, y_valid = self._split_data(X, y)\n\n return self._train_ensemble(X_train, y_train, X_valid, y_valid, self.task, 1, time_limit=time_limit)\n\n def predict(self, X):\n \"\"\"\n Predict regression target for samples in X.\n\n Parameters\n ----------\n X : array-like of shape (n_samples, n_features)\n The input samples\n\n Returns\n -------\n y : ndarray of shape (n_samples,)\n The predicted values\n \"\"\"\n\n with mitra_deterministic_context():\n if isinstance(X, pd.DataFrame):\n X = X.values\n\n preds = []\n for trainer in self.trainers:\n preds.append(trainer.predict(self.X, self.y, X))\n\n return sum(preds) / len(preds) # Averaging ensemble predictions\n\n\n@contextlib.contextmanager\ndef mitra_deterministic_context():\n \"\"\"Context manager to set deterministic settings only for Mitra operations.\"\"\"\n yield\n"
},
{
"path": "tabular/src/autogluon/tabular/models/realmlp/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/realmlp/realmlp_model.py",
"content": "\"\"\"\nCode Adapted from TabArena: https://github.com/autogluon/tabarena/blob/main/tabarena/tabarena/benchmark/models/ag/realmlp/realmlp_model.py\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nimport math\nimport time\nfrom contextlib import contextmanager\nfrom typing import Literal\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.impute import SimpleImputer\n\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular import __version__\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nlogger = logging.getLogger(__name__)\n\n\n@contextmanager\ndef set_logger_level(logger_name: str, level: int):\n _logger = logging.getLogger(logger_name)\n old_level = _logger.level\n _logger.setLevel(level)\n try:\n yield\n finally:\n _logger.setLevel(old_level)\n\n\n# pip install pytabkit\nclass RealMLPModel(AbstractTorchModel):\n \"\"\"\n RealMLP is an improved multilayer perception (MLP) model\n through a bag of tricks and better default hyperparameters.\n\n RealMLP is the top performing method overall on TabArena-v0.1: https://tabarena.ai\n\n Paper: Better by Default: Strong Pre-Tuned MLPs and Boosted Trees on Tabular Data\n Authors: David HolzmÃŧller, LÊo Grinsztajn, Ingo Steinwart\n Codebase: https://github.com/dholzmueller/pytabkit\n License: Apache-2.0\n\n .. versionadded:: 1.4.0\n \"\"\"\n\n ag_key = \"REALMLP\"\n ag_name = \"RealMLP\"\n ag_priority = 75\n seed_name = \"random_state\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._imputer = None\n self._features_to_impute = None\n self._features_to_keep = None\n self._indicator_columns = None\n self._features_bool = None\n self._bool_to_cat = None\n\n def get_model_cls(self, default_hyperparameters: Literal[\"td\", \"td_s\"] = \"td\"):\n from pytabkit import (\n RealMLP_TD_Classifier,\n RealMLP_TD_Regressor,\n RealMLP_TD_S_Classifier,\n RealMLP_TD_S_Regressor,\n )\n\n assert default_hyperparameters in [\"td\", \"td_s\"]\n if self.problem_type in [\"binary\", \"multiclass\"]:\n if default_hyperparameters == \"td\":\n model_cls = RealMLP_TD_Classifier\n else:\n model_cls = RealMLP_TD_S_Classifier\n else:\n if default_hyperparameters == \"td\":\n model_cls = RealMLP_TD_Regressor\n else:\n model_cls = RealMLP_TD_S_Regressor\n return model_cls\n\n def get_device(self) -> str:\n return self.model.device\n\n def _set_device(self, device: str):\n self.model.to(device)\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame = None,\n y_val: pd.Series = None,\n time_limit: float = None,\n num_cpus: int = 1,\n num_gpus: float = 0,\n verbosity: int = 2,\n **kwargs,\n ):\n start_time = time.time()\n\n try:\n import pytabkit\n import torch\n except ImportError as err:\n logger.log(\n 40,\n f\"\\tFailed to import pytabkit/torch! To use the ReaLMLP model, \"\n f\"do: `pip install autogluon.tabular[realmlp]=={__version__}`.\",\n )\n raise err\n\n if verbosity == 0:\n _lightning_log_level = logging.ERROR\n elif verbosity <= 2:\n _lightning_log_level = logging.WARNING\n else:\n _lightning_log_level = logging.INFO\n\n # FIXME: code assume we only see one GPU in the fit process.\n device = \"cpu\" if num_gpus == 0 else \"cuda:0\"\n if (device == \"cuda:0\") and (not torch.cuda.is_available()):\n raise AssertionError(\n \"Fit specified to use GPU, but CUDA is not available on this machine. \"\n \"Please switch to CPU usage instead.\",\n )\n\n hyp = self._get_model_params()\n\n default_hyperparameters = hyp.pop(\"default_hyperparameters\", \"td\")\n\n model_cls = self.get_model_cls(default_hyperparameters=default_hyperparameters)\n\n metric_map = {\n \"roc_auc\": \"1-auc_ovr_alt\",\n \"accuracy\": \"class_error\",\n \"balanced_accuracy\": \"1-balanced_accuracy\",\n \"log_loss\": \"cross_entropy\",\n \"rmse\": \"rmse\",\n \"root_mean_squared_error\": \"rmse\",\n \"r2\": \"rmse\",\n \"mae\": \"mae\",\n \"mean_average_error\": \"mae\",\n }\n\n val_metric_name = metric_map.get(self.stopping_metric.name, None)\n\n init_kwargs = dict()\n\n if val_metric_name is not None:\n init_kwargs[\"val_metric_name\"] = val_metric_name\n\n # TODO: Make this smarter? Maybe use `eval_metric.needs_pred`\n if hyp[\"use_ls\"] is not None and isinstance(hyp[\"use_ls\"], str) and hyp[\"use_ls\"] == \"auto\":\n if val_metric_name is None:\n hyp[\"use_ls\"] = False\n elif val_metric_name in [\"cross_entropy\", \"1-auc_ovr_alt\"]:\n hyp[\"use_ls\"] = False\n else:\n hyp[\"use_ls\"] = None\n\n if X_val is None:\n hyp[\"use_early_stopping\"] = False\n hyp[\"val_fraction\"] = 0\n\n bool_to_cat = hyp.pop(\"bool_to_cat\", True)\n impute_bool = hyp.pop(\"impute_bool\", True)\n name_categories = hyp.pop(\"name_categories\", True)\n\n n_features = len(X.columns)\n if (\n \"predict_batch_size\" in hyp\n and isinstance(hyp[\"predict_batch_size\"], str)\n and hyp[\"predict_batch_size\"] == \"auto\"\n ):\n # simple heuristic to avoid OOM during inference time\n # note: this isn't fool-proof, and ignores the actual memory availability of the machine.\n # note: this is based on an assumption of 32 GB of memory available on the instance\n # default is 1024\n hyp[\"predict_batch_size\"] = max(min(int(8192 * 200 / n_features), 8192), 64)\n\n self.model = model_cls(\n n_threads=num_cpus,\n device=device,\n **init_kwargs,\n **hyp,\n )\n\n X = self.preprocess(X, y=y, is_train=True, bool_to_cat=bool_to_cat, impute_bool=impute_bool)\n\n # FIXME: In rare cases can cause exceptions if name_categories=False, unknown why\n extra_fit_kwargs = {}\n if name_categories:\n cat_col_names = X.select_dtypes(include=\"category\").columns.tolist()\n extra_fit_kwargs[\"cat_col_names\"] = cat_col_names\n\n if X_val is not None:\n X_val = self.preprocess(X_val)\n\n with set_logger_level(\"lightning.pytorch\", _lightning_log_level):\n self.model = self.model.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n time_to_fit_in_seconds=time_limit - (time.time() - start_time) if time_limit is not None else None,\n **extra_fit_kwargs,\n )\n\n def _predict_proba(self, X, **kwargs) -> np.ndarray:\n with set_logger_level(\"lightning.pytorch\", logging.WARNING):\n return super()._predict_proba(X=X, kwargs=kwargs)\n\n # TODO: Move missing indicator + mean fill to a generic preprocess flag available to all models\n # FIXME: bool_to_cat is a hack: Maybe move to abstract model?\n def _preprocess(\n self, X: pd.DataFrame, is_train: bool = False, bool_to_cat: bool = False, impute_bool: bool = True, **kwargs\n ) -> pd.DataFrame:\n \"\"\"\n Imputes missing values via the mean and adds indicator columns for numerical features.\n Converts indicator columns to categorical features to avoid them being treated as numerical by RealMLP.\n \"\"\"\n X = super()._preprocess(X, **kwargs)\n\n # FIXME: is copy needed?\n X = X.copy(deep=True)\n if is_train:\n self._bool_to_cat = bool_to_cat\n self._features_bool = self._feature_metadata.get_features(required_special_types=[\"bool\"])\n if impute_bool: # Technically this should do nothing useful because bools will never have NaN\n self._features_to_impute = self._feature_metadata.get_features(valid_raw_types=[\"int\", \"float\"])\n self._features_to_keep = self._feature_metadata.get_features(invalid_raw_types=[\"int\", \"float\"])\n else:\n self._features_to_impute = self._feature_metadata.get_features(\n valid_raw_types=[\"int\", \"float\"], invalid_special_types=[\"bool\"]\n )\n self._features_to_keep = [\n f for f in self._feature_metadata.get_features() if f not in self._features_to_impute\n ]\n if self._features_to_impute:\n self._imputer = SimpleImputer(strategy=\"mean\", add_indicator=True)\n self._imputer.fit(X=X[self._features_to_impute])\n self._indicator_columns = [\n c for c in self._imputer.get_feature_names_out() if c not in self._features_to_impute\n ]\n if self._imputer is not None:\n X_impute = self._imputer.transform(X=X[self._features_to_impute])\n X_impute = pd.DataFrame(X_impute, index=X.index, columns=self._imputer.get_feature_names_out())\n if self._indicator_columns:\n # FIXME: Use CategoryFeatureGenerator? Or tell the model which is category\n # TODO: Add to features_bool?\n X_impute[self._indicator_columns] = X_impute[self._indicator_columns].astype(\"category\")\n X = pd.concat([X[self._features_to_keep], X_impute], axis=1)\n if self._bool_to_cat and self._features_bool:\n # FIXME: Use CategoryFeatureGenerator? Or tell the model which is category\n X[self._features_bool] = X[self._features_bool].astype(\"category\")\n return X\n\n def _set_default_params(self):\n default_params = dict(\n # Don't use early stopping by default, seems to work well without\n use_early_stopping=False,\n early_stopping_additive_patience=40,\n early_stopping_multiplicative_patience=3,\n # verdict: use_ls=\"auto\" is much better than None.\n use_ls=\"auto\",\n # verdict: no impact, but makes more sense to be False.\n impute_bool=False,\n # verdict: name_categories=True avoids random exceptions being raised in rare cases\n name_categories=True,\n # verdict: bool_to_cat=True is equivalent to False in terms of quality, but can be slightly faster in training time\n # and slightly slower in inference time\n bool_to_cat=True,\n # verdict: \"td\" is better than \"td_s\"\n default_hyperparameters=\"td\", # options [\"td\", \"td_s\"]\n predict_batch_size=\"auto\", # if auto, uses AutoGluon's heuristic to set a value between 8192 and 64.\n )\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n def _get_default_stopping_metric(self):\n return self.eval_metric\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n\n return num_cpus, num_gpus\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n **kwargs,\n ) -> int:\n \"\"\"\n Heuristic memory estimate that correlates strongly with RealMLP's more sophisticated method\n\n More comprehensive memory estimate logic:\n\n ```python\n from typing import Any\n\n from pytabkit.models.alg_interfaces.nn_interfaces import NNAlgInterface\n from pytabkit.models.data.data import DictDataset, TensorInfo\n from pytabkit.models.sklearn.default_params import DefaultParams\n\n def estimate_realmlp_cpu_ram_gb(hparams: dict[str, Any], n_numerical: int, cat_sizes: list[int], n_classes: int,\n n_samples: int):\n params = copy.copy(DefaultParams.RealMLP_TD_CLASS if n_classes > 0 else DefaultParams.RealMLP_TD_REG)\n params.update(hparams)\n\n ds = DictDataset(tensors=None, tensor_infos=dict(x_cont=TensorInfo(feat_shape=[n_numerical]),\n x_cat=TensorInfo(cat_sizes=cat_sizes),\n y=TensorInfo(cat_sizes=[n_classes])), device='cpu',\n n_samples=n_samples)\n\n alg_interface = NNAlgInterface(**params)\n res = alg_interface.get_required_resources(ds, n_cv=1, n_refit=0, n_splits=1, split_seeds=[0], n_train=n_samples)\n return res.cpu_ram_gb\n ```\n\n \"\"\"\n if hyperparameters is None:\n hyperparameters = {}\n plr_hidden_1 = hyperparameters.get(\"plr_hidden_1\", 16)\n plr_hidden_2 = hyperparameters.get(\"plr_hidden_2\", 4)\n hidden_width = hyperparameters.get(\"hidden_width\", 256)\n\n num_features = len(X.columns)\n columns_mem_est = num_features * 8e5\n\n hidden_1_weight = 0.13\n hidden_2_weight = 0.42\n width_factor = math.sqrt(hidden_width / 256 + 0.6)\n\n columns_mem_est_hidden_1 = columns_mem_est * hidden_1_weight * plr_hidden_1 / 16 * width_factor\n columns_mem_est_hidden_2 = columns_mem_est * hidden_2_weight * plr_hidden_2 / 16 * width_factor\n columns_mem_est = columns_mem_est_hidden_1 + columns_mem_est_hidden_2\n\n dataset_size_mem_est = 5 * get_approximate_df_mem_usage(X).sum() # roughly 5x DataFrame memory size\n baseline_overhead_mem_est = 3e8 # 300 MB generic overhead\n\n mem_estimate = dataset_size_mem_est + columns_mem_est + baseline_overhead_mem_est\n\n return mem_estimate\n\n @classmethod\n def _class_tags(cls) -> dict:\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self) -> dict:\n # TODO: Need to add train params support, track best epoch\n # How to mirror RealMLP learning rate scheduler while forcing stopping at a specific epoch?\n tags = {\"can_refit_full\": False}\n return tags\n"
},
{
"path": "tabular/src/autogluon/tabular/models/rf/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/rf/compilers/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/rf/compilers/native.py",
"content": "import os\nimport pickle\n\n\nclass AbstractNativeCompiler:\n name = \"native\"\n save_in_pkl = True\n\n @staticmethod\n def can_compile():\n return True\n\n @staticmethod\n def compile(model, path: str, input_types=None):\n \"\"\"\n Compile the trained model for faster inference.\n\n Parameters\n ----------\n model\n The native model that is expected to be compiled.\n path : str\n The path for saving the compiled model.\n input_types : list, default=None\n A list of tuples containing shape and element type info, e.g. [((1, 14), np.float32),].\n The list would be used as the input data for the model.\n The compiler would optimize the model to perform best with the given input type.\n \"\"\"\n AbstractNativeCompiler.save(model, path)\n return model\n\n @staticmethod\n def save(model, path: str):\n os.makedirs(os.path.dirname(path), exist_ok=True)\n with open(os.path.join(path, \"model_native.pkl\"), \"wb\") as fp:\n fp.write(pickle.dumps(model))\n\n @staticmethod\n def load(path: str):\n with open(os.path.join(path, \"model_native.pkl\"), \"rb\") as fp:\n pkl = fp.read()\n return pickle.loads(pkl)\n\n\nRFNativeCompiler = AbstractNativeCompiler\n"
},
{
"path": "tabular/src/autogluon/tabular/models/rf/compilers/onnx.py",
"content": "import os\n\nimport numpy as np\n\n\nclass InferenceSessionWrapper:\n \"\"\"\n Wrap around InferenceSession in onnxruntime, since it cannot be pickled.\n See https://github.com/microsoft/onnxruntime/issues/10097\n \"\"\"\n\n def __init__(self, onnx_bytes):\n import onnxruntime as rt\n\n self.sess = rt.InferenceSession(onnx_bytes.SerializeToString(), providers=[\"CPUExecutionProvider\"])\n\n def run(self, *args):\n return self.sess.run(*args)\n\n def get_inputs(self, *args):\n return self.sess.get_inputs(*args)\n\n def get_outputs(self, *args):\n return self.sess.get_outputs(*args)\n\n def __getstate__(self):\n # No need to duplicate the model parameters here.\n return {}\n\n def __setstate__(self, values):\n pass\n\n\nclass RFOnnxPredictor:\n def __init__(self, model):\n self.sess = InferenceSessionWrapper(model)\n self.num_classes = self.sess.get_outputs()[-1].shape[1]\n\n def predict(self, X):\n \"\"\"Run the model with the input and return the result.\"\"\"\n input_name = self.sess.get_inputs()[0].name\n label_name = self.sess.get_outputs()[0].name\n return self.sess.run([label_name], {input_name: X})[0].squeeze()\n\n def predict_proba(self, X):\n \"\"\"Run the model with the input, and return probabilities as result.\"\"\"\n input_name = self.sess.get_inputs()[0].name\n label_name = self.sess.get_outputs()[1].name\n pred_proba = self.sess.run([label_name], {input_name: X})[0]\n pred_proba = np.array([[r[i] for i in range(self.num_classes)] for r in pred_proba])\n return pred_proba\n\n\nclass RFOnnxCompiler:\n name = \"onnx\"\n save_in_pkl = False\n\n @staticmethod\n def can_compile():\n \"\"\"Verify whether the required package has been installed.\"\"\"\n try:\n import onnxruntime\n import skl2onnx\n\n return True\n except ImportError:\n return False\n\n @staticmethod\n def compile(model, path: str, input_types=None):\n \"\"\"\n Compile the trained model for faster inference.\n\n Parameters\n ----------\n model\n The native model that is expected to be compiled.\n path : str\n The path for saving the compiled model.\n input_types : list, default=None\n A list of tuples containing shape and element type info, e.g. [((1, 14), np.float32),].\n The list would be used as the input data for the model.\n The compiler would optimize the model to perform best with the given input type.\n \"\"\"\n if input_types is None or not isinstance(input_types[0], tuple):\n raise RuntimeError(\"input_types argument should contain at least one tuple, e.g. [((1, 14), np.float32)]\")\n if isinstance(model, RFOnnxPredictor):\n return model\n\n from skl2onnx import convert_sklearn\n from skl2onnx.common.data_types import FloatTensorType\n from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n\n input_shape = list(input_types[0][0])\n initial_type = [(\"float_input\", FloatTensorType(input_shape))]\n\n # Without ZipMap\n # See http://onnx.ai/sklearn-onnx/auto_examples/plot_convert_zipmap.html#without-zipmap\n options = {}\n if isinstance(model, (RandomForestClassifier, ExtraTreesClassifier)):\n options = {id(model): {\"zipmap\": False}}\n\n # Convert the model to onnx\n onnx_model = convert_sklearn(model, initial_types=initial_type, options=options)\n predictor = RFOnnxPredictor(model=onnx_model)\n RFOnnxCompiler.save(onnx_model, path)\n return predictor\n\n @staticmethod\n def save(model, path: str) -> str:\n \"\"\"Save the compiled model into onnx file format.\"\"\"\n file_path = os.path.join(path, \"model.onnx\")\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n with open(file_path, \"wb\") as f:\n f.write(model.SerializeToString())\n return os.path.join(path, \"model.onnx\")\n\n @staticmethod\n def load(path: str) -> RFOnnxPredictor:\n \"\"\"Load from the path that contains an onnx file.\"\"\"\n import onnx\n\n onnx_bytes = onnx.load(os.path.join(path, \"model.onnx\"))\n return RFOnnxPredictor(model=onnx_bytes)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/rf/rf_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport math\nimport pickle\nimport sys\nimport time\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.constants import MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils.exceptions import NotEnoughMemoryError, TimeLimitExceeded\nfrom autogluon.core.utils.utils import normalize_pred_probas\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\n\nfrom .compilers.native import RFNativeCompiler\nfrom .compilers.onnx import RFOnnxCompiler\n\nlogger = logging.getLogger(__name__)\n\n\nclass RFModel(AbstractModel):\n \"\"\"\n Random Forest model (scikit-learn): https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html\n \"\"\"\n\n ag_key = \"RF\"\n ag_name = \"RandomForest\"\n ag_priority = 80\n seed_name = \"random_state\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._feature_generator = None\n self._daal = False # Whether daal4py backend is being used\n self._num_features_post_process = None\n\n # noinspection PyUnresolvedReferences\n def _get_model_type(self):\n if self.problem_type == QUANTILE:\n from .rf_quantile import RandomForestQuantileRegressor\n\n return RandomForestQuantileRegressor\n if self.params_aux.get(\"use_daal\", False):\n # Disabled by default because OOB score does not yet work properly\n try:\n # FIXME: sklearnex OOB score is broken, returns biased predictions. Without this optimization, can't compute Efficient OOF.\n # Refer to https://github.com/intel/scikit-learn-intelex/issues/933\n # Current workaround: Forcibly set oob_score=True during fit to compute OOB during train time.\n # Downsides:\n # 1. Slows down training slightly by forcing computation of OOB even if OOB is not needed (such as in medium_quality)\n # 2. Makes computing the correct pred_time_val difficult, as the time is instead added to the fit_time,\n # and we would need to waste extra time to compute the proper pred_time_val post-fit.\n # Therefore with sklearnex enabled, pred_time_val is incorrect.\n from sklearnex.ensemble import RandomForestClassifier, RandomForestRegressor\n\n logger.log(15, \"\\tUsing sklearnex RF backend...\")\n self._daal = True\n except:\n from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n\n self._daal = False\n else:\n from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n\n self._daal = False\n if self.problem_type in [REGRESSION, SOFTCLASS]:\n return RandomForestRegressor\n else:\n return RandomForestClassifier\n\n # TODO: X.fillna -inf? Add extra is_missing column?\n def _preprocess(self, X, **kwargs):\n X = super()._preprocess(X, **kwargs)\n if self._feature_generator is None:\n self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0)\n self._feature_generator.fit(X=X)\n if self._feature_generator.features_in:\n X = X.copy()\n X[self._feature_generator.features_in] = self._feature_generator.transform(X=X)\n X = X.fillna(0).to_numpy(dtype=np.float32)\n return X\n\n def _set_default_params(self):\n default_params = {\n # TODO: 600 is much better, but increases info leakage in stacking -> therefore 300 is ~equal in stack ensemble final quality.\n # Consider adding targeted noise to OOF to avoid info leakage, or increase `min_samples_leaf`.\n \"n_estimators\": 300,\n # Cap leaf nodes to 15000 to avoid large datasets using unreasonable amounts of memory/disk for RF/XT.\n # Ensures that memory and disk usage of RF model with 300 n_estimators is at most ~500 MB for binary/regression, ~200 MB per class for multiclass.\n # This has no effect on datasets with <=15000 rows, and minimal to no impact on datasets with <50000 rows.\n # For large datasets, will often make the model worse, but will significantly speed up inference speed and massively reduce memory and disk usage.\n # For example, when left uncapped, RF can use 5 GB of disk for a regression dataset with 2M rows.\n # Multiply by the 8 RF/XT models in config for best quality / high quality and this is 40 GB of tree models, which is unreasonable.\n # This size scales linearly with number of rows.\n \"max_leaf_nodes\": 15000,\n \"n_jobs\": -1,\n \"bootstrap\": True, # Required for OOB estimates, setting to False will raise exception if bagging.\n # TODO: min_samples_leaf=5 is too large on most problems, however on some datasets it helps a lot (airlines likes >40 min_samples_leaf, adult likes 2 much better than 1)\n # This value would need to be tuned per dataset, likely very worthwhile.\n # Higher values = less OOF info leak, default = 1, which maximizes info leak.\n # 'min_samples_leaf': 5, # Significantly reduces info leakage to stacker models. Never use the default/1 when using as base model.\n # 'oob_score': True, # Disabled by default as it is better to do it post-fit via custom logic.\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n # TODO: Add in documentation that Categorical default is the first index\n # TODO: enable HPO for RF models\n def _get_default_searchspace(self):\n spaces = {\n # 'n_estimators': Int(lower=10, upper=1000, default=300),\n # 'max_features': Categorical(['auto', 0.5, 0.25]),\n # 'criterion': Categorical(['gini', 'entropy']),\n }\n return spaces\n\n def _get_num_trees_per_estimator(self) -> int:\n return self._get_num_trees_per_estimator_static(problem_type=self.problem_type, num_classes=self.num_classes)\n\n @classmethod\n def _get_num_trees_per_estimator_static(cls, problem_type: str, num_classes: int | None) -> int:\n # Very rough guess to size of a single tree before training\n if problem_type in [MULTICLASS, SOFTCLASS]:\n if num_classes is None:\n num_trees_per_estimator = 10 # Guess since it wasn't passed in, could also check y for a better value\n else:\n num_trees_per_estimator = num_classes\n else:\n num_trees_per_estimator = 1\n return num_trees_per_estimator\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n problem_type: str = None,\n num_classes: int = 1,\n **kwargs,\n ) -> int:\n if hyperparameters is None:\n hyperparameters = {}\n n_estimators_final = hyperparameters.get(\"n_estimators\", 300)\n if isinstance(n_estimators_final, int):\n n_estimators = n_estimators_final\n else: # if search space\n n_estimators = 40\n num_trees_per_estimator = cls._get_num_trees_per_estimator_static(\n problem_type=problem_type, num_classes=num_classes\n )\n bytes_per_estimator = num_trees_per_estimator * len(X) / 60000 * 1e6 # Underestimates by 3x on ExtraTrees\n expected_memory_usage = int(bytes_per_estimator * n_estimators)\n return expected_memory_usage\n\n def _validate_fit_memory_usage(\n self,\n mem_error_threshold: float = 0.5,\n mem_warning_threshold: float = 0.4,\n mem_size_threshold: int = 1e7,\n **kwargs,\n ):\n return super()._validate_fit_memory_usage(\n mem_error_threshold=mem_error_threshold,\n mem_warning_threshold=mem_warning_threshold,\n mem_size_threshold=mem_size_threshold,\n **kwargs,\n )\n\n def _expected_mem_usage(self, n_estimators_final, bytes_per_estimator):\n available_mem = ResourceManager.get_available_virtual_mem()\n return n_estimators_final * bytes_per_estimator / available_mem\n\n def _fit(self, X, y, num_cpus=-1, time_limit=None, sample_weight=None, **kwargs):\n time_start = time.time()\n\n model_cls = self._get_model_type()\n\n max_memory_usage_ratio = self.params_aux[\"max_memory_usage_ratio\"]\n params = self._get_model_params()\n if \"n_jobs\" not in params:\n params[\"n_jobs\"] = num_cpus\n n_estimators_final = params[\"n_estimators\"]\n\n n_estimators_minimum = min(40, n_estimators_final)\n n_estimators_test = min(4, max(1, math.floor(n_estimators_minimum / 5)))\n\n X = self.preprocess(X, y=y)\n n_estimator_increments = [n_estimators_final]\n\n num_trees_per_estimator = self._get_num_trees_per_estimator()\n bytes_per_estimator = num_trees_per_estimator * len(X) / 60000 * 1e6 # Underestimates by 3x on ExtraTrees\n expected_memory_usage = self._expected_mem_usage(n_estimators_final, bytes_per_estimator)\n\n if n_estimators_final > n_estimators_test * 2:\n if self.problem_type == MULTICLASS:\n n_estimator_increments = [n_estimators_test, n_estimators_final]\n params[\"warm_start\"] = True\n else:\n if expected_memory_usage > (\n 0.05 * max_memory_usage_ratio\n ): # Somewhat arbitrary, consider finding a better value, should it scale by cores?\n # Causes ~10% training slowdown, so try to avoid if memory is not an issue\n n_estimator_increments = [n_estimators_test, n_estimators_final]\n params[\"warm_start\"] = True\n\n params[\"n_estimators\"] = n_estimator_increments[0]\n if self._daal:\n if params.get(\"warm_start\", False):\n params[\"warm_start\"] = False\n # FIXME: This is inefficient but sklearnex doesn't support computing oob_score after training\n params[\"oob_score\"] = True\n\n model = model_cls(**params)\n\n time_train_start = time.time()\n for i, n_estimators in enumerate(n_estimator_increments):\n if i != 0:\n if params.get(\"warm_start\", False):\n model.n_estimators = n_estimators\n else:\n params[\"n_estimators\"] = n_estimators\n model = model_cls(**params)\n model = model.fit(X, y, sample_weight=sample_weight)\n if (i == 0) and (len(n_estimator_increments) > 1):\n time_elapsed = max(\n time.time() - time_train_start, 0.001\n ) # avoid it being too small and being truncated to 0\n model_size_bytes = 0\n for estimator in model.estimators_: # Uses far less memory than pickling the entire forest at once\n model_size_bytes += sys.getsizeof(pickle.dumps(estimator))\n expected_final_model_size_bytes = model_size_bytes * (n_estimators_final / model.n_estimators)\n available_mem = ResourceManager.get_available_virtual_mem()\n model_memory_ratio = expected_final_model_size_bytes / available_mem\n\n ideal_memory_ratio = 0.15 * max_memory_usage_ratio\n n_estimators_ideal = min(\n n_estimators_final, math.floor(ideal_memory_ratio / model_memory_ratio * n_estimators_final)\n )\n\n if n_estimators_final > n_estimators_ideal:\n if n_estimators_ideal < n_estimators_minimum:\n logger.warning(\n f\"\\tWarning: Model is expected to require {round(model_memory_ratio * 100, 2)}% of available memory...\"\n )\n raise NotEnoughMemoryError # don't train full model to avoid OOM error\n logger.warning(\n f\"\\tWarning: Reducing model 'n_estimators' from {n_estimators_final} -> {n_estimators_ideal} due to low memory. Expected memory usage reduced from {round(model_memory_ratio * 100, 2)}% -> {round(ideal_memory_ratio * 100, 2)}% of available memory...\"\n )\n\n if time_limit is not None:\n time_expected = time_train_start - time_start + (time_elapsed * n_estimators_ideal / n_estimators)\n n_estimators_time = math.floor(\n (time_limit - time_train_start + time_start) * n_estimators / time_elapsed\n )\n if n_estimators_time < n_estimators_ideal:\n if n_estimators_time < n_estimators_minimum:\n logger.warning(\n f\"\\tWarning: Model is expected to require {round(time_expected, 1)}s to train, which exceeds the maximum time limit of {round(time_limit, 1)}s, skipping model...\"\n )\n raise TimeLimitExceeded\n logger.warning(\n f\"\\tWarning: Reducing model 'n_estimators' from {n_estimators_ideal} -> {n_estimators_time} due to low time. Expected time usage reduced from {round(time_expected, 1)}s -> {round(time_limit, 1)}s...\"\n )\n n_estimators_ideal = n_estimators_time\n\n for j in range(len(n_estimator_increments)):\n if n_estimator_increments[j] > n_estimators_ideal:\n n_estimator_increments[j] = n_estimators_ideal\n self.model = model\n self.params_trained[\"n_estimators\"] = self.model.n_estimators\n\n # TODO: Remove this after simplifying _predict_proba to reduce code duplication. This is only present for SOFTCLASS support.\n def _predict_proba(self, X, **kwargs):\n X = self.preprocess(X, **kwargs)\n\n if self.problem_type == REGRESSION:\n return self.model.predict(X)\n elif self.problem_type == SOFTCLASS:\n return self.model.predict(X)\n elif self.problem_type == QUANTILE:\n return self.model.predict(X, quantile_levels=self.quantile_levels)\n\n y_pred_proba = self.model.predict_proba(X)\n return self._convert_proba_to_unified_form(y_pred_proba)\n\n def predict_proba_oof(self, X, normalize=None, **kwargs):\n \"\"\"X should be the same X passed to `.fit`\"\"\"\n y_oof_pred_proba = self._predict_proba_oof(X=X, **kwargs)\n if normalize is None:\n normalize = self.normalize_pred_probas\n if normalize:\n y_oof_pred_proba = normalize_pred_probas(y_oof_pred_proba, self.problem_type)\n y_oof_pred_proba = y_oof_pred_proba.astype(np.float32)\n return y_oof_pred_proba\n\n def _is_sklearn_1(self) -> bool:\n \"\"\"Returns True if the trained model is from sklearn>=1.0\"\"\"\n return callable(getattr(self.model, \"_set_oob_score_and_attributes\", None))\n\n def _model_supports_oob_pred_proba(self) -> bool:\n \"\"\"Returns True if model supports computing out-of-bag prediction probabilities\"\"\"\n # TODO: Remove `_set_oob_score` after sklearn version requirement is >=1.0\n return callable(getattr(self.model, \"_set_oob_score\", None)) or self._is_sklearn_1()\n\n # FIXME: Unknown if this works with quantile regression\n def _predict_proba_oof(self, X, y, **kwargs):\n if not self.model.bootstrap:\n raise ValueError(\n \"Forest models must set `bootstrap=True` to compute out-of-fold predictions via out-of-bag predictions.\"\n )\n\n oob_is_not_set = (\n getattr(self.model, \"oob_decision_function_\", None) is None\n and getattr(self.model, \"oob_prediction_\", None) is None\n )\n\n if oob_is_not_set and self._daal:\n raise AssertionError(\"DAAL forest backend does not support out-of-bag predictions.\")\n\n # TODO: This can also be done via setting `oob_score=True` in model params,\n # but getting the correct `pred_time_val` that way is not easy, since we can't time the internal call.\n if oob_is_not_set and self._model_supports_oob_pred_proba():\n X = self.preprocess(X)\n\n if getattr(self.model, \"n_classes_\", None) is not None:\n if self.model.n_outputs_ == 1:\n self.model.n_classes_ = [self.model.n_classes_]\n from sklearn.tree._tree import DOUBLE, DTYPE\n from sklearn.utils.validation import check_X_y\n\n X, y = check_X_y(X, y, multi_output=True, accept_sparse=\"csc\", dtype=DTYPE)\n if y.ndim == 1:\n # reshape is necessary to preserve the data contiguity against vs\n # [:, np.newaxis] that does not.\n y = np.reshape(y, (-1, 1))\n if getattr(y, \"dtype\", None) != DOUBLE or not y.flags.contiguous:\n y = np.ascontiguousarray(y, dtype=DOUBLE)\n if self._is_sklearn_1():\n # sklearn >= 1.0\n # TODO: Can instead do `_compute_oob_predictions` but requires post-processing. Skips scoring func.\n self.model._set_oob_score_and_attributes(X, y)\n else:\n # sklearn < 1.0\n # TODO: Remove once sklearn < 1.0 support is dropped\n self.model._set_oob_score(X, y)\n if getattr(self.model, \"n_classes_\", None) is not None:\n if self.model.n_outputs_ == 1:\n self.model.n_classes_ = self.model.n_classes_[0]\n\n if getattr(self.model, \"oob_decision_function_\", None) is not None:\n y_oof_pred_proba = self.model.oob_decision_function_\n self.model.oob_decision_function_ = None # save memory\n elif getattr(self.model, \"oob_prediction_\", None) is not None:\n y_oof_pred_proba = self.model.oob_prediction_\n self.model.oob_prediction_ = None # save memory\n else:\n raise AssertionError(f\"Model class {type(self.model)} does not support out-of-fold prediction generation.\")\n\n # TODO: Regression does not return NaN for missing rows, instead it sets them to 0. This makes life hard.\n # The below code corrects the missing rows to NaN instead of 0.\n # Don't bother if >60 trees, near impossible to have missing\n # If using 68% of data for training, chance of missing for each row is 1 in 11 billion.\n if self.problem_type == REGRESSION and self.model.n_estimators <= 60:\n from sklearn.ensemble._forest import _generate_unsampled_indices, _get_n_samples_bootstrap\n\n n_samples = len(y)\n\n n_predictions = np.zeros(n_samples)\n n_samples_bootstrap = _get_n_samples_bootstrap(n_samples, self.model.max_samples)\n for estimator in self.model.estimators_:\n unsampled_indices = _generate_unsampled_indices(estimator.random_state, n_samples, n_samples_bootstrap)\n n_predictions[unsampled_indices] += 1\n missing_row_mask = n_predictions == 0\n y_oof_pred_proba[missing_row_mask] = np.nan\n\n # fill missing prediction rows with average of non-missing rows\n if np.isnan(np.sum(y_oof_pred_proba)):\n if len(y_oof_pred_proba.shape) == 1:\n col_mean = np.nanmean(y_oof_pred_proba)\n y_oof_pred_proba[np.isnan(y_oof_pred_proba)] = col_mean\n else:\n col_mean = np.nanmean(y_oof_pred_proba, axis=0)\n inds = np.where(np.isnan(y_oof_pred_proba))\n y_oof_pred_proba[inds] = np.take(col_mean, inds[1])\n\n return self._convert_proba_to_unified_form(y_oof_pred_proba)\n\n def _get_maximum_resources(self) -> dict[str, int | float]:\n # no GPU support\n return {\"num_gpus\": 0}\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, problem_type=None, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(problem_type=problem_type, **kwargs)\n if problem_type != QUANTILE: # use_child_oof not supported in quantile regression\n extra_ag_args_ensemble = {\"use_child_oof\": True}\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\", \"softclass\"]\n\n @classmethod\n def _class_tags(cls):\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self):\n # `can_refit_full=True` because final n_estimators is communicated at end of `_fit`:\n # `self.params_trained['n_estimators'] = self.model.n_estimators`\n tags = {\"can_refit_full\": True}\n if self.problem_type == QUANTILE:\n tags[\"valid_oof\"] = False # not supported in quantile regression\n else:\n tags[\"valid_oof\"] = True\n return tags\n\n @classmethod\n def _valid_compilers(cls):\n return [RFNativeCompiler, RFOnnxCompiler]\n\n @classmethod\n def _default_compiler(cls):\n return RFNativeCompiler\n"
},
{
"path": "tabular/src/autogluon/tabular/models/rf/rf_quantile.py",
"content": "# The original implementation in this file was based on scikit-garden that comes under the following license.\n# The current version of the code has been modified beyond its original version.\n\n# New BSD License\n\n# Copyright (c) 2016 - scikit-garden developers.\n\n# All rights reserved.\n\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions are met:\n\n# a. Redistributions of source code must retain the above copyright notice,\n# this list of conditions and the following disclaimer.\n\n# b. Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n\n# c. Neither the name of the scikit-garden developers nor the names of\n# its contributors may be used to endorse or promote products\n# derived from this software without specific prior written\n# permission.\n\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR\n# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH\n# DAMAGE.\n\nimport logging\nfrom functools import partial\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.ensemble._forest import ForestRegressor\nfrom sklearn.tree import BaseDecisionTree, DecisionTreeRegressor, ExtraTreeRegressor\nfrom sklearn.utils import check_array, check_random_state, check_X_y\n\nlogger = logging.getLogger(__name__)\n\n\ndef weighted_percentile(a, q, weights=None, sorter=None, is_filtered=False):\n \"\"\"\n Returns the weighted percentile of a at q given weights.\n\n Parameters\n ----------\n a: array-like, shape=(n_samples,)\n samples at which the quantile.\n q: int\n quantile between 0 and 100.\n weights: array-like, shape=(n_samples,)\n weights[i] is the weight given to point a[i] while computing the\n quantile. If weights[i] is zero, a[i] is simply ignored during the\n percentile computation.\n sorter: array-like, shape=(n_samples,)\n If provided, assume that a[sorter] is sorted.\n is_filtered: bool\n If True, weights is assumed to contain only non-zero values.\n\n Returns\n -------\n percentile: float\n Weighted percentile of a at q.\n\n References\n ----------\n 1. https://en.wikipedia.org/wiki/Percentile#The_Weighted_Percentile_method\n\n Notes\n -----\n Note that weighted_percentile(a, q) is not equivalent to\n np.percentile(a, q). This is because in np.percentile\n sorted(a)[i] is assumed to be at quantile 0.0, while here we assume\n sorted(a)[i] is given a weight of 1.0 / len(a), hence it is at the\n 1.0 / len(a)th quantile.\n \"\"\"\n if weights is None:\n weights = np.ones_like(a)\n if q > 100 or q < 0:\n raise ValueError(\"q should be in-between 0 and 100, got %d\" % q)\n\n a = np.asarray(a, dtype=np.float32)\n weights = np.asarray(weights, dtype=np.float32)\n if len(a) != len(weights):\n raise ValueError(\"a and weights should have the same length.\")\n\n if sorter is not None:\n a = a[sorter]\n weights = weights[sorter]\n\n if not is_filtered:\n nz = weights != 0\n a = a[nz]\n weights = weights[nz]\n\n if sorter is None:\n sorted_indices = np.argsort(a)\n sorted_a = a[sorted_indices]\n sorted_weights = weights[sorted_indices]\n else:\n sorted_a = a\n sorted_weights = weights\n\n # Step 1\n sorted_cum_weights = np.cumsum(sorted_weights)\n total = sorted_cum_weights[-1]\n\n # Step 2\n partial_sum = 100.0 / total * (sorted_cum_weights - sorted_weights / 2.0)\n start = np.searchsorted(partial_sum, q) - 1\n if start == len(sorted_cum_weights) - 1:\n return sorted_a[-1]\n if start == -1:\n return sorted_a[0]\n\n # Step 3.\n fraction = (q - partial_sum[start]) / (partial_sum[start + 1] - partial_sum[start])\n return sorted_a[start] + fraction * (sorted_a[start + 1] - sorted_a[start])\n\n\nclass BaseTreeQuantileRegressor(BaseDecisionTree):\n def predict(self, X, quantile=None, check_input=False):\n \"\"\"\n Predict regression value for X.\n\n Parameters\n ----------\n X : array-like or sparse matrix of shape = [n_samples, n_features]\n The input samples. Internally, it will be converted to\n ``dtype=np.float32`` and if a sparse matrix is provided\n to a sparse ``csr_matrix``.\n\n quantile : int, optional\n Value ranging from 0 to 100. By default, the mean is returned.\n\n check_input : boolean, (default=True)\n Allow to bypass several input checking.\n Don't use this parameter unless you know what you do.\n\n Returns\n -------\n y : array of shape = [n_samples]\n If quantile is set to None, then return E(Y | X). Else return\n y such that F(Y=y | x) = quantile.\n \"\"\"\n # apply method requires X to be of dtype np.float32\n X = check_array(X, dtype=np.float32, accept_sparse=\"csc\")\n if quantile is None:\n return super(BaseTreeQuantileRegressor, self).predict(X, check_input=check_input)\n\n quantiles = np.zeros(X.shape[0])\n X_leaves = self.apply(X)\n unique_leaves = np.unique(X_leaves)\n for leaf in unique_leaves:\n quantiles[X_leaves == leaf] = weighted_percentile(self.y_train_[self.y_train_leaves_ == leaf], quantile)\n return quantiles\n\n def fit(self, X, y, sample_weight=None, check_input=True):\n \"\"\"\n Build a decision tree classifier from the training set (X, y).\n\n Parameters\n ----------\n X : array-like or sparse matrix, shape = [n_samples, n_features]\n The training input samples. Internally, it will be converted to\n ``dtype=np.float32`` and if a sparse matrix is provided\n to a sparse ``csc_matrix``.\n\n y : array-like, shape = [n_samples] or [n_samples, n_outputs]\n The target values (class labels) as integers or strings.\n\n sample_weight : array-like, shape = [n_samples] or None\n Sample weights. If None, then samples are equally weighted. Splits\n that would create child nodes with net zero or negative weight are\n ignored while searching for a split in each node. Splits are also\n ignored if they would result in any single class carrying a\n negative weight in either child node.\n\n check_input : boolean, (default=True)\n Allow to bypass several input checking.\n Don't use this parameter unless you know what you do.\n\n Returns\n -------\n self : object\n Returns self.\n \"\"\"\n # y passed from a forest is 2-D. This is to silence the\n # annoying data-conversion warnings.\n y = np.asarray(y)\n if np.ndim(y) == 2 and y.shape[1] == 1:\n y = np.ravel(y)\n\n # apply method requires X to be of dtype np.float32\n X, y = check_X_y(X, y, accept_sparse=\"csc\", dtype=np.float32, multi_output=False)\n super(BaseTreeQuantileRegressor, self).fit(X, y, sample_weight=sample_weight, check_input=check_input)\n self.y_train_ = y\n\n # Stores the leaf nodes that the samples lie in.\n self.y_train_leaves_ = self.tree_.apply(X)\n return self\n\n\nclass DecisionTreeQuantileRegressor(BaseTreeQuantileRegressor, DecisionTreeRegressor):\n \"\"\"A decision tree regressor that provides quantile estimates.\n\n Parameters\n ----------\n criterion : string, optional (default=\"squared_error\")\n The function to measure the quality of a split. Supported criteria\n are \"squared_error\" for the mean squared error, which is equal to variance\n reduction as feature selection criterion, and \"mae\" for the mean\n absolute error.\n .. versionadded:: 0.18\n Mean Absolute Error (MAE) criterion.\n\n splitter : string, optional (default=\"best\")\n The strategy used to choose the split at each node. Supported\n strategies are \"best\" to choose the best split and \"random\" to choose\n the best random split.\n\n max_features : int, float, string or None, optional (default=None)\n The number of features to consider when looking for the best split:\n - If int, then consider `max_features` features at each split.\n - If float, then `max_features` is a percentage and\n `int(max_features * n_features)` features are considered at each\n split.\n - If \"auto\", then `max_features=n_features`.\n - If \"sqrt\", then `max_features=sqrt(n_features)`.\n - If \"log2\", then `max_features=log2(n_features)`.\n - If None, then `max_features=n_features`.\n Note: the search for a split does not stop until at least one\n valid partition of the node samples is found, even if it requires to\n effectively inspect more than ``max_features`` features.\n\n max_depth : int or None, optional (default=None)\n The maximum depth of the tree. If None, then nodes are expanded until\n all leaves are pure or until all leaves contain less than\n min_samples_split samples.\n\n min_samples_split : int, float, optional (default=2)\n The minimum number of samples required to split an internal node:\n - If int, then consider `min_samples_split` as the minimum number.\n - If float, then `min_samples_split` is a percentage and\n `ceil(min_samples_split * n_samples)` are the minimum\n number of samples for each split.\n .. versionchanged:: 0.18\n Added float values for percentages.\n\n min_samples_leaf : int, float, optional (default=1)\n The minimum number of samples required to be at a leaf node:\n - If int, then consider `min_samples_leaf` as the minimum number.\n - If float, then `min_samples_leaf` is a percentage and\n `ceil(min_samples_leaf * n_samples)` are the minimum\n number of samples for each node.\n .. versionchanged:: 0.18\n Added float values for percentages.\n\n max_leaf_nodes : int or None, optional (default=None)\n Grow a tree with ``max_leaf_nodes`` in best-first fashion.\n Best nodes are defined as relative reduction in impurity.\n If None then unlimited number of leaf nodes.\n\n random_state : int, RandomState instance or None, optional (default=None)\n If int, random_state is the seed used by the random number generator;\n If RandomState instance, random_state is the random number generator;\n If None, the random number generator is the RandomState instance used\n by `np.random`.\n\n presort : bool, optional (default=False)\n Whether to presort the data to speed up the finding of best splits in\n fitting. For the default settings of a decision tree on large\n datasets, setting this to true may slow down the training process.\n When using either a smaller dataset or a restricted depth, this may\n speed up the training.\n\n Attributes\n ----------\n feature_importances_ : array of shape = [n_features]\n The feature importances.\n The higher, the more important the feature.\n The importance of a feature is computed as the\n (normalized) total reduction of the criterion brought\n by that feature. It is also known as the Gini importance [4]_.\n\n max_features_ : int,\n The inferred value of max_features.\n\n n_features_ : int\n The number of features when ``fit`` is performed.\n\n n_outputs_ : int\n The number of outputs when ``fit`` is performed.\n\n tree_ : Tree object\n The underlying Tree object.\n\n y_train_ : array-like\n Train target values.\n\n y_train_leaves_ : array-like.\n Cache the leaf nodes that each training sample falls into.\n y_train_leaves_[i] is the leaf that y_train[i] ends up at.\n \"\"\"\n\n def __init__(\n self,\n criterion=\"squared_error\",\n splitter=\"best\",\n max_depth=None,\n min_samples_split=2,\n min_samples_leaf=1,\n max_features=None,\n random_state=None,\n max_leaf_nodes=None,\n ):\n super(DecisionTreeQuantileRegressor, self).__init__(\n criterion=criterion,\n splitter=splitter,\n max_depth=max_depth,\n min_samples_split=min_samples_split,\n min_samples_leaf=min_samples_leaf,\n max_features=max_features,\n max_leaf_nodes=max_leaf_nodes,\n random_state=random_state,\n )\n\n\nclass ExtraTreeQuantileRegressor(BaseTreeQuantileRegressor, ExtraTreeRegressor):\n def __init__(\n self,\n criterion=\"squared_error\",\n splitter=\"random\",\n max_depth=None,\n min_samples_split=2,\n min_samples_leaf=1,\n max_features=1.0,\n random_state=None,\n max_leaf_nodes=None,\n ):\n super(ExtraTreeQuantileRegressor, self).__init__(\n criterion=criterion,\n splitter=splitter,\n max_depth=max_depth,\n min_samples_split=min_samples_split,\n min_samples_leaf=min_samples_leaf,\n max_features=max_features,\n max_leaf_nodes=max_leaf_nodes,\n random_state=random_state,\n )\n\n\ndef generate_sample_indices(random_state, n_samples):\n \"\"\"\n Generates bootstrap indices for each tree fit.\n\n Parameters\n ----------\n random_state: int, RandomState instance or None\n If int, random_state is the seed used by the random number generator.\n If RandomState instance, random_state is the random number generator.\n If None, the random number generator is the RandomState instance used\n by np.random.\n n_samples: int\n Number of samples to generate from each tree.\n\n Returns\n -------\n sample_indices: array-like, shape=(n_samples), dtype=np.int32\n Sample indices.\n \"\"\"\n random_instance = check_random_state(random_state)\n sample_indices = random_instance.randint(0, n_samples, n_samples)\n return sample_indices\n\n\ndef get_weighted_neighbors_dataframe(X_leaves, y_train_leaves, y_train, y_weights):\n \"\"\"For each test sample, get the list of weighted targets that are assigned to the same leaf by at least one estimator.\n\n Parameters\n ----------\n X_leaves : array, shape [n_test, n_estimators]\n Index of the leave assigned to each test sample by each estimator.\n y_train_leaves : array, shape [n_estimators, n_train]\n Index of the leave assigned to each training sample by each estimator.\n y_train : array, shape [n_train]\n Values of training samples.\n y_weights : array, shape [n_estimators, n_train]\n Weight assigned to each training sample by each estimator.\n\n Returns\n -------\n weighted_neighbors_dataframe : pd.DataFrame\n Dataframe that contains weighted neighbors of each item in the test set.\n Columns:\n item_id: ID of each item in the test set\n y: Target value encountered in the same leaf as the item\n weight: Weight assigned to each target value\n \"\"\"\n assert X_leaves.shape[1] == y_train_leaves.shape[0] == y_weights.shape[0]\n assert y_train_leaves.shape[1] == y_train.shape[0] == y_weights.shape[1]\n\n num_test, num_trees = X_leaves.shape\n _, num_train = y_train_leaves.shape\n\n tree_index_x = np.arange(num_trees)[None].repeat([num_test], axis=0)\n item_index_x = np.arange(num_test)[:, None].repeat([num_trees], axis=1)\n df_x = pd.DataFrame(\n {\n \"item_id\": item_index_x.ravel(),\n \"tree_id\": tree_index_x.ravel(),\n \"leaf\": X_leaves.ravel(),\n }\n )\n tree_index_y = np.arange(num_trees)[:, None].repeat([num_train], axis=1)\n target_y = y_train[None].repeat([num_trees], axis=0)\n df_y = pd.DataFrame(\n {\n \"tree_id\": tree_index_y.ravel(),\n \"leaf\": y_train_leaves.ravel(),\n \"y\": target_y.ravel(),\n \"weight\": y_weights.ravel(),\n }\n )\n samples_with_neighbors = pd.merge(df_x, df_y, on=[\"tree_id\", \"leaf\"])\n return samples_with_neighbors.groupby([\"item_id\", \"y\"]).sum().reset_index()[[\"item_id\", \"y\", \"weight\"]]\n\n\ndef get_quantiles(neighbors_df, quantile_levels):\n \"\"\"Compute predicted quantiles for the given sample.\n\n Parameters\n ----------\n neighbors_df : pd.DataFrame\n DataFrame with columns y (target values for each sample) and weight (weight assigned to each sample)\n quantile_levels : List[float]\n List of quantiles to predict between 0.0 and 1.0\n\n Returns\n -------\n quantiles : array, shape [len(quantile_levels)]\n Predicted quantiles.\n \"\"\"\n result = []\n for q in quantile_levels:\n result.append(\n weighted_percentile(\n neighbors_df.y,\n int(q * 100),\n neighbors_df.weight,\n sorter=slice(None), # targets are already sorted, so no sorting required\n is_filtered=True,\n )\n )\n return result\n\n\nclass BaseForestQuantileRegressor(ForestRegressor):\n def fit(self, X, y, sample_weight=None):\n \"\"\"\n Build a forest from the training set (X, y).\n\n Parameters\n ----------\n X : array-like or sparse matrix, shape = [n_samples, n_features]\n The training input samples. Internally, it will be converted to\n ``dtype=np.float32`` and if a sparse matrix is provided\n to a sparse ``csc_matrix``.\n y : array-like, shape = [n_samples] or [n_samples, n_outputs]\n The target values (class labels) as integers or strings.\n sample_weight : array-like, shape = [n_samples] or None\n Sample weights. If None, then samples are equally weighted. Splits\n that would create child nodes with net zero or negative weight are\n ignored while searching for a split in each node. Splits are also\n ignored if they would result in any single class carrying a\n negative weight in either child node.\n\n Returns\n -------\n self : object\n Returns self.\n \"\"\"\n logger.warning(\n f\"\\tWARNING: {self.__class__.__name__} are experimental for quantile regression. \"\n f\"They may change or be removed without warning in future releases.\"\n )\n if sample_weight is not None:\n logger.warning(f\"\\tWARNING: {self.__class__.__name__} ignores sample_weight.\")\n\n # apply method requires X to be of dtype np.float32\n X, y = check_X_y(X, y, accept_sparse=\"csc\", dtype=np.float32, multi_output=False)\n super(BaseForestQuantileRegressor, self).fit(X, y)\n\n self.y_train_ = y\n self.y_train_leaves_ = -np.ones((self.n_estimators, len(y)), dtype=np.int32)\n self.y_weights_ = np.zeros_like((self.y_train_leaves_), dtype=np.float32)\n\n for i, est in enumerate(self.estimators_):\n if self.bootstrap:\n bootstrap_indices = generate_sample_indices(est.random_state, len(y))\n else:\n bootstrap_indices = np.arange(len(y))\n\n est_weights = np.bincount(bootstrap_indices, minlength=len(y))\n # FIXME: When updating from scikit-learn 1.3.2 to 1.4.0, BaseTreeQuantileRegressor.fit is not called\n # Re-calculating y_train_ and y_train_leaves_ to resolve this issue\n est.y_train_ = y\n est.y_train_leaves_ = est.tree_.apply(X)\n y_train_leaves = est.y_train_leaves_\n # Normalize the bootstrap weights such that the total weight of each leaf sums up to 1\n # Relabel leaves starting from zero in order to efficiently count the total sum per leaf with bincount\n leaves_starting_from_zero = np.unique(y_train_leaves, return_inverse=True)[1]\n weight_per_leaf = np.bincount(leaves_starting_from_zero, weights=est_weights)\n self.y_weights_[i] = est_weights / weight_per_leaf[leaves_starting_from_zero]\n\n self.y_train_leaves_[i, bootstrap_indices] = y_train_leaves[bootstrap_indices]\n return self\n\n def predict(self, X, quantile_levels=None):\n \"\"\"\n Predict regression value for X.\n\n Parameters\n ----------\n X : array-like or sparse matrix of shape = [n_samples, n_features]\n The input samples. Internally, it will be converted to\n ``dtype=np.float32`` and if a sparse matrix is provided\n to a sparse ``csr_matrix``.\n quantile_levels : List[float], optional\n List of quantiles (between 0.0 and 1.0) to predict. If not provided, mean is returned.\n\n Returns\n -------\n y : array\n If quantile_levels is None, then y contains E(Y | X) and has shape [n_samples].\n Otherwise, y contains the predicted quantiles and has shape [n_samples, len(quantile_levels)]\n \"\"\"\n # apply method requires X to be of dtype np.float32\n X = check_array(X, dtype=np.float32, accept_sparse=\"csc\")\n if quantile_levels is None:\n return super(BaseForestQuantileRegressor, self).predict(X)\n elif isinstance(quantile_levels, float):\n quantile_levels = [quantile_levels]\n\n X_leaves = self.apply(X)\n\n samples_with_weighted_neighbors = get_weighted_neighbors_dataframe(\n X_leaves=X_leaves, y_train_leaves=self.y_train_leaves_, y_train=self.y_train_, y_weights=self.y_weights_\n )\n quantile_preds = samples_with_weighted_neighbors.groupby(\"item_id\").apply(\n partial(get_quantiles, quantile_levels=quantile_levels), include_groups=False\n )\n return np.stack(quantile_preds.values.tolist())\n\n\nclass RandomForestQuantileRegressor(BaseForestQuantileRegressor):\n \"\"\"\n A random forest regressor that provides quantile estimates.\n A random forest is a meta estimator that fits a number of classifying\n decision trees on various sub-samples of the dataset and use averaging\n to improve the predictive accuracy and control over-fitting.\n The sub-sample size is always the same as the original\n input sample size but the samples are drawn with replacement if\n `bootstrap=True` (default).\n\n Parameters\n ----------\n n_estimators : integer, optional (default=10)\n The number of trees in the forest.\n criterion : string, optional (default=\"squared_error\")\n The function to measure the quality of a split. Supported criteria\n are \"squared_error\" for the mean squared error, which is equal to variance\n reduction as feature selection criterion, and \"mae\" for the mean\n absolute error.\n .. versionadded:: 0.18\n Mean Absolute Error (MAE) criterion.\n max_features : int, float, string or None, optional (default=\"auto\")\n The number of features to consider when looking for the best split:\n - If int, then consider `max_features` features at each split.\n - If float, then `max_features` is a percentage and\n `int(max_features * n_features)` features are considered at each\n split.\n - If \"auto\", then `max_features=n_features`.\n - If \"sqrt\", then `max_features=sqrt(n_features)`.\n - If \"log2\", then `max_features=log2(n_features)`.\n - If None, then `max_features=n_features`.\n Note: the search for a split does not stop until at least one\n valid partition of the node samples is found, even if it requires to\n effectively inspect more than ``max_features`` features.\n max_depth : integer or None, optional (default=None)\n The maximum depth of the tree. If None, then nodes are expanded until\n all leaves are pure or until all leaves contain less than\n min_samples_split samples.\n min_samples_split : int, float, optional (default=2)\n The minimum number of samples required to split an internal node:\n - If int, then consider `min_samples_split` as the minimum number.\n - If float, then `min_samples_split` is a percentage and\n `ceil(min_samples_split * n_samples)` are the minimum\n number of samples for each split.\n .. versionchanged:: 0.18\n Added float values for percentages.\n min_samples_leaf : int, float, optional (default=1)\n The minimum number of samples required to be at a leaf node:\n - If int, then consider `min_samples_leaf` as the minimum number.\n - If float, then `min_samples_leaf` is a percentage and\n `ceil(min_samples_leaf * n_samples)` are the minimum\n number of samples for each node.\n .. versionchanged:: 0.18\n Added float values for percentages.\n max_leaf_nodes : int or None, optional (default=None)\n Grow trees with ``max_leaf_nodes`` in best-first fashion.\n Best nodes are defined as relative reduction in impurity.\n If None then unlimited number of leaf nodes.\n bootstrap : boolean, optional (default=True)\n Whether bootstrap samples are used when building trees.\n oob_score : bool, optional (default=False)\n whether to use out-of-bag samples to estimate\n the R^2 on unseen data.\n n_jobs : integer, optional (default=1)\n The number of jobs to run in parallel for both `fit` and `predict`.\n If -1, then the number of jobs is set to the number of cores.\n random_state : int, RandomState instance or None, optional (default=None)\n If int, random_state is the seed used by the random number generator;\n If RandomState instance, random_state is the random number generator;\n If None, the random number generator is the RandomState instance used\n by `np.random`.\n verbose : int, optional (default=0)\n Controls the verbosity of the tree building process.\n warm_start : bool, optional (default=False)\n When set to ``True``, reuse the solution of the previous call to fit\n and add more estimators to the ensemble, otherwise, just fit a whole\n new forest.\n Attributes\n ----------\n estimators_ : list of DecisionTreeQuantileRegressor\n The collection of fitted sub-estimators.\n feature_importances_ : array of shape = [n_features]\n The feature importances (the higher, the more important the feature).\n n_features_ : int\n The number of features when ``fit`` is performed.\n n_outputs_ : int\n The number of outputs when ``fit`` is performed.\n oob_score_ : float\n Score of the training dataset obtained using an out-of-bag estimate.\n oob_prediction_ : array of shape = [n_samples]\n Prediction computed with out-of-bag estimate on the training set.\n y_train_ : array-like, shape=(n_samples,)\n Cache the target values at fit time.\n y_weights_ : array-like, shape=(n_estimators, n_samples)\n y_weights_[i, j] is the weight given to sample ``j` while\n estimator ``i`` is fit. If bootstrap is set to True, this\n reduces to a 2-D array of ones.\n y_train_leaves_ : array-like, shape=(n_estimators, n_samples)\n y_train_leaves_[i, j] provides the leaf node that y_train_[i]\n ends up when estimator j is fit. If y_train_[i] is given\n a weight of zero when estimator j is fit, then the value is -1.\n\n References\n ----------\n .. [1] Nicolai Meinshausen, Quantile Regression Forests\n http://www.jmlr.org/papers/volume7/meinshausen06a/meinshausen06a.pdf\n \"\"\"\n\n def __init__(\n self,\n n_estimators=10,\n criterion=\"squared_error\",\n max_depth=None,\n min_samples_split=2,\n min_samples_leaf=1,\n max_features=1.0,\n max_leaf_nodes=None,\n bootstrap=True,\n oob_score=False,\n n_jobs=1,\n random_state=None,\n verbose=0,\n warm_start=False,\n max_samples=None,\n ):\n super(RandomForestQuantileRegressor, self).__init__(\n DecisionTreeQuantileRegressor(),\n n_estimators=n_estimators,\n estimator_params=(\n \"criterion\",\n \"max_depth\",\n \"min_samples_split\",\n \"min_samples_leaf\",\n \"max_features\",\n \"max_leaf_nodes\",\n \"random_state\",\n ),\n bootstrap=bootstrap,\n oob_score=oob_score,\n n_jobs=n_jobs,\n random_state=random_state,\n verbose=verbose,\n warm_start=warm_start,\n max_samples=max_samples,\n )\n\n self.criterion = criterion\n self.max_depth = max_depth\n self.min_samples_split = min_samples_split\n self.min_samples_leaf = min_samples_leaf\n self.max_features = max_features\n self.max_leaf_nodes = max_leaf_nodes\n\n\nclass ExtraTreesQuantileRegressor(BaseForestQuantileRegressor):\n \"\"\"\n An extra-trees regressor that provides quantile estimates.\n This class implements a meta estimator that fits a number of\n randomized decision trees (a.k.a. extra-trees) on various sub-samples\n of the dataset and use averaging to improve the predictive accuracy\n and control over-fitting.\n\n Parameters\n ----------\n n_estimators : integer, optional (default=10)\n The number of trees in the forest.\n criterion : string, optional (default=\"squared_error\")\n The function to measure the quality of a split. Supported criteria\n are \"squared_error\" for the mean squared error, which is equal to variance\n reduction as feature selection criterion, and \"mae\" for the mean\n absolute error.\n .. versionadded:: 0.18\n Mean Absolute Error (MAE) criterion.\n max_features : int, float, string or None, optional (default=\"auto\")\n The number of features to consider when looking for the best split:\n - If int, then consider `max_features` features at each split.\n - If float, then `max_features` is a percentage and\n `int(max_features * n_features)` features are considered at each\n split.\n - If \"auto\", then `max_features=n_features`.\n - If \"sqrt\", then `max_features=sqrt(n_features)`.\n - If \"log2\", then `max_features=log2(n_features)`.\n - If None, then `max_features=n_features`.\n Note: the search for a split does not stop until at least one\n valid partition of the node samples is found, even if it requires to\n effectively inspect more than ``max_features`` features.\n max_depth : integer or None, optional (default=None)\n The maximum depth of the tree. If None, then nodes are expanded until\n all leaves are pure or until all leaves contain less than\n min_samples_split samples.\n min_samples_split : int, float, optional (default=2)\n The minimum number of samples required to split an internal node:\n - If int, then consider `min_samples_split` as the minimum number.\n - If float, then `min_samples_split` is a percentage and\n `ceil(min_samples_split * n_samples)` are the minimum\n number of samples for each split.\n .. versionchanged:: 0.18\n Added float values for percentages.\n min_samples_leaf : int, float, optional (default=1)\n The minimum number of samples required to be at a leaf node:\n - If int, then consider `min_samples_leaf` as the minimum number.\n - If float, then `min_samples_leaf` is a percentage and\n `ceil(min_samples_leaf * n_samples)` are the minimum\n number of samples for each node.\n .. versionchanged:: 0.18\n Added float values for percentages.\n max_leaf_nodes : int or None, optional (default=None)\n Grow trees with ``max_leaf_nodes`` in best-first fashion.\n Best nodes are defined as relative reduction in impurity.\n If None then unlimited number of leaf nodes.\n bootstrap : boolean, optional (default=False)\n Whether bootstrap samples are used when building trees.\n oob_score : bool, optional (default=False)\n Whether to use out-of-bag samples to estimate the R^2 on unseen data.\n n_jobs : integer, optional (default=1)\n The number of jobs to run in parallel for both `fit` and `predict`.\n If -1, then the number of jobs is set to the number of cores.\n random_state : int, RandomState instance or None, optional (default=None)\n If int, random_state is the seed used by the random number generator;\n If RandomState instance, random_state is the random number generator;\n If None, the random number generator is the RandomState instance used\n by `np.random`.\n verbose : int, optional (default=0)\n Controls the verbosity of the tree building process.\n warm_start : bool, optional (default=False)\n When set to ``True``, reuse the solution of the previous call to fit\n and add more estimators to the ensemble, otherwise, just fit a whole\n new forest.\n\n Attributes\n ----------\n estimators_ : list of ExtraTreeQuantileRegressor\n The collection of fitted sub-estimators.\n feature_importances_ : array of shape = [n_features]\n The feature importances (the higher, the more important the feature).\n n_features_ : int\n The number of features when ``fit`` is performed.\n n_outputs_ : int\n The number of outputs when ``fit`` is performed.\n oob_score_ : float\n Score of the training dataset obtained using an out-of-bag estimate.\n oob_prediction_ : array of shape = [n_samples]\n Prediction computed with out-of-bag estimate on the training set.\n y_train_ : array-like, shape=(n_samples,)\n Cache the target values at fit time.\n y_weights_ : array-like, shape=(n_estimators, n_samples)\n y_weights_[i, j] is the weight given to sample ``j` while\n estimator ``i`` is fit. If bootstrap is set to True, this\n reduces to a 2-D array of ones.\n y_train_leaves_ : array-like, shape=(n_estimators, n_samples)\n y_train_leaves_[i, j] provides the leaf node that y_train_[i]\n ends up when estimator j is fit. If y_train_[i] is given\n a weight of zero when estimator j is fit, then the value is -1.\n\n References\n ----------\n .. [1] Nicolai Meinshausen, Quantile Regression Forests\n http://www.jmlr.org/papers/volume7/meinshausen06a/meinshausen06a.pdf\n \"\"\"\n\n def __init__(\n self,\n n_estimators=10,\n criterion=\"squared_error\",\n max_depth=None,\n min_samples_split=2,\n min_samples_leaf=1,\n max_features=1.0,\n max_leaf_nodes=None,\n bootstrap=True,\n oob_score=False,\n n_jobs=1,\n random_state=None,\n verbose=0,\n warm_start=False,\n max_samples=None,\n ):\n super(ExtraTreesQuantileRegressor, self).__init__(\n ExtraTreeQuantileRegressor(),\n n_estimators=n_estimators,\n estimator_params=(\n \"criterion\",\n \"max_depth\",\n \"min_samples_split\",\n \"min_samples_leaf\",\n \"max_features\",\n \"max_leaf_nodes\",\n \"random_state\",\n ),\n bootstrap=bootstrap,\n oob_score=oob_score,\n n_jobs=n_jobs,\n random_state=random_state,\n verbose=verbose,\n warm_start=warm_start,\n max_samples=max_samples,\n )\n\n self.criterion = criterion\n self.max_depth = max_depth\n self.min_samples_split = min_samples_split\n self.min_samples_leaf = min_samples_leaf\n self.max_features = max_features\n self.max_leaf_nodes = max_leaf_nodes\n"
},
{
"path": "tabular/src/autogluon/tabular/models/rf/rf_rapids_model.py",
"content": "import logging\n\nfrom autogluon.common.utils.try_import import try_import_rapids_cuml\nfrom autogluon.core.constants import REGRESSION, SOFTCLASS\n\nfrom .._utils.rapids_utils import RapidsModelMixin\nfrom .rf_model import RFModel\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Improve memory safety\n# TODO: Respect time limit\n# TODO: Depending on max_depth parameter, RFRapidsModel is slower than RFModel.\n# A lower max_depth (e.g., 16) results in a RFRapidsModel that is faster than RFModel,\n# but a higher max_depth (e.g., approximating unlimited depth)\n# results in a RFRapidsModel that is significantly slower than RFModel.\n# Refer to https://github.com/rapidsai/cuml/issues/1977\nclass RFRapidsModel(RapidsModelMixin, RFModel):\n \"\"\"\n RAPIDS Random Forest model : https://rapids.ai/start.html\n\n NOTE: This code is experimental, it is recommend to not use this unless you are a developer.\n This was tested on rapids-21.06 via:\n\n conda create -n rapids-21.06 -c rapidsai -c nvidia -c conda-forge rapids=21.06 python=3.8 cudatoolkit=11.2\n conda activate rapids-21.06\n pip install --pre autogluon.tabular[all]\n \"\"\"\n\n def _get_model_type(self):\n try_import_rapids_cuml()\n from cuml.ensemble import RandomForestClassifier, RandomForestRegressor\n\n if self.problem_type in [REGRESSION, SOFTCLASS]:\n return RandomForestRegressor\n else:\n return RandomForestClassifier\n\n def _set_default_params(self):\n default_params = {\n \"n_estimators\": 300,\n \"max_depth\": 99, # RAPIDS does not allow unlimited depth, so this approximates it.\n \"random_state\": 0,\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _fit(self, X, y, **kwargs):\n X = self.preprocess(X, y=y)\n self.model = self._get_model_type()(**self._get_model_params())\n self.model = self.model.fit(X, y)\n self.params_trained[\"n_estimators\"] = self.model.n_estimators\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabdpt/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabdpt/tabdpt_model.py",
"content": "from __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nif TYPE_CHECKING:\n import numpy as np\n import pandas as pd\n\n\n# FIXME: Nick:\n# TODO: batch_size is linear to memory usage\n# 512 default\n# should be less for very large datasets\n# 128 batch_size on Bioresponse -> 12 GB VRAM\n# Train Data Rows: 2500\n# Train Data Columns: 1776\n# Problem Type: binary\n# FIXME: Just set context_size = infinity, everything is way faster, memory usage is way lower, etc.\n# Train Data Rows: 100000\n# Train Data Columns: 10\n# binary\n# only takes 6.7 GB during inference with batch_size = 512\n# FIXME: Make it work when loading on CPU?\n# FIXME: Can we run 8 in parallel to speed up?\n# TODO: clip_sigma == 1 is terrible, clip_sigma == 16 maybe very good? What about higher values?\n# clip_sigma >= 16 is roughly all equivalent\n# FIXME: TabDPT stores self.X_test for no reason\n# FIXME: TabDPT creates faiss_knn even if it is never used. Better if `context_size=None` means it is never created.\n# TODO: unit test\n# TODO: memory estimate\nclass TabDPTModel(AbstractTorchModel):\n ag_key = \"TABDPT\"\n ag_name = \"TabDPT\"\n seed_name = \"seed\"\n ag_priority = 50\n default_random_seed = 0\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._feature_generator = None\n self._predict_hps = None\n self._use_flash_og = None\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n num_cpus: int = 1,\n num_gpus: int = 0,\n **kwargs,\n ):\n from torch.cuda import is_available\n\n device = \"cuda\" if num_gpus != 0 else \"cpu\"\n if (device == \"cuda\") and (not is_available()):\n raise AssertionError(\n \"Fit specified to use GPU, but CUDA is not available on this machine. \"\n \"Please switch to CPU usage instead.\",\n )\n from tabdpt import TabDPTClassifier, TabDPTRegressor\n\n model_cls = TabDPTClassifier if self.problem_type in [BINARY, MULTICLASS] else TabDPTRegressor\n fit_params, self._predict_hps = self._get_tabdpt_params(num_gpus=num_gpus)\n\n X = self.preprocess(X, y=y)\n y = y.to_numpy()\n self.model = model_cls(\n device=device,\n **fit_params,\n )\n self.model.fit(X=X, y=y)\n\n def _get_tabdpt_params(self, num_gpus: float) -> tuple[dict, dict]:\n model_params = self._get_model_params()\n\n valid_predict_params = (self.seed_name, \"context_size\", \"permute_classes\", \"temperature\", \"n_ensembles\")\n\n predict_params = {}\n for hp in valid_predict_params:\n if hp in model_params:\n predict_params[hp] = model_params.pop(hp)\n predict_params.setdefault(self.seed_name, self.default_random_seed)\n predict_params.setdefault(\"context_size\", None)\n\n supported_predict_params = (\n (self.seed_name, \"context_size\", \"n_ensembles\", \"permute_classes\", \"temperature\")\n if self.problem_type in [BINARY, MULTICLASS]\n else (self.seed_name, \"context_size\", \"n_ensembles\")\n )\n predict_params = {key: val for key, val in predict_params.items() if key in supported_predict_params}\n\n fit_params = model_params\n\n fit_params.setdefault(\"verbose\", False)\n fit_params.setdefault(\"compile\", False)\n if fit_params.get(\"use_flash\", True):\n fit_params[\"use_flash\"] = self._use_flash(num_gpus=num_gpus)\n return fit_params, predict_params\n\n @staticmethod\n def _use_flash(num_gpus: float) -> bool:\n \"\"\"Detect if torch's native flash attention is available on the current machine.\"\"\"\n if num_gpus == 0:\n return False\n\n import torch\n\n if not torch.cuda.is_available():\n return False\n\n if not torch.backends.cuda.is_flash_attention_available():\n return False\n\n device = torch.device(\"cuda:0\")\n capability = torch.cuda.get_device_capability(device)\n\n return capability != (7, 5)\n\n def _post_fit(self, **kwargs):\n super()._post_fit(**kwargs)\n self._use_flash_og = self.model.use_flash\n return self\n\n def get_device(self) -> str:\n return self.model.device\n\n def _set_device(self, device: str):\n self.model.to(device)\n if device == \"cpu\":\n self.model.use_flash = False\n self.model.model.use_flash = False\n else:\n self.model.use_flash = self._use_flash_og\n self.model.model.use_flash = self._use_flash_og\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n\n return num_cpus, num_gpus\n\n def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]:\n return {\n \"num_cpus\": 1,\n \"num_gpus\": 0.5 if is_gpu_available else 0,\n }\n\n def _predict_proba(self, X, **kwargs) -> np.ndarray:\n X = self.preprocess(X, **kwargs)\n\n if self.problem_type in [REGRESSION]:\n y_pred = self.model.predict(X, **self._predict_hps)\n return y_pred\n\n y_pred_proba = self.model.ensemble_predict_proba(X, **self._predict_hps)\n return self._convert_proba_to_unified_form(y_pred_proba)\n\n def _preprocess(self, X: pd.DataFrame, **kwargs) -> pd.DataFrame:\n \"\"\"TabDPT requires numpy array as input.\"\"\"\n X = super()._preprocess(X, **kwargs)\n if self._feature_generator is None:\n self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0)\n self._feature_generator.fit(X=X)\n if self._feature_generator.features_in:\n X = X.copy()\n X[self._feature_generator.features_in] = self._feature_generator.transform(X=X)\n return X.to_numpy()\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n @classmethod\n def _class_tags(cls):\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self) -> dict:\n return {\"can_refit_full\": True}\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_rows\": 100000, # TODO: Test >100k rows\n \"max_features\": 2500, # TODO: Test >2500 features\n \"max_classes\": 10, # TODO: Test >10 classes\n }\n )\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\n \"refit_folds\": True,\n }\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n # FIXME: This is copied from TabPFN, but TabDPT is not the same\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict | None = None,\n **kwargs,\n ) -> int:\n \"\"\"Heuristic memory estimate based on TabPFN's memory estimate logic in:\n https://github.com/PriorLabs/TabPFN/blob/57a2efd3ebdb3886245e4d097cefa73a5261a969/src/tabpfn/model/memory.py#L147.\n\n This is based on GPU memory usage, but hopefully with overheads it also approximates CPU memory usage.\n \"\"\"\n # TODO: update, this is not correct anymore, consider using internal TabPFN functions directly.\n features_per_group = 3 # Based on TabPFNv2 default (unused)\n n_layers = 12 # Based on TabPFNv2 default\n embedding_size = 192 # Based on TabPFNv2 default\n dtype_byte_size = 2 # Based on TabPFNv2 default\n\n model_mem = 14489108 # Based on TabPFNv2 default\n\n n_samples, n_features = X.shape[0], min(X.shape[1], 500)\n n_feature_groups = (n_features) / features_per_group + 1 # TODO: Unsure how to calculate this\n\n X_mem = n_samples * n_feature_groups * dtype_byte_size\n activation_mem = n_samples * n_feature_groups * embedding_size * n_layers * dtype_byte_size\n\n baseline_overhead_mem_est = 1e9 # 1 GB generic overhead\n\n # Add some buffer to each term + 1 GB overhead to be safe\n memory_estimate = model_mem + 4 * X_mem + 2 * activation_mem + baseline_overhead_mem_est\n\n # TabDPT memory estimation is very inaccurate because it is using TabPFN memory estimate. Double it to be safe.\n memory_estimate = memory_estimate * 2\n\n # Note: This memory estimate is way off if `context_size` is not None\n return int(memory_estimate)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabicl/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabicl/tabicl_model.py",
"content": "\"\"\"\nCode Adapted from TabArena: https://github.com/autogluon/tabarena/blob/main/tabarena/tabarena/benchmark/models/ag/tabicl/tabicl_model.py\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\n\nimport pandas as pd\n\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular import __version__\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Verify if crashes when weights are not yet downloaded and fit in parallel\nclass TabICLModel(AbstractTorchModel):\n \"\"\"\n TabICL is a foundation model for tabular data using in-context learning\n that is scalable to larger datasets than TabPFNv2. It is pretrained purely on synthetic data.\n\n The default TabICL version used is TabICLv2.\n\n TabICL is one of the top performing methods overall on TabArena-v0.1: https://tabarena.ai\n TabICLv2 significantly improves upon TabICLv1, and achieves very strong performance on TabArena.\n\n Paper: TabICL: A Tabular Foundation Model for In-Context Learning on Large Data\n Authors: Jingang Qu, David HolzmÃŧller, GaÃĢl Varoquaux, Marine Le Morvan\n\n Paper: TabICLv2: A better, faster, scalable, and open tabular foundation model\n Authors: Jingang Qu, David HolzmÃŧller, GaÃĢl Varoquaux, Marine Le Morvan\n\n Codebase: https://github.com/soda-inria/tabicl\n License: BSD-3-Clause\n\n .. versionadded:: 1.4.0\n \"\"\"\n\n ag_key = \"TABICL\"\n ag_name = \"TabICL\"\n\n default_classification_model: str | None = \"tabicl-classifier-v2-20260212.ckpt\"\n default_regression_model: str | None = \"tabicl-regressor-v2-20260212.ckpt\"\n\n ag_priority = 65\n seed_name = \"random_state\"\n\n def get_model_cls(self):\n if self.problem_type in [\"binary\", \"multiclass\"]:\n from tabicl import TabICLClassifier\n\n model_cls = TabICLClassifier\n else:\n from tabicl import TabICLRegressor\n\n model_cls = TabICLRegressor\n return model_cls\n\n @staticmethod\n def _get_batch_size(n_cells: int):\n if n_cells <= 4_000_000:\n return 8\n elif n_cells <= 6_000_000:\n return 4\n else:\n return 2\n\n def get_checkpoint_version(self, hyperparameter: dict) -> str:\n clf_checkpoint = self.default_classification_model\n reg_checkpoint = self.default_regression_model\n\n # Resolve HPO\n if \"checkpoint_version\" in hyperparameter:\n if isinstance(hyperparameter[\"checkpoint_version\"], str):\n clf_checkpoint = hyperparameter[\"checkpoint_version\"]\n reg_checkpoint = hyperparameter[\"checkpoint_version\"]\n elif isinstance(hyperparameter[\"checkpoint_version\"], tuple):\n clf_checkpoint = hyperparameter[\"checkpoint_version\"][0]\n reg_checkpoint = hyperparameter[\"checkpoint_version\"][1]\n else:\n raise ValueError(\n \"checkpoint_version hyperparameter must be either a string or a tuple of two strings (clf, reg).\"\n )\n\n if self.problem_type in [\"binary\", \"multiclass\"]:\n return clf_checkpoint\n\n return reg_checkpoint\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n num_cpus: int = 1,\n num_gpus: int = 0,\n **kwargs,\n ):\n try:\n import tabicl\n except ImportError as err:\n logger.log(\n 40,\n f\"\\tFailed to import tabicl! To use the TabICL model, \"\n f\"do: `pip install autogluon.tabular[tabicl]=={__version__}`.\",\n )\n raise err\n\n from torch.cuda import is_available\n\n device = \"cuda\" if num_gpus != 0 else \"cpu\"\n if (device == \"cuda\") and (not is_available()):\n # FIXME: warn instead and switch to CPU.\n raise AssertionError(\n \"Fit specified to use GPU, but CUDA is not available on this machine. \"\n \"Please switch to CPU usage instead.\",\n )\n\n model_cls = self.get_model_cls()\n hyp = self._get_model_params()\n hyp[\"batch_size\"] = hyp.get(\"batch_size\", self._get_batch_size(X.shape[0] * X.shape[1]))\n self.model = model_cls(\n **hyp,\n device=device,\n n_jobs=num_cpus,\n )\n X = self.preprocess(X, y=y)\n self.model = self.model.fit(\n X=X,\n y=y,\n )\n\n def get_device(self) -> str:\n return self.model.device_.type\n\n # TODO: Better to have an official TabICL method for this\n def _set_device(self, device: str):\n device = self.to_torch_device(device)\n self.model.device_ = device\n self.model.device = self.model.device_.type\n self.model.model_ = self.model.model_.to(self.model.device_)\n self.model.inference_config_.COL_CONFIG.device = self.model.device_\n self.model.inference_config_.ROW_CONFIG.device = self.model.device_\n self.model.inference_config_.ICL_CONFIG.device = self.model.device_\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n # TODO: Instead of caps, should we subsample for large datasets?\n \"max_rows\": 1000000, # TODO: What should be the cap? 1 million rows works, but unsure if it is good\n \"max_features\": 2000, # TODO: What should be the cap? 10k features works, but unsure if it is good\n \"max_batch_size\": 1024, # avoid excessive VRAM usage\n }\n )\n return default_auxiliary_params\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n return num_cpus, num_gpus\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n **kwargs,\n ) -> int:\n \"\"\"\n Heuristic memory estimate that is very primitive.\n Can be vastly improved.\n \"\"\"\n if hyperparameters is None:\n hyperparameters = {}\n\n dataset_size_mem_est = 3 * get_approximate_df_mem_usage(X).sum() # roughly 3x DataFrame memory size\n baseline_overhead_mem_est = 1e9 # 1 GB generic overhead\n\n n_rows = X.shape[0]\n n_features = X.shape[1]\n batch_size = hyperparameters.get(\"batch_size\", cls._get_batch_size(X.shape[0] * X.shape[1]))\n embedding_dim = 128\n bytes_per_float = 4\n model_mem_estimate = 2 * batch_size * embedding_dim * bytes_per_float * (4 + n_rows) * n_features\n\n model_mem_estimate *= 1.3 # add 30% buffer\n\n # FIXME: Likely this is overly conservative now, figure out more accurate memory estimate for TabICLv2\n # Early testing shows that cutting this in half is safe.\n # TODO: Observed memory spikes above expected values on large datasets, increasing mem estimate to compensate\n model_mem_estimate *= 2.0 # Note: 1.5 is not large enough, still gets OOM\n\n mem_estimate = model_mem_estimate + dataset_size_mem_est + baseline_overhead_mem_est\n\n return mem_estimate\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n \"\"\"\n Set fold_fitting_strategy to sequential_local,\n as parallel folding crashes if model weights aren't pre-downloaded.\n \"\"\"\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\n # FIXME: If parallel, uses way more memory, seems to behave incorrectly, so we force sequential.\n \"fold_fitting_strategy\": \"sequential_local\",\n \"refit_folds\": True, # Better to refit the model for faster inference and similar quality as the bag.\n }\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n @classmethod\n def _class_tags(cls) -> dict:\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self) -> dict:\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabm/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabm/_tabm_internal.py",
"content": "\"\"\"Partially adapted from pytabkit's TabM implementation.\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nimport math\nimport random\nimport time\nfrom typing import TYPE_CHECKING, Any, Literal\n\nimport numpy as np\nimport pandas as pd\nimport scipy\nimport torch\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import OrdinalEncoder, QuantileTransformer\nfrom sklearn.utils.validation import check_is_fitted\n\nfrom autogluon.core.metrics import compute_metric\n\nfrom . import rtdl_num_embeddings, tabm_reference\nfrom .tabm_reference import make_parameter_groups\n\nif TYPE_CHECKING:\n from autogluon.core.metrics import Scorer\n\nTaskType = Literal[\"regression\", \"binclass\", \"multiclass\"]\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_tabm_auto_batch_size(n_train: int) -> int:\n # by Yury Gorishniy, inferred from the choices in the TabM paper.\n if n_train < 2_800:\n return 32\n if n_train < 4_500:\n return 64\n if n_train < 6_400:\n return 128\n if n_train < 32_000:\n return 256\n if n_train < 108_000:\n return 512\n return 1024\n\n\nclass RTDLQuantileTransformer(BaseEstimator, TransformerMixin):\n # adapted from pytabkit\n def __init__(\n self,\n noise=1e-5,\n random_state=None,\n n_quantiles=1000,\n subsample=1_000_000_000,\n output_distribution=\"normal\",\n ):\n self.noise = noise\n self.random_state = random_state\n self.n_quantiles = n_quantiles\n self.subsample = subsample\n self.output_distribution = output_distribution\n\n def fit(self, X, y=None):\n # Calculate the number of quantiles based on data size\n n_quantiles = max(min(X.shape[0] // 30, self.n_quantiles), 10)\n\n # Initialize QuantileTransformer\n normalizer = QuantileTransformer(\n output_distribution=self.output_distribution,\n n_quantiles=n_quantiles,\n subsample=self.subsample,\n random_state=self.random_state,\n )\n\n # Add noise if required\n X_modified = self._add_noise(X) if self.noise > 0 else X\n\n # Fit the normalizer\n normalizer.fit(X_modified)\n # show that it's fitted\n self.normalizer_ = normalizer\n\n return self\n\n def transform(self, X, y=None):\n check_is_fitted(self)\n return self.normalizer_.transform(X)\n\n def _add_noise(self, X):\n return X + np.random.default_rng(self.random_state).normal(0.0, 1e-5, X.shape).astype(X.dtype)\n\n\nclass TabMOrdinalEncoder(BaseEstimator, TransformerMixin):\n # encodes missing and unknown values to a value one larger than the known values\n def __init__(self):\n # No fitted attributes here â only parameters\n pass\n\n def fit(self, X, y=None):\n X = pd.DataFrame(X)\n\n # Fit internal OrdinalEncoder with NaNs preserved for now\n self.encoder_ = OrdinalEncoder(\n handle_unknown=\"use_encoded_value\",\n unknown_value=np.nan,\n encoded_missing_value=np.nan,\n )\n self.encoder_.fit(X)\n\n # Cardinalities = number of known categories per column\n self.cardinalities_ = [len(cats) for cats in self.encoder_.categories_]\n\n return self\n\n def transform(self, X):\n check_is_fitted(self, [\"encoder_\", \"cardinalities_\"])\n\n X = pd.DataFrame(X)\n X_enc = self.encoder_.transform(X)\n\n # Replace np.nan (unknown or missing) with cardinality value\n for col_idx, cardinality in enumerate(self.cardinalities_):\n mask = np.isnan(X_enc[:, col_idx])\n X_enc[mask, col_idx] = cardinality\n\n return X_enc.astype(int)\n\n def get_cardinalities(self):\n check_is_fitted(self, [\"cardinalities_\"])\n return self.cardinalities_\n\n\nclass TabMImplementation:\n def __init__(self, early_stopping_metric: Scorer, **config):\n self.config = config\n self.early_stopping_metric = early_stopping_metric\n\n self.ord_enc_ = None\n self.num_prep_ = None\n self.cat_col_names_ = None\n self.n_classes_ = None\n self.task_type_ = None\n self.device_ = None\n self.has_num_cols = None\n\n def fit(\n self,\n X_train: pd.DataFrame,\n y_train: pd.Series,\n X_val: pd.DataFrame,\n y_val: pd.Series,\n cat_col_names: list[Any],\n time_to_fit_in_seconds: float | None = None,\n ):\n start_time = time.time()\n\n if X_val is None or len(X_val) == 0:\n raise ValueError(\"Training without validation set is currently not implemented\")\n seed: int | None = self.config.get(\"random_state\", None)\n if seed is not None:\n torch.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n if \"n_threads\" in self.config:\n torch.set_num_threads(self.config[\"n_threads\"])\n\n # -- Meta parameters\n problem_type = self.config[\"problem_type\"]\n task_type: TaskType = \"binclass\" if problem_type == \"binary\" else problem_type\n n_train = len(X_train)\n n_classes = None\n device = self.config[\"device\"]\n device = torch.device(device)\n self.task_type_ = task_type\n self.device_ = device\n self.cat_col_names_ = cat_col_names\n\n # -- Hyperparameters\n arch_type = self.config.get(\"arch_type\", \"tabm-mini\")\n num_emb_type = self.config.get(\"num_emb_type\", \"pwl\")\n n_epochs = self.config.get(\"n_epochs\", 1_000_000_000)\n patience = self.config.get(\"patience\", 16)\n batch_size = self.config.get(\"batch_size\", \"auto\")\n compile_model = self.config.get(\"compile_model\", False)\n lr = self.config.get(\"lr\", 2e-3)\n d_embedding = self.config.get(\"d_embedding\", 16)\n d_block = self.config.get(\"d_block\", 512)\n dropout = self.config.get(\"dropout\", 0.1)\n tabm_k = self.config.get(\"tabm_k\", 32)\n allow_amp = self.config.get(\"allow_amp\", False)\n n_blocks = self.config.get(\"n_blocks\", \"auto\")\n num_emb_n_bins = self.config.get(\"num_emb_n_bins\", 48)\n eval_batch_size = self.config.get(\"eval_batch_size\", 1024)\n share_training_batches = self.config.get(\"share_training_batches\", False)\n weight_decay = self.config.get(\"weight_decay\", 3e-4)\n # this is the search space default but not the example default (which is 'none')\n gradient_clipping_norm = self.config.get(\"gradient_clipping_norm\", 1.0)\n\n # -- Verify HPs\n num_emb_n_bins = min(num_emb_n_bins, n_train - 1)\n if n_train <= 2:\n num_emb_type = \"none\" # there is no valid number of bins for piecewise linear embeddings\n if batch_size == \"auto\":\n batch_size = get_tabm_auto_batch_size(n_train=n_train)\n\n # -- Preprocessing\n ds_parts = dict()\n self.ord_enc_ = (\n TabMOrdinalEncoder()\n ) # Unique ordinal encoder -> replaces nan and missing values with the cardinality\n self.ord_enc_.fit(X_train[self.cat_col_names_])\n # TODO: fix transformer to be able to work with empty input data like the sklearn default\n self.num_prep_ = Pipeline(\n steps=[\n (\"qt\", RTDLQuantileTransformer(random_state=self.config.get(\"random_state\", None))),\n (\"imp\", SimpleImputer(add_indicator=True)),\n ]\n )\n self.has_num_cols = bool(set(X_train.columns) - set(cat_col_names))\n for part, X, y in [(\"train\", X_train, y_train), (\"val\", X_val, y_val)]:\n tensors = dict()\n\n tensors[\"x_cat\"] = torch.as_tensor(self.ord_enc_.transform(X[cat_col_names]), dtype=torch.long)\n\n if self.has_num_cols:\n x_cont_np = X.drop(columns=cat_col_names).to_numpy(dtype=np.float32)\n if part == \"train\":\n self.num_prep_.fit(x_cont_np)\n tensors[\"x_cont\"] = torch.as_tensor(self.num_prep_.transform(x_cont_np))\n else:\n tensors[\"x_cont\"] = torch.empty((len(X), 0), dtype=torch.float32)\n\n if task_type == \"regression\":\n tensors[\"y\"] = torch.as_tensor(y.to_numpy(np.float32))\n if part == \"train\":\n n_classes = 0\n else:\n tensors[\"y\"] = torch.as_tensor(y.to_numpy(np.int32), dtype=torch.long)\n if part == \"train\":\n n_classes = tensors[\"y\"].max().item() + 1\n\n ds_parts[part] = tensors\n\n part_names = [\"train\", \"val\"]\n cat_cardinalities = self.ord_enc_.get_cardinalities()\n self.n_classes_ = n_classes\n\n # filter out numerical columns with only a single value\n # -> AG also does this already but preprocessing might create constant columns again\n x_cont_train = ds_parts[\"train\"][\"x_cont\"]\n self.num_col_mask_ = ~torch.all(x_cont_train == x_cont_train[0:1, :], dim=0)\n for part in part_names:\n ds_parts[part][\"x_cont\"] = ds_parts[part][\"x_cont\"][:, self.num_col_mask_]\n # tensor infos are not correct anymore, but might not be used either\n for part in part_names:\n for tens_name in ds_parts[part]:\n ds_parts[part][tens_name] = ds_parts[part][tens_name].to(device)\n\n # update\n n_cont_features = ds_parts[\"train\"][\"x_cont\"].shape[1]\n\n Y_train = ds_parts[\"train\"][\"y\"].clone()\n if task_type == \"regression\":\n self.y_mean_ = ds_parts[\"train\"][\"y\"].mean().item()\n self.y_std_ = ds_parts[\"train\"][\"y\"].std(correction=0).item()\n\n Y_train = (Y_train - self.y_mean_) / (self.y_std_ + 1e-30)\n\n # the | operator joins dicts (like update() but not in-place)\n data = {\n part: dict(x_cont=ds_parts[part][\"x_cont\"], y=ds_parts[part][\"y\"])\n | (dict(x_cat=ds_parts[part][\"x_cat\"]) if ds_parts[part][\"x_cat\"].shape[1] > 0 else dict())\n for part in part_names\n }\n\n # adapted from https://github.com/yandex-research/tabm/blob/main/example.ipynb\n\n # Automatic mixed precision (AMP)\n # torch.float16 is implemented for completeness,\n # but it was not tested in the project,\n # so torch.bfloat16 is used by default.\n amp_dtype = (\n torch.bfloat16\n if torch.cuda.is_available() and torch.cuda.is_bf16_supported()\n else torch.float16\n if torch.cuda.is_available()\n else None\n )\n # Changing False to True will result in faster training on compatible hardware.\n amp_enabled = allow_amp and amp_dtype is not None\n grad_scaler = torch.cuda.amp.GradScaler() if amp_dtype is torch.float16 else None # type: ignore\n\n # fmt: off\n logger.log(15, f\"Device: {device.type.upper()}\"\n f\"\\nAMP: {amp_enabled} (dtype: {amp_dtype})\"\n f\"\\ntorch.compile: {compile_model}\",\n )\n # fmt: on\n\n bins = (\n None\n if num_emb_type != \"pwl\" or n_cont_features == 0\n else rtdl_num_embeddings.compute_bins(data[\"train\"][\"x_cont\"], n_bins=num_emb_n_bins)\n )\n\n model = tabm_reference.Model(\n n_num_features=n_cont_features,\n cat_cardinalities=cat_cardinalities,\n n_classes=n_classes if n_classes > 0 else None,\n backbone={\n \"type\": \"MLP\",\n \"n_blocks\": n_blocks if n_blocks != \"auto\" else (3 if bins is None else 2),\n \"d_block\": d_block,\n \"dropout\": dropout,\n },\n bins=bins,\n num_embeddings=(\n None\n if bins is None\n else {\n \"type\": \"PiecewiseLinearEmbeddings\",\n \"d_embedding\": d_embedding,\n \"activation\": False,\n \"version\": \"B\",\n }\n ),\n arch_type=arch_type,\n k=tabm_k,\n share_training_batches=share_training_batches,\n ).to(device)\n optimizer = torch.optim.AdamW(make_parameter_groups(model), lr=lr, weight_decay=weight_decay)\n\n if compile_model:\n # NOTE\n # `torch.compile` is intentionally called without the `mode` argument\n # (mode=\"reduce-overhead\" caused issues during training with torch==2.0.1).\n model = torch.compile(model)\n evaluation_mode = torch.no_grad\n else:\n evaluation_mode = torch.inference_mode\n\n @torch.autocast(device.type, enabled=amp_enabled, dtype=amp_dtype) # type: ignore[code]\n def apply_model(part: str, idx: torch.Tensor) -> torch.Tensor:\n return (\n model(\n data[part][\"x_cont\"][idx],\n data[part][\"x_cat\"][idx] if \"x_cat\" in data[part] else None,\n )\n .squeeze(-1) # Remove the last dimension for regression tasks.\n .float()\n )\n\n # TODO: use BCELoss for binary classification\n base_loss_fn = torch.nn.functional.mse_loss if task_type == \"regression\" else torch.nn.functional.cross_entropy\n\n def loss_fn(y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:\n # TabM produces k predictions per object. Each of them must be trained separately.\n # (regression) y_pred.shape == (batch_size, k)\n # (classification) y_pred.shape == (batch_size, k, n_classes)\n k = y_pred.shape[1]\n return base_loss_fn(\n y_pred.flatten(0, 1),\n y_true.repeat_interleave(k) if model.share_training_batches else y_true,\n )\n\n @evaluation_mode()\n def evaluate(part: str) -> float:\n model.eval()\n\n # When using torch.compile, you may need to reduce the evaluation batch size.\n y_pred: np.ndarray = (\n torch.cat(\n [\n apply_model(part, idx)\n for idx in torch.arange(len(data[part][\"y\"]), device=device).split(\n eval_batch_size,\n )\n ],\n )\n .cpu()\n .numpy()\n )\n if task_type == \"regression\":\n # Transform the predictions back to the original label space.\n y_pred = y_pred * self.y_std_ + self.y_mean_\n\n # Compute the mean of the k predictions.\n average_logits = self.config.get(\"average_logits\", False)\n if average_logits:\n y_pred = y_pred.mean(1)\n if task_type != \"regression\":\n # For classification, the mean must be computed in the probability space.\n y_pred = scipy.special.softmax(y_pred, axis=-1)\n if not average_logits:\n y_pred = y_pred.mean(1)\n\n return compute_metric(\n y=data[part][\"y\"].cpu().numpy(),\n metric=self.early_stopping_metric,\n y_pred=y_pred if task_type == \"regression\" else y_pred.argmax(1),\n y_pred_proba=y_pred[:, 1] if task_type == \"binclass\" else y_pred,\n silent=True,\n )\n\n math.ceil(n_train / batch_size)\n best = {\n \"val\": -math.inf,\n # 'test': -math.inf,\n \"epoch\": -1,\n }\n best_params = [p.clone() for p in model.parameters()]\n # Early stopping: the training stops when\n # there are more than `patience` consecutive bad updates.\n remaining_patience = patience\n\n try:\n if self.config.get(\"verbosity\", 0) >= 1:\n from tqdm.std import tqdm\n else:\n tqdm = lambda arr, desc: arr\n except ImportError:\n tqdm = lambda arr, desc: arr\n\n logger.log(15, \"-\" * 88 + \"\\n\")\n for epoch in range(n_epochs):\n # check time limit\n if epoch > 0 and time_to_fit_in_seconds is not None:\n pred_time_after_next_epoch = (epoch + 1) / epoch * (time.time() - start_time)\n if pred_time_after_next_epoch >= time_to_fit_in_seconds:\n break\n\n batches = (\n torch.randperm(n_train, device=device).split(batch_size)\n if model.share_training_batches\n else [\n x.transpose(0, 1).flatten()\n for x in torch.rand((model.k, n_train), device=device).argsort(dim=1).split(batch_size, dim=1)\n ]\n )\n\n for batch_idx in tqdm(batches, desc=f\"Epoch {epoch}\"):\n model.train()\n optimizer.zero_grad()\n loss = loss_fn(apply_model(\"train\", batch_idx), Y_train[batch_idx])\n\n # added from https://github.com/yandex-research/tabm/blob/main/bin/model.py\n if gradient_clipping_norm is not None and gradient_clipping_norm != \"none\":\n if grad_scaler is not None:\n grad_scaler.unscale_(optimizer)\n torch.nn.utils.clip_grad.clip_grad_norm_(\n model.parameters(),\n gradient_clipping_norm,\n )\n\n if grad_scaler is None:\n loss.backward()\n optimizer.step()\n else:\n grad_scaler.scale(loss).backward() # type: ignore\n grad_scaler.step(optimizer) # Ignores grad scaler might skip steps; should not break anything\n grad_scaler.update()\n\n val_score = evaluate(\"val\")\n logger.log(15, f\"(val) {val_score:.4f}\")\n\n if val_score > best[\"val\"]:\n logger.log(15, \"đ¸ New best epoch! đ¸\")\n # best = {'val': val_score, 'test': test_score, 'epoch': epoch}\n best = {\"val\": val_score, \"epoch\": epoch}\n remaining_patience = patience\n with torch.no_grad():\n for bp, p in zip(best_params, model.parameters()):\n bp.copy_(p)\n else:\n remaining_patience -= 1\n\n if remaining_patience < 0:\n break\n\n logger.log(15, \"\\n\\nResult:\")\n logger.log(15, str(best))\n\n logger.log(15, \"Restoring best model\")\n with torch.no_grad():\n for bp, p in zip(best_params, model.parameters()):\n p.copy_(bp)\n\n self.model_ = model\n\n def predict_raw(self, X: pd.DataFrame) -> torch.Tensor:\n self.model_.eval()\n\n tensors = dict()\n tensors[\"x_cat\"] = torch.as_tensor(self.ord_enc_.transform(X[self.cat_col_names_]), dtype=torch.long).to(\n self.device_,\n )\n tensors[\"x_cont\"] = torch.as_tensor(\n self.num_prep_.transform(X.drop(columns=X[self.cat_col_names_]).to_numpy(dtype=np.float32))\n if self.has_num_cols\n else np.empty((len(X), 0), dtype=np.float32),\n ).to(self.device_)\n\n tensors[\"x_cont\"] = tensors[\"x_cont\"][:, self.num_col_mask_]\n\n eval_batch_size = self.config.get(\"eval_batch_size\", 1024)\n with torch.no_grad():\n y_pred: torch.Tensor = torch.cat(\n [\n self.model_(\n tensors[\"x_cont\"][idx],\n tensors[\"x_cat\"][idx] if tensors[\"x_cat\"].numel() != 0 else None,\n )\n .squeeze(-1) # Remove the last dimension for regression tasks.\n .float()\n for idx in torch.arange(tensors[\"x_cont\"].shape[0], device=self.device_).split(\n eval_batch_size,\n )\n ],\n )\n if self.task_type_ == \"regression\":\n # Transform the predictions back to the original label space.\n y_pred = y_pred * self.y_std_ + self.y_mean_\n y_pred = y_pred.mean(1)\n # y_pred = y_pred.unsqueeze(-1) # add extra \"features\" dimension\n else:\n average_logits = self.config.get(\"average_logits\", False)\n if average_logits:\n y_pred = y_pred.mean(1)\n else:\n # For classification, the mean must be computed in the probability space.\n y_pred = torch.log(torch.softmax(y_pred, dim=-1).mean(1) + 1e-30)\n\n return y_pred.cpu()\n\n def predict(self, X: pd.DataFrame) -> np.ndarray:\n y_pred = self.predict_raw(X)\n if self.task_type_ == \"regression\":\n return y_pred.numpy()\n return y_pred.argmax(dim=-1).numpy()\n\n def predict_proba(self, X: pd.DataFrame) -> np.ndarray:\n probas = torch.softmax(self.predict_raw(X), dim=-1).numpy()\n if probas.shape[1] == 2:\n probas = probas[:, 1]\n return probas\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabm/rtdl_num_embeddings.py",
"content": "# taken from https://github.com/yandex-research/rtdl-num-embeddings/blob/main/package/rtdl_num_embeddings.py\n\"\"\"On Embeddings for Numerical Features in Tabular Deep Learning.\"\"\"\n\nfrom __future__ import annotations\n\n__version__ = \"0.0.12\"\n\n__all__ = [\n \"LinearEmbeddings\",\n \"LinearReLUEmbeddings\",\n \"PeriodicEmbeddings\",\n \"PiecewiseLinearEmbeddings\",\n \"PiecewiseLinearEncoding\",\n \"compute_bins\",\n]\n\n\nimport math\nimport warnings\nfrom typing import Any, Literal, Optional, Union\n\ntry:\n import sklearn.tree as sklearn_tree\nexcept ImportError:\n sklearn_tree = None\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nfrom torch.nn.parameter import Parameter\n\ntry:\n from tqdm import tqdm\nexcept ImportError:\n tqdm = None\n\n\ndef _check_input_shape(x: Tensor, expected_n_features: int) -> None:\n if x.ndim < 1:\n raise ValueError(f\"The input must have at least one dimension, however: {x.ndim=}\")\n if x.shape[-1] != expected_n_features:\n raise ValueError(\n f\"The last dimension of the input was expected to be {expected_n_features}, however, {x.shape[-1]=}\"\n )\n\n\nclass LinearEmbeddings(nn.Module):\n \"\"\"Linear embeddings for continuous features.\n\n **Shape**\n\n - Input: `(*, n_features)`\n - Output: `(*, n_features, d_embedding)`\n\n **Examples**\n\n >>> batch_size = 2\n >>> n_cont_features = 3\n >>> x = torch.randn(batch_size, n_cont_features)\n >>> d_embedding = 4\n >>> m = LinearEmbeddings(n_cont_features, d_embedding)\n >>> m.get_output_shape()\n torch.Size([3, 4])\n >>> m(x).shape\n torch.Size([2, 3, 4])\n \"\"\"\n\n def __init__(self, n_features: int, d_embedding: int) -> None:\n \"\"\"\n Args:\n n_features: the number of continuous features.\n d_embedding: the embedding size.\n \"\"\"\n if n_features <= 0:\n raise ValueError(f\"n_features must be positive, however: {n_features=}\")\n if d_embedding <= 0:\n raise ValueError(f\"d_embedding must be positive, however: {d_embedding=}\")\n\n super().__init__()\n self.weight = Parameter(torch.empty(n_features, d_embedding))\n self.bias = Parameter(torch.empty(n_features, d_embedding))\n self.reset_parameters()\n\n def reset_parameters(self) -> None:\n d_rqsrt = self.weight.shape[1] ** -0.5\n nn.init.uniform_(self.weight, -d_rqsrt, d_rqsrt)\n nn.init.uniform_(self.bias, -d_rqsrt, d_rqsrt)\n\n def get_output_shape(self) -> torch.Size:\n \"\"\"Get the output shape without the batch dimensions.\"\"\"\n return self.weight.shape\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n _check_input_shape(x, self.weight.shape[0])\n return torch.addcmul(self.bias, self.weight, x[..., None])\n\n\nclass LinearReLUEmbeddings(nn.Module):\n \"\"\"Simple non-linear embeddings for continuous features.\n\n **Shape**\n\n - Input: `(*, n_features)`\n - Output: `(*, n_features, d_embedding)`\n\n **Examples**\n\n >>> batch_size = 2\n >>> n_cont_features = 3\n >>> x = torch.randn(batch_size, n_cont_features)\n >>>\n >>> d_embedding = 32\n >>> m = LinearReLUEmbeddings(n_cont_features, d_embedding)\n >>> m.get_output_shape()\n torch.Size([3, 32])\n >>> m(x).shape\n torch.Size([2, 3, 32])\n \"\"\"\n\n def __init__(self, n_features: int, d_embedding: int = 32) -> None:\n \"\"\"\n Args:\n n_features: the number of continuous features.\n d_embedding: the embedding size.\n \"\"\"\n super().__init__()\n self.linear = LinearEmbeddings(n_features, d_embedding)\n self.activation = nn.ReLU()\n\n def get_output_shape(self) -> torch.Size:\n \"\"\"Get the output shape without the batch dimensions.\"\"\"\n return self.linear.weight.shape\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n x = self.linear(x)\n x = self.activation(x)\n return x\n\n\nclass _Periodic(nn.Module):\n \"\"\"\n NOTE: THIS MODULE SHOULD NOT BE USED DIRECTLY.\n\n Technically, this is a linear embedding without bias followed by\n the periodic activations. The scale of the initialization\n (defined by the `sigma` argument) plays an important role.\n \"\"\"\n\n def __init__(self, n_features: int, k: int, sigma: float) -> None:\n if sigma <= 0.0:\n raise ValueError(f\"sigma must be positive, however: {sigma=}\")\n\n super().__init__()\n self._sigma = sigma\n self.weight = Parameter(torch.empty(n_features, k))\n self.reset_parameters()\n\n def reset_parameters(self):\n \"\"\"Reset the parameters.\"\"\"\n # NOTE[DIFF]\n # Here, extreme values (~0.3% probability) are explicitly avoided just in case.\n # In the paper, there was no protection from extreme values.\n bound = self._sigma * 3\n nn.init.trunc_normal_(self.weight, 0.0, self._sigma, a=-bound, b=bound)\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n _check_input_shape(x, self.weight.shape[0])\n x = 2 * math.pi * self.weight * x[..., None]\n x = torch.cat([torch.cos(x), torch.sin(x)], -1)\n return x\n\n\n# _NLinear is a simplified copy of delu.nn.NLinear:\n# https://yura52.github.io/delu/stable/api/generated/delu.nn.NLinear.html\nclass _NLinear(nn.Module):\n \"\"\"N *separate* linear layers for N feature embeddings.\n\n In other words,\n each feature embedding is transformed by its own dedicated linear layer.\n \"\"\"\n\n def __init__(self, n: int, in_features: int, out_features: int, bias: bool = True) -> None:\n super().__init__()\n self.weight = Parameter(torch.empty(n, in_features, out_features))\n self.bias = Parameter(torch.empty(n, out_features)) if bias else None\n self.reset_parameters()\n\n def reset_parameters(self):\n \"\"\"Reset the parameters.\"\"\"\n d_in_rsqrt = self.weight.shape[-2] ** -0.5\n nn.init.uniform_(self.weight, -d_in_rsqrt, d_in_rsqrt)\n if self.bias is not None:\n nn.init.uniform_(self.bias, -d_in_rsqrt, d_in_rsqrt)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n \"\"\"Do the forward pass.\"\"\"\n if x.ndim != 3:\n raise ValueError(\n \"_NLinear supports only inputs with exactly one batch dimension,\"\n \" so `x` must have a shape like (BATCH_SIZE, N_FEATURES, D_EMBEDDING).\"\n )\n assert x.shape[-(self.weight.ndim - 1) :] == self.weight.shape[:-1]\n\n x = x.transpose(0, 1)\n x = x @ self.weight\n x = x.transpose(0, 1)\n if self.bias is not None:\n x = x + self.bias\n return x\n\n\nclass PeriodicEmbeddings(nn.Module):\n \"\"\"Embeddings for continuous features based on periodic activations.\n\n See README for details.\n\n **Shape**\n\n - Input: `(*, n_features)`\n - Output: `(*, n_features, d_embedding)`\n\n **Examples**\n\n >>> batch_size = 2\n >>> n_cont_features = 3\n >>> x = torch.randn(batch_size, n_cont_features)\n >>>\n >>> d_embedding = 24\n >>> m = PeriodicEmbeddings(n_cont_features, d_embedding, lite=False)\n >>> m.get_output_shape()\n torch.Size([3, 24])\n >>> m(x).shape\n torch.Size([2, 3, 24])\n >>>\n >>> m = PeriodicEmbeddings(n_cont_features, d_embedding, lite=True)\n >>> m.get_output_shape()\n torch.Size([3, 24])\n >>> m(x).shape\n torch.Size([2, 3, 24])\n >>>\n >>> # PL embeddings.\n >>> m = PeriodicEmbeddings(n_cont_features, d_embedding=8, activation=False, lite=False)\n >>> m.get_output_shape()\n torch.Size([3, 8])\n >>> m(x).shape\n torch.Size([2, 3, 8])\n \"\"\" # noqa: E501\n\n def __init__(\n self,\n n_features: int,\n d_embedding: int = 24,\n *,\n n_frequencies: int = 48,\n frequency_init_scale: float = 0.01,\n activation: bool = True,\n lite: bool,\n ) -> None:\n \"\"\"\n Args:\n n_features: the number of features.\n d_embedding: the embedding size.\n n_frequencies: the number of frequencies for each feature.\n (denoted as \"k\" in Section 3.3 in the paper).\n frequency_init_scale: the initialization scale for the first linear layer\n (denoted as \"sigma\" in Section 3.3 in the paper).\n **This is an important hyperparameter**, see README for details.\n activation: if `False`, the ReLU activation is not applied.\n Must be `True` if ``lite=True``.\n lite: if True, the outer linear layer is shared between all features.\n See README for details.\n \"\"\"\n super().__init__()\n self.periodic = _Periodic(n_features, n_frequencies, frequency_init_scale)\n self.linear: Union[nn.Linear, _NLinear]\n if lite:\n # NOTE[DIFF]\n # The lite variation was introduced in a different paper\n # (about the TabR model).\n if not activation:\n raise ValueError(\"lite=True is allowed only when activation=True\")\n self.linear = nn.Linear(2 * n_frequencies, d_embedding)\n else:\n self.linear = _NLinear(n_features, 2 * n_frequencies, d_embedding)\n self.activation = nn.ReLU() if activation else None\n\n def get_output_shape(self) -> torch.Size:\n \"\"\"Get the output shape without the batch dimensions.\"\"\"\n n_features = self.periodic.weight.shape[0]\n d_embedding = (\n self.linear.weight.shape[0] if isinstance(self.linear, nn.Linear) else self.linear.weight.shape[-1]\n )\n return torch.Size((n_features, d_embedding))\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n x = self.periodic(x)\n x = self.linear(x)\n if self.activation is not None:\n x = self.activation(x)\n return x\n\n\ndef _check_bins(bins: list[Tensor]) -> None:\n if not bins:\n raise ValueError(\"The list of bins must not be empty\")\n for i, feature_bins in enumerate(bins):\n if not isinstance(feature_bins, Tensor):\n raise ValueError(f\"bins must be a list of PyTorch tensors. However, for {i=}: {type(bins[i])=}\")\n if feature_bins.ndim != 1:\n raise ValueError(\n f\"Each item of the bin list must have exactly one dimension. However, for {i=}: {bins[i].ndim=}\"\n )\n if len(feature_bins) < 2:\n raise ValueError(f\"All features must have at least two bin edges. However, for {i=}: {len(bins[i])=}\")\n if not feature_bins.isfinite().all():\n raise ValueError(\n f\"Bin edges must not contain nan/inf/-inf. However, this is not true for the {i}-th feature\"\n )\n if (feature_bins[:-1] >= feature_bins[1:]).any():\n raise ValueError(\n f\"Bin edges must be sorted. However, the for the {i}-th feature, the bin edges are not sorted\"\n )\n # Commented out due to spaming warnings.\n # if len(feature_bins) == 2:\n # warnings.warn(\n # f'The {i}-th feature has just two bin edges, which means only one bin.'\n # ' Strictly speaking, using a single bin for the'\n # ' piecewise-linear encoding should not break anything,'\n # ' but it is the same as using sklearn.preprocessing.MinMaxScaler'\n # )\n\n\ndef compute_bins(\n X: torch.Tensor,\n n_bins: int = 48,\n *,\n tree_kwargs: Optional[dict[str, Any]] = None,\n y: Optional[Tensor] = None,\n regression: Optional[bool] = None,\n verbose: bool = False,\n) -> list[Tensor]:\n \"\"\"Compute the bin boundaries for `PiecewiseLinearEncoding` and `PiecewiseLinearEmbeddings`.\n\n **Usage**\n\n Compute bins using quantiles (Section 3.2.1 in the paper):\n\n >>> X_train = torch.randn(10000, 2)\n >>> bins = compute_bins(X_train)\n\n Compute bins using decision trees (Section 3.2.2 in the paper):\n\n >>> X_train = torch.randn(10000, 2)\n >>> y_train = torch.randn(len(X_train))\n >>> bins = compute_bins(\n ... X_train,\n ... y=y_train,\n ... regression=True,\n ... tree_kwargs={'min_samples_leaf': 64, 'min_impurity_decrease': 1e-4},\n ... )\n\n Args:\n X: the training features.\n n_bins: the number of bins.\n tree_kwargs: keyword arguments for `sklearn.tree.DecisionTreeRegressor`\n (if ``regression=True``) or `sklearn.tree.DecisionTreeClassifier`\n (if ``regression=False``).\n NOTE: requires ``scikit-learn>=1.0,>2`` to be installed.\n y: the training labels (must be provided if ``tree`` is not None).\n regression: whether the labels are regression labels\n (must be provided if ``tree`` is not None).\n verbose: if True and ``tree_kwargs`` is not None, than ``tqdm``\n (must be installed) will report the progress while fitting trees.\n\n Returns:\n A list of bin edges for all features. For one feature:\n\n - the maximum possible number of bin edges is ``n_bins + 1``.\n - the minimum possible number of bin edges is ``1``.\n \"\"\" # noqa: E501\n if not isinstance(X, Tensor):\n raise ValueError(f\"X must be a PyTorch tensor, however: {type(X)=}\")\n if X.ndim != 2:\n raise ValueError(f\"X must have exactly two dimensions, however: {X.ndim=}\")\n if X.shape[0] < 2:\n raise ValueError(f\"X must have at least two rows, however: {X.shape[0]=}\")\n if X.shape[1] < 1:\n raise ValueError(f\"X must have at least one column, however: {X.shape[1]=}\")\n if not X.isfinite().all():\n raise ValueError(\"X must not contain nan/inf/-inf.\")\n if (X == X[0]).all(dim=0).any():\n raise ValueError(\n \"All columns of X must have at least two distinct values.\"\n \" However, X contains columns with just one distinct value.\"\n )\n if n_bins <= 1 or n_bins >= len(X):\n raise ValueError(f\"n_bins must be more than 1, but less than len(X), however: {n_bins=}, {len(X)=}\")\n\n if tree_kwargs is None:\n if y is not None or regression is not None or verbose:\n raise ValueError(\n \"If tree_kwargs is None, then y must be None, regression must be None and verbose must be False\"\n )\n\n _upper = 2**24 # 16_777_216\n if len(X) > _upper:\n warnings.warn(\n f\"Computing quantile-based bins for more than {_upper} million objects\"\n \" may not be possible due to the limitation of PyTorch\"\n \" (for details, see https://github.com/pytorch/pytorch/issues/64947;\"\n \" if that issue is successfully resolved, this warning may be irrelevant).\" # noqa\n \" As a workaround, subsample the data, i.e. instead of\"\n \"\\ncompute_bins(X, ...)\"\n \"\\ndo\"\n \"\\ncompute_bins(X[torch.randperm(len(X), device=X.device)[:16_777_216]], ...)\" # noqa\n \"\\nOn CUDA, the computation can still fail with OOM even after\"\n \" subsampling. If this is the case, try passing features by groups:\"\n \"\\nbins = sum(\"\n \"\\n compute_bins(X[:, idx], ...)\"\n \"\\n for idx in torch.arange(len(X), device=X.device).split(group_size),\" # noqa\n \"\\n start=[]\"\n \"\\n)\"\n \"\\nAnother option is to perform the computation on CPU:\"\n \"\\ncompute_bins(X.cpu(), ...)\"\n )\n del _upper\n\n # NOTE[DIFF]\n # The code below is more correct than the original implementation,\n # because the original implementation contains an unintentional divergence\n # from what is written in the paper. That divergence affected only the\n # quantile-based embeddings, but not the tree-based embeddings.\n # For historical reference, here is the original, less correct, implementation:\n # https://github.com/yandex-research/tabular-dl-num-embeddings/blob/c1d9eb63c0685b51d7e1bc081cdce6ffdb8886a8/bin/train4.py#L612C30-L612C30\n # (explanation: limiting the number of quantiles by the number of distinct\n # values is NOT the same as removing identical quantiles after computing them).\n bins = [q.unique() for q in torch.quantile(X, torch.linspace(0.0, 1.0, n_bins + 1).to(X), dim=0).T]\n _check_bins(bins)\n return bins\n\n else:\n if sklearn_tree is None:\n raise RuntimeError(\"The scikit-learn package is missing. See README.md for installation instructions\")\n if y is None or regression is None:\n raise ValueError(\"If tree_kwargs is not None, then y and regression must not be None\")\n if y.ndim != 1:\n raise ValueError(f\"y must have exactly one dimension, however: {y.ndim=}\")\n if len(y) != len(X):\n raise ValueError(f\"len(y) must be equal to len(X), however: {len(y)=}, {len(X)=}\")\n if y is None or regression is None:\n raise ValueError(\"If tree_kwargs is not None, then y and regression must not be None\")\n if \"max_leaf_nodes\" in tree_kwargs:\n raise ValueError(\n 'tree_kwargs must not contain the key \"max_leaf_nodes\" (it will be set to n_bins automatically).'\n )\n\n if verbose:\n if tqdm is None:\n raise ImportError(\"If verbose is True, tqdm must be installed\")\n tqdm_ = tqdm\n else:\n tqdm_ = lambda x: x # noqa: E731\n\n if X.device.type != \"cpu\" or y.device.type != \"cpu\":\n warnings.warn(\n \"Computing tree-based bins involves the conversion of the input PyTorch\"\n \" tensors to NumPy arrays. The provided PyTorch tensors are not\"\n \" located on CPU, so the conversion has some overhead.\",\n UserWarning,\n )\n X_numpy = X.cpu().numpy()\n y_numpy = y.cpu().numpy()\n bins = []\n for column in tqdm_(X_numpy.T):\n feature_bin_edges = [float(column.min()), float(column.max())]\n tree = (\n (sklearn_tree.DecisionTreeRegressor if regression else sklearn_tree.DecisionTreeClassifier)(\n max_leaf_nodes=n_bins, **tree_kwargs\n )\n .fit(column.reshape(-1, 1), y_numpy)\n .tree_\n )\n for node_id in range(tree.node_count):\n # The following condition is True only for split nodes. Source:\n # https://scikit-learn.org/1.0/auto_examples/tree/plot_unveil_tree_structure.html#tree-structure\n if tree.children_left[node_id] != tree.children_right[node_id]:\n feature_bin_edges.append(float(tree.threshold[node_id]))\n bins.append(torch.as_tensor(feature_bin_edges).unique())\n _check_bins(bins)\n return [x.to(device=X.device, dtype=X.dtype) for x in bins]\n\n\nclass _PiecewiseLinearEncodingImpl(nn.Module):\n \"\"\"Piecewise-linear encoding.\n\n NOTE: THIS CLASS SHOULD NOT BE USED DIRECTLY.\n In particular, this class does *not* add any positional information\n to feature encodings. Thus, for Transformer-like models,\n `PiecewiseLinearEmbeddings` is the only valid option.\n\n Note:\n This is the *encoding* module, not the *embedding* module,\n so it only implements Equation 1 (Figure 1) from the paper,\n and does not have trainable parameters.\n\n **Shape**\n\n * Input: ``(*, n_features)``\n * Output: ``(*, n_features, max_n_bins)``,\n where ``max_n_bins`` is the maximum number of bins over all features:\n ``max_n_bins = max(len(b) - 1 for b in bins)``.\n\n To understand the output structure,\n consider a feature with the number of bins ``n_bins``.\n Formally, its piecewise-linear encoding is a vector of the size ``n_bins``\n that looks as follows::\n\n x_ple = [1, ..., 1, (x - this_bin_left_edge) / this_bin_width, 0, ..., 0]\n\n However, this class will instead produce a vector of the size ``max_n_bins``::\n\n x_ple_actual = [*x_ple[:-1], *zeros(max_n_bins - n_bins), x_ple[-1]]\n\n In other words:\n\n * The last encoding component is **always** located in the end,\n even if ``n_bins == 1`` (i.e. even if it is the only component).\n * The leading ``n_bins - 1`` components are located in the beginning.\n * Everything in-between is always set to zeros (like \"padding\", but in the middle).\n\n This implementation is *significantly* faster than the original one.\n It relies on two key observations:\n\n * The piecewise-linear encoding is just\n a non-trainable linear transformation followed by a clamp-based activation.\n Pseudocode: `PiecewiseLinearEncoding(x) = Activation(Linear(x))`.\n The parameters of the linear transformation are defined by the bin edges.\n * Aligning the *last* encoding channel across all features\n allows applying the aforementioned activation simultaneously to all features\n without the loop over features.\n \"\"\"\n\n weight: Tensor\n \"\"\"The weight of the linear transformation mentioned in the class docstring.\"\"\"\n\n bias: Tensor\n \"\"\"The bias of the linear transformation mentioned in the class docstring.\"\"\"\n\n single_bin_mask: Optional[Tensor]\n \"\"\"The indicators of the features with only one bin.\"\"\"\n\n mask: Optional[Tensor]\n \"\"\"The indicators of the \"valid\" (i.e. \"non-padding\") part of the encoding.\"\"\"\n\n def __init__(self, bins: list[Tensor]) -> None:\n \"\"\"\n Args:\n bins: the bins computed by `compute_bins`.\n \"\"\"\n assert len(bins) > 0\n super().__init__()\n\n n_features = len(bins)\n n_bins = [len(x) - 1 for x in bins]\n max_n_bins = max(n_bins)\n\n self.register_buffer(\"weight\", torch.zeros(n_features, max_n_bins))\n self.register_buffer(\"bias\", torch.zeros(n_features, max_n_bins))\n\n single_bin_mask = torch.tensor(n_bins) == 1\n self.register_buffer(\"single_bin_mask\", single_bin_mask if single_bin_mask.any() else None)\n\n self.register_buffer(\n \"mask\",\n # The mask is needed if features have different number of bins.\n None\n if all(len(x) == len(bins[0]) for x in bins)\n else torch.row_stack(\n [\n torch.cat(\n [\n # The number of bins for this feature, minus 1:\n torch.ones((len(x) - 1) - 1, dtype=torch.bool),\n # Unused components (always zeros):\n torch.zeros(max_n_bins - (len(x) - 1), dtype=torch.bool),\n # The last bin:\n torch.ones(1, dtype=torch.bool),\n ]\n )\n # x is a tensor containing the bin bounds for a given feature.\n for x in bins\n ]\n ),\n )\n\n for i, bin_edges in enumerate(bins):\n # Formally, the piecewise-linear encoding of one feature looks as follows:\n # `[1, ..., 1, (x - this_bin_left_edge) / this_bin_width, 0, ..., 0]`\n # The linear transformation based on the weight and bias defined below\n # implements the expression in the middle before the clipping to [0, 1].\n # Note that the actual encoding layout produced by this class\n # is slightly different. See the docstring of this class for details.\n bin_width = bin_edges.diff()\n w = 1.0 / bin_width\n b = -bin_edges[:-1] / bin_width\n # The last encoding component:\n self.weight[i, -1] = w[-1]\n self.bias[i, -1] = b[-1]\n # The leading encoding components:\n self.weight[i, : n_bins[i] - 1] = w[:-1]\n self.bias[i, : n_bins[i] - 1] = b[:-1]\n # All in-between components will always be zeros,\n # because the weight and bias are initialized with zeros.\n\n def get_max_n_bins(self) -> int:\n return self.weight.shape[-1]\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n x = torch.addcmul(self.bias, self.weight, x[..., None])\n if x.shape[-1] > 1:\n x = torch.cat(\n [\n x[..., :1].clamp_max(1.0),\n x[..., 1:-1].clamp(0.0, 1.0),\n (\n x[..., -1:].clamp_min(0.0)\n if self.single_bin_mask is None\n else torch.where(\n # For features with only one bin,\n # the whole \"piecewise-linear\" encoding effectively behaves\n # like mix-max scaling\n # (assuming that the edges of the single bin\n # are the minimum and maximum feature values).\n self.single_bin_mask[..., None],\n x[..., -1:],\n x[..., -1:].clamp_min(0.0),\n )\n ),\n ],\n dim=-1,\n )\n return x\n\n\nclass PiecewiseLinearEncoding(nn.Module):\n \"\"\"Piecewise-linear encoding.\n\n See README for detailed explanation.\n\n **Shape**\n\n - Input: ``(*, n_features)``\n - Output: ``(*, total_n_bins)``,\n where ``total_n_bins`` is the total number of bins for all features:\n ``total_n_bins = sum(len(b) - 1 for b in bins)``.\n\n Technically, the output of this module is the flattened output\n of `_PiecewiseLinearEncoding` with all \"padding\" values removed.\n \"\"\"\n\n def __init__(self, bins: list[Tensor]) -> None:\n \"\"\"\n Args:\n bins: the bins computed by `compute_bins`.\n \"\"\"\n super().__init__()\n self.impl = _PiecewiseLinearEncodingImpl(bins)\n\n def get_output_shape(self) -> torch.Size:\n \"\"\"Get the output shape without the batch dimensions.\"\"\"\n total_n_bins = (\n self.impl.weight.shape.numel() if self.impl.mask is None else int(self.impl.mask.long().sum().cpu().item())\n )\n return torch.Size((total_n_bins,))\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n x = self.impl(x)\n return x.flatten(-2) if self.impl.mask is None else x[:, self.impl.mask]\n\n\nclass PiecewiseLinearEmbeddings(nn.Module):\n \"\"\"Piecewise-linear embeddings.\n\n **Shape**\n\n - Input: ``(batch_size, n_features)``\n - Output: ``(batch_size, n_features, d_embedding)``\n \"\"\"\n\n def __init__(\n self,\n bins: list[Tensor],\n d_embedding: int,\n *,\n activation: bool,\n version: Literal[None, \"A\", \"B\"] = None,\n ) -> None:\n \"\"\"\n Args:\n bins: the bins computed by `compute_bins`.\n d_embedding: the embedding size.\n activation: if True, the ReLU activation is additionally applied in the end.\n version: the preset for various implementation details, such as\n parametrization and initialization. See README for details.\n \"\"\"\n if d_embedding <= 0:\n raise ValueError(f\"d_embedding must be a positive integer, however: {d_embedding=}\")\n _check_bins(bins)\n if version is None:\n warnings.warn(\n 'The `version` argument is not provided, so version=\"A\" will be used'\n \" for backward compatibility.\"\n \" See README for recommendations regarding `version`.\"\n \" In future, omitting this argument will result in an exception.\"\n )\n version = \"A\"\n\n super().__init__()\n n_features = len(bins)\n # NOTE[DIFF]\n # version=\"B\" was introduced in a different paper (about the TabM model).\n is_version_B = version == \"B\"\n\n self.linear0 = LinearEmbeddings(n_features, d_embedding) if is_version_B else None\n self.impl = _PiecewiseLinearEncodingImpl(bins)\n self.linear = _NLinear(\n len(bins),\n self.impl.get_max_n_bins(),\n d_embedding,\n # For the version \"B\", the bias is already presented in self.linear0.\n bias=not is_version_B,\n )\n if is_version_B:\n # Because of the following line, at initialization,\n # the whole embedding behaves like a linear embedding.\n # The piecewise-linear component is incrementally learnt during training.\n nn.init.zeros_(self.linear.weight)\n self.activation = nn.ReLU() if activation else None\n\n def get_output_shape(self) -> torch.Size:\n \"\"\"Get the output shape without the batch dimensions.\"\"\"\n n_features = self.linear.weight.shape[0]\n d_embedding = self.linear.weight.shape[2]\n return torch.Size((n_features, d_embedding))\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Do the forward pass.\"\"\"\n if x.ndim != 2:\n raise ValueError(\"For now, only inputs with exactly one batch dimension are supported.\")\n\n x_linear = None if self.linear0 is None else self.linear0(x)\n\n x_ple = self.impl(x)\n x_ple = self.linear(x_ple)\n if self.activation is not None:\n x_ple = self.activation(x_ple)\n return x_ple if x_linear is None else x_linear + x_ple\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabm/tabm_model.py",
"content": "\"\"\"\nCode Adapted from TabArena: https://github.com/autogluon/tabarena/blob/main/tabarena/tabarena/benchmark/models/ag/tabm/tabm_model.py\nPartially adapted from pytabkit's TabM implementation.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nimport time\n\nimport pandas as pd\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular import __version__\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass TabMModel(AbstractTorchModel):\n \"\"\"\n TabM is an efficient ensemble of MLPs that is trained simultaneously with mostly shared parameters.\n\n TabM is one of the top performing methods overall on TabArena-v0.1: https://tabarena.ai\n\n Paper: TabM: Advancing Tabular Deep Learning with Parameter-Efficient Ensembling\n Authors: Yury Gorishniy, Akim Kotelnikov, Artem Babenko\n Codebase: https://github.com/yandex-research/tabm\n License: Apache-2.0\n\n Partially adapted from pytabkit's TabM implementation.\n\n .. versionadded:: 1.4.0\n \"\"\"\n\n ag_key = \"TABM\"\n ag_name = \"TabM\"\n ag_priority = 85\n seed_name = \"random_state\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._imputer = None\n self._features_to_impute = None\n self._features_to_keep = None\n self._indicator_columns = None\n self._features_bool = None\n self._bool_to_cat = None\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame = None,\n y_val: pd.Series = None,\n time_limit: float | None = None,\n num_cpus: int = 1,\n num_gpus: float = 0,\n **kwargs,\n ):\n start_time = time.time()\n\n try:\n # imports various dependencies such as torch\n from torch.cuda import is_available\n\n from ._tabm_internal import TabMImplementation\n except ImportError as err:\n logger.log(\n 40,\n f\"\\tFailed to import tabm! To use the TabM model, \"\n f\"do: `pip install autogluon.tabular[tabm]=={__version__}`.\",\n )\n raise err\n\n device = \"cpu\" if num_gpus == 0 else \"cuda\"\n if (device == \"cuda\") and (not is_available()):\n # FIXME: warn instead and switch to CPU.\n raise AssertionError(\n \"Fit specified to use GPU, but CUDA is not available on this machine. \"\n \"Please switch to CPU usage instead.\",\n )\n\n if X_val is None:\n from autogluon.core.utils import generate_train_test_split\n\n X, X_val, y, y_val = generate_train_test_split(\n X=X,\n y=y,\n problem_type=self.problem_type,\n test_size=0.2,\n random_state=0,\n )\n\n hyp = self._get_model_params()\n bool_to_cat = hyp.pop(\"bool_to_cat\", True)\n\n X = self.preprocess(X, y=y, is_train=True, bool_to_cat=bool_to_cat)\n if X_val is not None:\n X_val = self.preprocess(X_val)\n\n self.model = TabMImplementation(\n n_threads=num_cpus,\n device=device,\n problem_type=self.problem_type,\n early_stopping_metric=self.stopping_metric,\n **hyp,\n )\n\n self.model.fit(\n X_train=X,\n y_train=y,\n X_val=X_val,\n y_val=y_val,\n cat_col_names=X.select_dtypes(include=\"category\").columns.tolist(),\n time_to_fit_in_seconds=time_limit - (time.time() - start_time) if time_limit is not None else None,\n )\n\n # FIXME: bool_to_cat is a hack: Maybe move to abstract model?\n def _preprocess(\n self,\n X: pd.DataFrame,\n is_train: bool = False,\n bool_to_cat: bool = False,\n **kwargs,\n ) -> pd.DataFrame:\n \"\"\"Imputes missing values via the mean and adds indicator columns for numerical features.\n Converts indicator columns to categorical features to avoid them being treated as numerical by RealMLP.\n \"\"\"\n X = super()._preprocess(X, **kwargs)\n\n if is_train:\n self._bool_to_cat = bool_to_cat\n self._features_bool = self._feature_metadata.get_features(required_special_types=[\"bool\"])\n if self._bool_to_cat and self._features_bool:\n # FIXME: Use CategoryFeatureGenerator? Or tell the model which is category\n X = X.copy(deep=True)\n X[self._features_bool] = X[self._features_bool].astype(\"category\")\n\n return X\n\n def get_device(self) -> str:\n return self.model.device_.type\n\n def _set_device(self, device: str):\n device = self.to_torch_device(device)\n self.model.device_ = device\n self.model.model_ = self.model.model_.to(device)\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n def _get_default_stopping_metric(self):\n return self.eval_metric\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n return num_cpus, num_gpus\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n num_classes: int | None = 1,\n **kwargs,\n ) -> int:\n \"\"\"\n Heuristic memory estimate that correlates strongly with RealMLP\n \"\"\"\n if num_classes is None:\n num_classes = 1\n if hyperparameters is None:\n hyperparameters = {}\n\n cat_sizes = []\n for col in X.select_dtypes(include=[\"category\", \"object\"]):\n if isinstance(X[col], pd.CategoricalDtype):\n # Use .cat.codes for category dtype\n unique_codes = X[col].cat.codes.unique()\n else:\n # For object dtype, treat unique strings as codes\n unique_codes = X[col].astype(\"category\").cat.codes.unique()\n cat_sizes.append(len(unique_codes))\n\n n_numerical = len(X.select_dtypes(include=[\"number\"]).columns)\n\n # TODO: This estimates very high memory usage,\n # we probably need to adjust batch size automatically to compensate\n mem_estimate_bytes = cls._estimate_tabm_ram(\n hyperparameters=hyperparameters,\n n_numerical=n_numerical,\n cat_sizes=cat_sizes,\n n_classes=num_classes,\n n_samples=len(X),\n )\n\n return mem_estimate_bytes\n\n @classmethod\n def _estimate_tabm_ram(\n cls,\n hyperparameters: dict,\n n_numerical: int,\n cat_sizes: list[int],\n n_classes: int,\n n_samples: int,\n ) -> int:\n num_emb_n_bins = hyperparameters.get(\"num_emb_n_bins\", 48)\n d_embedding = hyperparameters.get(\"d_embedding\", 16)\n d_block = hyperparameters.get(\"d_block\", 512)\n # not completely sure if this is hidden blocks or all blocks, taking the safe option below\n n_blocks = hyperparameters.get(\"n_blocks\", \"auto\")\n if isinstance(n_blocks, str) and n_blocks == \"auto\":\n n_blocks = 3\n batch_size = hyperparameters.get(\"batch_size\", \"auto\")\n if isinstance(batch_size, str) and batch_size == \"auto\":\n batch_size = cls.get_tabm_auto_batch_size(n_samples=n_samples)\n tabm_k = hyperparameters.get(\"tabm_k\", 32)\n predict_batch_size = hyperparameters.get(\"eval_batch_size\", 1024)\n\n # not completely sure\n n_params_num_emb = n_numerical * (num_emb_n_bins + 1) * d_embedding\n n_params_mlp = (\n (n_numerical + sum(cat_sizes)) * d_embedding * (d_block + tabm_k)\n + (n_blocks - 1) * d_block**2\n + n_blocks * d_block\n + d_block * (1 + max(1, n_classes))\n )\n # 4 bytes per float, up to 5 copies of parameters (1 standard, 1 .grad, 2 adam, 1 best_epoch)\n mem_params = 4 * 5 * (n_params_num_emb + n_params_mlp)\n\n # compute number of floats in forward pass (per batch element)\n # todo: numerical embedding layer (not sure if this is entirely correct)\n n_floats_forward = n_numerical * (num_emb_n_bins + d_embedding)\n # before and after scale\n n_floats_forward += 2 * (sum(cat_sizes) + n_numerical * d_embedding)\n # 2 for pre-act, post-act\n n_floats_forward += n_blocks * 2 * d_block + 2 * max(1, n_classes)\n # 2 for forward and backward, 4 bytes per float\n mem_forward_backward = 4 * max(batch_size * 2, predict_batch_size) * n_floats_forward * tabm_k\n # * 8 is pessimistic for the long tensors in the forward pass, 4 would probably suffice\n\n mem_ds = n_samples * (4 * n_numerical + 8 * len(cat_sizes))\n\n # some safety constants and offsets (the 5 is probably excessive)\n mem_total = 5 * mem_ds + 1.2 * mem_forward_backward + 1.2 * mem_params + 0.3 * (1024**3)\n\n return mem_total\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_batch_size\": 16384, # avoid excessive VRAM usage\n }\n )\n return default_auxiliary_params\n\n @classmethod\n def get_tabm_auto_batch_size(cls, n_samples: int) -> int:\n # by Yury Gorishniy, inferred from the choices in the TabM paper.\n if n_samples < 2_800:\n return 32\n if n_samples < 4_500:\n return 64\n if n_samples < 6_400:\n return 128\n if n_samples < 32_000:\n return 256\n if n_samples < 108_000:\n return 512\n return 1024\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"reset_torch_threads\": True,\n }\n\n def _more_tags(self) -> dict:\n # TODO: Need to add train params support, track best epoch\n # How to force stopping at a specific epoch?\n return {\"can_refit_full\": False}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabm/tabm_reference.py",
"content": "# License: https://github.com/yandex-research/tabm/blob/main/LICENSE\n\n# NOTE\n# The minimum required versions of the dependencies are specified in README.md.\n\nfrom __future__ import annotations\n\nimport itertools\nfrom typing import Any, Literal, Union\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\n\nfrom . import rtdl_num_embeddings\nfrom .rtdl_num_embeddings import _Periodic\n\n\n# ======================================================================================\n# Initialization\n# ======================================================================================\ndef init_rsqrt_uniform_(x: Tensor, d: int) -> Tensor:\n assert d > 0\n d_rsqrt = d**-0.5\n return nn.init.uniform_(x, -d_rsqrt, d_rsqrt)\n\n\n@torch.inference_mode()\ndef init_random_signs_(x: Tensor) -> Tensor:\n return x.bernoulli_(0.5).mul_(2).add_(-1)\n\n\n# ======================================================================================\n# Modules\n# ======================================================================================\nclass NLinear(nn.Module):\n \"\"\"N linear layers applied in parallel to N disjoint parts of the input.\n\n **Shape**\n\n - Input: ``(B, N, in_features)``\n - Output: ``(B, N, out_features)``\n\n The i-th linear layer is applied to the i-th matrix of the shape (B, in_features).\n\n Technically, this is a simplified version of delu.nn.NLinear:\n https://yura52.github.io/delu/stable/api/generated/delu.nn.NLinear.html.\n The difference is that this layer supports only 3D inputs\n with exactly one batch dimension. By contrast, delu.nn.NLinear supports\n any number of batch dimensions.\n \"\"\"\n\n def __init__(self, n: int, in_features: int, out_features: int, bias: bool = True) -> None:\n super().__init__()\n self.weight = nn.Parameter(torch.empty(n, in_features, out_features))\n self.bias = nn.Parameter(torch.empty(n, out_features)) if bias else None\n self.reset_parameters()\n\n def reset_parameters(self):\n d = self.weight.shape[-2]\n init_rsqrt_uniform_(self.weight, d)\n if self.bias is not None:\n init_rsqrt_uniform_(self.bias, d)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n assert x.ndim == 3\n assert x.shape[-(self.weight.ndim - 1) :] == self.weight.shape[:-1]\n\n x = x.transpose(0, 1)\n x = x @ self.weight\n x = x.transpose(0, 1)\n if self.bias is not None:\n x = x + self.bias\n return x\n\n\nclass OneHotEncoding0d(nn.Module):\n # Input: (*, n_cat_features=len(cardinalities))\n # Output: (*, sum(cardinalities))\n\n def __init__(self, cardinalities: list[int]) -> None:\n super().__init__()\n self._cardinalities = cardinalities\n\n def forward(self, x: Tensor) -> Tensor:\n assert x.ndim >= 1\n assert x.shape[-1] == len(self._cardinalities)\n\n return torch.cat(\n [\n # NOTE\n # This is a quick hack to support out-of-vocabulary categories.\n #\n # Recall that lib.data.transform_cat encodes categorical features\n # as follows:\n # - In-vocabulary values receive indices from `range(cardinality)`.\n # - All out-of-vocabulary values (i.e. new categories in validation\n # and test data that are not presented in the training data)\n # receive the index `cardinality`.\n #\n # As such, the line below will produce the standard one-hot encoding for\n # known categories, and the all-zeros encoding for unknown categories.\n # This may not be the best approach to deal with unknown values,\n # but should be enough for our purposes.\n nn.functional.one_hot(x[..., i], cardinality + 1)[..., :-1]\n for i, cardinality in enumerate(self._cardinalities)\n ],\n -1,\n )\n\n\nclass ScaleEnsemble(nn.Module):\n def __init__(\n self,\n k: int,\n d: int,\n *,\n init: Literal[\"ones\", \"normal\", \"random-signs\"],\n ) -> None:\n super().__init__()\n self.weight = nn.Parameter(torch.empty(k, d))\n self._weight_init = init\n self.reset_parameters()\n\n def reset_parameters(self) -> None:\n if self._weight_init == \"ones\":\n nn.init.ones_(self.weight)\n elif self._weight_init == \"normal\":\n nn.init.normal_(self.weight)\n elif self._weight_init == \"random-signs\":\n init_random_signs_(self.weight)\n else:\n raise ValueError(f\"Unknown weight_init: {self._weight_init}\")\n\n def forward(self, x: Tensor) -> Tensor:\n assert x.ndim >= 2\n return x * self.weight\n\n\nclass LinearEfficientEnsemble(nn.Module):\n \"\"\"\n This layer is a more configurable version of the \"BatchEnsemble\" layer\n from the paper\n \"BatchEnsemble: An Alternative Approach to Efficient Ensemble and Lifelong Learning\"\n (link: https://arxiv.org/abs/2002.06715).\n\n First, this layer allows to select only some of the \"ensembled\" parts:\n - the input scaling (r_i in the BatchEnsemble paper)\n - the output scaling (s_i in the BatchEnsemble paper)\n - the output bias (not mentioned in the BatchEnsemble paper,\n but is presented in public implementations)\n\n Second, the initialization of the scaling weights is configurable\n through the `scaling_init` argument.\n\n NOTE\n The term \"adapter\" is used in the TabM paper only to tell the story.\n The original BatchEnsemble paper does NOT use this term. So this class also\n avoids the term \"adapter\".\n \"\"\"\n\n r: Union[None, Tensor]\n s: Union[None, Tensor]\n bias: Union[None, Tensor]\n\n def __init__(\n self,\n in_features: int,\n out_features: int,\n bias: bool = True,\n *,\n k: int,\n ensemble_scaling_in: bool,\n ensemble_scaling_out: bool,\n ensemble_bias: bool,\n scaling_init: Literal[\"ones\", \"random-signs\"],\n ):\n assert k > 0\n if ensemble_bias:\n assert bias\n super().__init__()\n\n self.weight = nn.Parameter(torch.empty(out_features, in_features))\n self.register_parameter(\n \"r\",\n (nn.Parameter(torch.empty(k, in_features)) if ensemble_scaling_in else None), # type: ignore[code]\n )\n self.register_parameter(\n \"s\",\n (nn.Parameter(torch.empty(k, out_features)) if ensemble_scaling_out else None), # type: ignore[code]\n )\n self.register_parameter(\n \"bias\",\n (\n nn.Parameter(torch.empty(out_features)) # type: ignore[code]\n if bias and not ensemble_bias\n else nn.Parameter(torch.empty(k, out_features))\n if ensemble_bias\n else None\n ),\n )\n\n self.in_features = in_features\n self.out_features = out_features\n self.k = k\n self.scaling_init = scaling_init\n\n self.reset_parameters()\n\n def reset_parameters(self):\n init_rsqrt_uniform_(self.weight, self.in_features)\n scaling_init_fn = {\"ones\": nn.init.ones_, \"random-signs\": init_random_signs_}[self.scaling_init]\n if self.r is not None:\n scaling_init_fn(self.r)\n if self.s is not None:\n scaling_init_fn(self.s)\n if self.bias is not None:\n bias_init = torch.empty(\n # NOTE: the shape of bias_init is (out_features,) not (k, out_features).\n # It means that all biases have the same initialization.\n # This is similar to having one shared bias plus\n # k zero-initialized non-shared biases.\n self.out_features,\n dtype=self.weight.dtype,\n device=self.weight.device,\n )\n bias_init = init_rsqrt_uniform_(bias_init, self.in_features)\n with torch.inference_mode():\n self.bias.copy_(bias_init)\n\n def forward(self, x: Tensor) -> Tensor:\n # x.shape == (B, K, D)\n assert x.ndim == 3\n\n # >>> The equation (5) from the BatchEnsemble paper (arXiv v2).\n if self.r is not None:\n x = x * self.r\n x = x @ self.weight.T\n if self.s is not None:\n x = x * self.s\n # <<<\n\n if self.bias is not None:\n x = x + self.bias\n return x\n\n\nclass MLP(nn.Module):\n def __init__(\n self,\n *,\n d_in: Union[None, int] = None,\n d_out: Union[None, int] = None,\n n_blocks: int,\n d_block: int,\n dropout: float,\n activation: str = \"ReLU\",\n ) -> None:\n super().__init__()\n\n d_first = d_block if d_in is None else d_in\n self.blocks = nn.ModuleList(\n [\n nn.Sequential(\n nn.Linear(d_first if i == 0 else d_block, d_block),\n getattr(nn, activation)(),\n nn.Dropout(dropout),\n )\n for i in range(n_blocks)\n ]\n )\n self.output = None if d_out is None else nn.Linear(d_block, d_out)\n\n def forward(self, x: Tensor) -> Tensor:\n for block in self.blocks:\n x = block(x)\n if self.output is not None:\n x = self.output(x)\n return x\n\n\ndef make_efficient_ensemble(module: nn.Module, EnsembleLayer, **kwargs) -> None:\n \"\"\"Replace linear layers with efficient ensembles of linear layers.\n\n NOTE\n In the paper, there are no experiments with networks with normalization layers.\n Perhaps, their trainable weights (the affine transformations) also need\n \"ensemblification\" as in the paper about \"FiLM-Ensemble\".\n Additional experiments are required to make conclusions.\n \"\"\"\n for name, submodule in list(module.named_children()):\n if isinstance(submodule, nn.Linear):\n module.add_module(\n name,\n EnsembleLayer(\n in_features=submodule.in_features,\n out_features=submodule.out_features,\n bias=submodule.bias is not None,\n **kwargs,\n ),\n )\n else:\n make_efficient_ensemble(submodule, EnsembleLayer, **kwargs)\n\n\ndef _get_first_ensemble_layer(backbone: MLP) -> LinearEfficientEnsemble:\n if isinstance(backbone, MLP):\n return backbone.blocks[0][0] # type: ignore[code]\n else:\n raise RuntimeError(f\"Unsupported backbone: {backbone}\")\n\n\n@torch.inference_mode()\ndef _init_first_adapter(\n weight: Tensor,\n distribution: Literal[\"normal\", \"random-signs\"],\n init_sections: list[int],\n) -> None:\n \"\"\"Initialize the first adapter.\n\n NOTE\n The `init_sections` argument is a historical artifact that accidentally leaked\n from irrelevant experiments to the final models. Perhaps, the code related\n to `init_sections` can be simply removed, but this was not tested.\n \"\"\"\n assert weight.ndim == 2\n assert weight.shape[1] == sum(init_sections)\n\n if distribution == \"normal\":\n init_fn_ = nn.init.normal_\n elif distribution == \"random-signs\":\n init_fn_ = init_random_signs_\n else:\n raise ValueError(f\"Unknown distribution: {distribution}\")\n\n section_bounds = [0, *torch.tensor(init_sections).cumsum(0).tolist()]\n for i in range(len(init_sections)):\n # NOTE\n # As noted above, this section-based initialization is an arbitrary historical\n # artifact. Consider the first adapter of one ensemble member.\n # This adapter vector is implicitly split into \"sections\",\n # where one section corresponds to one feature. The code below ensures that\n # the adapter weights in one section are initialized with the same random value\n # from the given distribution.\n w = torch.empty((len(weight), 1), dtype=weight.dtype, device=weight.device)\n init_fn_(w)\n weight[:, section_bounds[i] : section_bounds[i + 1]] = w\n\n\n_CUSTOM_MODULES = {\n # https://docs.python.org/3/library/stdtypes.html#definition.__name__\n CustomModule.__name__: CustomModule\n for CustomModule in [\n rtdl_num_embeddings.LinearEmbeddings,\n rtdl_num_embeddings.LinearReLUEmbeddings,\n rtdl_num_embeddings.PeriodicEmbeddings,\n rtdl_num_embeddings.PiecewiseLinearEmbeddings,\n MLP,\n ]\n}\n\n\ndef make_module(type: str, *args, **kwargs) -> nn.Module:\n Module = getattr(nn, type, None)\n if Module is None:\n Module = _CUSTOM_MODULES[type]\n return Module(*args, **kwargs)\n\n\n# ======================================================================================\n# Optimization\n# ======================================================================================\ndef default_zero_weight_decay_condition(\n module_name: str, module: nn.Module, parameter_name: str, parameter: nn.Parameter\n):\n del module_name, parameter\n return parameter_name.endswith(\"bias\") or isinstance(\n module,\n (\n nn.BatchNorm1d,\n nn.LayerNorm,\n nn.InstanceNorm1d,\n rtdl_num_embeddings.LinearEmbeddings,\n rtdl_num_embeddings.LinearReLUEmbeddings,\n _Periodic,\n ),\n )\n\n\ndef make_parameter_groups(\n module: nn.Module,\n zero_weight_decay_condition=default_zero_weight_decay_condition,\n custom_groups: Union[None, list[dict[str, Any]]] = None,\n) -> list[dict[str, Any]]:\n if custom_groups is None:\n custom_groups = []\n custom_params = frozenset(itertools.chain.from_iterable(group[\"params\"] for group in custom_groups))\n assert len(custom_params) == sum(len(group[\"params\"]) for group in custom_groups), (\n \"Parameters in custom_groups must not intersect\"\n )\n zero_wd_params = frozenset(\n p\n for mn, m in module.named_modules()\n for pn, p in m.named_parameters()\n if p not in custom_params and zero_weight_decay_condition(mn, m, pn, p)\n )\n default_group = {\"params\": [p for p in module.parameters() if p not in custom_params and p not in zero_wd_params]}\n return [\n default_group,\n {\"params\": list(zero_wd_params), \"weight_decay\": 0.0},\n *custom_groups,\n ]\n\n\n# ======================================================================================\n# The model\n# ======================================================================================\nclass Model(nn.Module):\n \"\"\"MLP & TabM.\"\"\"\n\n def __init__(\n self,\n *,\n n_num_features: int,\n cat_cardinalities: list[int],\n n_classes: Union[None, int],\n backbone: dict,\n bins: Union[None, list[Tensor]], # For piecewise-linear encoding/embeddings.\n num_embeddings: Union[None, dict] = None,\n arch_type: Literal[\n # Plain feed-forward network without any kind of ensembling.\n \"plain\",\n #\n # TabM\n \"tabm\",\n #\n # TabM-mini\n \"tabm-mini\",\n #\n # TabM-packed\n \"tabm-packed\",\n #\n # TabM. The first adapter is initialized from the normal distribution.\n # This variant was not used in the paper, but it may be useful in practice.\n \"tabm-normal\",\n #\n # TabM-mini. The adapter is initialized from the normal distribution.\n # This variant was not used in the paper.\n \"tabm-mini-normal\",\n ],\n k: Union[None, int] = None,\n share_training_batches: bool = True,\n ) -> None:\n # >>> Validate arguments.\n assert n_num_features >= 0\n assert n_num_features or cat_cardinalities\n if arch_type == \"plain\":\n assert k is None\n assert share_training_batches, 'If `arch_type` is set to \"plain\", then `simple` must remain True'\n else:\n assert k is not None\n assert k > 0\n\n super().__init__()\n\n # >>> Continuous (numerical) features\n first_adapter_sections = [] # See the comment in `_init_first_adapter`.\n\n if n_num_features == 0:\n assert bins is None\n self.num_module = None\n d_num = 0\n\n elif num_embeddings is None:\n assert bins is None\n self.num_module = None\n d_num = n_num_features\n first_adapter_sections.extend(1 for _ in range(n_num_features))\n\n else:\n if bins is None:\n self.num_module = make_module(**num_embeddings, n_features=n_num_features)\n else:\n assert num_embeddings[\"type\"].startswith(\"PiecewiseLinearEmbeddings\")\n self.num_module = make_module(**num_embeddings, bins=bins)\n d_num = n_num_features * num_embeddings[\"d_embedding\"]\n first_adapter_sections.extend(num_embeddings[\"d_embedding\"] for _ in range(n_num_features))\n\n # >>> Categorical features\n self.cat_module = OneHotEncoding0d(cat_cardinalities) if cat_cardinalities else None\n first_adapter_sections.extend(cat_cardinalities)\n d_cat = sum(cat_cardinalities)\n\n # >>> Backbone\n d_flat = d_num + d_cat\n self.minimal_ensemble_adapter = None\n # Any backbone can be here but we provide only MLP\n self.backbone = make_module(d_in=d_flat, **backbone)\n\n if arch_type != \"plain\":\n assert k is not None\n first_adapter_init = (\n None\n if arch_type == \"tabm-packed\"\n else \"normal\"\n if arch_type in (\"tabm-mini-normal\", \"tabm-normal\")\n # For other arch_types, the initialization depends\n # on the presence of num_embeddings.\n else \"random-signs\"\n if num_embeddings is None\n else \"normal\"\n )\n\n if arch_type in (\"tabm\", \"tabm-normal\"):\n # Like BatchEnsemble, but all multiplicative adapters,\n # except for the very first one, are initialized with ones.\n assert first_adapter_init is not None\n make_efficient_ensemble(\n self.backbone,\n LinearEfficientEnsemble,\n k=k,\n ensemble_scaling_in=True,\n ensemble_scaling_out=True,\n ensemble_bias=True,\n scaling_init=\"ones\",\n )\n _init_first_adapter(\n _get_first_ensemble_layer(self.backbone).r, # type: ignore[code]\n first_adapter_init,\n first_adapter_sections,\n )\n\n elif arch_type in (\"tabm-mini\", \"tabm-mini-normal\"):\n # MiniEnsemble\n assert first_adapter_init is not None\n self.minimal_ensemble_adapter = ScaleEnsemble(\n k,\n d_flat,\n init=\"random-signs\" if num_embeddings is None else \"normal\",\n )\n _init_first_adapter(\n self.minimal_ensemble_adapter.weight, # type: ignore[code]\n first_adapter_init,\n first_adapter_sections,\n )\n\n elif arch_type == \"tabm-packed\":\n # Packed ensemble.\n # In terms of the Packed Ensembles paper by Laurent et al.,\n # TabM-packed is PackedEnsemble(alpha=k, M=k, gamma=1).\n assert first_adapter_init is None\n make_efficient_ensemble(self.backbone, NLinear, n=k)\n\n else:\n raise ValueError(f\"Unknown arch_type: {arch_type}\")\n\n # >>> Output\n d_block = backbone[\"d_block\"]\n d_out = 1 if n_classes is None else n_classes\n self.output = (\n nn.Linear(d_block, d_out) if arch_type == \"plain\" else NLinear(k, d_block, d_out) # type: ignore[code]\n )\n\n # >>>\n self.arch_type = arch_type\n self.k = k\n self.share_training_batches = share_training_batches\n\n def forward(self, x_num: Union[None, Tensor] = None, x_cat: Union[None, Tensor] = None) -> Tensor:\n x = []\n if x_num is not None:\n x.append(x_num if self.num_module is None else self.num_module(x_num))\n if x_cat is None:\n assert self.cat_module is None\n else:\n assert self.cat_module is not None\n x.append(self.cat_module(x_cat).float())\n x = torch.column_stack([x_.flatten(1, -1) for x_ in x])\n\n if self.k is not None:\n if self.share_training_batches or not self.training:\n # (B, D) -> (B, K, D)\n x = x[:, None].expand(-1, self.k, -1)\n else:\n # (B * K, D) -> (B, K, D)\n x = x.reshape(len(x) // self.k, self.k, *x.shape[1:])\n if self.minimal_ensemble_adapter is not None:\n x = self.minimal_ensemble_adapter(x)\n else:\n assert self.minimal_ensemble_adapter is None\n\n x = self.backbone(x)\n x = self.output(x)\n if self.k is None:\n # Adjust the output shape for plain networks to make them compatible\n # with the rest of the script (loss, metrics, predictions, ...).\n # (B, D_OUT) -> (B, 1, D_OUT)\n x = x[:, None]\n return x\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/LICENSE",
"content": "MIT License\n\nCopyright (c) 2024 Felix den Breejen, Nick Erickson\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/README.md",
"content": "The code located in this directory (`autogluon.tabular.models.tabpfnmix._internal`) is based on the code for TabForestPFN: https://github.com/FelixdenBreejen/TabForestPFN, forked when the codebase was on [this commit](https://github.com/FelixdenBreejen/TabForestPFN/tree/53114795d3c96f87348a7ccbb675665e9d3e5243).\n\nThe TabForestPFN codebase was originally created as an implementation of the paper \"[Why In-Context Learning Transformers are Tabular Data Classifiers](https://arxiv.org/pdf/2405.13396)\" by authors Felix den Breejen, Sangmin Bae, Stephen Cha, Se-Young Yun.\n\nThe codebase provides convenient functionality surrounding conducting fine-tuning and inference on tabular transformer models, which we leverage in the TabPFNMix model.\n\nFor more information on the TabPFNMix model, refer to the HuggingFace model repositories:\n\n1. https://huggingface.co/autogluon/tabpfn-mix-1.0-classifier\n2. https://huggingface.co/autogluon/tabpfn-mix-1.0-regressor\n\nThe following changes to the original codebase have been made by Nick Erickson (@innixma) and Xiyuan Zhang (@xiyuanzh):\n\n1. Improved model early stopping logic to properly load the best epoch's weights at the end of the fit call.\n2. Removed all code that is unused for model fine-tuning and inference. Therefore, all code related to benchmarking and pre-training have been removed.\n3. Removed all dependencies unrelated to model fine-tuning and inference.\n4. Optimized fine-tuning checkpoints to use in-memory checkpoints.\n5. Added CPU support\n6. Added torch thread control\n7. Added custom metric support\n8. Optimized metric calculation to avoid calculating unnecessary metrics\n9. Optimized fine-tuning speed by avoiding calculating metrics for train data by default\n10. Replacing several pieces of custom functionality with AutoGluon utility equivalents to reduce code duplication\n11. Only checkpoint when a new best iteration is found, rather than each iteration\n12. Various cosmetic, logging, typing, docstring, and formatting changes\n13. Added random seed control for reproducible results\n14. Reduced memory usage and disk usage for inference by 5x by deleting unnecessary checkpoint and optimizer objects\n15. Added time_limit support\n16. Added support for fine-tuning without validation data\n17. Added HuggingFace Hub `from_pretrained` support\n18. Added regression support\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/config/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/config/config_run.py",
"content": "from __future__ import annotations\n\nfrom dataclasses import dataclass\n\nimport torch\n\nfrom ..core.enums import Task\n\n\n@dataclass\nclass ConfigRun:\n task: Task\n hyperparams: dict\n seed: int = 0\n device: torch.device = None\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/callbacks.py",
"content": "\"\"\"\nThese are not callbacks, but if you look for callback most likely you look for these\n\"\"\"\n\nimport io\nfrom pathlib import Path\n\nimport numpy as np\nimport torch\n\n\nclass EarlyStopping:\n def __init__(self, patience=10, delta=0.0001):\n self.patience = patience\n self.counter = 0\n self.best_score = None\n self.early_stop = False\n self.delta = delta\n\n def __call__(self, val_loss):\n score = -val_loss\n\n if self.best_score is None:\n self.best_score = score\n elif score < self.best_score + self.delta:\n self.counter += 1\n if self.counter >= self.patience:\n self.early_stop = True\n else:\n self.best_score = score\n self.counter = 0\n\n def we_should_stop(self):\n return self.early_stop\n\n\nclass Checkpoint:\n def __init__(self, dirname: Path = None, id: str = None, in_memory: bool = True, save_best: bool = True):\n self.dirname = dirname\n self.id = id\n self.curr_best_loss = np.inf\n self.save_best = save_best\n self.in_memory = in_memory\n if not self.in_memory:\n assert self.dirname is not None\n assert self.id is not None\n self.path = Path(self.dirname) / f\"params_{self.id}.pt\"\n self.path.parent.mkdir(exist_ok=True)\n else:\n self.path = None\n\n self.buffer = io.BytesIO()\n self.best_model = None\n self.best_epoch = None\n\n def reset(self):\n self.curr_best_loss = np.inf\n self.best_model = None\n self.best_epoch = None\n self.buffer = io.BytesIO()\n\n def __call__(self, net, loss: float, epoch: int):\n if loss < self.curr_best_loss:\n self.curr_best_loss = loss\n self.best_model = net.state_dict()\n self.best_epoch = epoch\n if self.save_best:\n self.save()\n\n def save(self):\n if self.in_memory:\n self.buffer = io.BytesIO()\n torch.save(self.best_model, self.buffer) # nosec B614\n else:\n torch.save(self.best_model, self.path) # nosec B614\n\n def load(self):\n if self.in_memory:\n # Reset the buffer's position to the beginning\n self.buffer.seek(0)\n return torch.load(self.buffer, weights_only=True) # nosec B614\n else:\n return torch.load(self.path) # nosec B614\n\n\nclass EpochStatistics:\n def __init__(self) -> None:\n self.n = 0\n self.loss = 0\n self.score = 0\n\n def update(self, loss, score, n):\n self.n += n\n self.loss += loss * n\n self.score += score * n\n\n def get(self):\n return self.loss / self.n, self.score / self.n\n\n\nclass TrackOutput:\n def __init__(self) -> None:\n self.y_true: list[np.ndarray] = []\n self.y_pred: list[np.ndarray] = []\n\n def update(self, y_true: np.ndarray, y_pred: np.ndarray):\n self.y_true.append(y_true)\n self.y_pred.append(y_pred)\n\n def get(self):\n return np.concatenate(self.y_true, axis=0), np.concatenate(self.y_pred, axis=0)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/collator.py",
"content": "from typing import Optional\n\nimport torch\n\n\nclass CollatorWithPadding:\n def __init__(self, pad_to_n_support_samples: Optional[int]) -> None:\n self.pad_to_n_support_samples = pad_to_n_support_samples\n\n def __call__(self, batch: list[dict[str, torch.Tensor]]) -> dict[str, torch.Tensor]:\n if self.pad_to_n_support_samples is not None:\n assert all(dataset[\"x_support\"].shape[0] <= self.pad_to_n_support_samples for dataset in batch)\n self.n_support_samples = self.pad_to_n_support_samples\n else:\n self.n_support_samples = max(dataset[\"x_support\"].shape[0] for dataset in batch)\n\n max_query_samples = max(dataset[\"x_query\"].shape[0] for dataset in batch)\n\n n_support_features = batch[0][\"x_support\"].shape[1]\n n_query_features = batch[0][\"x_query\"].shape[1]\n y_dtype = batch[0][\"y_support\"].dtype\n\n batch_size = len(batch)\n\n tensor_dict = {\n \"x_support\": torch.zeros((batch_size, self.n_support_samples, n_support_features), dtype=torch.float32),\n \"y_support\": torch.zeros((batch_size, self.n_support_samples), dtype=y_dtype),\n \"x_query\": torch.zeros((batch_size, max_query_samples, n_query_features), dtype=torch.float32),\n \"y_query\": torch.zeros((batch_size, max_query_samples), dtype=y_dtype),\n }\n\n for i, dataset in enumerate(batch):\n tensor_dict[\"x_support\"][i, : dataset[\"x_support\"].shape[0], :] = dataset[\"x_support\"]\n tensor_dict[\"y_support\"][i, : dataset[\"y_support\"].shape[0]] = dataset[\"y_support\"]\n tensor_dict[\"x_query\"][i, : dataset[\"x_query\"].shape[0], :] = dataset[\"x_query\"]\n tensor_dict[\"y_query\"][i, : dataset[\"y_query\"].shape[0]] = dataset[\"y_query\"]\n\n return tensor_dict\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/dataset_split.py",
"content": "import numpy as np\nfrom numpy.random import Generator\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\nfrom .enums import Task\n\n\ndef make_dataset_split(\n x: np.ndarray, y: np.ndarray, task: Task, random_state: Generator = None\n) -> tuple[np.ndarray, ...]:\n # Splits the dataset into train and validation sets with ratio 80/20\n\n if task == Task.CLASSIFICATION and np.min(np.bincount(y)) >= 5:\n # stratification needs have at least 5 samples in each class if split is 80/20\n return make_stratified_dataset_split(x, y, rng=random_state)\n else:\n return make_standard_dataset_split(x, y, rng=random_state)\n\n\ndef make_stratified_dataset_split(x, y, rng: Generator = None):\n # Stratify doesn't shuffle the data, so we shuffle it first\n permutation = rng.permutation(len(y))\n x, y = x[permutation], y[permutation]\n\n skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=rng.integers(low=0, high=1000000))\n indices = next(skf.split(x, y))\n x_t_train, x_t_valid = x[indices[0]], x[indices[1]]\n y_t_train, y_t_valid = y[indices[0]], y[indices[1]]\n\n return x_t_train, x_t_valid, y_t_train, y_t_valid\n\n\ndef make_standard_dataset_split(x, y, rng: Generator = None):\n return train_test_split(x, y, test_size=0.2, random_state=rng.integers(low=0, high=1000000))\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/enums.py",
"content": "class Task:\n CLASSIFICATION = \"classification\"\n REGRESSION = \"regression\"\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/get_loss.py",
"content": "import torch\n\nfrom .enums import Task\n\n\ndef get_loss(task: Task):\n if task == Task.REGRESSION:\n return torch.nn.MSELoss()\n else:\n return torch.nn.CrossEntropyLoss()\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/get_optimizer.py",
"content": "import torch\nfrom torch.optim import SGD, Adam, AdamW\n\n\ndef get_optimizer(hyperparams: dict, model: torch.nn.Module) -> torch.optim.Optimizer:\n optimizer: torch.optim.Optimizer\n\n if hyperparams[\"optimizer\"] == \"adam\":\n optimizer = Adam(\n model.parameters(), lr=hyperparams[\"lr\"], betas=(0.9, 0.999), weight_decay=hyperparams[\"weight_decay\"]\n )\n elif hyperparams[\"optimizer\"] == \"adamw\":\n optimizer = AdamW(\n model.parameters(), lr=hyperparams[\"lr\"], betas=(0.9, 0.999), weight_decay=hyperparams[\"weight_decay\"]\n )\n elif hyperparams[\"optimizer\"] == \"sgd\":\n optimizer = SGD(model.parameters(), lr=hyperparams[\"lr\"], weight_decay=hyperparams[\"weight_decay\"])\n else:\n raise ValueError(\"Optimizer not recognized\")\n\n return optimizer\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/get_scheduler.py",
"content": "import torch\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n\ndef get_scheduler(hyperparams: dict, optimizer: torch.optim.Optimizer):\n if hyperparams[\"lr_scheduler\"]:\n scheduler = ReduceLROnPlateau(optimizer, patience=hyperparams[\"lr_scheduler_patience\"], min_lr=0, factor=0.2)\n else:\n scheduler = ReduceLROnPlateau(optimizer, patience=10000000, min_lr=0, factor=0.2)\n\n return scheduler\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/trainer_finetune.py",
"content": "from __future__ import annotations\n\nimport logging\nimport time\n\nimport einops\nimport numpy as np\nimport torch\nfrom numpy.random import Generator\nfrom sklearn.base import BaseEstimator\n\nfrom autogluon.core.metrics import Scorer\n\nfrom ..config.config_run import ConfigRun\nfrom ..data.dataset_finetune import DatasetFinetune, DatasetFinetuneGenerator\nfrom ..data.preprocessor import Preprocessor\nfrom ..results.prediction_metrics import PredictionMetrics\nfrom .callbacks import Checkpoint, EarlyStopping, TrackOutput\nfrom .collator import CollatorWithPadding\nfrom .enums import Task\nfrom .get_loss import get_loss\nfrom .get_optimizer import get_optimizer\nfrom .get_scheduler import get_scheduler\nfrom .y_transformer import create_y_transformer\n\nlogger = logging.getLogger(__name__)\n\n\nclass TrainerFinetune(BaseEstimator):\n def __init__(\n self,\n cfg: ConfigRun,\n model: torch.nn.Module,\n n_classes: int,\n stopping_metric: Scorer,\n use_best_epoch: bool = True,\n compute_train_metrics: bool = False,\n ) -> None:\n self.cfg = cfg\n self.model = model\n self.model.to(self.cfg.device)\n self.n_classes = n_classes\n\n self.loss = get_loss(self.cfg.task)\n self.optimizer = get_optimizer(self.cfg.hyperparams, self.model)\n self.scheduler = get_scheduler(self.cfg.hyperparams, self.optimizer)\n self.use_best_epoch = use_best_epoch\n self.compute_train_metrics = compute_train_metrics\n\n self.early_stopping = EarlyStopping(patience=self.cfg.hyperparams[\"early_stopping_patience\"])\n self.preprocessor = Preprocessor(\n use_quantile_transformer=self.cfg.hyperparams[\"use_quantile_transformer\"],\n use_feature_count_scaling=self.cfg.hyperparams[\"use_feature_count_scaling\"],\n max_features=self.cfg.hyperparams[\"n_features\"],\n task=self.cfg.task,\n )\n\n self.stopping_metric = stopping_metric\n self.best_epoch = None\n\n def set_random_seed(self) -> Generator:\n torch.manual_seed(self.cfg.seed)\n np.random.seed(self.cfg.seed)\n rng = np.random.default_rng(seed=self.cfg.seed)\n return rng\n\n def reset_optimizer(self):\n self.optimizer = get_optimizer(self.cfg.hyperparams, self.model)\n self.scheduler = get_scheduler(self.cfg.hyperparams, self.optimizer)\n\n def train(\n self,\n x_train: np.ndarray,\n y_train: np.ndarray,\n x_val: np.ndarray = None,\n y_val: np.ndarray = None,\n time_limit: float = None,\n ):\n time_start = time.time()\n if self.optimizer is None:\n self.reset_optimizer()\n # FIXME: Figure out best way to seed model\n rng = self.set_random_seed()\n use_val = x_val is not None\n\n checkpoint = Checkpoint(save_best=self.use_best_epoch, in_memory=True)\n\n self.preprocessor.fit(x_train, y_train)\n x_train = self.preprocessor.transform(x_train)\n\n if use_val:\n x_val = self.preprocessor.transform(x_val)\n self.y_transformer = create_y_transformer(y_train, self.cfg.task)\n\n dataset_train_generator = DatasetFinetuneGenerator(\n self.cfg,\n x=x_train,\n y=self.y_transformer.transform(y_train),\n task=self.cfg.task,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n split=0.8,\n random_state=rng,\n )\n\n if use_val:\n dataset_valid = DatasetFinetune(\n self.cfg,\n x_support=x_train,\n y_support=self.y_transformer.transform(y_train),\n x_query=x_val,\n y_query=y_val,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n )\n loader_valid = self.make_loader(dataset_valid, training=False)\n else:\n loader_valid = None\n\n if use_val and self.use_best_epoch:\n checkpoint.reset()\n\n max_epochs = self.cfg.hyperparams[\"max_epochs\"]\n\n epoch = 0\n if max_epochs != 0 and use_val:\n metrics_valid = self.test_epoch(loader_valid, y_val)\n\n log_msg = f\"Epoch 000\"\n if self.compute_train_metrics:\n log_msg += f\" | Train error: -.---- | Train score: -.---- |\"\n if metrics_valid is not None:\n log_msg += f\" | Val error: {metrics_valid.loss:.4f} | Val score: {metrics_valid.score:.4f}\"\n\n logger.log(15, log_msg)\n if self.use_best_epoch:\n checkpoint(self.model, metrics_valid.loss, epoch=0)\n\n if time_limit is not None:\n time_cur = time.time()\n time_elapsed = time_cur - time_start\n time_left = time_limit - time_elapsed\n if time_left < (time_elapsed * 3 + 3):\n # Fine-tuning an epoch will take longer than this, so triple the time required\n logger.log(15, \"Early stopping due to running out of time...\")\n max_epochs = 0\n\n for epoch in range(1, max_epochs + 1):\n dataset_train = next(dataset_train_generator)\n loader_train = self.make_loader(dataset_train, training=True)\n\n metrics_train = self.train_epoch(loader_train, return_metrics=self.compute_train_metrics)\n if use_val:\n metrics_valid = self.test_epoch(loader_valid, y_val)\n else:\n metrics_valid = None\n\n log_msg = f\"Epoch {epoch:03d}\"\n if metrics_train is not None:\n log_msg += f\" | Train error: {metrics_train.loss:.4f} | Train score: {metrics_train.score:.4f}\"\n if metrics_valid is not None:\n log_msg += f\" | Val error: {metrics_valid.loss:.4f} | Val score: {metrics_valid.score:.4f}\"\n\n logger.log(15, log_msg)\n if metrics_valid is not None:\n if self.use_best_epoch:\n checkpoint(self.model, metrics_valid.loss, epoch=epoch)\n\n self.early_stopping(metrics_valid.loss)\n if self.early_stopping.we_should_stop():\n logger.info(\"Early stopping\")\n break\n self.scheduler.step(\n metrics_valid.loss\n ) # TODO: Make scheduler work properly during refit with no val data, to mimic scheduler in OG fit\n\n if time_limit is not None:\n time_cur = time.time()\n time_elapsed = time_cur - time_start\n\n time_per_epoch = time_elapsed / epoch\n time_left = time_limit - time_elapsed\n if time_left < (time_per_epoch + 3):\n logger.log(15, \"Early stopping due to running out of time...\")\n break\n\n if use_val and self.use_best_epoch and checkpoint.best_model is not None:\n # TODO: Can do a trick: Skip saving and loading best epoch if best epoch is the final epoch, will save around ~0.5 seconds\n self.best_epoch = checkpoint.best_epoch\n self.model.load_state_dict(checkpoint.load())\n else:\n self.best_epoch = epoch\n\n def minimize_for_inference(self):\n # delete unnecessary objects for inference\n self.optimizer = None\n self.scheduler = None\n\n def train_epoch(\n self, dataloader: torch.utils.data.DataLoader, return_metrics: bool = False\n ) -> PredictionMetrics | None:\n \"\"\"\n\n Parameters\n ----------\n dataloader\n return_metrics: bool = False\n If True, will calculate and return metrics on the train data.\n Note that this can slow down training speed by >10%.\n\n Returns\n -------\n\n \"\"\"\n assert self.optimizer is not None\n self.model.train()\n\n if return_metrics:\n output_tracker = TrackOutput()\n else:\n output_tracker = None\n\n for batch in dataloader:\n self.optimizer.zero_grad()\n\n x_support = batch[\"x_support\"].to(self.cfg.device)\n y_support = batch[\"y_support\"].to(self.cfg.device)\n x_query = batch[\"x_query\"].to(self.cfg.device)\n y_query = batch[\"y_query\"].to(self.cfg.device)\n\n if self.cfg.task == Task.REGRESSION:\n x_support, y_support, x_query, y_query = (\n x_support.float(),\n y_support.float(),\n x_query.float(),\n y_query.float(),\n )\n\n y_hat = self.model(x_support, y_support, x_query)\n\n if self.cfg.task == Task.REGRESSION:\n y_hat = y_hat[0, :, 0]\n else:\n y_hat = y_hat[0, :, : self.n_classes]\n\n y_query = y_query[0, :]\n\n loss = self.loss(y_hat, y_query)\n loss.backward()\n self.optimizer.step()\n\n if return_metrics:\n output_tracker.update(einops.asnumpy(y_query), einops.asnumpy(y_hat))\n\n if return_metrics:\n y_true, y_pred = output_tracker.get()\n y_pred = self.y_transformer.inverse_transform(y_pred)\n prediction_metrics = PredictionMetrics.from_prediction(\n y_pred, y_true, self.cfg.task, metric=self.stopping_metric\n )\n return prediction_metrics\n else:\n return None\n\n def test_epoch(self, dataloader: torch.utils.data.DataLoader, y_test: np.ndarray) -> PredictionMetrics:\n # FIXME: test_epoch might be better if it uses the for loop logic with n_ensembles\n y_hat = self.predict_epoch(dataloader)\n y_hat_finish = self.y_transformer.inverse_transform(y_hat)\n\n prediction_metrics = PredictionMetrics.from_prediction(\n y_hat_finish, y_test, self.cfg.task, metric=self.stopping_metric\n )\n return prediction_metrics\n\n def _get_memory_size(self) -> int:\n import gc\n import pickle\n import sys\n\n gc.collect() # Try to avoid OOM error\n return sys.getsizeof(pickle.dumps(self, protocol=4))\n\n def predict(self, x_support: np.ndarray, y_support: np.ndarray, x_query: np.ndarray) -> np.ndarray:\n \"\"\"\n Give a prediction for the query set.\n \"\"\"\n\n x_support = self.preprocessor.transform(x_support)\n x_query = self.preprocessor.transform(x_query)\n\n dataset = DatasetFinetune(\n self.cfg,\n x_support=x_support,\n y_support=self.y_transformer.transform(y_support),\n x_query=x_query,\n y_query=None,\n max_samples_support=self.cfg.hyperparams[\"max_samples_support\"],\n max_samples_query=self.cfg.hyperparams[\"max_samples_query\"],\n )\n\n loader = self.make_loader(dataset, training=False)\n\n y_hat_ensembles = []\n\n for _ in range(self.cfg.hyperparams[\"n_ensembles\"]):\n y_hat = self.predict_epoch(loader)\n y_hat_ensembles.append(y_hat)\n\n y_hat_ensembled = sum(y_hat_ensembles) / self.cfg.hyperparams[\"n_ensembles\"]\n y_hat_finish = self.y_transformer.inverse_transform(y_hat_ensembled)\n\n return y_hat_finish\n\n def predict_epoch(self, dataloader: torch.utils.data.DataLoader) -> np.ndarray:\n \"\"\"\n Returns the predictions for the data in the dataloader.\n The predictions are in the original state as they come from the model, i.e. not transformed.\n \"\"\"\n\n self.model.eval()\n\n y_hat_list = []\n\n with torch.no_grad():\n for batch in dataloader:\n x_support = batch[\"x_support\"].to(self.cfg.device)\n y_support = batch[\"y_support\"].to(self.cfg.device)\n x_query = batch[\"x_query\"].to(self.cfg.device)\n\n if self.cfg.task == Task.REGRESSION:\n y_support = y_support.float()\n\n y_hat = self.model(x_support, y_support, x_query)\n\n if self.cfg.task == Task.REGRESSION:\n y_hat = y_hat[0, :, 0]\n else:\n y_hat = y_hat[0, :, : self.n_classes]\n\n y_hat_list.append(einops.asnumpy(y_hat))\n\n y_hat = np.concatenate(y_hat_list, axis=0)\n return y_hat\n\n def make_loader(self, dataset, training):\n return torch.utils.data.DataLoader(\n dataset,\n batch_size=1,\n shuffle=training,\n pin_memory=True,\n num_workers=0,\n drop_last=False,\n collate_fn=CollatorWithPadding(pad_to_n_support_samples=None),\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/core/y_transformer.py",
"content": "import numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import FunctionTransformer\nfrom sklearn.preprocessing import QuantileTransformer\n\nfrom .enums import Task\n\n\ndef create_y_transformer(y_train: np.ndarray, task: Task) -> TransformerMixin:\n # The y_transformer transforms the target variable to a normal distribution\n # This should be used for the y variable when training a regression model,\n # but when testing the model, we want to inverse transform the predictions\n\n if task == Task.REGRESSION:\n y_transformer = QuantileTransformer1D(output_distribution=\"uniform\")\n y_transformer.fit(y_train)\n return y_transformer\n else:\n # Identity\n return FunctionTransformer()\n\n\nclass QuantileTransformer1D(BaseEstimator, TransformerMixin):\n def __init__(self, output_distribution=\"normal\") -> None:\n self.quantile_transformer = QuantileTransformer(output_distribution=output_distribution)\n\n def fit(self, x: np.ndarray):\n self.quantile_transformer.fit(x[:, None])\n return self\n\n def transform(self, x: np.ndarray):\n return self.quantile_transformer.transform(x[:, None])[:, 0]\n\n def inverse_transform(self, x: np.ndarray):\n return self.quantile_transformer.inverse_transform(x[:, None])[:, 0]\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/data/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/data/dataset_finetune.py",
"content": "from typing import Optional\n\nimport numpy as np\nimport torch\n\nfrom ..config.config_run import ConfigRun\nfrom ..core.dataset_split import make_dataset_split\nfrom ..core.enums import Task\n\n\nclass DatasetFinetune(torch.utils.data.Dataset):\n \"\"\"\n The main goal of this class is to generate a dataset for fine-tuning.\n The input data are the full (x_support, y_support, x_query, y_query)\n But these arrays are too large to be pushed through the model at once.\n So here we split query the data into chunks if the query data is too large.\n If the support data is too large, we randomly sample from it.\n Furthermore, we transition from numpy to tensors.\n \"\"\"\n\n def __init__(\n self,\n cfg: ConfigRun,\n x_support: np.ndarray,\n y_support: np.ndarray,\n x_query: np.ndarray,\n y_query: Optional[np.ndarray],\n max_samples_support: int,\n max_samples_query: int,\n ):\n \"\"\"\n :param: max_features: number of features the tab pfn model has been trained on\n \"\"\"\n\n self.cfg = cfg\n\n self.x_support = x_support\n self.y_support = y_support\n self.x_query = x_query\n self.y_query = y_query\n\n if self.y_query is None:\n self.y_query = np.zeros((self.x_query.shape[0],)) - 1\n\n self.max_samples_support = max_samples_support\n self.max_samples_query = max_samples_query\n\n self.x_queries = self.split_in_chunks(self.x_query, max_samples_query)\n self.y_queries = self.split_in_chunks(self.y_query, max_samples_query)\n\n self.n_samples_support = self.x_support.shape[0]\n\n # We push the whole training data through the model, unless it is too large\n self.support_size = min(self.max_samples_support, self.n_samples_support)\n\n def __len__(self):\n return len(self.x_queries)\n\n def __getitem__(self, idx):\n support_indices = np.random.choice(self.n_samples_support, size=self.support_size, replace=False)\n\n x_support = self.x_support[support_indices]\n y_support = self.y_support[support_indices]\n\n x_support_tensor = torch.as_tensor(x_support)\n y_support_tensor = torch.as_tensor(y_support)\n x_query_tensor = torch.as_tensor(self.x_queries[idx])\n y_query_tensor = torch.as_tensor(self.y_queries[idx])\n\n return {\n \"x_support\": x_support_tensor,\n \"y_support\": y_support_tensor,\n \"x_query\": x_query_tensor,\n \"y_query\": y_query_tensor,\n }\n\n def split_in_chunks(self, x: np.ndarray, batch_size: int) -> list[np.ndarray]:\n \"\"\"\n Splits the data into chunks of size batch_size\n \"\"\"\n\n n_chunks = int(np.ceil(x.shape[0] / batch_size))\n x_chunks = []\n\n for i in range(n_chunks):\n x_chunks.append(x[i * batch_size : (i + 1) * batch_size])\n\n return x_chunks\n\n\ndef DatasetFinetuneGenerator(\n cfg: ConfigRun,\n x: np.ndarray,\n y: np.ndarray,\n task: Task,\n max_samples_support: int,\n max_samples_query: int,\n split: float,\n random_state=None,\n):\n \"\"\"\n The dataset fine-tune generator is a generator that yields a dataset for fine-tuning.\n The idea is to split the training dataset into a support and query set.\n Every single iteration, the generator yields a different support and query set split.\n The dataset made always has exactly one batch.\n \"\"\"\n\n while True:\n x_support, x_query, y_support, y_query = make_dataset_split(x=x, y=y, task=task, random_state=random_state)\n n_samples_support = x_support.shape[0]\n n_samples_query = x_query.shape[0]\n\n support_size = min(max_samples_support, n_samples_support)\n query_size = min(max_samples_query, n_samples_query)\n\n dataset_finetune = DatasetFinetune(\n cfg=cfg,\n x_support=x_support[:support_size],\n y_support=y_support[:support_size],\n x_query=x_query[:query_size],\n y_query=y_query[:query_size],\n max_samples_support=max_samples_support,\n max_samples_query=max_samples_query,\n )\n\n yield dataset_finetune\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/data/preprocessor.py",
"content": "import logging\n\nimport numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_selection import SelectKBest, f_classif, f_regression\nfrom sklearn.preprocessing import QuantileTransformer\n\nfrom ..core.enums import Task\n\nlogger = logging.getLogger(__name__)\n\n\nclass Preprocessor(TransformerMixin, BaseEstimator):\n \"\"\"\n This class is used to preprocess the data before it is pushed through the model.\n The preprocessor assures that the data has the right shape and is normalized,\n This way the model always gets the same input distribution,\n no matter whether the input data is synthetic or real.\n\n \"\"\"\n\n def __init__(\n self,\n max_features: int,\n use_quantile_transformer: bool,\n use_feature_count_scaling: bool,\n task: Task,\n ):\n self.max_features = max_features\n self.use_quantile_transformer = use_quantile_transformer\n self.use_feature_count_scaling = use_feature_count_scaling\n self.task = task\n\n def fit(self, X: np.ndarray, y: np.ndarray):\n self.compute_pre_nan_mean(X)\n X = self.impute_nan_features_with_mean(X)\n\n self.determine_which_features_are_singular(X)\n X = self.cutoff_singular_features(X, self.singular_features)\n\n self.determine_which_features_to_select(X, y)\n X = self.select_features(X)\n\n if self.use_quantile_transformer:\n n_obs, n_features = X.shape\n n_quantiles = min(n_obs, 1000)\n self.quantile_transformer = QuantileTransformer(n_quantiles=n_quantiles, output_distribution=\"normal\")\n X = self.quantile_transformer.fit_transform(X)\n\n self.mean, self.std = self.calc_mean_std(X)\n X = self.normalize_by_mean_std(X, self.mean, self.std)\n\n assert np.isnan(X).sum() == 0, \"There are NaNs in the data after preprocessing\"\n\n return self\n\n def transform(self, X: np.ndarray):\n X = self.cutoff_singular_features(X, self.singular_features)\n X = self.impute_nan_features_with_mean(X)\n X = self.select_features(X)\n\n if self.use_quantile_transformer:\n X = self.quantile_transformer.transform(X)\n\n X = self.normalize_by_mean_std(X, self.mean, self.std)\n\n if self.use_feature_count_scaling:\n X = self.normalize_by_feature_count(X, self.max_features)\n\n X = self.extend_feature_dim_to_max_features(X, self.max_features)\n\n assert np.isnan(X).sum() == 0, \"There are NaNs in the data after preprocessing\"\n\n return X\n\n def determine_which_features_are_singular(self, x: np.ndarray) -> None:\n self.singular_features = np.array([len(np.unique(x_col)) for x_col in x.T]) == 1\n\n def determine_which_features_to_select(self, x: np.ndarray, y: np.ndarray) -> None:\n if x.shape[1] > self.max_features:\n logger.info(\n f\"A maximum of {self.max_features} features are allowed, but the dataset has {x.shape[1]} features. A subset of {self.max_features} are selected using SelectKBest\"\n )\n\n if self.task == Task.CLASSIFICATION:\n self.select_k_best = SelectKBest(k=self.max_features, score_func=f_classif)\n else: # Task.REGRESSION\n self.select_k_best = SelectKBest(k=self.max_features, score_func=f_regression)\n self.select_k_best.fit(x, y)\n\n def compute_pre_nan_mean(self, x: np.ndarray) -> None:\n \"\"\"\n Computes the mean of the data before the NaNs are imputed\n \"\"\"\n self.pre_nan_mean = np.nanmean(x, axis=0)\n\n def impute_nan_features_with_mean(self, x: np.ndarray) -> np.ndarray:\n inds = np.where(np.isnan(x))\n x[inds] = np.take(self.pre_nan_mean, inds[1])\n return x\n\n def select_features(self, x: np.ndarray) -> np.ndarray:\n if x.shape[1] > self.max_features:\n x = self.select_k_best.transform(x)\n\n return x\n\n def cutoff_singular_features(self, x: np.ndarray, singular_features: np.ndarray) -> np.ndarray:\n if singular_features.any():\n x = x[:, ~singular_features]\n\n return x\n\n def calc_mean_std(self, x: np.ndarray) -> tuple[np.ndarray, np.ndarray]:\n \"\"\"\n Calculates the mean and std of the training data\n \"\"\"\n mean = x.mean(axis=0)\n std = x.std(axis=0)\n return mean, std\n\n def normalize_by_mean_std(self, x: np.ndarray, mean: np.ndarray, std: np.ndarray) -> np.ndarray:\n \"\"\"\n Normalizes the data by the mean and std\n \"\"\"\n\n x = (x - mean) / std\n return x\n\n def normalize_by_feature_count(self, x: np.ndarray, max_features) -> np.ndarray:\n \"\"\"\n An interesting way of normalization by the tabPFN paper\n \"\"\"\n\n x = x * max_features / x.shape[1]\n return x\n\n def extend_feature_dim_to_max_features(self, x: np.ndarray, max_features) -> np.ndarray:\n \"\"\"\n Increases the number of features to the number of features the model has been trained on\n \"\"\"\n added_zeros = np.zeros((x.shape[0], max_features - x.shape[1]), dtype=np.float32)\n x = np.concatenate([x, added_zeros], axis=1)\n return x\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/models/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/models/foundation/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/models/foundation/embedding.py",
"content": "import einops\nimport torch\nimport torch.nn as nn\n\n\nclass FoundationEmbeddingX(torch.nn.Module):\n def __init__(\n self,\n dim: int,\n n_features: int,\n ) -> None:\n super().__init__()\n\n self.dim = dim\n self.n_features = n_features\n\n self.x_embedding = nn.Linear(n_features, dim)\n\n def forward(self, x_support: torch.Tensor, x_query__: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = x_support.shape[0]\n n_obs_support = x_support.shape[1]\n n_obs_query__ = x_query__.shape[1]\n\n x_support = self.x_embedding(x_support)\n x_query__ = self.x_embedding(x_query__)\n\n return x_support, x_query__\n\n\nclass FoundationEmbeddingYFloat(torch.nn.Module):\n def __init__(\n self,\n dim: int,\n ) -> None:\n super().__init__()\n\n self.dim = dim\n\n self.y_embedding = nn.Linear(1, dim)\n\n def forward(self, y_support: torch.Tensor, n_obs_query: int) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = y_support.shape[0]\n\n y_support = y_support.type(torch.float32)\n y_support = einops.rearrange(y_support, \"b n -> b n 1\")\n\n y_support = self.y_embedding(y_support)\n y_query = torch.zeros((batch_size, n_obs_query, self.dim), device=y_support.device, dtype=torch.float32)\n\n return y_support, y_query\n\n\nclass FoundationEmbeddingYInteger(torch.nn.Module):\n def __init__(\n self,\n n_classes: int,\n dim: int,\n ) -> None:\n super().__init__()\n\n self.n_classes = n_classes\n self.dim = dim\n\n self.y_embedding = nn.Embedding(n_classes, dim)\n self.y_padding = nn.Embedding(1, dim, padding_idx=0) # padding is modeled as a separate class\n self.y_mask = nn.Embedding(1, dim) # masking is also modeled as a separate class\n\n def forward(self, y_support: torch.Tensor, n_obs_query: int) -> tuple[torch.Tensor, torch.Tensor]:\n batch_size = y_support.shape[0]\n n_obs_support = y_support.shape[1]\n\n # padding is given as -100. We turn the padding into a 'separate class'\n # because padding is ignored in the attention, this should make no difference whatsoever\n\n y_support_pad = y_support == -100\n\n y_sup = torch.zeros((batch_size, n_obs_support, self.dim), device=y_support.device, dtype=torch.float32)\n y_sup[y_support_pad] = self.y_padding(y_support[y_support_pad] + 100)\n y_sup[~y_support_pad] = self.y_embedding(y_support[~y_support_pad])\n\n y_query = torch.zeros((batch_size, n_obs_query), device=y_support.device, dtype=torch.int64)\n y_query = self.y_mask(y_query)\n\n return y_sup, y_query\n\n\nclass FoundationObservationEmbedding(torch.nn.Module):\n def __init__(self, dim: int) -> None:\n super().__init__()\n\n self.dim = dim\n self.max_dim = 2**16\n self.embedding = nn.Embedding(self.max_dim, dim)\n\n def forward(self, batch_size: int, n_obs: int) -> torch.Tensor:\n assert n_obs <= self.max_dim, f\"Number of observations is too large. Max is {self.max_dim}, got {n_obs}\"\n\n # Take a random embedding from the pool of embeddings\n weights = torch.ones((batch_size, self.max_dim), dtype=torch.float32, device=self.embedding.weight.device)\n indices = torch.multinomial(weights, num_samples=n_obs, replacement=False)\n x = self.embedding(indices)\n\n return x\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/models/foundation/foundation_transformer.py",
"content": "import einops\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom huggingface_hub import PyTorchModelHubMixin\n\nfrom ...core.enums import Task\nfrom .embedding import FoundationEmbeddingX, FoundationEmbeddingYFloat, FoundationEmbeddingYInteger\n\n\nclass FoundationTransformer(nn.Module, PyTorchModelHubMixin):\n def __init__(\n self,\n n_features: int,\n n_classes: int,\n dim: int,\n n_layers: int,\n n_heads: int,\n attn_dropout: float,\n y_as_float_embedding: bool,\n task: str = Task.CLASSIFICATION,\n ) -> None:\n super().__init__()\n\n self.n_features = n_features\n self.n_classes = n_classes\n self.dim = dim\n self.n_layers = n_layers\n self.n_heads = n_heads\n self.attn_dropout = attn_dropout\n self.y_as_float_embedding = y_as_float_embedding\n self.task = task\n\n self.x_embedding = FoundationEmbeddingX(dim, n_features)\n\n if self.y_as_float_embedding:\n self.y_embedding = FoundationEmbeddingYFloat(dim)\n else:\n self.y_embedding = FoundationEmbeddingYInteger(n_classes, dim)\n\n self.layers = nn.ModuleList([])\n\n for _ in range(n_layers):\n att = MultiheadAttention(dim, n_heads)\n\n self.layers.append(\n nn.ModuleDict(\n {\n \"layer_norm1\": nn.LayerNorm(dim),\n \"attention\": att,\n \"layer_norm2\": nn.LayerNorm(dim),\n \"linear1\": nn.Linear(dim, dim * 4),\n \"linear2\": nn.Linear(dim * 4, dim),\n }\n )\n )\n\n self.final_layer1 = nn.Linear(dim, dim * 4)\n if self.task == Task.CLASSIFICATION:\n self.final_layer2 = nn.Linear(dim * 4, n_classes)\n elif self.task == Task.REGRESSION:\n self.final_layer2 = nn.Linear(dim * 4, 1)\n self.init_weights()\n\n def init_weights(self):\n for module_dict in self.layers:\n # module_dict['attention'].init_weights()\n nn.init.zeros_(module_dict[\"linear2\"].weight)\n nn.init.zeros_(module_dict[\"linear2\"].bias)\n\n def forward(self, x_support: torch.Tensor, y_support: torch.Tensor, x_query: torch.Tensor):\n \"\"\"\n x_support is (batch_size, n_observations_support, n_features)\n y_support is (batch_size, n_observations_support)\n\n x_query is (batch_size, n_observations_query, n_features)\n\n returns:\n\n y_query is (batch_size, n_observations_query, n_classes)\n\n syntax:\n b = batch size\n n = number of observations\n d = dimension of embedding\n c = number of classes\n \"\"\"\n\n x_query__ = x_query\n\n batch_size = x_support.shape[0]\n n_obs_support = x_support.shape[1]\n n_obs_query__ = x_query__.shape[1]\n\n padding_mask = torch.zeros((batch_size, n_obs_support), dtype=torch.bool, device=x_support.device)\n padding_mask[y_support == -100] = True\n\n x_support, x_query__ = self.x_embedding(x_support, x_query__)\n y_support, y_query__ = self.y_embedding(y_support, n_obs_query__)\n\n support = x_support + y_support\n query__ = x_query__ + y_query__\n\n x, pack = einops.pack((support, query__), \"b * d\")\n\n for module_dict in self.layers:\n x_residual = x\n support, query__ = einops.unpack(x, pack, \"b * d\")\n att_support = module_dict[\"attention\"](support, support, support, key_padding_mask=padding_mask)\n att_query__ = module_dict[\"attention\"](query__, support, support, key_padding_mask=padding_mask)\n x = einops.pack((att_support, att_query__), \"b * d\")[0]\n x = x_residual + x\n x = module_dict[\"layer_norm1\"](x)\n x_residual = x\n x = module_dict[\"linear1\"](x)\n x = torch.nn.functional.gelu(x)\n x = module_dict[\"linear2\"](x)\n x = x_residual + x\n x = module_dict[\"layer_norm2\"](x)\n\n x = self.final_layer1(x)\n x = F.gelu(x)\n x = self.final_layer2(x)\n\n support, query__ = einops.unpack(x, pack, \"b * c\")\n\n return query__\n\n\nclass MultiheadAttention(torch.nn.Module):\n def __init__(self, dim: int, n_heads: int) -> None:\n super().__init__()\n\n self.use_flash_attention = False\n self.dim = dim\n self.n_heads = n_heads\n\n self.att = nn.MultiheadAttention(dim, n_heads, dropout=0.0, batch_first=True)\n\n def init_weights(self):\n pass\n # nn.init.zeros_(self.att.out_proj.weight)\n # nn.init.zeros_(self.att.out_proj.bias)\n\n def forward(\n self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, key_padding_mask: torch.Tensor\n ) -> torch.Tensor:\n \"\"\"\n b = batch size\n n = number of samples (dataset size)\n h = heads\n d = dimension of embedding\n\n query is (b, n, d)\n key is (b, n, d)\n value is (b, n, d)\n\n attention weights will be (b, h, n, n)\n output will be (b, n, d)\n \"\"\"\n\n output = self.att(query, key, value, key_padding_mask=key_padding_mask)[0]\n return output\n\n\nclass SwiGLU(nn.Module):\n def forward(self, x):\n x, gate = x.chunk(2, dim=-1)\n return F.silu(gate) * x\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/results/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/results/prediction_metrics.py",
"content": "from dataclasses import dataclass\n\nimport numpy as np\nimport scipy\n\nfrom autogluon.core.metrics import Scorer\n\nfrom ..core.enums import Task\n\n\n@dataclass\nclass PredictionMetrics:\n task: Task\n loss: float\n score: float\n metrics: dict[str, float]\n\n @classmethod\n def from_prediction(cls, y_pred: np.ndarray, y_true: np.ndarray, task: Task, metric: Scorer):\n loss, score, metrics = compute_metrics(y_pred, y_true, task, metric=metric)\n\n return PredictionMetrics(task=task, loss=loss, score=score, metrics=metrics)\n\n\ndef compute_metrics(y_pred: np.ndarray, y_true: np.ndarray, task: Task, metric: Scorer) -> tuple[float, float, dict]:\n if task == Task.CLASSIFICATION:\n return compute_classification_metrics(y_pred, y_true, metric=metric)\n else:\n return compute_regression_metrics(y_pred, y_true, metric=metric)\n\n\ndef compute_classification_metrics(\n y_pred: np.ndarray, y_true: np.ndarray, metric: Scorer\n) -> tuple[float, float, dict]:\n # predictions are assumed to be log-probabilities\n\n if metric.needs_pred or metric.needs_class:\n y_pred_class = np.argmax(y_pred, axis=1)\n metric_score = metric(y_true, y_pred_class)\n else:\n y_pred_proba = scipy.special.softmax(y_pred, axis=1)\n metric_score = metric(y_true, y_pred_proba)\n\n metric_error = metric.convert_score_to_error(metric_score)\n\n return metric_error, metric_score, {metric.name: metric_score}\n\n\ndef compute_regression_metrics(y_pred: np.ndarray, y_true: np.ndarray, metric: Scorer) -> tuple[float, float, dict]:\n metric_score = metric(y_true, y_pred)\n metric_error = metric.convert_score_to_error(metric_score)\n\n return metric_error, metric_score, {metric.name: metric_score}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/tabpfnmix_classifier.py",
"content": "from __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport torch\nfrom sklearn.base import BaseEstimator, ClassifierMixin\n\nfrom .core.dataset_split import make_stratified_dataset_split\nfrom .core.trainer_finetune import TrainerFinetune\nfrom .models.foundation.foundation_transformer import FoundationTransformer\n\n\n# FIXME: Delete this class, it isn't needed\n# TODO: test_epoch might be better if it uses the for loop logic with n_ensembles during finetuning to better estimate val score\n# TODO: To mitigate val overfitting, can fit multiple random seeds at same time and pick same epoch for all of them, track average performance on epoch.\n# TODO: Test shuffling the data and see if it makes TabPFNv2 worse, same with TabForestPFN\nclass TabPFNMixClassifier(BaseEstimator, ClassifierMixin):\n def __init__(\n self,\n n_classes,\n cfg,\n split_val,\n model_path: str = None,\n weights_path: str | Path = None,\n stopping_metric=None,\n use_best_epoch: bool = True,\n ):\n if weights_path is not None:\n weights_path = str(Path(weights_path))\n\n if model_path is not None:\n model = FoundationTransformer.from_pretrained(model_path)\n assert model.task == cfg.task, (\n f\"The pretrained model '{model_path}' is for task {model.task}, but the problem type is for task {cfg.task}...\"\n )\n else:\n model = FoundationTransformer(\n n_features=cfg.hyperparams[\"n_features\"],\n n_classes=cfg.hyperparams[\"n_classes\"],\n dim=cfg.hyperparams[\"dim\"],\n n_layers=cfg.hyperparams[\"n_layers\"],\n n_heads=cfg.hyperparams[\"n_heads\"],\n attn_dropout=cfg.hyperparams[\"attn_dropout\"],\n y_as_float_embedding=cfg.hyperparams[\"y_as_float_embedding\"],\n task=cfg.task,\n )\n if weights_path is not None:\n model.load_state_dict(torch.load(weights_path, weights_only=True)) # nosec B614\n\n self.split_val = split_val\n self.trainer = TrainerFinetune(\n cfg, model, n_classes=n_classes, stopping_metric=stopping_metric, use_best_epoch=use_best_epoch\n )\n super().__init__()\n\n def fit(\n self,\n X: np.ndarray,\n y: np.ndarray,\n X_val: np.ndarray = None,\n y_val: np.ndarray = None,\n time_limit: float = None,\n ):\n # FIXME: Should X and y be preprocessed for inference efficiency? Yes.\n self.X_ = X # FIXME: Optimize storage of X and y? Is this redundant? Is X and y saving done multiple times during pickle?\n self.y_ = y\n\n if X_val is not None and y_val is not None:\n X_train, X_valid, y_train, y_valid = X, X_val, y, y_val\n elif self.split_val:\n X_train, X_valid, y_train, y_valid = make_stratified_dataset_split(X, y) # FIXME: Add random seed\n else:\n X_train, X_valid, y_train, y_valid = X, None, y, None\n self.trainer.train(x_train=X_train, y_train=y_train, x_val=X_valid, y_val=y_valid, time_limit=time_limit)\n\n return self\n\n # FIXME: Avoid preprocessing self.X_ and self.y_ each predict call\n def predict(self, X):\n logits = self.trainer.predict(self.X_, self.y_, X)\n return logits.argmax(axis=1)\n\n # FIXME: Avoid preprocessing self.X_ and self.y_ each predict_proba call\n def predict_proba(self, X):\n logits = self.trainer.predict(self.X_, self.y_, X)\n return np.exp(logits) / np.exp(logits).sum(axis=1)[:, None]\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/_internal/tabpfnmix_regressor.py",
"content": "from __future__ import annotations\n\nfrom pathlib import Path\n\nimport numpy as np\nimport torch\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nfrom .core.dataset_split import make_stratified_dataset_split\nfrom .core.trainer_finetune import TrainerFinetune\nfrom .models.foundation.foundation_transformer import FoundationTransformer\n\n\n# FIXME: Delete this class, it isn't needed\n# TODO: test_epoch might be better if it uses the for loop logic with n_ensembles during finetuning to better estimate val score\n# TODO: To mitigate val overfitting, can fit multiple random seeds at same time and pick same epoch for all of them, track average performance on epoch.\n# TODO: Test shuffling the data and see if it makes TabPFNv2 worse, same with TabForestPFN\nclass TabPFNMixRegressor(BaseEstimator, RegressorMixin):\n def __init__(\n self,\n n_classes,\n cfg,\n split_val,\n model_path: str = None,\n weights_path: str | Path = None,\n stopping_metric=None,\n use_best_epoch: bool = True,\n ):\n self.cfg = cfg\n\n if weights_path is not None:\n weights_path = str(Path(weights_path))\n\n if model_path is not None:\n model = FoundationTransformer.from_pretrained(model_path)\n assert model.task == cfg.task, (\n f\"The pretrained model '{model_path}' is for task {model.task}, but the problem type is for task {cfg.task}...\"\n )\n else:\n model = FoundationTransformer(\n n_features=cfg.hyperparams[\"n_features\"],\n n_classes=cfg.hyperparams[\"n_classes\"],\n dim=cfg.hyperparams[\"dim\"],\n n_layers=cfg.hyperparams[\"n_layers\"],\n n_heads=cfg.hyperparams[\"n_heads\"],\n attn_dropout=cfg.hyperparams[\"attn_dropout\"],\n y_as_float_embedding=cfg.hyperparams[\"y_as_float_embedding\"],\n task=cfg.task,\n )\n if weights_path is not None:\n model.load_state_dict(torch.load(weights_path, weights_only=True)) # nosec B614\n\n self.split_val = split_val\n self.trainer = TrainerFinetune(\n cfg, model, n_classes=n_classes, stopping_metric=stopping_metric, use_best_epoch=use_best_epoch\n )\n super().__init__()\n\n def fit(\n self,\n X: np.ndarray,\n y: np.ndarray,\n X_val: np.ndarray = None,\n y_val: np.ndarray = None,\n time_limit: float = None,\n ):\n # FIXME: Should X and y be preprocessed for inference efficiency? Yes.\n self.X_ = X # FIXME: Optimize storage of X and y? Is this redundant? Is X and y saving done multiple times during pickle?\n self.y_ = y\n\n if X_val is not None and y_val is not None:\n X_train, X_valid, y_train, y_valid = X, X_val, y, y_val\n elif self.split_val:\n X_train, X_valid, y_train, y_valid = make_stratified_dataset_split(X, y) # FIXME: Add random seed\n else:\n X_train, X_valid, y_train, y_valid = X, None, y, None\n self.trainer.train(x_train=X_train, y_train=y_train, x_val=X_valid, y_val=y_valid, time_limit=time_limit)\n\n return self\n\n # FIXME: Avoid preprocessing self.X_ and self.y_ each predict call\n def predict(self, X):\n logits = self.trainer.predict(self.X_, self.y_, X)\n return logits\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnmix/tabpfnmix_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport os\nimport time\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_torch\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils import generate_train_test_split\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass TabPFNMixModel(AbstractModel):\n \"\"\"\n [Experimental model] Can be changed/removed without warning in future releases.\n\n TabPFNMix is based off of the TabPFN and TabForestPFN models.\n\n We recommend using Mitra instead, as it is an improved version of TabPFNMix.\n\n It is a tabular transformer model pre-trained on purely synthetic data.\n\n It currently has several limitations:\n 1. Does not support regression\n 2. Does not support >10 classes\n 3. Does not support GPU\n\n For more information, refer to the `./_internals/README.md` file.\n\n .. versionadded:: 1.2.0\n \"\"\"\n\n ag_key = \"TABPFNMIX\"\n ag_name = \"TabPFNMix\"\n ag_priority = 45\n seed_name = \"random_state\"\n\n weights_file_name = \"model.pt\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._feature_generator = None\n self._weights_saved = False\n\n def _get_model_type(self):\n from ._internal.tabpfnmix_classifier import TabPFNMixClassifier\n from ._internal.tabpfnmix_regressor import TabPFNMixRegressor\n\n if self.problem_type in [\"binary\", \"multiclass\"]:\n model_cls = TabPFNMixClassifier\n elif self.problem_type in [\"regression\"]:\n model_cls = TabPFNMixRegressor\n else:\n raise AssertionError(f\"TabPFN does not support problem_type='{self.problem_type}'\")\n return model_cls\n\n def _set_default_params(self):\n \"\"\"Specifies hyperparameter values to use by default\"\"\"\n default_params = {\n # most important hyperparameters. Only set `n_estimators>1` if `max_epochs>1`, else there will be no benefit.\n # model_path, # most important, defines huggingface model path\n \"model_path_classifier\": \"autogluon/tabpfn-mix-1.0-classifier\", # if specified, overrides model_path for classification problems, set to None to ignore.\n \"model_path_regressor\": \"autogluon/tabpfn-mix-1.0-regressor\", # if specified, overrides model_path for regression problems, set to None to ignore.\n # weights_path, # most important, defines weights location (overrides huggingface weights if specified)\n # weights_path_classifier, # if specified, overrides weights_path for classification problems\n # weights_path_regressor, # if specified, overrides weights_path for regression problems\n \"n_ensembles\": 1, # FIXME: RENAME: n_estimators\n \"max_epochs\": 0, # fine-tuning epochs. Will do pure in-context learning if 0.\n # next most important hyperparameters\n \"lr\": 1.0e-05,\n \"max_samples_query\": 1024, # larger = slower but better quality on datasets with at least this many validation samples\n \"max_samples_support\": 8196, # larger = slower but better quality on datasets with at least this many training samples\n # other hyperparameters\n \"early_stopping_patience\": 40, # TODO: Figure out optimal value\n \"linear_attention\": True,\n \"lr_scheduler\": False,\n \"lr_scheduler_patience\": 30,\n \"optimizer\": \"adamw\",\n \"use_feature_count_scaling\": True,\n \"use_quantile_transformer\": True,\n \"weight_decay\": 0,\n # architecture hyperparameters, recommended to keep as default unless using a custom pre-trained backbone\n \"n_classes\": 10,\n \"n_features\": 100,\n \"n_heads\": 4,\n \"n_layers\": 12,\n \"attn_dropout\": 0.0,\n \"dim\": 512,\n \"y_as_float_embedding\": True,\n # utility parameters, recommended to keep as default\n \"split_val\": False,\n \"use_best_epoch\": True,\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n # FIXME: Make X_val, y_val = None work well, currently it uses X and y as validation, should instead skip validation entirely\n # FIXME: Use `params_trained` for refit optimal epochs\n # FIXME: Handle model weights download\n # FIXME: GPU support?\n # FIXME: Save model weights to file instead of pickling?\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame = None,\n y_val: pd.Series = None,\n time_limit: float = None,\n num_cpus: int = 1,\n num_gpus: float = 0,\n **kwargs,\n ):\n time_start = time.time()\n try_import_torch()\n import torch\n\n from ._internal.config.config_run import ConfigRun\n\n ag_params = self._get_ag_params()\n max_classes = ag_params.get(\"max_classes\")\n if max_classes is not None and self.num_classes is not None and self.num_classes > max_classes:\n # TODO: Move to earlier stage when problem_type is checked\n raise AssertionError(\n f\"Max allowed classes for the model is {max_classes}, but found {self.num_classes} classes.\"\n )\n\n params = self._get_model_params()\n random_state = params.pop(self.seed_name, self.default_random_seed)\n sample_rows = ag_params.get(\"sample_rows\", None)\n sample_rows_val = ag_params.get(\"sample_rows_val\", None)\n max_rows = ag_params.get(\"max_rows\", None)\n\n # TODO: Make max_rows generic\n if max_rows is not None and isinstance(max_rows, (int, float)) and len(X) > max_rows:\n raise AssertionError(\n f\"Skipping model due to X having more rows than `ag.max_rows={max_rows}` (len(X)={len(X)})\"\n )\n\n # TODO: Make sample_rows generic\n if sample_rows is not None and isinstance(sample_rows, int) and len(X) > sample_rows:\n X, y = self._subsample_data(X=X, y=y, num_rows=sample_rows, random_state=random_state)\n\n # TODO: Make sample_rows generic\n if (\n X_val is not None\n and y_val is not None\n and sample_rows_val is not None\n and isinstance(sample_rows_val, int)\n and len(X_val) > sample_rows_val\n ):\n X_val, y_val = self._subsample_data(X=X_val, y=y_val, num_rows=sample_rows_val, random_state=random_state)\n\n from ._internal.core.enums import Task\n\n if self.problem_type in [REGRESSION, QUANTILE]:\n task = Task.REGRESSION\n n_classes = 0\n else:\n task = Task.CLASSIFICATION\n n_classes = self.num_classes\n\n if num_gpus > 0:\n device = \"cuda:0\"\n else:\n device = \"cpu\"\n\n model_path = None\n if task == Task.CLASSIFICATION:\n if \"model_path_classifier\" in params and params[\"model_path_classifier\"] is not None:\n model_path = params[\"model_path_classifier\"]\n elif task == Task.REGRESSION:\n if \"model_path_regressor\" in params and params[\"model_path_regressor\"] is not None:\n model_path = params[\"model_path_regressor\"]\n if model_path is None:\n model_path = params.get(\"model_path\", None)\n\n weights_path = None\n if task == Task.CLASSIFICATION:\n if \"weights_path_classifier\" in params and params[\"weights_path_classifier\"] is not None:\n weights_path = Path(params[\"weights_path_classifier\"])\n elif task == Task.REGRESSION:\n if \"weights_path_regressor\" in params and params[\"weights_path_regressor\"] is not None:\n weights_path = Path(params[\"weights_path_regressor\"])\n if weights_path is None:\n if \"weights_path\" in params and params[\"weights_path\"] is not None:\n weights_path = Path(params[\"weights_path\"])\n\n if weights_path is None and model_path is None:\n logger.log(15, \"\\tNo model_path or weights_path specified, fitting model from random initialization...\")\n elif weights_path is not None:\n logger.log(15, f'\\tLoading pre-trained weights from file... (weights_path=\"{weights_path}\")')\n\n cfg = ConfigRun(hyperparams=params, task=task, device=device, seed=random_state)\n\n if cfg.hyperparams[\"max_epochs\"] == 0 and cfg.hyperparams[\"n_ensembles\"] != 1:\n logger.log(\n 30,\n f\"WARNING: max_epochs should be > 0 if n_ensembles > 1, otherwise there will be zero quality benefit with slower inference. \"\n f\"(max_epochs={cfg.hyperparams['max_epochs']}, n_ensembles={cfg.hyperparams['n_ensembles']})\",\n )\n\n X = self.preprocess(X, y=y)\n y = y.values\n if X_val is not None:\n X_val = self.preprocess(X_val)\n y_val = y_val.values\n\n # FIXME: What if an exception occurs or timeout occurs after updating threads? Can we add logic to ensure torch num_threads are reset?\n need_to_reset_torch_threads = False\n torch_threads_og = None\n if num_cpus is not None and isinstance(num_cpus, (int, float)):\n torch_threads_og = torch.get_num_threads()\n if torch_threads_og != num_cpus:\n # reset torch threads back to original value after fit\n torch.set_num_threads(num_cpus)\n need_to_reset_torch_threads = True\n\n model_cls = self._get_model_type()\n\n if time_limit is not None:\n time_cur = time.time()\n time_left = time_limit - (time_cur - time_start)\n if time_left <= 0:\n raise TimeLimitExceeded(\n f\"No time remaining to fit model (time_limit={time_limit:.2f}s, time_left={time_left:.2f}s)\"\n )\n time_limit = time_left\n\n self.model = model_cls(\n cfg=cfg,\n n_classes=n_classes,\n split_val=params[\"split_val\"],\n model_path=model_path,\n weights_path=weights_path,\n stopping_metric=self.stopping_metric,\n use_best_epoch=params[\"use_best_epoch\"],\n )\n self.model.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n time_limit=time_limit,\n )\n\n # Ensure refit_full uses the same number of max_epochs as the original's best\n self.params_trained[\"max_epochs\"] = self.model.trainer.best_epoch\n self.params_trained[\"ag.max_rows\"] = None # This ensures we don't raise an exception during refit_full\n\n # reduce memory and disk usage by 3x\n self.model.trainer.minimize_for_inference()\n\n if need_to_reset_torch_threads:\n torch.set_num_threads(torch_threads_og)\n\n return self\n\n # TODO: Make this generic by creating a generic `preprocess_train` and putting this logic prior to `_preprocess`.\n def _subsample_data(\n self, X: pd.DataFrame, y: pd.Series, num_rows: int, random_state: int | None = 0\n ) -> (pd.DataFrame, pd.Series):\n num_rows_to_drop = len(X) - num_rows\n X, _, y, _ = generate_train_test_split(\n X=X,\n y=y,\n problem_type=self.problem_type,\n test_size=num_rows_to_drop,\n random_state=random_state,\n min_cls_count_train=1,\n )\n return X, y\n\n def _preprocess(self, X: pd.DataFrame, **kwargs) -> np.ndarray:\n \"\"\"\n Converts categorical to label encoded integers\n Keeps missing values, as TabPFN automatically handles missing values internally.\n \"\"\"\n X = super()._preprocess(X, **kwargs)\n if self._feature_generator is None:\n # FIXME: Check if this is needed, never actually tried removing it, copy pasted from TabPFNModel implementation in AG\n self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0)\n self._feature_generator.fit(X=X)\n if self._feature_generator.features_in:\n X = X.copy()\n X[self._feature_generator.features_in] = self._feature_generator.transform(X=X)\n X = X.values.astype(np.float64)\n return X\n\n # FIXME: Switch to `torch.save(_model_weights.state_dict(), PATH)`, need to reinitialize the model though...\n def save(self, path: str = None, verbose=True) -> str:\n _model_weights = None\n if self.model is not None:\n _model_weights = self.model.trainer.model\n self.model.trainer.model = None\n self._weights_saved = True\n path = super().save(path=path, verbose=verbose)\n if _model_weights is not None:\n import torch\n\n os.makedirs(self.path, exist_ok=True)\n torch.save(_model_weights, self.weights_path)\n self.model.trainer.model = _model_weights\n return path\n\n # FIXME: Switch to `weights_only=True`, need to reinitialize the model though...\n @classmethod\n def load(cls, path: str, reset_paths=False, verbose=True):\n model: TabPFNMixModel = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n\n if model._weights_saved:\n import torch\n\n model.model.trainer.model = torch.load(model.weights_path, weights_only=False) # nosec B614\n model._weights_saved = False\n return model\n\n @property\n def weights_path(self) -> str:\n return os.path.join(self.path, self.weights_file_name)\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_classes\": 10,\n }\n )\n return default_auxiliary_params\n\n def _get_maximum_resources(self) -> dict[str, int | float]:\n # torch model trains slower when utilizing virtual cores and this issue scale up when the number of cpu cores increases\n return {\"num_cpus\": ResourceManager.get_cpu_count(only_physical_cores=True)}\n\n def _get_default_resources(self) -> tuple[int, float]:\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n def get_minimum_ideal_resources(self) -> dict[str, int | float]:\n return {\"num_cpus\": 4}\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n # TODO: This is wildly inaccurate, find a better estimation\n # TODO: Fitting 8 in parallel causes many OOM errors with 32 GB of memory on relatively small datasets, so each model is using over 4 GB of memory\n # TODO: Fitting 4 in parallel still causes many OOM errors with 32 GB of memory on relatively small datasets, so each model is using over 8 GB of memory\n # The below logic returns a minimum of 8.8 GB, to avoid OOM errors\n data_mem_usage = 5 * get_approximate_df_mem_usage(X).sum() # rough estimate\n model_size = (\n 160 * 1e6\n ) # model weights are ~160 MB # TODO: Avoid hardcoding, we can derive from the model itself?\n model_mem_usage = (\n model_size * 5\n ) # Account for 1x copy being fit, 1x copy checkpointed, 2x for optimizer, and 1x for overhead\n model_fit_usage = model_size * 50 # TODO: This is a placeholder large value to try to avoid OOM errors\n mem_usage_estimate = data_mem_usage + model_mem_usage + model_fit_usage\n return mem_usage_estimate\n\n @classmethod\n def _class_tags(cls) -> dict:\n return {\n \"can_estimate_memory_usage_static\": True,\n }\n\n def _ag_params(self) -> set:\n return {\"max_classes\", \"max_rows\", \"sample_rows\", \"sample_rows_val\"}\n\n def _more_tags(self) -> dict:\n tags = {\"can_refit_full\": True}\n return tags\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnv2/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabpfnv2/tabpfnv2_5_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport os\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nimport numpy as np\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\nfrom autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel\n\nif TYPE_CHECKING:\n import pandas as pd\n\nlogger = logging.getLogger(__name__)\n\n_HAS_LOGGED_TABPFN_LICENSE: bool = False\n_HAS_LOGGED_TABPFN_NONCOMMERICAL: bool = False\n_HAS_LOGGED_TABPFN_CPU_WARNING: bool = False\n\n\nclass TabPFNModel(AbstractTorchModel):\n \"\"\"TabPFN-2.5 is a tabular foundation model that is developed and maintained by PriorLabs: https://priorlabs.ai/.\n\n This class is an abstract template for various TabPFN versions as subclasses.\n\n Paper: Accurate predictions on small data with a tabular foundation model\n Authors: Noah Hollmann, Samuel MÃŧller, Lennart Purucker, Arjun Krishnakumar, Max KÃļrfer, Shi Bin Hoo, Robin Tibor Schirrmeister & Frank Hutter\n Codebase: https://github.com/PriorLabs/TabPFN\n License: https://github.com/PriorLabs/TabPFN/blob/main/LICENSE\n\n .. versionadded:: 1.5.0\n \"\"\"\n\n ag_key = \"NOTSET\"\n ag_name = \"NOTSET\"\n ag_priority = 40\n seed_name = \"random_state\"\n\n custom_model_dir: str | None = None\n default_classification_model: str | None = \"NOTSET\"\n default_regression_model: str | None = \"NOTSET\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._feature_generator = None\n self._cat_features = None\n self._cat_indices = None\n\n def _preprocess(self, X: pd.DataFrame, is_train=False, **kwargs) -> pd.DataFrame:\n X = super()._preprocess(X, **kwargs)\n\n if is_train:\n self._cat_indices = []\n\n # X will be the training data.\n self._feature_generator = LabelEncoderFeatureGenerator(verbosity=0)\n self._feature_generator.fit(X=X)\n\n # This converts categorical features to numeric via stateful label encoding.\n if self._feature_generator.features_in:\n X = X.copy()\n X[self._feature_generator.features_in] = self._feature_generator.transform(X=X)\n\n if is_train:\n # Detect/set cat features and indices\n if self._cat_features is None:\n self._cat_features = self._feature_generator.features_in[:]\n self._cat_indices = [X.columns.get_loc(col) for col in self._cat_features]\n\n return X\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n num_cpus: int = 1,\n num_gpus: int = 0,\n time_limit: float | None = None,\n verbosity: int = 2,\n **kwargs,\n ):\n if not self.params_aux.get(\"model_telemetry\", False):\n self.disable_tabpfn_telemetry()\n\n from tabpfn import TabPFNClassifier, TabPFNRegressor\n from tabpfn.model.loading import resolve_model_path\n from torch.cuda import is_available\n\n is_classification = self.problem_type in [\"binary\", \"multiclass\"]\n\n model_base = TabPFNClassifier if is_classification else TabPFNRegressor\n\n device = \"cuda\" if num_gpus != 0 else \"cpu\"\n if (device == \"cuda\") and (not is_available()):\n raise AssertionError(\n \"Fit specified to use GPU, but CUDA is not available on this machine. \"\n \"Please switch to CPU usage instead.\",\n )\n\n if verbosity >= 2:\n # logs \"Built with PriorLabs-TabPFN\"\n self._log_license(device=device)\n self._log_cpu_warning(device=device)\n\n X = self.preprocess(X, y=y, is_train=True)\n\n hps = self._get_model_params()\n hps[\"device\"] = device\n hps[\"n_jobs\"] = num_cpus # FIXME: remove this, it doesn't do anything, use n_preprocessing_jobs??\n hps[\"categorical_features_indices\"] = self._cat_indices\n\n # Resolve preprocessing\n if \"preprocessing/scaling\" in hps:\n hps[\"inference_config/PREPROCESS_TRANSFORMS\"] = [\n {\n \"name\": scaler,\n \"global_transformer_name\": hps.pop(\"preprocessing/global\", None),\n \"categorical_name\": hps.pop(\"preprocessing/categoricals\", \"numeric\"),\n \"append_original\": hps.pop(\"preprocessing/append_original\", True),\n }\n for scaler in hps[\"preprocessing/scaling\"]\n ]\n for k in [\n \"preprocessing/scaling\",\n \"preprocessing/categoricals\",\n \"preprocessing/append_original\",\n \"preprocessing/global\",\n ]:\n hps.pop(k, None)\n\n # Remove task specific HPs\n if is_classification:\n hps.pop(\"inference_config/REGRESSION_Y_PREPROCESS_TRANSFORMS\", None)\n else:\n hps.pop(\"balance_probabilities\", None)\n\n # Resolve model_path\n if self.custom_model_dir is not None:\n model_dir = Path(self.custom_model_dir)\n else:\n _, model_dir, _, _ = resolve_model_path(\n model_path=None,\n which=\"classifier\" if is_classification else \"regressor\",\n )\n model_dir = model_dir[0]\n clf_path, reg_path = hps.pop(\n \"zip_model_path\",\n [self.default_classification_model, self.default_regression_model],\n )\n model_path = clf_path if is_classification else reg_path\n if model_path is not None:\n hps[\"model_path\"] = model_dir / model_path\n\n # Resolve inference_config\n inference_config = {\n _k: v for k, v in hps.items() if k.startswith(\"inference_config/\") and (_k := k.split(\"/\")[-1])\n }\n if inference_config:\n hps[\"inference_config\"] = inference_config\n for k in list(hps.keys()):\n if k.startswith(\"inference_config/\"):\n del hps[k]\n\n # Model and fit\n self.model = model_base(**hps)\n self.model = self.model.fit(\n X=X,\n y=y,\n )\n\n def _predict_proba(self, X, **kwargs) -> np.ndarray:\n if not self.params_aux.get(\"model_telemetry\", False):\n self.disable_tabpfn_telemetry()\n\n if self.problem_type == \"quantile\":\n y_pred = self.model.predict(\n X,\n output_type=\"quantiles\",\n quantiles=self.quantile_levels,\n )\n return np.column_stack(y_pred)\n\n return super()._predict_proba(X=X, kwargs=kwargs)\n\n def _get_default_resources(self) -> tuple[int, int]:\n # Use only physical cores for better performance based on benchmarks\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n\n num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True))\n\n return num_cpus, num_gpus\n\n def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]:\n return {\n \"num_cpus\": 1,\n \"num_gpus\": 1 if is_gpu_available else 0,\n }\n\n def _set_default_params(self):\n default_params = {\n \"ignore_pretraining_limits\": True, # to ignore warnings and size limits\n }\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def get_device(self) -> str:\n return self.model.devices_[0].type\n\n def _set_device(self, device: str):\n self.model.to(device)\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\"]\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_rows\": 100_000,\n \"max_features\": 2000,\n \"max_classes\": 10,\n \"model_telemetry\": False,\n }\n )\n return default_auxiliary_params\n\n @classmethod\n def _get_default_ag_args_ensemble(cls, **kwargs) -> dict:\n \"\"\"Set fold_fitting_strategy to sequential_local,\n as parallel folding crashes if model weights aren't pre-downloaded.\n \"\"\"\n default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs)\n extra_ag_args_ensemble = {\n # FIXME: Find a work-around to avoid crash if parallel and weights are not downloaded\n \"fold_fitting_strategy\": \"sequential_local\",\n \"refit_folds\": True, # Better to refit the model for faster inference and similar quality as the bag.\n }\n default_ag_args_ensemble.update(extra_ag_args_ensemble)\n return default_ag_args_ensemble\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def disable_tabpfn_telemetry(cls):\n os.environ[\"TABPFN_DISABLE_TELEMETRY\"] = \"1\"\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict | None = None,\n **kwargs,\n ) -> int:\n \"\"\"Heuristic memory estimate based on TabPFN's memory estimate logic in:\n https://github.com/PriorLabs/TabPFN/blob/57a2efd3ebdb3886245e4d097cefa73a5261a969/src/tabpfn/model/memory.py#L147.\n\n This is based on GPU memory usage, but hopefully with overheads it also approximates CPU memory usage.\n \"\"\"\n # TODO: update, this is not correct anymore, consider using internal TabPFN functions directly.\n features_per_group = 3 # Based on TabPFNv2 default (unused)\n n_layers = 12 # Based on TabPFNv2 default\n embedding_size = 192 # Based on TabPFNv2 default\n dtype_byte_size = 2 # Based on TabPFNv2 default\n\n model_mem = 14489108 # Based on TabPFNv2 default\n\n n_samples, n_features = X.shape[0], min(X.shape[1], 2000)\n n_feature_groups = (n_features) / features_per_group + 1 # TODO: Unsure how to calculate this\n\n X_mem = n_samples * n_feature_groups * dtype_byte_size\n activation_mem = n_samples * n_feature_groups * embedding_size * n_layers * dtype_byte_size\n\n baseline_overhead_mem_est = 1e9 # 1 GB generic overhead\n\n # Add some buffer to each term + 1 GB overhead to be safe\n return int(model_mem + 4 * X_mem + 2 * activation_mem + baseline_overhead_mem_est)\n\n @classmethod\n def _class_tags(cls):\n return {\"can_estimate_memory_usage_static\": True}\n\n def _more_tags(self) -> dict:\n return {\"can_refit_full\": True}\n\n @staticmethod\n def extra_checkpoints_for_tuning(problem_type: str) -> list[str]:\n raise NotImplementedError(\"This method must be implemented in the subclass.\")\n\n def _log_license(self, device: str):\n pass\n\n def _log_cpu_warning(self, device: str):\n global _HAS_LOGGED_TABPFN_CPU_WARNING\n if not _HAS_LOGGED_TABPFN_CPU_WARNING:\n if device == \"cpu\":\n logger.log(\n 20, \"\\tRunning TabPFN on CPU. This can be very slow. It is recommended to run TabPFN on a GPU.\"\n )\n _HAS_LOGGED_TABPFN_CPU_WARNING = True\n\n\nclass RealTabPFNv25Model(TabPFNModel):\n \"\"\"RealTabPFN-v2.5 version: https://priorlabs.ai/technical-reports/tabpfn-2-5-model-report.\n\n We name this model RealTabPFN-v2.5 as its default checkpoints were trained on\n real-world datasets, following the naming conventions of Prior Labs.\n The extra checkpoints include models trained on only synthetic datasets as well.\n\n .. versionadded:: 1.5.0\n \"\"\"\n\n ag_key = \"REALTABPFN-V2.5\"\n ag_name = \"RealTabPFN-v2.5\"\n\n default_classification_model: str | None = \"tabpfn-v2.5-classifier-v2.5_default.ckpt\"\n default_regression_model: str | None = \"tabpfn-v2.5-regressor-v2.5_default.ckpt\"\n\n @staticmethod\n def extra_checkpoints_for_tuning(problem_type: str) -> list[str]:\n \"\"\"The list of checkpoints to use for hyperparameter tuning.\"\"\"\n if problem_type == \"classification\":\n return [\n \"tabpfn-v2.5-classifier-v2.5_default-2.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_large-features-L.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_large-features-XL.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_large-samples.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_real-large-features.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_real-large-samples-and-features.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_real.ckpt\",\n \"tabpfn-v2.5-classifier-v2.5_variant.ckpt\",\n ]\n\n return [\n \"tabpfn-v2.5-regressor-v2.5_low-skew.ckpt\",\n \"tabpfn-v2.5-regressor-v2.5_quantiles.ckpt\",\n \"tabpfn-v2.5-regressor-v2.5_real-variant.ckpt\",\n \"tabpfn-v2.5-regressor-v2.5_real.ckpt\",\n \"tabpfn-v2.5-regressor-v2.5_small-samples.ckpt\",\n \"tabpfn-v2.5-regressor-v2.5_variant.ckpt\",\n ]\n\n def _log_license(self, device: str):\n global _HAS_LOGGED_TABPFN_NONCOMMERICAL\n if not _HAS_LOGGED_TABPFN_NONCOMMERICAL:\n logger.log(\n 30,\n \"\\tWarning: TabPFN-2.5 is a NONCOMMERCIAL model. \"\n \"Usage of this artifact (including through AutoGluon) is not permitted \"\n \"for commercial tasks unless granted explicit permission \"\n \"by the model authors (PriorLabs).\",\n ) # Aligning with TabPFNv25 license\n _HAS_LOGGED_TABPFN_NONCOMMERICAL = True # Avoid repeated logging\n\n\nclass RealTabPFNv2Model(TabPFNModel):\n \"\"\"RealTabPFN-v2 version\n\n We name this model RealTabPFN-v2 as its default checkpoints were trained on\n real-world datasets, following the naming conventions of Prior Labs.\n The extra checkpoints include models trained on only synthetic datasets as well.\n\n .. versionadded:: 1.5.0\n \"\"\"\n\n ag_key = \"REALTABPFN-V2\"\n ag_name = \"RealTabPFN-v2\"\n\n # TODO: Verify if this is the same as the \"default\" ckpt\n default_classification_model: str | None = \"tabpfn-v2-classifier-finetuned-zk73skhh.ckpt\"\n default_regression_model: str | None = \"tabpfn-v2-regressor-v2_default.ckpt\"\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n default_auxiliary_params.update(\n {\n \"max_rows\": 10_000,\n \"max_features\": 500,\n \"max_classes\": 10,\n \"max_batch_size\": 10000, # TabPFN seems to cryptically error if predicting on 100,000 samples.\n }\n )\n return default_auxiliary_params\n\n def _log_license(self, device: str):\n global _HAS_LOGGED_TABPFN_LICENSE\n if not _HAS_LOGGED_TABPFN_LICENSE:\n logger.log(20, \"\\tBuilt with PriorLabs-TabPFN\") # Aligning with TabPFNv2 license requirements\n _HAS_LOGGED_TABPFN_LICENSE = True # Avoid repeated logging\n\n # FIXME: Avoid code dupe. This one has 500 features max, 2.5 has 2000.\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict | None = None,\n **kwargs,\n ) -> int:\n \"\"\"Heuristic memory estimate based on TabPFN's memory estimate logic in:\n https://github.com/PriorLabs/TabPFN/blob/57a2efd3ebdb3886245e4d097cefa73a5261a969/src/tabpfn/model/memory.py#L147.\n\n This is based on GPU memory usage, but hopefully with overheads it also approximates CPU memory usage.\n \"\"\"\n # TODO: update, this is not correct anymore, consider using internal TabPFN functions directly.\n features_per_group = 3 # Based on TabPFNv2 default (unused)\n n_layers = 12 # Based on TabPFNv2 default\n embedding_size = 192 # Based on TabPFNv2 default\n dtype_byte_size = 2 # Based on TabPFNv2 default\n\n model_mem = 14489108 # Based on TabPFNv2 default\n\n n_samples, n_features = X.shape[0], min(X.shape[1], 500)\n n_feature_groups = (n_features) / features_per_group + 1 # TODO: Unsure how to calculate this\n\n X_mem = n_samples * n_feature_groups * dtype_byte_size\n activation_mem = n_samples * n_feature_groups * embedding_size * n_layers * dtype_byte_size\n\n baseline_overhead_mem_est = 1e9 # 1 GB generic overhead\n\n # Add some buffer to each term + 1 GB overhead to be safe\n return int(model_mem + 4 * X_mem + 2 * activation_mem + baseline_overhead_mem_est)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabprep/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabprep/prep_lgb_model.py",
"content": "from __future__ import annotations\n\nfrom ..lgb.lgb_model import LGBModel\nfrom .prep_mixin import ModelAgnosticPrepMixin\n\n\nclass PrepLGBModel(ModelAgnosticPrepMixin, LGBModel):\n ag_key = \"GBM_PREP\"\n ag_name = \"LightGBMPrep\"\n\n @classmethod\n def _estimate_memory_usage_static(cls, **kwargs) -> int:\n memory_usage = super()._estimate_memory_usage_static(**kwargs)\n # FIXME: 1.5 runs OOM on kddcup09_appetency fold 2 repeat 0 prep_LightGBM_r49_BAG_L1\n return memory_usage * 2.0 # FIXME: For some reason this underestimates mem usage without this\n\n @classmethod\n def _estimate_memory_usage_static_lite(cls, **kwargs) -> int:\n memory_usage = super()._estimate_memory_usage_static_lite(**kwargs)\n # FIXME: 1.5 runs OOM on kddcup09_appetency fold 2 repeat 0 prep_LightGBM_r49_BAG_L1\n return memory_usage * 2.0 # FIXME: For some reason this underestimates mem usage without this\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabprep/prep_mixin.py",
"content": "from __future__ import annotations\n\nimport logging\nfrom typing import Type\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.features import BulkFeatureGenerator\nfrom autogluon.features.generators.abstract import AbstractFeatureGenerator\nfrom autogluon.features.registry._ag_feature_generator_registry import ag_feature_generator_registry\n\nlogger = logging.getLogger(__name__)\n\n\ndef _recursive_expand_prep_param(prep_param: tuple | list[list | tuple]) -> list[tuple]:\n if isinstance(prep_param, list):\n if len(prep_param) == 0:\n param_type = \"list\"\n elif len(prep_param) == 2:\n if isinstance(prep_param[0], (str, AbstractFeatureGenerator)):\n param_type = \"generator\"\n else:\n param_type = \"list\"\n else:\n param_type = \"list\"\n elif isinstance(prep_param, tuple):\n param_type = \"generator\"\n else:\n raise ValueError(f\"Invalid value for prep_param: {prep_param}\")\n if param_type == \"list\":\n out = []\n for p in prep_param:\n out += _recursive_expand_prep_param(p)\n return out\n elif param_type == \"generator\":\n return [prep_param]\n else:\n raise ValueError(f\"Invalid value for prep_param: {prep_param}\")\n\n\n# FIXME: Why is preprocessing twice as slow per fold when bagging LightGBM??? Need to investigate. Try sequential fold fit\n# TODO: Why is `prep_params` a dict instead of a list?\nclass ModelAgnosticPrepMixin:\n def _estimate_dtypes_after_preprocessing(self, X: pd.DataFrame, **kwargs) -> int:\n prep_params = self._get_ag_params().get(\"prep_params\", None)\n if prep_params is None:\n prep_params = []\n\n # FIXME: Temporarily simplify for memory calculation\n prep_params = _recursive_expand_prep_param(prep_params)\n\n X_nunique = X.nunique().values\n n_categorical = X.select_dtypes(exclude=[np.number]).shape[1]\n n_numeric = X.loc[:, X_nunique > 2].select_dtypes(include=[np.number]).shape[1]\n n_binary = (\n X.loc[:, X_nunique <= 2].select_dtypes(include=[np.number]).shape[1]\n ) # NOTE: It can happen that features have less than two unique values if cleaning is applied before the bagging, i.e. Bioresponse\n\n assert n_numeric + n_categorical + n_binary == X.shape[1] # NOTE: FOr debugging, to be removed later\n for preprocessor_cls_name, init_params in prep_params:\n if isinstance(preprocessor_cls_name, str):\n prep_cls = ag_feature_generator_registry.key_to_cls(key=preprocessor_cls_name)\n else:\n prep_cls = preprocessor_cls_name\n feature_generator = prep_cls(target_type=self.problem_type, **init_params)\n\n if hasattr(feature_generator, \"estimate_new_dtypes\"):\n n_numeric, n_categorical, n_binary = feature_generator.estimate_new_dtypes(\n n_numeric, n_categorical, n_binary, num_classes=self.num_classes\n )\n else:\n pass # FIXME: Need to implement\n\n return n_numeric, n_categorical, n_binary\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n n_numeric, n_categorical, n_binary = self._estimate_dtypes_after_preprocessing(X=X, **kwargs)\n\n if hasattr(self, \"_estimate_memory_usage_static_lite\"):\n return self._estimate_memory_usage_static_lite(\n num_samples=X.shape[0],\n num_features=n_numeric + n_categorical + n_binary,\n num_bytes_per_cell=4,\n hyperparameters=hyperparameters,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n **kwargs,\n )\n\n # TODO: Replace with memory estimation logic based on no. of features instead of dataframe generation\n shape = X.shape[0]\n df_lst = []\n if n_numeric > 0:\n X_estimate = np.random.random(size=[shape, n_numeric]).astype(np.float32)\n X_estimate_numeric = pd.DataFrame(X_estimate)\n df_lst.append(X_estimate_numeric)\n if n_categorical > 0:\n cardinality = int(X.select_dtypes(exclude=[np.number]).nunique().mean())\n X_estimate = np.random.randint(0, cardinality, [shape, n_categorical]).astype(\"str\")\n X_estimate_cat = pd.DataFrame(X_estimate)\n df_lst.append(X_estimate_cat)\n if n_binary > 0:\n X_estimate = np.random.randint(0, 2, [shape, n_binary]).astype(np.int8)\n X_estimate_binary = pd.DataFrame(X_estimate)\n df_lst.append(X_estimate_binary)\n X = pd.concat(df_lst, ignore_index=True, axis=1)\n\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n def _init_preprocessor(\n self,\n preprocessor_cls: Type[AbstractFeatureGenerator] | str,\n init_params: dict | None,\n ) -> AbstractFeatureGenerator:\n if isinstance(preprocessor_cls, str):\n preprocessor_cls = ag_feature_generator_registry.key_to_cls(preprocessor_cls)\n if init_params is None:\n init_params = {}\n _init_params = dict(\n verbosity=0,\n random_state=self.random_seed, # FIXME: Not a generic param\n target_type=self.problem_type, # FIXME: Not a generic param\n )\n _init_params.update(**init_params)\n return preprocessor_cls(\n **_init_params,\n )\n\n def _recursive_init_preprocessors(self, prep_param: tuple | list[list | tuple]):\n if isinstance(prep_param, list):\n if len(prep_param) == 0:\n param_type = \"list\"\n elif len(prep_param) == 2:\n if isinstance(prep_param[0], (str, AbstractFeatureGenerator)):\n param_type = \"generator\"\n else:\n param_type = \"list\"\n else:\n param_type = \"list\"\n elif isinstance(prep_param, tuple):\n param_type = \"generator\"\n else:\n raise ValueError(f\"Invalid value for prep_param: {prep_param}\")\n\n if param_type == \"list\":\n out = []\n for i, p in enumerate(prep_param):\n out.append(self._recursive_init_preprocessors(p))\n return out\n elif param_type == \"generator\":\n assert len(prep_param) == 2\n preprocessor_cls = prep_param[0]\n init_params = prep_param[1]\n return self._init_preprocessor(preprocessor_cls=preprocessor_cls, init_params=init_params)\n else:\n raise ValueError(f\"Invalid value for prep_param: {prep_param}\")\n\n def get_preprocessor(self) -> AbstractFeatureGenerator | None:\n ag_params = self._get_ag_params()\n prep_params = ag_params.get(\"prep_params\", None)\n passthrough_types = ag_params.get(\"prep_params.passthrough_types\", None)\n if prep_params is None:\n return None\n if not prep_params:\n return None\n\n preprocessors = self._recursive_init_preprocessors(prep_param=prep_params)\n if len(preprocessors) == 0:\n return None\n if len(preprocessors) == 1 and isinstance(preprocessors[0], AbstractFeatureGenerator):\n return preprocessors[0]\n else:\n preprocessor = BulkFeatureGenerator(\n generators=preprocessors,\n # TODO: \"false_recursive\" technically can slow down inference, but need to optimize `True` first\n # Refer to `Bioresponse` dataset where setting to `True` -> 200s fit time vs `false_recursive` -> 1s fit time\n remove_unused_features=\"false_recursive\",\n post_drop_duplicates=True,\n passthrough=True,\n passthrough_types=passthrough_types,\n verbosity=0,\n )\n return preprocessor\n\n def _preprocess(self, X: pd.DataFrame, y=None, is_train: bool = False, **kwargs):\n if is_train:\n self.preprocessor = self.get_preprocessor()\n if self.preprocessor is not None:\n assert y is not None, (\n f\"y must be specified to fit preprocessors... Likely the inheriting class isn't passing `y` in its `preprocess` call.\"\n )\n # FIXME: add `post_drop_useless`, example: anneal has many useless features\n feature_metadata_in = self._feature_metadata\n X = self.preprocessor.fit_transform(X, y, feature_metadata_in=feature_metadata_in)\n # FIXME: Nick: This is incorrect because it strips away special dtypes. Need to do this properly by fixing in the preprocessors\n feature_metadata_in = self.preprocessor.feature_metadata\n self._feature_metadata = feature_metadata_in\n self._features_internal = self._feature_metadata.get_features()\n else:\n if self.preprocessor is not None:\n X = self.preprocessor.transform(X)\n\n return super()._preprocess(X, y=y, is_train=is_train, **kwargs)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/compilers/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/compilers/native.py",
"content": "import os\nimport pickle\n\n\nclass AbstractNativeCompiler:\n name = \"native\"\n save_in_pkl = True\n\n @staticmethod\n def can_compile():\n return True\n\n @staticmethod\n def compile(model, path: str, input_types=None):\n \"\"\"\n Compile the trained model for faster inference.\n\n Parameters\n ----------\n model\n The native model that is expected to be compiled.\n path : str\n The path for saving the compiled model.\n input_types : list, default=None\n A list of tuples containing shape and element type info, e.g. [((1, 14), np.float32),].\n The list would be used as the input data for the model.\n The compiler would optimize the model to perform best with the given input type.\n \"\"\"\n AbstractNativeCompiler.save(model, path)\n\n @staticmethod\n def save(model, path: str):\n os.makedirs(os.path.dirname(path), exist_ok=True)\n with open(os.path.join(path, \"model_native.pkl\"), \"wb\") as fp:\n fp.write(pickle.dumps(model))\n\n @staticmethod\n def load(path: str):\n pkl = None\n with open(os.path.join(path, \"model_native.pkl\"), \"rb\") as fp:\n pkl = fp.read()\n return pickle.loads(pkl)\n\n\nTabularNeuralNetTorchNativeCompiler = AbstractNativeCompiler\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/compilers/onnx.py",
"content": "import os\n\nimport numpy as np\n\n\ndef quantile_transformer_shape_calculator(operator):\n op = operator.raw_operator\n input_type = operator.inputs[0].type.__class__\n input_dim = operator.inputs[0].type.shape[0]\n output_type = input_type([input_dim, op.quantiles_.shape[1]])\n operator.outputs[0].type = output_type\n\n\ndef quantile_transformer_converter(scope, operator, container):\n from scipy.stats import norm\n from skl2onnx.algebra.onnx_ops import (\n OnnxAbs,\n OnnxArgMin,\n OnnxCast,\n OnnxConcat,\n OnnxGatherElements,\n OnnxMatMul,\n OnnxReshape,\n OnnxSplit,\n OnnxSub,\n )\n from skl2onnx.common.data_types import guess_numpy_type\n\n op = operator.raw_operator\n opv = container.target_opset\n out = operator.outputs\n\n # We retrieve the unique input.\n X = operator.inputs[0]\n\n # In most case, computation happen in floats.\n # But it might be with double. ONNX is very strict\n # about types, every constant should have the same\n # type as the input.\n dtype = guess_numpy_type(X.type)\n batch_size = X.type.shape[0]\n n_quantiles = op.n_quantiles_\n\n # We tell in ONNX language how to compute the unique output.\n # op_version=opv tells which opset is requested\n C = op.quantiles_.astype(dtype)\n if opv < 18:\n C_col = OnnxSplit(C, axis=1, output_names=[f\"C_col{x}\" for x in range(op.n_features_in_)], op_version=opv)\n else:\n C_col = OnnxSplit(\n C,\n axis=1,\n num_outputs=C.shape[1],\n output_names=[f\"C_col{x}\" for x in range(op.n_features_in_)],\n op_version=opv,\n )\n C_col.add_to(scope, container)\n if opv < 18:\n X_col = OnnxSplit(X, axis=1, output_names=[f\"X_col{x}\" for x in range(op.n_features_in_)], op_version=opv)\n else:\n X_col = OnnxSplit(\n X,\n axis=1,\n num_outputs=X.type.shape[1],\n output_names=[f\"X_col{x}\" for x in range(op.n_features_in_)],\n op_version=opv,\n )\n X_col.add_to(scope, container)\n Y_col = []\n for feature_idx in range(op.n_features_in_):\n # This implements\n # > X_col[isfinite_mask] = np.interp(X_col_finite, self.references_, quantiles)\n #\n # Specifically, for yi = np.interp(xi, x, y), we implement\n # > def nearest_interp(xi, x, y):\n # > idx = np.abs(x - xi[:,None])\n # > return y[idx.argmin(axis=1)]\n # See https://stackoverflow.com/questions/21002799/extraploation-with-nearest-method-in-python\n repeat = OnnxMatMul(\n OnnxReshape(X_col.outputs[feature_idx], np.array([batch_size, 1], dtype=np.int64), op_version=opv),\n np.ones(shape=(1, n_quantiles)).astype(dtype),\n )\n sub = OnnxSub(\n repeat,\n OnnxReshape(C_col.outputs[feature_idx], np.array([1, n_quantiles], dtype=np.int64), op_version=opv),\n op_version=opv,\n output_names=[f\"sub_col{feature_idx}\"],\n )\n idx = OnnxAbs(sub, op_version=opv, output_names=[f\"idx_col{feature_idx}\"])\n argmin = OnnxArgMin(\n OnnxReshape(idx, np.array([batch_size, n_quantiles], dtype=np.int64), op_version=opv),\n axis=1,\n op_version=opv,\n output_names=[f\"argmin_col{feature_idx}\"],\n )\n references = np.clip(norm.ppf(op.references_), -5.2, 5.2).astype(dtype)\n cst = np.broadcast_to(references, (batch_size, n_quantiles))\n argmin_reshaped = OnnxReshape(\n argmin, np.array([batch_size, 1], dtype=np.int64), output_names=[f\"reshape_col{feature_idx}\"]\n )\n ref = OnnxGatherElements(\n cst, argmin_reshaped, axis=1, op_version=opv, output_names=[f\"gathernd_col{feature_idx}\"]\n )\n ref_reshape = OnnxReshape(ref, np.array([batch_size, 1], dtype=np.int64), output_names=[f\"Y_col{feature_idx}\"])\n ref_cast = OnnxCast(ref_reshape, to=1, op_version=opv, output_names=[f\"ref_cast{feature_idx}\"])\n Y_col.append(ref_cast)\n Y = OnnxConcat(*Y_col, axis=1, op_version=opv, output_names=out[:1])\n Y.add_to(scope, container)\n\n\ndef onehot_handle_unknown_transformer_shape_calculator(operator):\n op = operator.raw_operator\n input_type = operator.inputs[0].type.__class__\n input_dim = operator.inputs[0].type.shape[0]\n output_type = input_type([input_dim, sum([len(c) for c in op.categories_])])\n operator.outputs[0].type = output_type\n\n\ndef onehot_handle_unknown_transformer_converter(scope, operator, container):\n return _encoder_handle_unknown_transformer_converter(scope, operator, container, \"onehot_\")\n\n\ndef ordinal_handle_unknown_transformer_shape_calculator(operator):\n op = operator.raw_operator\n input_type = operator.inputs[0].type.__class__\n input_dim = operator.inputs[0].type.shape[0]\n output_type = input_type([input_dim, len(op.categories_len_)])\n operator.outputs[0].type = output_type\n\n\ndef ordinal_handle_unknown_transformer_converter(scope, operator, container):\n return _encoder_handle_unknown_transformer_converter(scope, operator, container, \"ordinal_\")\n\n\ndef _encoder_handle_unknown_transformer_converter(scope, operator, container, name_prefix):\n from skl2onnx.algebra.onnx_ops import (\n OnnxAbs,\n OnnxArgMin,\n OnnxCast,\n OnnxConcat,\n OnnxMatMul,\n OnnxOneHot,\n OnnxReshape,\n OnnxSplit,\n OnnxSub,\n )\n from skl2onnx.common.data_types import guess_numpy_type\n\n op = operator.raw_operator\n opv = container.target_opset\n out = operator.outputs\n\n # We retrieve the unique input.\n X = operator.inputs[0]\n\n # In most case, computation happen in floats.\n # But it might be with double. ONNX is very strict\n # about types, every constant should have the same\n # type as the input.\n dtype = guess_numpy_type(X.type)\n batch_size = X.type.shape[0]\n num_categories = len(op.categories_)\n\n C_col = op.categories_\n if opv < 18:\n X_col = OnnxSplit(\n X, axis=1, output_names=[f\"{name_prefix}X_col{x}\" for x in range(num_categories)], op_version=opv\n )\n else:\n X_col = OnnxSplit(\n X,\n axis=1,\n num_outputs=X.type.shape[1],\n output_names=[f\"{name_prefix}X_col{x}\" for x in range(num_categories)],\n op_version=opv,\n )\n X_col.add_to(scope, container)\n Y_col = []\n for feature_idx in range(num_categories):\n # This implements\n # > X_col = np.searchsorted(X_col_finite, self.references_)\n #\n # Specifically, for yi = np.searchsorted(xi, x), we implement\n # > def searchsorted(xi, x, y):\n # > idx = np.abs(x - xi[:,None])\n # > return idx.argmin(axis=1)\n num_classes = len(C_col[feature_idx])\n repeat = OnnxMatMul(\n OnnxReshape(X_col.outputs[feature_idx], np.array([batch_size, 1], dtype=np.int64), op_version=opv),\n np.ones(shape=(1, num_classes)).astype(dtype),\n op_version=opv,\n )\n sub = OnnxSub(\n repeat,\n OnnxReshape(C_col[feature_idx].astype(dtype), np.array([1, num_classes], dtype=np.int64), op_version=opv),\n op_version=opv,\n output_names=[f\"{name_prefix}sub_col{feature_idx}\"],\n )\n idx = OnnxAbs(sub, op_version=opv, output_names=[f\"{name_prefix}idx_col{feature_idx}\"])\n argmin = OnnxArgMin(\n OnnxReshape(idx, np.array([batch_size, num_classes], dtype=np.int64), op_version=opv),\n axis=1,\n op_version=opv,\n output_names=[f\"{name_prefix}argmin_col{feature_idx}\"],\n )\n if name_prefix.startswith(\"onehot\"):\n onehot = OnnxOneHot(\n argmin,\n np.array([num_classes]).astype(np.int64), # number of classes\n np.array([0, 1]).astype(dtype), # [off_value, on_value]\n axis=1,\n op_version=opv,\n output_names=[f\"{name_prefix}onehot_col{feature_idx}\"],\n )\n onehot_reshaped = OnnxReshape(\n onehot,\n np.array([batch_size, num_classes], dtype=np.int64),\n output_names=[f\"{name_prefix}Y_col{feature_idx}\"],\n op_version=opv,\n )\n onehot_cast = OnnxCast(\n onehot_reshaped, to=1, op_version=opv, output_names=[f\"{name_prefix}onehot_cast{feature_idx}\"]\n )\n Y_col.append(onehot_cast)\n else:\n argmin_reshaped = OnnxReshape(\n argmin,\n np.array([batch_size, 1], dtype=np.int64),\n output_names=[f\"{name_prefix}Y_col{feature_idx}\"],\n op_version=opv,\n )\n argmin_cast = OnnxCast(\n argmin_reshaped, to=1, op_version=opv, output_names=[f\"{name_prefix}argmin_cast{feature_idx}\"]\n )\n Y_col.append(argmin_cast)\n Y = OnnxConcat(*Y_col, axis=1, op_version=opv, output_names=out[:1])\n Y.add_to(scope, container)\n\n\nclass InferenceSessionWrapper:\n \"\"\"\n Wrap around InferenceSession in onnxruntime, since it cannot be pickled.\n See https://github.com/microsoft/onnxruntime/issues/10097\n \"\"\"\n\n def __init__(self, onnx_bytes):\n import onnxruntime as ort\n\n self.sess = ort.InferenceSession(onnx_bytes.SerializeToString(), providers=[\"CPUExecutionProvider\"])\n\n def run(self, *args):\n return self.sess.run(*args)\n\n def get_inputs(self, *args):\n return self.sess.get_inputs(*args)\n\n def get_outputs(self, *args):\n return self.sess.get_outputs(*args)\n\n def __getstate__(self):\n # No need to duplicate the model parameters here.\n return {}\n\n def __setstate__(self, values):\n pass\n\n\nclass TabularNeuralNetTorchOnnxTransformer:\n def __init__(self, model):\n self.sess = InferenceSessionWrapper(model)\n self.batch_size = self.sess.get_inputs()[0].shape[0]\n self.onnx_input_names = [x.name for x in self.sess.get_inputs()]\n self.input_names = [] # raw_name\n\n def transform(self, X):\n \"\"\"Run the model with the input and return the result.\"\"\"\n if not self.input_names:\n raw_names = list(X.columns)\n onnx_names = [n.replace(\"-\", \"_\").replace(\".\", \"_\") for n in raw_names]\n onnx_to_raw = {o: r for o, r in zip(onnx_names, raw_names)}\n self.input_names = [onnx_to_raw[oname] for oname in self.onnx_input_names]\n\n input_dict = {}\n input_arr = X[self.input_names].astype(np.float32).to_numpy()\n input_size = input_arr.shape[0]\n inputs = []\n if input_size > self.batch_size:\n indices = list(np.arange(self.batch_size, input_size, self.batch_size)) + [input_size]\n inputs = np.split(input_arr, indices)\n else:\n inputs = [input_arr]\n outputs = []\n for input_arr in inputs:\n input_size = input_arr.shape[0]\n if input_size < self.batch_size:\n # padding\n pad_size = self.batch_size - input_size\n pad_shape = list(input_arr.shape)\n pad_shape[0] = pad_size\n pad_arr = np.zeros(shape=tuple(pad_shape), dtype=np.float32)\n input_arr = np.concatenate([input_arr, pad_arr])\n for idx, name in enumerate(self.onnx_input_names):\n input_dict[name] = input_arr[:, idx].reshape(self.batch_size, 1)\n label_name = self.sess.get_outputs()[0].name\n output_arr = self.sess.run([label_name], input_dict)[0]\n if input_size < self.batch_size:\n # remove padding\n output_arr = output_arr[:input_size, :]\n outputs.append(output_arr)\n outputs = np.concatenate(outputs)\n return outputs\n\n\nclass TabularNeuralNetTorchOnnxCompiler:\n name = \"onnx\"\n save_in_pkl = True\n\n @staticmethod\n def can_compile():\n \"\"\"Verify whether the required package has been installed.\"\"\"\n try:\n import onnxruntime\n import skl2onnx\n\n return True\n except ImportError:\n return False\n\n @staticmethod\n def compile(model, path: str, input_types=None):\n \"\"\"\n Compile the trained model for faster inference.\n\n Parameters\n ----------\n model\n The native model that is expected to be compiled.\n \"\"\"\n if isinstance(model, TabularNeuralNetTorchOnnxTransformer):\n return model\n import skl2onnx\n from skl2onnx import convert_sklearn, update_registered_converter\n from skl2onnx.common.data_types import FloatTensorType\n from sklearn.pipeline import Pipeline\n from sklearn.preprocessing import QuantileTransformer\n\n from ..utils.categorical_encoders import (\n OneHotMergeRaresHandleUnknownEncoder,\n OrdinalMergeRaresHandleUnknownEncoder,\n )\n\n update_registered_converter(\n QuantileTransformer,\n \"SklearnQuantileTransformer\",\n quantile_transformer_shape_calculator,\n quantile_transformer_converter,\n )\n update_registered_converter(\n OneHotMergeRaresHandleUnknownEncoder,\n \"OneHotMergeRaresHandleUnknownEncoder\",\n onehot_handle_unknown_transformer_shape_calculator,\n onehot_handle_unknown_transformer_converter,\n )\n update_registered_converter(\n OrdinalMergeRaresHandleUnknownEncoder,\n \"OrdinalMergeRaresHandleUnknownEncoder\",\n ordinal_handle_unknown_transformer_shape_calculator,\n ordinal_handle_unknown_transformer_converter,\n )\n\n if input_types is None or not isinstance(input_types[0], tuple):\n raise RuntimeError(\"input_types argument should contain at least one tuple, e.g. [((1, 14), np.float32)]\")\n pipeline = Pipeline(\n steps=[\n (\"processor\", model[0]),\n ]\n )\n\n for idx, input_type in enumerate(input_types):\n input_types[idx] = (input_type[0], FloatTensorType(input_type[1]))\n\n onnx_model = convert_sklearn(pipeline, initial_types=input_types)\n\n predictor = TabularNeuralNetTorchOnnxTransformer(model=onnx_model)\n TabularNeuralNetTorchOnnxCompiler.save(onnx_model, path)\n return predictor\n\n @staticmethod\n def save(model, path: str) -> str:\n \"\"\"Save the compiled model into onnx file format.\"\"\"\n os.makedirs(os.path.dirname(path), exist_ok=True)\n with open(os.path.join(path, \"model.onnx\"), \"wb\") as f:\n f.write(model.SerializeToString())\n return os.path.join(path, \"model.onnx\")\n\n @staticmethod\n def load(path: str) -> TabularNeuralNetTorchOnnxTransformer:\n \"\"\"Load from the path that contains an onnx file.\"\"\"\n import onnx\n\n onnx_bytes = onnx.load(os.path.join(path, \"model.onnx\"))\n return TabularNeuralNetTorchOnnxTransformer(model=onnx_bytes)\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/hyperparameters/parameters.py",
"content": "\"\"\"Default (fixed) hyperparameter values used in Tabular Neural Network models.\nA value of None typically indicates an adaptive value for the hyperparameter will be chosen based on the data.\n\"\"\"\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\n\n\ndef get_fixed_params(framework):\n \"\"\"Parameters that currently cannot be searched during HPO\"\"\"\n fixed_params = {}\n # TODO: v1.2 Change default epochs_wo_improve to \"auto\", so that None can mean no early stopping.\n pytorch_fixed_params = {\n \"num_epochs\": 1000, # maximum number of epochs (passes over full dataset) for training NN\n \"epochs_wo_improve\": None, # we terminate training if validation performance hasn't improved in the last 'epochs_wo_improve' # of epochs\n }\n return merge_framework_params(framework=framework, shared_params=fixed_params, pytorch_params=pytorch_fixed_params)\n\n\ndef get_hyper_params(framework):\n \"\"\"Parameters that currently can be tuned during HPO\"\"\"\n hyper_params = {\n ## Hyperparameters for neural net architecture:\n \"activation\": \"relu\", # Activation function\n # Options: ['relu', 'softrelu', 'tanh', 'softsign'], options for pytorch: ['relu', 'elu', 'tanh']\n \"embedding_size_factor\": 1.0, # scaling factor to adjust size of embedding layers (float > 0)\n # Options: range[0.01 - 100] on log-scale\n \"embed_exponent\": 0.56, # exponent used to determine size of embedding layers based on # categories.\n \"max_embedding_dim\": 100, # maximum size of embedding layer for a single categorical feature (int > 0).\n ## Regression-specific hyperparameters:\n \"y_range\": None, # Tuple specifying whether Y is constrained to (min_y, max_y). Can be = (-np.inf, np.inf).\n # If None, inferred based on training labels. Note: MUST be None for classification tasks!\n \"y_range_extend\": 0.05, # Only used to extend size of inferred y_range when y_range = None.\n ## Hyperparameters for neural net training:\n \"dropout_prob\": 0.1, # dropout probability, = 0 turns off Dropout.\n # Options: range(0.0, 0.5)\n \"optimizer\": \"adam\", # Which optimizer to use for training\n # Options include: ['adam','sgd']\n \"learning_rate\": 3e-4, # learning rate used for NN training (float > 0)\n \"weight_decay\": 1e-6, # weight decay regularizer (float > 0)\n ## Hyperparameters for data processing:\n \"proc.embed_min_categories\": 4, # apply embedding layer to categorical features with at least this many levels. Features with fewer levels are one-hot encoded. Choose big value to avoid use of Embedding layers\n # Options: [3,4,10, 100, 1000]\n \"proc.impute_strategy\": \"median\", # strategy argument of sklearn.SimpleImputer() used to impute missing numeric values\n # Options: ['median', 'mean', 'most_frequent']\n \"proc.max_category_levels\": 100, # maximum number of allowed levels per categorical feature\n # Options: [10, 100, 200, 300, 400, 500, 1000, 10000]\n \"proc.skew_threshold\": 0.99, # numerical features whose absolute skewness is greater than this receive special power-transform preprocessing. Choose big value to avoid using power-transforms\n # Options: [0.2, 0.3, 0.5, 0.8, 0.9, 0.99, 0.999, 1.0, 10.0, 100.0]\n \"use_ngram_features\": False, # If False, will drop automatically generated ngram features from language features. This results in worse model quality but far faster inference and training times.\n # Options: [True, False]\n }\n pytorch_hyper_params = {\n \"num_layers\": 4, # number of layers\n # Options: [2, 3, 4, 5]\n \"hidden_size\": 128, # number of hidden units in each layer\n # Options: [128, 256, 512]\n \"max_batch_size\": 512, # maximum batch-size, actual batch size may be slightly smaller.\n \"use_batchnorm\": False, # whether or not to utilize batch normalization\n # Options: [True, False]\n \"loss_function\": \"auto\", # Pytorch loss function minimized during training\n # Example options for regression: nn.MSELoss(), nn.L1Loss()\n }\n return merge_framework_params(framework=framework, shared_params=hyper_params, pytorch_params=pytorch_hyper_params)\n\n\ndef get_quantile_hyper_params(framework):\n \"\"\"Parameters that currently can be searched during HPO\"\"\"\n hyper_params = get_hyper_params(framework)\n new_hyper_params = {\n \"gamma\": 5.0, # margin loss weight which helps ensure noncrossing quantile estimates\n # Options: range(0.1, 10.0)\n \"alpha\": 0.01, # used for smoothing huber pinball loss\n }\n hyper_params.update(new_hyper_params)\n return hyper_params\n\n\n# Note: params for original NNTabularModel were:\n# weight_decay=0.01, dropout_prob = 0.1, batch_size = 2048, lr = 1e-2, epochs=30, layers= [200, 100] (semi-equivalent to our layers = [100],numeric_embed_dim=200)\ndef get_default_param(problem_type, framework, num_classes=None):\n if problem_type == BINARY:\n return get_param_binary(framework)\n elif problem_type == MULTICLASS:\n return get_param_multiclass(framework=framework, num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_param_regression(framework)\n elif problem_type == QUANTILE:\n return get_param_quantile(framework)\n else:\n return get_param_binary(framework)\n\n\ndef get_param_binary(framework):\n params = get_fixed_params(framework)\n params.update(get_hyper_params(framework))\n return params\n\n\ndef get_param_multiclass(framework, num_classes):\n return get_param_binary(framework) # Use same hyperparameters as for binary classification for now.\n\n\ndef get_param_regression(framework):\n return get_param_binary(framework) # Use same hyperparameters as for binary classification for now.\n\n\ndef get_param_quantile(framework):\n if framework != \"pytorch\":\n raise ValueError(\"Only pytorch tabular neural network is currently supported for quantile regression.\")\n params = get_fixed_params(framework)\n params.update(get_quantile_hyper_params(framework))\n return params\n\n\ndef merge_framework_params(framework, shared_params, pytorch_params):\n if framework == \"pytorch\":\n shared_params.update(pytorch_params)\n else:\n raise ValueError(\"framework must be 'pytorch'\")\n return shared_params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/hyperparameters/searchspaces.py",
"content": "\"\"\"Default (fixed) hyperparameter search spaces used in Tabular Neural Network models.\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\n\nfrom .parameters import merge_framework_params\n\n\ndef get_default_searchspace(problem_type, framework, num_classes=None):\n params = {\n \"learning_rate\": space.Real(1e-4, 3e-2, default=3e-4, log=True),\n \"weight_decay\": space.Real(1e-12, 0.1, default=1e-6, log=True),\n \"dropout_prob\": space.Categorical(0.1, 0.0, 0.5, 0.2, 0.3, 0.4),\n \"embedding_size_factor\": space.Categorical(1.0, 0.5, 1.5, 0.7, 0.6, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4),\n \"proc.embed_min_categories\": space.Categorical(4, 3, 10, 100, 1000),\n \"proc.impute_strategy\": space.Categorical(\"median\", \"mean\", \"most_frequent\"),\n \"proc.max_category_levels\": space.Categorical(100, 10, 20, 200, 300, 400, 500, 1000, 10000),\n \"proc.skew_threshold\": space.Categorical(0.99, 0.2, 0.3, 0.5, 0.8, 0.9, 0.999, 1.0, 10.0, 100.0),\n }\n pytorch_params = {\n \"use_batchnorm\": space.Categorical(False, True),\n \"num_layers\": space.Categorical(2, 3, 4),\n \"hidden_size\": space.Categorical(128, 256, 512),\n \"activation\": space.Categorical(\"relu\", \"elu\"),\n }\n params = merge_framework_params(framework=framework, shared_params=params, pytorch_params=pytorch_params)\n if problem_type == QUANTILE:\n problem_params = get_searchspace_quantile(framework)\n elif problem_type == BINARY:\n problem_params = get_searchspace_binary(framework)\n elif problem_type == MULTICLASS:\n problem_params = get_searchspace_multiclass(framework, num_classes=num_classes)\n elif problem_type == REGRESSION:\n problem_params = get_searchspace_regression(framework)\n params.update(problem_params)\n return params.copy()\n\n\ndef get_searchspace_multiclass(framework, num_classes):\n return {}\n\n\ndef get_searchspace_binary(framework):\n return {}\n\n\ndef get_searchspace_regression(framework):\n params = {\n \"weight_decay\": space.Real(1e-12, 1.0, default=1e-6, log=True),\n }\n pytorch_params = {\n \"activation\": space.Categorical(\"relu\", \"elu\", \"tanh\"),\n }\n return merge_framework_params(framework=framework, shared_params=params, pytorch_params=pytorch_params)\n\n\ndef get_searchspace_quantile(framework):\n if framework != \"pytorch\":\n raise ValueError(\"Only pytorch tabular neural network is currently supported for quantile regression.\")\n params = {\n \"activation\": space.Categorical(\"relu\", \"elu\", \"tanh\"),\n \"weight_decay\": space.Real(1e-12, 1.0, default=1e-6, log=True),\n \"gamma\": space.Real(0.1, 10.0, default=5.0),\n \"alpha\": space.Categorical(0.001, 0.01, 0.1, 1.0),\n }\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/torch/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/torch/tabular_nn_torch.py",
"content": "from __future__ import annotations\n\nimport io\nimport json\nimport logging\nimport os\nimport random\nimport time\nimport warnings\nfrom copy import deepcopy\nfrom typing import TYPE_CHECKING, Dict, Union\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT, S_TEXT_AS_CATEGORY, S_TEXT_NGRAM\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_torch\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\nfrom autogluon.core.hpo.constants import RAY_BACKEND\nfrom autogluon.core.metrics import Scorer\nfrom autogluon.core.models._utils import get_early_stopping_rounds\nfrom autogluon.core.models.abstract.abstract_nn_model import AbstractNeuralNetworkModel\nfrom autogluon.core.utils.early_stopping import AdaptiveES, NoES, SimpleES\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\n\nfrom ..compilers.native import TabularNeuralNetTorchNativeCompiler\nfrom ..compilers.onnx import TabularNeuralNetTorchOnnxCompiler\nfrom ..hyperparameters.parameters import get_default_param\nfrom ..hyperparameters.searchspaces import get_default_searchspace\nfrom ..utils.data_preprocessor import create_preprocessor, get_feature_arraycol_map, get_feature_type_map\nfrom ..utils.nn_architecture_utils import infer_y_range\n\nif TYPE_CHECKING:\n from .tabular_torch_dataset import TabularTorchDataset\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: QuantileTransformer in pipelines accounts for majority of online inference time\nclass TabularNeuralNetTorchModel(AbstractNeuralNetworkModel):\n \"\"\"\n PyTorch neural network models for classification/regression with tabular data.\n\n Extra hyperparameter options:\n ag.early_stop : int | str, default = \"default\"\n Specifies the early stopping rounds. Defaults to an adaptive strategy. Recommended to keep default.\n \"\"\"\n\n ag_key = \"NN_TORCH\"\n ag_name = \"NeuralNetTorch\"\n ag_priority = 25\n seed_name = \"seed_value\"\n\n # Constants used throughout this class:\n unique_category_str = (\n np.nan\n ) # string used to represent missing values and unknown categories for categorical features.\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self.feature_arraycol_map = None\n self.feature_type_map = None\n self.features_to_drop = [] # may change between different bagging folds. TODO: consider just removing these from self._features_internal\n self.processor = None # data processor\n self.num_dataloading_workers = None\n self._architecture_desc = None\n self.optimizer = None\n self.device = None\n self.max_batch_size = None\n self._num_cpus_infer = None\n\n def _set_default_params(self):\n \"\"\"Specifies hyperparameter values to use by default\"\"\"\n default_params = get_default_param(problem_type=self.problem_type, framework=\"pytorch\")\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n ignored_type_group_special=[S_TEXT_NGRAM, S_TEXT_AS_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n def _get_default_searchspace(self):\n return get_default_searchspace(problem_type=self.problem_type, framework=\"pytorch\")\n\n def _get_num_net_outputs(self):\n if self.problem_type in [MULTICLASS, SOFTCLASS]:\n return self.num_classes\n elif self.problem_type == BINARY:\n return 2\n elif self.problem_type == REGRESSION:\n return 1\n elif self.problem_type == QUANTILE:\n return len(self.quantile_levels)\n else:\n raise ValueError(f\"Unknown problem_type: {self.problem_type}\")\n\n def _get_device(self, num_gpus):\n import torch\n\n if num_gpus is not None and num_gpus >= 1:\n if torch.cuda.is_available():\n device = torch.device(\"cuda\")\n logger.log(15, \"Training on GPU (CUDA)\")\n if num_gpus > 1:\n logger.warning(\n f\"{self.__class__.__name__} not yet able to use more than 1 GPU. 'num_gpus' is set to >1, but we will be using only 1 GPU.\"\n )\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n device = torch.device(\"mps\")\n logger.log(15, \"Training on GPU (MPS - Apple Silicon)\")\n if num_gpus > 1:\n logger.warning(\n f\"{self.__class__.__name__} on Apple Silicon can only use 1 GPU (MPS). 'num_gpus' is set to >1, but we will be using only 1 GPU.\"\n )\n else:\n device = torch.device(\"cpu\")\n logger.log(15, \"Training on CPU\")\n else:\n device = torch.device(\"cpu\")\n logger.log(15, \"Training on CPU\")\n return device\n\n def _set_net_defaults(self, train_dataset, params):\n params = params.copy()\n y_range_extend = params.pop(\"y_range_extend\", None)\n \"\"\" Sets dataset-adaptive default values to use for our neural network \"\"\"\n if self.problem_type in [REGRESSION, QUANTILE]:\n if params[\"y_range\"] is None:\n params[\"y_range\"] = infer_y_range(\n y_vals=train_dataset.data_list[train_dataset.label_index], y_range_extend=y_range_extend\n )\n return params\n\n def _get_default_loss_function(self):\n import torch\n\n if self.problem_type == REGRESSION:\n return torch.nn.L1Loss() # or torch.nn.MSELoss()\n elif self.problem_type in [BINARY, MULTICLASS]:\n return torch.nn.CrossEntropyLoss()\n elif self.problem_type == SOFTCLASS:\n return torch.nn.KLDivLoss() # compares log-probability prediction vs probability target.\n\n @staticmethod\n def _prepare_params(params):\n params = params.copy()\n\n processor_param_keys = {\n \"proc.embed_min_categories\",\n \"proc.impute_strategy\",\n \"proc.max_category_levels\",\n \"proc.skew_threshold\",\n \"use_ngram_features\",\n }\n processor_kwargs = {k: v for k, v in params.items() if k in processor_param_keys}\n for key in processor_param_keys:\n params.pop(key, None)\n\n optimizer_param_keys = {\"optimizer\", \"learning_rate\", \"weight_decay\"}\n optimizer_kwargs = {k: v for k, v in params.items() if k in optimizer_param_keys}\n for key in optimizer_param_keys:\n params.pop(key, None)\n\n fit_param_keys = {\"num_epochs\", \"epochs_wo_improve\"}\n fit_kwargs = {k: v for k, v in params.items() if k in fit_param_keys}\n for key in fit_param_keys:\n params.pop(key, None)\n\n loss_param_keys = {\"loss_function\", \"gamma\"}\n loss_kwargs = {k: v for k, v in params.items() if k in loss_param_keys}\n for key in loss_param_keys:\n params.pop(key, None)\n\n return processor_kwargs, optimizer_kwargs, fit_kwargs, loss_kwargs, params\n\n def _fit(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame = None,\n y_val: pd.Series = None,\n X_test: pd.DataFrame = None,\n y_test: pd.Series = None,\n time_limit: float = None,\n sample_weight=None,\n num_cpus: int = 1,\n num_gpus: float = 0,\n reporter=None,\n verbosity: int = 2,\n **kwargs,\n ):\n try_import_torch()\n import torch\n\n torch.set_num_threads(num_cpus)\n\n start_time = time.time()\n\n params = self._get_model_params()\n\n processor_kwargs, optimizer_kwargs, fit_kwargs, loss_kwargs, params = self._prepare_params(params=params)\n\n seed_value = params.pop(self.seed_name, self.default_random_seed)\n\n self._num_cpus_infer = params.pop(\"_num_cpus_infer\", 1)\n if seed_value is not None: # Set seeds\n random.seed(seed_value)\n np.random.seed(seed_value)\n torch.manual_seed(seed_value)\n\n if sample_weight is not None: # TODO: support\n logger.log(\n 15,\n f\"sample_weight not yet supported for {self.__class__.__name__},\"\n \" this model will ignore them in training.\",\n )\n\n if num_cpus is not None:\n self.num_dataloading_workers = max(1, int(num_cpus / 2.0))\n else:\n self.num_dataloading_workers = 1\n if self.num_dataloading_workers == 1:\n self.num_dataloading_workers = (\n 0 # TODO: verify 0 is typically faster and uses less memory than 1 in pytorch\n )\n self.num_dataloading_workers = 0 # TODO: >0 crashes on MacOS\n self.max_batch_size = params.pop(\"max_batch_size\", 512)\n\n train_dataset = self._generate_dataset(X=X, y=y, train_params=processor_kwargs, is_train=True)\n if X_val is not None and y_val is not None:\n val_dataset = self._generate_dataset(X=X_val, y=y_val)\n else:\n val_dataset = None\n if X_test is not None and y_test is not None:\n test_dataset = self._generate_dataset(X=X_test, y=y_test)\n else:\n test_dataset = None\n\n batch_size = params.pop(\"batch_size\", None)\n if batch_size is None:\n batch_size = min(int(2 ** (3 + np.floor(np.log10(len(X))))), self.max_batch_size, len(X))\n\n logger.log(\n 15,\n f\"Training data for {self.__class__.__name__} has: \"\n f\"{train_dataset.num_examples} examples, {train_dataset.num_features} features \"\n f\"({len(train_dataset.feature_groups['vector'])} vector, {len(train_dataset.feature_groups['embed'])} embedding)\",\n )\n\n self.device = self._get_device(num_gpus=num_gpus)\n\n self._get_net(train_dataset, params=params)\n self.optimizer = self._init_optimizer(**optimizer_kwargs)\n\n if time_limit is not None:\n time_elapsed = time.time() - start_time\n time_limit_orig = time_limit\n time_limit = time_limit - time_elapsed\n\n # if 60% of time was spent preprocessing, likely not enough time to train model\n if time_limit <= time_limit_orig * 0.4:\n raise TimeLimitExceeded\n\n # train network\n self._train_net(\n train_dataset=train_dataset,\n loss_kwargs=loss_kwargs,\n batch_size=batch_size,\n val_dataset=val_dataset,\n test_dataset=test_dataset,\n time_limit=time_limit,\n reporter=reporter,\n verbosity=verbosity,\n **fit_kwargs,\n )\n\n def _get_net(self, train_dataset, params):\n from .torch_network_modules import EmbedNet\n\n # set network params\n params = self._set_net_defaults(train_dataset, params)\n self.model = EmbedNet(\n problem_type=self.problem_type,\n num_net_outputs=self._get_num_net_outputs(),\n quantile_levels=self.quantile_levels,\n train_dataset=train_dataset,\n device=self.device,\n **params,\n )\n self.model = self.model.to(self.device)\n if not os.path.exists(self.path):\n os.makedirs(self.path)\n\n def _train_net(\n self,\n train_dataset: TabularTorchDataset,\n loss_kwargs: dict,\n batch_size: int,\n num_epochs: int,\n epochs_wo_improve: int,\n val_dataset: TabularTorchDataset = None,\n test_dataset: TabularTorchDataset = None,\n time_limit: float = None,\n reporter=None,\n verbosity: int = 2,\n ):\n import torch\n\n start_time = time.time()\n logging.debug(\"initializing neural network...\")\n self.model.init_params()\n logging.debug(\"initialized\")\n train_dataloader = train_dataset.build_loader(batch_size, self.num_dataloading_workers, is_test=False)\n\n if (\n isinstance(loss_kwargs.get(\"loss_function\", \"auto\"), str)\n and loss_kwargs.get(\"loss_function\", \"auto\") == \"auto\"\n ):\n loss_kwargs[\"loss_function\"] = self._get_default_loss_function()\n if epochs_wo_improve is not None:\n early_stopping_method = SimpleES(patience=epochs_wo_improve)\n else:\n early_stopping_method = self._get_early_stopping_strategy(num_rows_train=len(train_dataset))\n\n ag_params = self._get_ag_params()\n generate_curves = ag_params.get(\"generate_curves\", False)\n\n if generate_curves:\n scorers = ag_params.get(\"curve_metrics\", [self.eval_metric])\n use_curve_metric_error = ag_params.get(\"use_error_for_curve_metrics\", False)\n metric_names = [scorer.name for scorer in scorers]\n\n train_curves = {metric.name: [] for metric in scorers}\n val_curves = {metric.name: [] for metric in scorers}\n test_curves = {metric.name: [] for metric in scorers}\n\n # make copy of train_dataset to avoid interfering with train_dataloader\n curve_train_dataset = deepcopy(train_dataset)\n y_train = curve_train_dataset.get_labels()\n if y_train.ndim == 2 and y_train.shape[1] == 1:\n y_train = y_train.flatten()\n\n if test_dataset is not None:\n y_test = test_dataset.get_labels()\n if y_test.ndim == 2 and y_test.shape[1] == 1:\n y_test = y_test.flatten()\n else:\n y_test = None\n\n if val_dataset is not None:\n y_val = val_dataset.get_labels()\n if y_val.ndim == 2 and y_val.shape[1] == 1:\n y_val = y_val.flatten()\n else:\n y_val = None\n\n if verbosity <= 1:\n verbose_eval = False\n else:\n verbose_eval = True\n\n logger.log(15, \"Neural network architecture:\")\n logger.log(15, str(self.model))\n\n io_buffer = None\n if num_epochs == 0:\n # use dummy training loop that stops immediately\n # useful for using NN just for data preprocessing / debugging\n logger.log(20, \"Not training Tabular Neural Network since num_updates == 0\")\n\n # for each batch\n for batch_idx, data_batch in enumerate(train_dataloader):\n if batch_idx > 0:\n break\n loss = self.model.compute_loss(data_batch, **loss_kwargs)\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()\n return\n\n # start training loop:\n logger.log(15, f\"Training tabular neural network for up to {num_epochs} epochs...\")\n total_updates = 0\n num_updates_per_epoch = max(round(len(train_dataset) / batch_size) + 1, 1)\n update_to_check_time = min(10, max(1, int(num_updates_per_epoch / 5)))\n do_update = True\n epoch = 0\n best_epoch = 0\n best_val_metric = -np.inf # higher = better\n best_val_update = 0\n start_fit_time = time.time()\n if time_limit is not None:\n time_limit = time_limit - (start_fit_time - start_time)\n if time_limit <= 0:\n raise TimeLimitExceeded\n while do_update:\n time_start_epoch = time.time()\n time_cur = time_start_epoch\n total_train_loss = 0.0\n total_train_size = 0.0\n for batch_idx, data_batch in enumerate(train_dataloader):\n # forward\n loss = self.model.compute_loss(data_batch, **loss_kwargs)\n total_train_loss += loss.item()\n total_train_size += 1\n\n # update\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()\n total_updates += 1\n\n # time limit\n if time_limit is not None:\n time_cur_tmp = time.time()\n time_elapsed_batch = time_cur_tmp - time_cur\n time_cur = time_cur_tmp\n update_cur = batch_idx + 1\n if epoch == 0 and update_cur == update_to_check_time:\n time_elapsed_epoch = time_cur - time_start_epoch\n\n # v1 estimate is sensitive to fixed cost overhead at the start of training, such as torch initialization.\n # v2 fixes this, but we keep both and take the min to avoid potential cases where v2 is inaccurate due to an overly slow batch.\n estimated_time_v1 = (\n time_elapsed_epoch / update_cur * num_updates_per_epoch\n ) # Less accurate than v2, but never underestimates time\n estimated_time_v2 = time_elapsed_epoch + time_elapsed_batch * (\n num_updates_per_epoch - update_cur\n ) # Less likely to overestimate time\n estimated_time = min(estimated_time_v1, estimated_time_v2)\n if estimated_time > time_limit:\n logger.log(\n 30,\n f\"\\tNot enough time to train first epoch. \"\n f\"(Time Required: {round(estimated_time, 2)}s, Time Left: {round(time_limit, 2)}s)\",\n )\n raise TimeLimitExceeded\n time_elapsed = time_cur - start_fit_time\n if time_limit < time_elapsed:\n if epoch == 0:\n logger.log(\n 30,\n f\"\\tNot enough time to train first epoch. Stopped on Update {total_updates} (Epoch {epoch}))\",\n )\n raise TimeLimitExceeded\n logger.log(\n 15,\n f\"\\tRan out of time, stopping training early. (Stopped on Update {total_updates} (Epoch {epoch}))\",\n )\n do_update = False\n break\n\n if not do_update:\n break\n\n epoch += 1\n\n # learning curve generation\n if generate_curves:\n stop = self._generate_curves(\n train_curves=train_curves,\n val_curves=val_curves,\n test_curves=test_curves,\n scorers=scorers,\n best_epoch=best_epoch,\n use_curve_metric_error=use_curve_metric_error,\n train_dataset=curve_train_dataset,\n val_dataset=val_dataset,\n test_dataset=test_dataset,\n y_train=y_train,\n y_val=y_val,\n y_test=y_test,\n )\n\n if stop:\n break\n\n # validation\n if val_dataset is not None:\n is_best = False\n # compute validation score\n val_metric = self.score(X=val_dataset, y=y_val, metric=self.stopping_metric, _reset_threads=False)\n if not self._assert_valid_metric(metric=val_metric, best_epoch=best_epoch):\n break\n\n # update best validation\n if (val_metric >= best_val_metric) or best_epoch == 0:\n if val_metric > best_val_metric:\n is_best = True\n best_val_metric = val_metric\n io_buffer = io.BytesIO()\n torch.save(self.model.state_dict(), io_buffer)\n best_epoch = epoch\n best_val_update = total_updates\n early_stop = early_stopping_method.update(cur_round=epoch - 1, is_best=is_best)\n if verbose_eval:\n logger.log(\n 15,\n f\"Epoch {epoch} (Update {total_updates}).\\t\"\n f\"Train loss: {round(total_train_loss / total_train_size, 4)}, \"\n f\"Val {self.stopping_metric.name}: {round(val_metric, 4)}, \"\n f\"Best Epoch: {best_epoch}\",\n )\n\n if reporter is not None:\n reporter(\n epoch=total_updates,\n validation_performance=val_metric, # Higher val_metric = better\n train_loss=total_train_loss / total_train_size,\n eval_metric=self.eval_metric.name,\n greater_is_better=self.eval_metric.greater_is_better,\n )\n\n # no improvement\n if early_stop:\n break\n\n if epoch >= num_epochs:\n break\n\n if time_limit is not None:\n time_elapsed = time.time() - start_fit_time\n time_epoch_average = time_elapsed / max(epoch, 1) # avoid divide by 0\n time_left = time_limit - time_elapsed\n if time_left < time_epoch_average:\n logger.log(20, f\"\\tRan out of time, stopping training early. (Stopping on epoch {epoch})\")\n break\n\n if epoch == 0:\n raise AssertionError(\"0 epochs trained!\")\n\n if generate_curves:\n curves = {\"train\": train_curves}\n if val_dataset is not None:\n curves[\"val\"] = val_curves\n if test_dataset is not None:\n curves[\"test\"] = test_curves\n self.save_learning_curves(metrics=metric_names, curves=curves)\n\n # revert back to best model\n if val_dataset is not None:\n logger.log(\n 15,\n f\"Best model found on Epoch {best_epoch} (Update {best_val_update}). Val {self.stopping_metric.name}: {best_val_metric}\",\n )\n if io_buffer is not None:\n io_buffer.seek(0)\n self.model.load_state_dict(torch.load(io_buffer, weights_only=True))\n else:\n logger.log(15, f\"Best model found on Epoch {best_epoch} (Update {best_val_update}).\")\n self.params_trained[\"batch_size\"] = batch_size\n self.params_trained[\"num_epochs\"] = best_epoch\n\n def _get_early_stopping_strategy(self, num_rows_train: int):\n ag_early_stop = self._get_ag_params().get(\"early_stop\", \"default\")\n if ag_early_stop is None:\n early_stopping_method = NoES()\n elif isinstance(ag_early_stop, str) and ag_early_stop == \"default\":\n early_stopping_method = self._get_early_stop_default()\n elif isinstance(ag_early_stop, (str, tuple, list)):\n early_stopping_rounds = self._get_early_stopping_rounds(\n num_rows_train=num_rows_train, strategy=ag_early_stop\n )\n early_stopping_method = early_stopping_rounds[0](**early_stopping_rounds[1])\n elif isinstance(ag_early_stop, int):\n early_stopping_method = SimpleES(patience=ag_early_stop)\n else:\n raise ValueError(f\"Invalid `ag.early_stop` value specified: `{ag_early_stop}`\")\n return early_stopping_method\n\n def _get_early_stop_default(self):\n return AdaptiveES(adaptive_rate=0.5, adaptive_offset=20)\n\n def _get_early_stopping_rounds(self, num_rows_train, strategy=\"auto\"):\n return get_early_stopping_rounds(num_rows_train=num_rows_train, strategy=strategy)\n\n def _generate_curves(\n self,\n train_curves: dict,\n val_curves: dict,\n test_curves: dict,\n scorers: list[Scorer],\n best_epoch: int,\n use_curve_metric_error: bool,\n train_dataset: \"TabularTorchDataset\",\n val_dataset: \"TabularTorchDataset\",\n test_dataset: \"TabularTorchDataset\",\n y_train: np.ndarray,\n y_val: np.ndarray,\n y_test: np.ndarray,\n ) -> bool:\n \"\"\"\n Extends learning curve dictionaries across all metrics listed in scorers by one epoch.\n\n Returns:\n --------\n bool:\n Whether to break out of the neural net training loop.\n \"\"\"\n train_metrics = []\n val_metrics = []\n test_metrics = []\n\n for metric in scorers:\n train_metrics.append(self.score(X=train_dataset, y=y_train, metric=metric, _reset_threads=False))\n val_metrics += (\n [self.score(X=val_dataset, y=y_val, metric=metric, _reset_threads=False)]\n if val_dataset is not None\n else []\n )\n test_metrics += (\n [self.score(X=test_dataset, y=y_test, metric=metric, _reset_threads=False)]\n if test_dataset is not None\n else []\n )\n\n if use_curve_metric_error:\n train_metrics[-1] = metric.convert_score_to_error(train_metrics[-1])\n if val_dataset is not None:\n val_metrics[-1] = metric.convert_score_to_error(val_metrics[-1])\n if test_dataset is not None:\n test_metrics[-1] = metric.convert_score_to_error(test_metrics[-1])\n\n if (\n not self._assert_valid_metric(metric=train_metrics[-1], best_epoch=best_epoch)\n or (\n val_dataset is not None\n and not self._assert_valid_metric(metric=val_metrics[-1], best_epoch=best_epoch)\n )\n or (\n test_dataset is not None\n and not self._assert_valid_metric(metric=test_metrics[-1], best_epoch=best_epoch)\n )\n ):\n return True\n\n # update learning curve\n for i, metric in enumerate(scorers):\n train_curves[metric.name].append(float(train_metrics[i]))\n val_curves[metric.name] += [float(val_metrics[i])] if val_dataset is not None else []\n test_curves[metric.name] += [float(test_metrics[i])] if test_dataset is not None else []\n\n return False\n\n def _assert_valid_metric(self, metric: int | float, best_epoch: int) -> bool:\n \"\"\"\n Asserts that metric calculated is valid.\n\n Parameters:\n -----------\n metric: int or float\n the metric calculated\n best_epoch: int\n the best epoch encountered since training started\n\n Returns:\n --------\n Whether the metric is valid\n \"\"\"\n if np.isnan(metric):\n if best_epoch == 0:\n raise RuntimeError(\n f\"NaNs encountered in {self.__class__.__name__} training. \"\n \"Features/labels may be improperly formatted, \"\n \"or NN weights may have diverged.\"\n )\n else:\n logger.warning(\n f\"Warning: NaNs encountered in {self.__class__.__name__} training. \"\n \"Reverting model to last checkpoint without NaNs.\"\n )\n return False\n return True\n\n def _predict_proba(self, X, **kwargs):\n \"\"\"To align predict with abstract_model API.\n Preprocess here only refers to feature processing steps done by all AbstractModel objects,\n not tabularNN-specific preprocessing steps.\n If X is not DataFrame but instead TabularNNDataset object, we can still produce predictions,\n but cannot use preprocess in this case (needs to be already processed).\n \"\"\"\n from .tabular_torch_dataset import TabularTorchDataset\n\n if isinstance(X, TabularTorchDataset):\n return self._predict_tabular_data(new_data=X, process=False)\n elif isinstance(X, pd.DataFrame):\n X = self.preprocess(X, **kwargs)\n return self._predict_tabular_data(new_data=X, process=True)\n else:\n raise ValueError(\"X must be of type pd.DataFrame or TabularTorchDataset, not type: %s\" % type(X))\n\n def _predict_tabular_data(self, new_data, process=True):\n from .tabular_torch_dataset import TabularTorchDataset\n\n if process:\n new_data = self._process_test_data(new_data)\n if not isinstance(new_data, TabularTorchDataset):\n raise ValueError(\"new_data must of of type TabularTorchDataset if process=False\")\n val_dataloader = new_data.build_loader(self.max_batch_size, self.num_dataloading_workers, is_test=True)\n preds_dataset = []\n for data_batch in val_dataloader:\n preds_batch = self.model.predict(data_batch)\n preds_dataset.append(preds_batch)\n preds_dataset = np.concatenate(preds_dataset, 0)\n return preds_dataset\n\n def _generate_dataset(\n self, X: pd.DataFrame | TabularTorchDataset, y: pd.Series, train_params: dict = {}, is_train: bool = False\n ) -> TabularTorchDataset:\n \"\"\"\n Generate TabularTorchDataset from X and y.\n\n Params:\n -------\n X: pd.DataFrame | TabularTorchDataset\n The X data.\n y: pd.Series\n The y data.\n params: dict\n Parameters related to processing training data.\n is_train: bool\n Whether the X and y values are training data.\n\n Returns:\n --------\n TabularTorchDataset containing the contents of X and y.\n \"\"\"\n from .tabular_torch_dataset import TabularTorchDataset\n\n if is_train:\n impute_strategy = train_params[\"proc.impute_strategy\"]\n max_category_levels = train_params[\"proc.max_category_levels\"]\n skew_threshold = train_params[\"proc.skew_threshold\"]\n embed_min_categories = train_params[\"proc.embed_min_categories\"]\n use_ngram_features = train_params[\"use_ngram_features\"]\n\n if isinstance(X, TabularTorchDataset):\n dataset = X\n else:\n X = self.preprocess(X, y=y)\n dataset = self._process_train_data(\n df=X,\n labels=y,\n impute_strategy=impute_strategy,\n max_category_levels=max_category_levels,\n skew_threshold=skew_threshold,\n embed_min_categories=embed_min_categories,\n use_ngram_features=use_ngram_features,\n )\n else:\n if isinstance(X, TabularTorchDataset):\n dataset = X\n else:\n X = self.preprocess(X)\n dataset = self._process_test_data(df=X, labels=y)\n\n return dataset\n\n def _process_test_data(self, df, labels=None):\n \"\"\"Process train or test DataFrame into a form fit for neural network models.\n Args:\n df (pd.DataFrame): Data to be processed (X)\n labels (pd.Series): labels to be processed (y)\n Returns:\n Dataset object\n \"\"\"\n from .tabular_torch_dataset import TabularTorchDataset\n\n # sklearn processing n_quantiles warning\n warnings.filterwarnings(\"ignore\", module=\"sklearn.preprocessing\")\n if labels is not None and len(labels) != len(df):\n raise ValueError(\"Number of examples in Dataframe does not match number of labels\")\n if (\n self.processor is None\n or self._types_of_features is None\n or self.feature_arraycol_map is None\n or self.feature_type_map is None\n ):\n raise ValueError(\"Need to process training data before test data\")\n if self.features_to_drop:\n drop_cols = [col for col in df.columns if col in self.features_to_drop]\n if drop_cols:\n df = df.drop(columns=drop_cols)\n\n # self.feature_arraycol_map, self.feature_type_map have been previously set while processing training data.\n df = self.processor.transform(df)\n return TabularTorchDataset(df, self.feature_arraycol_map, self.feature_type_map, self.problem_type, labels)\n\n def _process_train_data(\n self,\n df,\n impute_strategy,\n max_category_levels,\n skew_threshold,\n embed_min_categories,\n use_ngram_features,\n labels,\n ):\n from .tabular_torch_dataset import TabularTorchDataset\n\n # sklearn processing n_quantiles warning\n warnings.filterwarnings(\"ignore\", module=\"sklearn.preprocessing\")\n if labels is None:\n raise ValueError(\"Attempting process training data without labels\")\n if len(labels) != len(df):\n raise ValueError(\"Number of examples in Dataframe does not match number of labels\")\n\n # dict with keys: : 'continuous', 'skewed', 'onehot', 'embed', values = column-names of df\n self._types_of_features, df = self._get_types_of_features(\n df,\n skew_threshold=skew_threshold,\n embed_min_categories=embed_min_categories,\n use_ngram_features=use_ngram_features,\n )\n logger.log(15, \"Tabular Neural Network treats features as the following types:\")\n logger.log(15, json.dumps(self._types_of_features, indent=4))\n logger.log(15, \"\\n\")\n if self.processor is not None:\n Warning(\n f\"Attempting to process training data for {self.__class__.__name__}, but previously already did this.\"\n )\n self.processor = create_preprocessor(\n impute_strategy=impute_strategy,\n max_category_levels=max_category_levels,\n unique_category_str=self.unique_category_str,\n continuous_features=self._types_of_features[\"continuous\"],\n skewed_features=self._types_of_features[\"skewed\"],\n onehot_features=self._types_of_features[\"onehot\"],\n embed_features=self._types_of_features[\"embed\"],\n bool_features=self._types_of_features[\"bool\"],\n )\n df = self.processor.fit_transform(df)\n # OrderedDict of feature-name -> list of column-indices in df corresponding to this feature\n self.feature_arraycol_map = get_feature_arraycol_map(\n processor=self.processor, max_category_levels=max_category_levels\n )\n num_array_cols = np.sum(\n [len(self.feature_arraycol_map[key]) for key in self.feature_arraycol_map]\n ) # should match number of columns in processed array\n if num_array_cols != df.shape[1]:\n raise ValueError(\n \"Error during one-hot encoding data processing for neural network. \"\n \"Number of columns in df array does not match feature_arraycol_map.\"\n )\n\n # OrderedDict of feature-name -> feature_type string (options: 'vector', 'embed')\n self.feature_type_map = get_feature_type_map(\n feature_arraycol_map=self.feature_arraycol_map, types_of_features=self._types_of_features\n )\n return TabularTorchDataset(df, self.feature_arraycol_map, self.feature_type_map, self.problem_type, labels)\n\n def _init_optimizer(self, optimizer, learning_rate, weight_decay):\n \"\"\"\n Set up optimizer needed for training.\n Network must first be initialized before this.\n \"\"\"\n import torch\n\n if optimizer == \"sgd\":\n optimizer = torch.optim.SGD(params=self.model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n elif optimizer == \"adam\":\n optimizer = torch.optim.Adam(params=self.model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n elif optimizer == \"adamw\":\n optimizer = torch.optim.AdamW(params=self.model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n else:\n raise ValueError(f\"Unknown optimizer specified: {optimizer}\")\n return optimizer\n\n def reduce_memory_size(self, remove_fit=True, requires_save=True, **kwargs):\n super().reduce_memory_size(remove_fit=remove_fit, requires_save=requires_save, **kwargs)\n if remove_fit and requires_save:\n self.optimizer = None\n\n def _get_default_stopping_metric(self):\n return self.eval_metric\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n **kwargs,\n ) -> int:\n return 5 * get_approximate_df_mem_usage(X).sum()\n\n def _get_maximum_resources(self) -> Dict[str, Union[int, float]]:\n # torch model trains slower when utilizing virtual cores and this issue scale up when the number of cpu cores increases\n return {\"num_cpus\": ResourceManager.get_cpu_count(only_physical_cores=True)}\n\n def _get_default_resources(self):\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n def save(self, path: str = None, verbose=True) -> str:\n import torch\n\n # Save on CPU to ensure the model can be loaded on a box without GPU\n if self.model is not None:\n self.model = self.model.to(torch.device(\"cpu\"))\n path = super().save(path, verbose)\n # Put the model back to the device after the save\n if self.model is not None:\n self.model.to(self.device)\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True):\n \"\"\"\n Loads the model from disk to memory.\n The loaded model will be on the same device it was trained on (cuda/mps);\n if the device is it's not available (trained on GPU, deployed on CPU),\n then `cpu` will be used.\n\n Parameters\n ----------\n path : str\n Path to the saved model, minus the file name.\n This should generally be a directory path ending with a '/' character (or appropriate path separator value depending on OS).\n The model file is typically located in os.path.join(path, cls.model_file_name).\n reset_paths : bool, default True\n Whether to reset the self.path value of the loaded model to be equal to path.\n It is highly recommended to keep this value as True unless accessing the original self.path value is important.\n If False, the actual valid path and self.path may differ, leading to strange behaviour and potential exceptions if the model needs to load any other files at a later time.\n verbose : bool, default True\n Whether to log the location of the loaded file.\n\n Returns\n -------\n model : cls\n Loaded model object.\n \"\"\"\n import torch\n\n model: TabularNeuralNetTorchModel = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n\n # Put the model on the same device it was train on (GPU/MPS) if it is available; otherwise use CPU\n if model.model is not None:\n original_device_type = model.device.type\n if \"cuda\" in original_device_type:\n # cuda: nvidia GPU\n device = torch.device(original_device_type if torch.cuda.is_available() else \"cpu\")\n elif \"mps\" in original_device_type:\n # mps: Apple Silicon\n device = torch.device(original_device_type if torch.backends.mps.is_available() else \"cpu\")\n else:\n device = torch.device(original_device_type)\n\n if verbose and (original_device_type != device.type):\n logger.log(\n 15,\n f\"Model is trained on {original_device_type}, but the device is not available - loading on {device.type}\",\n )\n\n model.device = device\n model.model = model.model.to(model.device)\n model.model.device = model.device\n\n # Compiled models handling\n if hasattr(model, \"_compiler\") and model._compiler and model._compiler.name != \"native\":\n model.model.eval()\n model.processor = model._compiler.load(path=model.path)\n return model\n\n def _get_hpo_backend(self):\n \"\"\"Choose which backend(Ray or Custom) to use for hpo\"\"\"\n return RAY_BACKEND\n\n def get_minimum_resources(self, is_gpu_available=False):\n minimum_resources = {\n \"num_cpus\": 1,\n }\n if is_gpu_available:\n # Our custom implementation does not support partial GPU. No gpu usage according to nvidia-smi when the `num_gpus` passed to fit is fractional`\n minimum_resources[\"num_gpus\"] = 1\n return minimum_resources\n\n @classmethod\n def _valid_compilers(cls):\n return [TabularNeuralNetTorchNativeCompiler, TabularNeuralNetTorchOnnxCompiler]\n\n @classmethod\n def _default_compiler(cls):\n return TabularNeuralNetTorchNativeCompiler\n\n def _ag_params(self) -> set:\n return {\"early_stop\", \"generate_curves\", \"curve_metrics\", \"use_error_for_curve_metrics\"}\n\n def _get_input_types(self, batch_size=None):\n input_types = []\n for f in self._features:\n input_types.append((f, [batch_size, 1]))\n return input_types\n\n def compile(self, compiler_configs=None):\n \"\"\"\n Compile the trained model for faster inference.\n\n This completely overrides the compile() in AbstractModel, since we won't\n overwrite self.model in the compilation process.\n Instead, self.processor would be converted from sklearn ColumnTransformer\n to its alternative counterpart.\n \"\"\"\n assert self.is_fit(), \"The model must be fit before calling the compile method.\"\n if compiler_configs is None:\n compiler_configs = {}\n # Take self.max_batch_size as default batch size, instead of None in AbstractModel\n batch_size = compiler_configs.get(\"batch_size\", self.max_batch_size)\n compiler_configs.update(batch_size=batch_size)\n super().compile(compiler_configs)\n\n def _compile(self, **kwargs):\n \"\"\"\n Take the compiler to perform actual compilation.\n\n This overrides the _compile() in AbstractModel, since we won't\n overwrite self.model in the compilation process.\n Instead, self.processor would be converted from sklearn ColumnTransformer\n to TabularNeuralNetTorchOnnxTransformer.\n \"\"\"\n from sklearn.compose._column_transformer import ColumnTransformer\n\n input_types = kwargs.get(\"input_types\", self._get_input_types(batch_size=self.max_batch_size))\n assert isinstance(self.processor, ColumnTransformer), (\n f\"unexpected processor type {type(self.processor)}, \"\n \"expecting processor type to be sklearn.compose._column_transformer.ColumnTransformer\"\n )\n self.processor = self._compiler.compile(\n model=(self.processor, self.model), path=self.path, input_types=input_types\n )\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\", \"softclass\"]\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"supports_learning_curves\": True,\n }\n\n def _more_tags(self):\n # `can_refit_full=True` because batch_size and num_epochs is communicated at end of `_fit`:\n # self.params_trained['batch_size'] = batch_size\n # self.params_trained['num_epochs'] = best_epoch\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/torch/tabular_torch_dataset.py",
"content": "import logging\nimport os\nimport random\n\nimport numpy as np\nimport torch\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\n\nlogger = logging.getLogger(__name__)\n\n\nclass TabularTorchDataset(torch.utils.data.IterableDataset):\n \"\"\"\n This class follows the structure of TabularNNDataset in tabular_nn_dataset.py,\n\n Class for preprocessing & storing/feeding data batches used by pytorch neural networks for tabular data.\n Assumes entire dataset can be loaded into numpy arrays.\n Original data table may contain numeric and categorical fields and missing values.\n\n Attributes:\n data_list (list[np.array]): Contains the raw data. Different indices in this list correspond to different\n types of inputs to the neural network (each is 2D array). All vector-valued\n (continuous & one-hot) features are concatenated together into a single index\n of the dataset.\n data_desc (list[str]): Describes the data type of each index of dataset\n (options: 'vector','embed_')\n embed_indices (list): which columns in dataset correspond to embed features (order matters!)\n vecfeature_col_map (dict): maps vector_feature_name -> columns of dataset._data[vector] array that\n contain the data for this feature\n feature_dataindex_map (dict): maps feature_name -> i such that dataset._data[i] = data array for\n this feature. Cannot be used for vector-valued features,\n instead use vecfeature_col_map\n feature_groups (dict): maps feature_type (ie. 'vector' or 'embed') to list of feature\n names of this type (empty list if there are no features of this type)\n vectordata_index (int): describes which element of the dataset._data list holds the vector data matrix\n (access via self.data_list[self.vectordata_index]); None if no vector features\n label_index (int): describing which element of the dataset._data list holds labels\n (access via self.data_list[self.label_index]); None if no labels\n num_categories_per_embedfeature (list): Number of categories for each embedding feature (order matters!)\n num_examples (int): number of examples in this dataset\n num_features (int): number of features (we only consider original variables as features, so num_features\n may not correspond to dimensionality of the data eg in the case of one-hot encoding)\n Note: Default numerical data-type is converted to float32.\n \"\"\"\n\n # hard-coded names for files. This file contains pickled torch.util.data.Dataset object\n DATAOBJ_SUFFIX = \"_tabdataset_torch.pt\"\n\n def __init__(self, processed_array, feature_arraycol_map, feature_type_map, problem_type, labels=None):\n \"\"\"Args:\n processed_array: 2D numpy array returned by preprocessor. Contains raw data of all features as columns\n feature_arraycol_map (OrderedDict): Mapsfeature-name -> list of column-indices in processed_array\n corresponding to this feature\n feature_type_map (OrderedDict): Maps feature-name -> feature_type string\n (options: 'vector', 'embed')\n problem_type (str): what prediction task this data is used for.\n labels (pd.Series): list of labels (y) if available\n \"\"\"\n self.problem_type = problem_type\n self.num_examples = processed_array.shape[0]\n self.num_features = len(feature_arraycol_map)\n if feature_arraycol_map.keys() != feature_type_map.keys():\n raise ValueError(\"feature_arraycol_map and feature_type_map must share same keys\")\n self.feature_groups = {\"vector\": [], \"embed\": []}\n self.feature_type_map = feature_type_map\n for feature in feature_type_map:\n if feature_type_map[feature] == \"vector\":\n self.feature_groups[\"vector\"].append(feature)\n elif feature_type_map[feature] == \"embed\":\n self.feature_groups[\"embed\"].append(feature)\n else:\n raise ValueError(\"unknown feature type: %s\" % feature)\n if labels is not None and len(labels) != self.num_examples:\n raise ValueError(\"number of labels and training examples do not match\")\n\n self.data_desc = []\n self.data_list = []\n self.label_index = None\n self.vectordata_index = None\n self.vecfeature_col_map = {}\n self.feature_dataindex_map = {}\n self.num_classes = None\n\n # numerical data\n if len(self.feature_groups[\"vector\"]) > 0:\n vector_inds = []\n for feature in feature_type_map:\n if feature_type_map[feature] == \"vector\":\n current_last_ind = len(vector_inds)\n vector_inds += feature_arraycol_map[feature]\n new_last_ind = len(vector_inds)\n self.vecfeature_col_map[feature] = list(range(current_last_ind, new_last_ind))\n self.data_list.append(processed_array[:, vector_inds].astype(\"float32\"))\n self.data_desc.append(\"vector\")\n self.vectordata_index = len(self.data_list) - 1\n\n # embedding data\n if len(self.feature_groups[\"embed\"]) > 0:\n for feature in feature_type_map:\n if feature_type_map[feature] == \"embed\":\n feature_colind = feature_arraycol_map[feature]\n self.data_list.append(processed_array[:, feature_colind].astype(\"int64\").flatten())\n self.data_desc.append(\"embed\")\n self.feature_dataindex_map[feature] = len(self.data_list) - 1\n\n # output (target) data\n if labels is not None:\n labels = np.array(labels)\n self.data_desc.append(\"label\")\n self.label_index = len(self.data_list)\n if self.problem_type == SOFTCLASS:\n self.num_classes = labels.shape[1]\n self.data_list.append(labels.astype(\"float32\"))\n else:\n if self.problem_type in [REGRESSION, QUANTILE] and labels.dtype != np.float32:\n labels = labels.astype(\"float32\") # Convert to proper float-type if not already\n elif self.problem_type in [BINARY, MULTICLASS]:\n self.num_classes = len(set(labels))\n labels = labels.astype(\"long\")\n self.data_list.append(labels.reshape(-1, 1))\n\n self.embed_indices = [i for i in range(len(self.data_desc)) if \"embed\" in self.data_desc[i]]\n self.num_categories_per_embed_feature = None\n self.num_categories_per_embedfeature = self.getNumCategoriesEmbeddings()\n\n self.has_vector_features = self.vectordata_index is not None\n self.has_embed_features = len(self.feature_groups[\"embed\"]) > 0\n\n def __iter__(self):\n \"\"\"\n Iterate through the iterable dataset, and return a subsample of it.\n\n This overrides the `__iter__` function in IterableDataset.\n This is typically useful when we are using :class:`torch.utils.data.DataLoader` to\n load the dataset.\n\n Returns a tuple containing (vector_features, embed_features, label).\n The length of the tuple depends on `has_vector_features` and `has_embed_features` attribute.\n \"\"\"\n idxarray = np.arange(self.num_examples)\n if self.shuffle:\n np.random.shuffle(idxarray)\n indices = range(0, self.num_examples, self.batch_size)\n for idx_start in indices:\n # Drop last batch\n if self.drop_last and (idx_start + self.batch_size) > self.num_examples:\n break\n idx = range(idx_start, min(self.num_examples, idx_start + self.batch_size))\n\n # Shuffle the index array to reorder the output sequence.\n # This should be consistent across different features (vector, embed and label).\n if self.shuffle:\n idx = idxarray[idx]\n\n # Generate a tuple that contains (vector_features, embed_features, label).\n # The length of the tuple depends on `has_vector_features`, `has_embed_features`, and\n # whether the label has been provided.\n output_list = []\n if self.has_vector_features:\n output_list.append(self.data_list[self.vectordata_index][idx])\n if self.has_embed_features:\n output_embed = []\n for i in self.embed_indices:\n output_embed.append(self.data_list[i][idx])\n output_list.append(output_embed)\n if self.label_index is not None:\n output_list.append(self.data_list[self.label_index][idx])\n yield tuple(output_list)\n\n def __len__(self):\n return self.num_examples\n\n def has_vector_features(self):\n \"\"\"Returns boolean indicating whether this dataset contains vector features\"\"\"\n return self.vectordata_index is not None\n\n def num_embed_features(self):\n \"\"\"Returns number of embed features in this dataset\"\"\"\n return len(self.feature_groups[\"embed\"])\n\n def num_vector_features(self):\n \"\"\"Number of vector features (each onehot feature counts = 1, regardless of how many categories)\"\"\"\n return len(self.feature_groups[\"vector\"])\n\n def get_labels(self):\n \"\"\"Returns numpy array of labels for this dataset\"\"\"\n if self.label_index is not None:\n return self.data_list[self.label_index]\n else:\n return None\n\n def getNumCategoriesEmbeddings(self):\n \"\"\"Returns number of categories for each embedding feature.\n Should only be applied to training data.\n If training data feature contains unique levels 1,...,n-1, there are actually n categories,\n since category n is reserved for unknown test-time categories.\n \"\"\"\n if self.num_categories_per_embed_feature is not None:\n return self.num_categories_per_embedfeature\n else:\n num_embed_feats = self.num_embed_features()\n num_categories_per_embedfeature = [0] * num_embed_feats\n for i in range(num_embed_feats):\n feat_i = self.feature_groups[\"embed\"][i]\n feat_i_data = self.get_feature_data(feat_i)\n num_categories_i = len(np.unique(feat_i_data)) # number of categories for ith feature\n num_categories_per_embedfeature[i] = (\n num_categories_i + 1\n ) # to account for unknown test-time categories\n return num_categories_per_embedfeature\n\n def get_feature_data(self, feature):\n \"\"\"Returns all data for this feature.\n Args:\n feature (str): name of feature of interest (in processed dataframe)\n \"\"\"\n nonvector_featuretypes = set([\"embed\"])\n if feature not in self.feature_type_map:\n raise ValueError(\"unknown feature encountered: %s\" % feature)\n if self.feature_type_map[feature] == \"vector\":\n vector_datamatrix = self.data_list[self.vectordata_index]\n feature_data = vector_datamatrix[:, self.vecfeature_col_map[feature]]\n elif self.feature_type_map[feature] in nonvector_featuretypes:\n feature_idx = self.feature_dataindex_map[feature]\n feature_data = self.data_list[feature_idx]\n else:\n raise ValueError(\"Unknown feature specified: \" % feature)\n return feature_data\n\n def save(self, file_prefix=\"\"):\n \"\"\"Additional naming changes will be appended to end of file_prefix (must contain full absolute path)\"\"\"\n dataobj_file = file_prefix + self.DATAOBJ_SUFFIX\n if not os.path.exists(os.path.dirname(dataobj_file)):\n os.makedirs(os.path.dirname(dataobj_file))\n torch.save(self, dataobj_file) # nosec B614\n logger.debug(\"TabularPyTorchDataset Dataset saved to a file: \\n %s\" % dataobj_file)\n\n @classmethod\n def load(cls, file_prefix=\"\"):\n \"\"\"Additional naming changes will be appended to end of file_prefix (must contain full absolute path)\"\"\"\n dataobj_file = file_prefix + cls.DATAOBJ_SUFFIX\n dataset: TabularTorchDataset = torch.load(dataobj_file) # nosec B614\n logger.debug(\"TabularNN Dataset loaded from a file: \\n %s\" % dataobj_file)\n return dataset\n\n def build_loader(self, batch_size, num_workers, is_test=False):\n # See https://pytorch.org/docs/stable/notes/randomness.html\n def worker_init_fn(worker_id):\n if is_test:\n worker_seed = torch.initial_seed() % 2**32\n np.random.seed(worker_seed)\n random.seed(worker_seed)\n else:\n np.random.seed(np.random.get_state()[1][0] + worker_id)\n\n self.batch_size = batch_size\n self.shuffle = False if is_test else True\n self.drop_last = False if is_test else True\n generator = torch.Generator().manual_seed(torch.initial_seed()) if is_test else None\n loader = torch.utils.data.DataLoader(\n self, num_workers=num_workers, batch_size=None, worker_init_fn=worker_init_fn, generator=generator\n ) # no collation\n return loader\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/torch/torch_network_modules.py",
"content": "import logging\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION, SOFTCLASS\n\nfrom ..utils.nn_architecture_utils import get_embed_sizes\n\nlogger = logging.getLogger(__name__)\n\n\nclass EmbedNet(nn.Module):\n \"\"\"\n y_range: Used specifically for regression. = None for classification.\n \"\"\"\n\n def __init__(\n self,\n problem_type,\n num_net_outputs=None,\n quantile_levels=None,\n train_dataset=None,\n architecture_desc=None,\n device=None,\n **kwargs,\n ):\n if (architecture_desc is None) and (train_dataset is None):\n raise ValueError(\"train_dataset cannot = None if architecture_desc=None\")\n super().__init__()\n self.problem_type = problem_type\n if self.problem_type == QUANTILE:\n self.register_buffer(\"quantile_levels\", torch.Tensor(quantile_levels).float().reshape(1, -1))\n self.device = torch.device(\"cpu\") if device is None else device\n if architecture_desc is None:\n params = self._set_params(**kwargs)\n # adpatively specify network architecture based on training dataset\n self.from_logits = False\n self.has_vector_features = train_dataset.has_vector_features\n self.has_embed_features = train_dataset.has_embed_features\n if self.has_embed_features:\n num_categs_per_feature = train_dataset.getNumCategoriesEmbeddings()\n embed_dims = get_embed_sizes(train_dataset, params, num_categs_per_feature)\n if self.has_vector_features:\n vector_dims = train_dataset.data_list[train_dataset.vectordata_index].shape[-1]\n else:\n # ignore train_dataset, params, etc. Recreate architecture based on description:\n self.architecture_desc = architecture_desc\n self.has_vector_features = architecture_desc[\"has_vector_features\"]\n self.has_embed_features = architecture_desc[\"has_embed_features\"]\n self.from_logits = architecture_desc[\"from_logits\"]\n params = architecture_desc[\"params\"]\n if self.has_embed_features:\n num_categs_per_feature = architecture_desc[\"num_categs_per_feature\"]\n embed_dims = architecture_desc[\"embed_dims\"]\n if self.has_vector_features:\n vector_dims = architecture_desc[\"vector_dims\"]\n # init input size\n input_size = 0\n\n # define embedding layer:\n if self.has_embed_features:\n self.embed_blocks = nn.ModuleList()\n for i in range(len(num_categs_per_feature)):\n self.embed_blocks.append(\n nn.Embedding(num_embeddings=num_categs_per_feature[i], embedding_dim=embed_dims[i])\n )\n input_size += embed_dims[i]\n\n # update input size\n if self.has_vector_features:\n input_size += vector_dims\n\n # activation\n act_fn = nn.Identity()\n if params[\"activation\"] == \"elu\":\n act_fn = nn.ELU()\n elif params[\"activation\"] == \"relu\":\n act_fn = nn.ReLU()\n elif params[\"activation\"] == \"tanh\":\n act_fn = nn.Tanh()\n\n layers = []\n if params[\"use_batchnorm\"]:\n layers.append(nn.BatchNorm1d(input_size))\n layers.append(nn.Linear(input_size, params[\"hidden_size\"]))\n layers.append(act_fn)\n for _ in range(params[\"num_layers\"] - 1):\n if params[\"use_batchnorm\"]:\n layers.append(nn.BatchNorm1d(params[\"hidden_size\"]))\n layers.append(nn.Dropout(params[\"dropout_prob\"]))\n layers.append(nn.Linear(params[\"hidden_size\"], params[\"hidden_size\"]))\n layers.append(act_fn)\n layers.append(nn.Linear(params[\"hidden_size\"], num_net_outputs))\n self.main_block = nn.Sequential(*layers)\n\n if self.problem_type in [REGRESSION, QUANTILE]: # set range for output\n y_range = params[\"y_range\"] # Used specifically for regression. = None for classification.\n self.y_constraint = None # determines if Y-predictions should be constrained\n if y_range is not None:\n if y_range[0] == -np.inf and y_range[1] == np.inf:\n self.y_constraint = None # do not worry about Y-range in this case\n elif y_range[0] >= 0 and y_range[1] == np.inf:\n self.y_constraint = \"nonnegative\"\n elif y_range[0] == -np.inf and y_range[1] <= 0:\n self.y_constraint = \"nonpositive\"\n else:\n self.y_constraint = \"bounded\"\n self.y_lower = y_range[0]\n self.y_upper = y_range[1]\n self.y_span = self.y_upper - self.y_lower\n\n if self.problem_type == QUANTILE:\n self.alpha = params[\"alpha\"] # for huber loss\n if self.problem_type == SOFTCLASS:\n self.log_softmax = torch.nn.LogSoftmax(dim=1)\n if self.problem_type in [BINARY, MULTICLASS, SOFTCLASS]:\n self.softmax = torch.nn.Softmax(dim=1)\n if architecture_desc is None: # Save Architecture description\n self.architecture_desc = {\n \"has_vector_features\": self.has_vector_features,\n \"has_embed_features\": self.has_embed_features,\n \"params\": params,\n \"num_net_outputs\": num_net_outputs,\n \"from_logits\": self.from_logits,\n }\n if self.has_embed_features:\n self.architecture_desc[\"num_categs_per_feature\"] = num_categs_per_feature\n self.architecture_desc[\"embed_dims\"] = embed_dims\n if self.has_vector_features:\n self.architecture_desc[\"vector_dims\"] = vector_dims\n\n def _set_params(\n self,\n num_layers=4,\n hidden_size=128,\n activation=\"relu\",\n use_batchnorm=False,\n dropout_prob=0.1,\n y_range=None,\n alpha=0.01,\n max_embedding_dim=100,\n embed_exponent=0.56,\n embedding_size_factor=1.0,\n ):\n return dict(\n num_layers=num_layers,\n hidden_size=hidden_size,\n activation=activation,\n use_batchnorm=use_batchnorm,\n dropout_prob=dropout_prob,\n y_range=y_range,\n alpha=alpha,\n max_embedding_dim=max_embedding_dim,\n embed_exponent=embed_exponent,\n embedding_size_factor=embedding_size_factor,\n )\n\n def init_params(self):\n for layer in self.children():\n if hasattr(layer, \"reset_parameters\"):\n layer.reset_parameters()\n\n def forward(self, data_batch):\n input_data = []\n input_offset = 0\n if self.has_vector_features:\n input_data.append(data_batch[0].to(self.device))\n input_offset += 1\n if self.has_embed_features:\n embed_data = data_batch[input_offset]\n for i in range(len(self.embed_blocks)):\n input_data.append(self.embed_blocks[i](embed_data[i].to(self.device)))\n\n if len(input_data) > 1:\n input_data = torch.cat(input_data, dim=1)\n else:\n input_data = input_data[0]\n\n output_data = self.main_block(input_data)\n if self.problem_type in [REGRESSION, QUANTILE]: # output with y-range\n if self.y_constraint is None:\n return output_data\n else:\n if self.y_constraint == \"nonnegative\":\n return self.y_lower + torch.abs(output_data)\n elif self.y_constraint == \"nonpositive\":\n return self.y_upper - torch.abs(output_data)\n else:\n return torch.sigmoid(output_data) * self.y_span + self.y_lower\n elif self.problem_type == SOFTCLASS:\n return self.log_softmax(output_data) # KLDivLoss takes in log-probabilities as predictions.\n else:\n return output_data\n\n def huber_pinball_loss(self, input_data, target_data):\n error_data = target_data.contiguous().reshape(-1, 1) - input_data\n if self.alpha == 0.0:\n loss_data = torch.max(self.quantile_levels * error_data, (self.quantile_levels - 1) * error_data)\n return loss_data.mean()\n\n loss_data = torch.where(\n torch.abs(error_data) < self.alpha,\n 0.5 * error_data * error_data,\n self.alpha * (torch.abs(error_data) - 0.5 * self.alpha),\n )\n loss_data /= self.alpha\n scale = torch.where(\n error_data >= 0,\n torch.ones_like(error_data) * self.quantile_levels,\n torch.ones_like(error_data) * (1 - self.quantile_levels),\n )\n loss_data *= scale\n return loss_data.mean()\n\n def margin_loss(self, input_data, margin_scale=0.0001):\n # number of samples\n batch_size, num_quantiles = input_data.size()\n\n # compute margin loss (batch_size x num_net_outputs(above) x num_net_outputs(below))\n error_data = input_data.unsqueeze(1) - input_data.unsqueeze(2)\n\n # margin data (num_quantiles x num_quantiles)\n margin_data = self.quantile_levels.permute(1, 0) - self.quantile_levels\n margin_data = torch.tril(margin_data, -1) * margin_scale\n\n # compute accumulated margin\n loss_data = torch.tril(error_data + margin_data, diagonal=-1)\n loss_data = loss_data.relu()\n loss_data = loss_data.sum() / float(batch_size * (num_quantiles * num_quantiles - num_quantiles) * 0.5)\n return loss_data\n\n def quantile_loss(self, predict_data, target_data, margin):\n if margin > 0.0:\n m_loss = self.margin_loss(predict_data)\n else:\n m_loss = 0.0\n h_loss = self.huber_pinball_loss(predict_data, target_data).mean()\n return h_loss + margin * m_loss\n\n def compute_loss(self, data_batch, loss_function=None, gamma=None):\n # train mode\n self.train()\n predict_data = self(data_batch)\n target_data = data_batch[-1].to(self.device)\n if self.problem_type in [BINARY, MULTICLASS]:\n target_data = target_data.type(\n torch.long\n ) # Windows default int type is int32. Need to explicit convert to Long.\n if self.problem_type == QUANTILE:\n return self.quantile_loss(predict_data, target_data, margin=gamma)\n if self.problem_type == SOFTCLASS:\n return loss_function(predict_data, target_data)\n else:\n target_data = target_data.flatten()\n if self.problem_type == REGRESSION:\n predict_data = predict_data.flatten()\n return loss_function(predict_data, target_data)\n\n def predict(self, input_data):\n self.eval()\n with torch.no_grad():\n predict_data = self(input_data)\n if self.problem_type == QUANTILE:\n predict_data = torch.sort(predict_data, -1)[0] # sorting ensures monotonicity of quantile estimates\n elif self.problem_type in [BINARY, MULTICLASS, SOFTCLASS]:\n predict_data = self.softmax(predict_data) # convert NN output to probability\n elif self.problem_type == REGRESSION:\n predict_data = predict_data.flatten()\n if self.problem_type == BINARY:\n predict_data = predict_data[:, 1]\n return predict_data.data.cpu().numpy()\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/utils/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/utils/categorical_encoders.py",
"content": "\"\"\"\nVariant of the sklearn OneHotEncoder and OrdinalEncoder that can handle unknown classes at test-time\nas well as binning of infrequent categories to limit the overall number of categories considered.\nUnknown categories are returned as None in inverse transforms. Always converts input list X to list of the same type elements first (string typically)\n\"\"\"\n\nimport copy\nfrom numbers import Integral\n\nimport numpy as np\nfrom packaging.version import parse as parse_version\nfrom scipy import sparse\nfrom sklearn import __version__ as _sklearn_version\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import check_array\nfrom sklearn.utils.validation import check_is_fitted\n\nfrom autogluon.features.generators import LabelEncoderFeatureGenerator\n\n__all__ = [\"OneHotMergeRaresHandleUnknownEncoder\", \"OrdinalMergeRaresHandleUnknownEncoder\"]\n\n\ndef _encode_numpy(values, uniques=None, encode=False, check_unknown=True):\n # only used in _encode below, see docstring there for details\n if uniques is None:\n if encode:\n uniques, encoded = np.unique(values, return_inverse=True)\n return uniques, encoded\n else:\n # unique sorts\n return np.unique(values)\n if encode:\n if check_unknown:\n diff = _encode_check_unknown(values, uniques)\n if diff:\n raise ValueError(\"y contains previously unseen labels: %s\" % str(diff))\n encoded = np.searchsorted(uniques, values)\n return uniques, encoded\n else:\n return uniques\n\n\ndef _encode_python(values, uniques=None, encode=False):\n # only used in _encode below, see docstring there for details\n if uniques is None:\n uniques = sorted(set(values))\n uniques = np.array(uniques, dtype=values.dtype)\n if encode:\n table = {val: i for i, val in enumerate(uniques)}\n try:\n encoded = np.array([table[v] for v in values])\n except KeyError as e:\n raise ValueError(\"y contains previously unseen labels: %s\" % str(e))\n return uniques, encoded\n else:\n return uniques\n\n\ndef _encode(values, uniques=None, encode=False, check_unknown=True):\n \"\"\"Helper function to factorize (find uniques) and encode values.\n Uses pure python method for object dtype, and numpy method for\n all other dtypes.\n The numpy method has the limitation that the `uniques` need to\n be sorted. Importantly, this is not checked but assumed to already be\n the case. The calling method needs to ensure this for all non-object\n values.\n Parameters\n ----------\n values : array\n Values to factorize or encode.\n uniques : array, optional\n If passed, uniques are not determined from passed values (this\n can be because the user specified categories, or because they\n already have been determined in fit).\n encode : bool, default False\n If True, also encode the values into integer codes based on `uniques`.\n check_unknown : bool, default True\n If True, check for values in ``values`` that are not in ``unique``\n and raise an error. This is ignored for object dtype, and treated as\n True in this case. This parameter is useful for\n _BaseEncoder._transform() to avoid calling _encode_check_unknown()\n twice.\n Returns\n -------\n uniques\n If ``encode=False``. The unique values are sorted if the `uniques`\n parameter was None (and thus inferred from the data).\n (uniques, encoded)\n If ``encode=True``.\n \"\"\"\n if values.dtype == object:\n try:\n res = _encode_python(values, uniques, encode)\n except TypeError:\n raise TypeError(\"argument must be a string or number\")\n return res\n else:\n return _encode_numpy(values, uniques, encode, check_unknown=check_unknown)\n\n\ndef _encode_check_unknown(values, uniques, return_mask=False):\n \"\"\"\n Helper function to check for unknowns in values to be encoded.\n Uses pure python method for object dtype, and numpy method for\n all other dtypes.\n Parameters\n ----------\n values : array\n Values to check for unknowns.\n uniques : array\n Allowed uniques values.\n return_mask : bool, default False\n If True, return a mask of the same shape as `values` indicating\n the valid values.\n Returns\n -------\n diff : list\n The unique values present in `values` and not in `uniques` (the\n unknown values).\n valid_mask : boolean array\n Additionally returned if ``return_mask=True``.\n \"\"\"\n if values.dtype == object:\n uniques_set = set(uniques)\n diff = list(set(values) - uniques_set)\n if return_mask:\n if diff:\n valid_mask = np.array([val in uniques_set for val in values])\n else:\n valid_mask = np.ones(len(values), dtype=bool)\n return diff, valid_mask\n else:\n return diff\n else:\n unique_values = np.unique(values)\n diff = list(np.setdiff1d(unique_values, uniques, assume_unique=True))\n if return_mask:\n if diff:\n valid_mask = np.isin(values, uniques)\n else:\n valid_mask = np.ones(len(values), dtype=bool)\n return diff, valid_mask\n else:\n return diff\n\n\nclass _BaseEncoder(BaseEstimator, TransformerMixin):\n \"\"\"\n Base class for encoders that includes the code to categorize and\n transform the input features.\n \"\"\"\n\n def _check_X(self, X):\n \"\"\"\n Perform custom check_array:\n - convert list of strings to object dtype\n - check for missing values for object dtype data (check_array does\n not do that)\n - return list of features (arrays): this list of features is\n constructed feature by feature to preserve the data types\n of pandas DataFrame columns, as otherwise information is lost\n and cannot be used, eg for the `categories_` attribute.\n\n \"\"\"\n if not (hasattr(X, \"iloc\") and getattr(X, \"ndim\", 0) == 2):\n # if not a dataframe, do normal check_array validation\n if parse_version(_sklearn_version) >= parse_version(\"1.6.0\"):\n X_temp = check_array(X, dtype=None, ensure_all_finite=False)\n else:\n X_temp = check_array(X, dtype=None, force_all_finite=False)\n if not hasattr(X, \"dtype\") and np.issubdtype(X_temp.dtype, np.str_):\n X = check_array(X, dtype=object)\n else:\n X = X_temp\n needs_validation = False\n else:\n # pandas dataframe, do validation later column by column, in order\n # to keep the dtype information to be used in the encoder.\n needs_validation = True\n\n n_samples, n_features = X.shape\n X_columns = []\n\n for i in range(n_features):\n Xi = self._get_feature(X, feature_idx=i)\n if parse_version(_sklearn_version) >= parse_version(\"1.6.0\"):\n Xi = check_array(Xi, ensure_2d=False, dtype=None, ensure_all_finite=needs_validation)\n else:\n Xi = check_array(Xi, ensure_2d=False, dtype=None, force_all_finite=needs_validation)\n X_columns.append(Xi)\n\n return X_columns, n_samples, n_features\n\n def _get_feature(self, X, feature_idx):\n if hasattr(X, \"iloc\"):\n # pandas dataframes\n return X.iloc[:, feature_idx]\n # numpy arrays, sparse arrays\n return X[:, feature_idx]\n\n def _fit(self, X, handle_unknown=\"error\"):\n X_list, n_samples, n_features = self._check_X(X)\n\n if self.categories != \"auto\":\n if len(self.categories) != n_features:\n raise ValueError(\"Shape mismatch: if categories is an array, it has to be of shape (n_features,).\")\n\n if self.max_levels is not None:\n if not isinstance(self.max_levels, Integral) or self.max_levels <= 0:\n raise ValueError(\"max_levels must be None or a strictly positive int, got {}.\".format(self.max_levels))\n\n self.categories_ = []\n self.infrequent_indices_ = []\n\n for i in range(n_features):\n Xi = X_list[i]\n if self.categories == \"auto\":\n cats = _encode(Xi)\n else:\n cats = np.array(self.categories[i], dtype=Xi.dtype)\n if Xi.dtype != object:\n if not np.all(np.sort(cats) == cats):\n raise ValueError(\"Unsorted categories are not supported for numerical categories\")\n if handle_unknown == \"error\":\n diff = _encode_check_unknown(Xi, cats)\n if diff:\n msg = \"Found unknown categories {0} in column {1} during fit\".format(diff, i)\n raise ValueError(msg)\n self.categories_.append(cats)\n\n if self.max_levels is not None:\n infrequent_indices = self._find_infrequent_category_indices(Xi)\n else:\n infrequent_indices = np.array([])\n self.infrequent_indices_.append(infrequent_indices)\n\n def _find_infrequent_category_indices(self, Xi):\n # TODO: this is using unique on X again. Ideally we should integrate\n # this into _encode()\n _, counts = np.unique(Xi, return_counts=True)\n return np.argsort(counts)[: -self.max_levels]\n\n def _transform(self, X, handle_unknown=\"error\"):\n X_list, n_samples, n_features = self._check_X(X)\n\n X_int = np.zeros((n_samples, n_features), dtype=int)\n X_mask = np.ones((n_samples, n_features), dtype=bool)\n\n if n_features != len(self.categories_):\n raise ValueError(\n \"The number of features in X is different to the number of \"\n \"features of the fitted data. The fitted data had {} features \"\n \"and the X has {} features.\".format(\n len(\n self.categories_,\n ),\n n_features,\n )\n )\n\n for i in range(n_features):\n Xi = X_list[i]\n diff, valid_mask = _encode_check_unknown(Xi, self.categories_[i], return_mask=True)\n\n if not np.all(valid_mask):\n if handle_unknown == \"error\":\n msg = \"Found unknown categories {0} in column {1} during transform\".format(diff, i)\n raise ValueError(msg)\n else:\n # Set the problematic rows to an acceptable value and\n # continue `The rows are marked `X_mask` and will be\n # removed later.\n X_mask[:, i] = valid_mask\n # cast Xi into the largest string type necessary\n # to handle different lengths of numpy strings\n if self.categories_[i].dtype.kind in (\"U\", \"S\") and self.categories_[i].itemsize > Xi.itemsize:\n Xi = Xi.astype(self.categories_[i].dtype)\n else:\n Xi = Xi.copy()\n\n Xi[~valid_mask] = self.categories_[i][0]\n # We use check_unknown=False, since _encode_check_unknown was\n # already called above.\n _, encoded = _encode(Xi, self.categories_[i], encode=True, check_unknown=False)\n X_int[:, i] = encoded\n\n # We need to take care of infrequent categories here. We want all the\n # infrequent categories to end up in a specific column, after all the\n # frequent ones. Let's say we have 4 categories with 2 infrequent\n # categories (and 2 frequent categories): we want the value in X_int\n # for the infrequent categories to be 2 (third and last column), and\n # the values for the frequent ones to be 0 and 1. The piece of code\n # below performs this mapping.\n # TODO: maybe integrate this part with the one above\n self._infrequent_mappings = {}\n huge_int = np.iinfo(X_int.dtype).max\n for feature_idx in range(n_features):\n if self.infrequent_indices_[feature_idx].size > 0:\n mapping = np.arange(len(self.categories_[feature_idx]))\n # Trick: set the infrequent cats columns to a very big int and\n # encode again.\n for ordinal_cat in self.infrequent_indices_[feature_idx]:\n mapping[ordinal_cat] = huge_int\n _, mapping = _encode_numpy(mapping, encode=True)\n\n # update X_int and save mapping for later (for dropping logic)\n X_int[:, feature_idx] = mapping[X_int[:, feature_idx]]\n self._infrequent_mappings[feature_idx] = mapping\n\n return X_int, X_mask\n\n def _more_tags(self):\n return {\"X_types\": [\"categorical\"]}\n\n def __sklearn_tags__(self):\n \"\"\"\n Returns a Tags object with scikit-learn estimator tags.\n\n This is the scikit-learn 1.6+ compatible way to define estimator tags,\n replacing the deprecated _more_tags method.\n\n Returns\n -------\n tags : sklearn.utils.Tags\n A Tags object containing all tag information.\n \"\"\"\n # lazily import to avoid crashing if sklearn<1.6\n from sklearn.utils import InputTags, Tags, TargetTags\n\n # Create the Tags object with appropriate settings\n tags = Tags(\n estimator_type=None, # This is a transformer, not a classifier/regressor\n target_tags=TargetTags(\n required=False # Target is not required for transformers\n ),\n input_tags=InputTags(\n categorical=True,\n string=True,\n ),\n array_api_support=False,\n no_validation=False,\n non_deterministic=False,\n requires_fit=True,\n )\n\n return tags\n\n\nclass OneHotMergeRaresHandleUnknownEncoder(_BaseEncoder):\n \"\"\"Encode categorical integer features as a one-hot numeric array.\n\n The input to this transformer should be an array-like of integers or\n strings, denoting the values taken on by categorical (discrete) features.\n The features are encoded using a one-hot (aka 'one-of-K' or 'dummy')\n encoding scheme. This creates a binary column for each category and\n returns a sparse matrix or dense array (depending on the ``sparse``\n parameter)\n\n By default, the encoder derives the categories based on the unique values\n in each feature. Alternatively, you can also specify the `categories`\n manually.\n\n Always uses handle_unknown='ignore' which maps unknown test-time categories to all zeros vector.\n\n Parameters\n ----------\n categories : 'auto' or a list of lists/arrays of values, default='auto'.\n Categories (unique values) per feature:\n\n - 'auto' : Determine categories automatically from the training data.\n - list : ``categories[i]`` holds the categories expected in the ith\n column. The passed categories should not mix strings and numeric\n values within a single feature, and should be sorted in case of\n numeric values.\n\n The used categories can be found in the ``categories_`` attribute.\n\n drop : 'first' or a list/array of shape (n_features,), default=None.\n Specifies a methodology to use to drop one of the categories per\n feature. This is useful in situations where perfectly collinear\n features cause problems, such as when feeding the resulting data\n into a neural network or an unregularized regression.\n\n - None : retain all features (the default).\n - 'first' : drop the first category in each feature. If only one\n category is present, the feature will be dropped entirely.\n - array : ``drop[i]`` is the category in feature ``X[:, i]`` that\n should be dropped. If ``drop[i]`` is an infrequent category, an\n error is raised: it is only possible to drop all of the infrequent\n categories, not just one of them.\n - 'infrequent' : drop the infrequent categories column (see\n ``max_levels`` parameter).\n\n sparse : boolean, default=True\n Will return sparse matrix if set True else will return an array.\n\n dtype : number type, default=float\n Desired dtype of output.\n\n max_levels : int, default=None\n One less than the maximum number of categories to keep (max_levels = 2 means we keep 3 distinct categories).\n Infrequent categories are grouped together and mapped into a single column, which counts as extra category.\n Unknown categories encountered at test time are mapped to all zeros vector.\n\n Attributes\n ----------\n categories_ : list of arrays\n The categories of each feature determined during fitting\n (in order of the features in X and corresponding with the output\n of ``transform``). This includes the category specified in ``drop``\n (if any).\n\n drop_idx_ : array of shape (n_features,)\n ``drop_idx_[i]`` is the index in ``categories_[i]`` of the category to\n be dropped for each feature. None if all the transformed features will\n be retained.\n\n infrequent_indices_: list of arrays of shape(n_infrequent_categories)\n ``infrequent_indices_[i]`` contains a list of indices in\n ``categories_[i]`` corresponding to the infrequent categories.\n\n \"\"\"\n\n def __init__(self, categories=\"auto\", drop=None, sparse=True, dtype=np.float64, max_levels=None):\n self.categories = categories\n self.sparse = sparse\n self.dtype = dtype\n self.handle_unknown = \"ignore\"\n self.drop = drop\n self.max_levels = max_levels\n self._label_encoder = None\n self._cat_cols = None\n\n def _validate_keywords(self):\n if self.handle_unknown not in (\"error\", \"ignore\"):\n msg = \"handle_unknown should be either 'error' or 'ignore', got {0}.\".format(self.handle_unknown)\n raise ValueError(msg)\n # If we have both dropped columns and ignored unknown\n # values, there will be ambiguous cells. This creates difficulties\n # in interpreting the model.\n if self.drop is not None and self.handle_unknown != \"error\":\n raise ValueError(\n \"`handle_unknown` must be 'error' when the drop parameter is \"\n \"specified, as both would create categories that are all \"\n \"zero.\"\n )\n\n def _compute_drop_idx(self):\n if self.drop is None:\n return None\n elif isinstance(self.drop, str) and self.drop in (\"first\", \"infrequent\"):\n return np.zeros(len(self.categories_), dtype=np.int_)\n elif not isinstance(self.drop, str):\n try:\n self.drop = np.asarray(self.drop, dtype=object)\n droplen = len(self.drop)\n except (ValueError, TypeError):\n msg = \"Wrong input for parameter `drop`. Expected 'first', None or array of objects, got {}\"\n raise ValueError(msg.format(type(self.drop)))\n if droplen != len(self.categories_):\n msg = \"`drop` should have length equal to the number of features ({}), got {}\"\n raise ValueError(msg.format(len(self.categories_), len(self.drop)))\n missing_drops = [(i, val) for i, val in enumerate(self.drop) if val not in self.categories_[i]]\n if any(missing_drops):\n msg = (\n \"The following categories were supposed to be \"\n \"dropped, but were not found in the training \"\n \"data.\\n{}\".format(\"\\n\".join([\"Category: {}, Feature: {}\".format(c, v) for c, v in missing_drops]))\n )\n raise ValueError(msg)\n return np.array(\n [np.where(cat_list == val)[0][0] for (val, cat_list) in zip(self.drop, self.categories_)],\n dtype=np.int_,\n )\n else:\n msg = \"Wrong input for parameter `drop`. Expected 'first', None or array of objects, got {}\"\n raise ValueError(msg.format(type(self.drop)))\n\n def _convert_cat_to_int(self, X):\n if self._cat_cols:\n X = copy.deepcopy(X)\n X[self._cat_cols] = self._label_encoder.transform(X[self._cat_cols])\n return X\n\n def fit(self, X, y=None):\n \"\"\"Fit OneHotEncoder to X.\n\n Parameters\n ----------\n X : array-like, shape [n_samples, n_features]\n The data to determine the categories of each feature.\n\n Returns\n -------\n self\n \"\"\"\n self._label_encoder = LabelEncoderFeatureGenerator(verbosity=0)\n self._cat_cols = list(X.select_dtypes(include=\"category\").columns)\n if self._cat_cols:\n self._label_encoder.fit(X=X[self._cat_cols])\n X = self._convert_cat_to_int(X=X)\n X = np.array(X).tolist() # converts all elements in X to the same type (i.e. cannot mix floats, ints, and str)\n self._validate_keywords()\n self._fit(X, handle_unknown=self.handle_unknown)\n self.drop_idx_ = self._compute_drop_idx()\n # check if user wants to manually drop a feature that is\n # infrequent: this is not allowed\n if self.drop is not None and not isinstance(self.drop, str):\n for feature_idx, (infrequent_indices, drop_idx) in enumerate(\n zip(self.infrequent_indices_, self.drop_idx_)\n ):\n if drop_idx in infrequent_indices:\n raise ValueError(\n \"Category {} of feature {} is infrequent and thus \"\n \"cannot be dropped. Use drop='infrequent' \"\n \"instead.\".format(self.categories_[feature_idx][drop_idx], feature_idx)\n )\n return self\n\n def transform(self, X):\n \"\"\"Transform X using one-hot encoding.\n\n Parameters\n ----------\n X : array-like, shape [n_samples, n_features]\n The data to encode.\n\n Returns\n -------\n X_out : sparse matrix if sparse=True else a 2-d array\n Transformed input.\n \"\"\"\n X = self._convert_cat_to_int(X=X)\n X = np.array(X).tolist() # converts all elements in X to the same type (i.e. cannot mix floats, ints, and str)\n check_is_fitted(self, \"categories_\")\n # validation of X happens in _check_X called by _transform\n X_int, X_mask = self._transform(X, handle_unknown=self.handle_unknown)\n n_samples, n_features = X_int.shape\n\n # n_columns indicates, for each feature, how many columns are used in\n # X_trans. By default this corresponds to the number of categories, but\n # will differ if we drop some of them, or if there are infrequent\n # categories (all mapped to the same column)\n n_columns = [len(cats) for cats in self.categories_]\n for feature_idx in range(n_features):\n n_infrequent = self.infrequent_indices_[feature_idx].size\n if n_infrequent > 0:\n # still add 1 for the infrequent column\n n_columns[feature_idx] += 1 - n_infrequent\n if self.drop is not None:\n # if drop is not None we always drop one column in general,\n # except when drop is 'infrequent' and there is no infrequent\n # category.\n n_columns[feature_idx] -= 1\n if isinstance(self.drop, str) and self.drop == \"infrequent\" and n_infrequent == 0:\n n_columns[feature_idx] += 1 # revert decrement from above\n\n if self.drop is not None:\n to_drop = self.drop_idx_.copy()\n if isinstance(self.drop, str):\n if self.drop == \"infrequent\":\n for feature_idx in range(n_features):\n if self.infrequent_indices_[feature_idx].size > 0:\n # drop the infrequent column (i.e. the last one)\n to_drop[feature_idx] = n_columns[feature_idx]\n else:\n # no infrequent category, use special marker -1\n # so that no dropping happens for this feature\n to_drop[feature_idx] = -1\n else:\n # self.drop is an array of categories. we need to remap the\n # dropped indexes if some of the categories are infrequent.\n # see _transform() for details about the mapping.\n for feature_idx in range(n_features):\n if self.infrequent_indices_[feature_idx].size > 0:\n mapping = self._infrequent_mappings[feature_idx]\n to_drop[feature_idx] = mapping[to_drop[feature_idx]]\n\n # We remove all the dropped categories from mask, and decrement\n # all categories that occur after them to avoid an empty column.\n to_drop = to_drop.reshape(1, -1)\n keep_cells = (X_int != to_drop) | (to_drop == -1)\n X_mask &= keep_cells\n X_int[(X_int > to_drop) & (to_drop != -1)] -= 1\n\n mask = X_mask.ravel()\n n_values = np.array([0] + n_columns)\n feature_indices = np.cumsum(n_values)\n indices = (X_int + feature_indices[:-1]).ravel()[mask]\n indptr = X_mask.sum(axis=1).cumsum()\n indptr = np.insert(indptr, 0, 0)\n data = np.ones(n_samples * n_features)[mask]\n\n out = sparse.csr_matrix((data, indices, indptr), shape=(n_samples, feature_indices[-1]), dtype=self.dtype)\n if not self.sparse:\n return out.toarray()\n else:\n return out\n\n def inverse_transform(self, X):\n \"\"\"Convert the back data to the original representation.\n\n In case unknown categories are encountered (all zeros in the\n one-hot encoding), ``None`` is used to represent this category.\n\n Parameters\n ----------\n X : array-like or sparse matrix, shape [n_samples, n_encoded_features]\n The transformed data.\n\n Returns\n -------\n X_tr : array-like, shape [n_samples, n_features]\n Inverse transformed array.\n\n \"\"\"\n check_is_fitted(self, \"categories_\")\n X = check_array(X, accept_sparse=\"csr\")\n\n n_samples, _ = X.shape\n n_features = len(self.categories_)\n if self.drop is None:\n n_transformed_features = sum(len(cats) for cats in self.categories_)\n else:\n n_transformed_features = sum(len(cats) - 1 for cats in self.categories_)\n\n # validate shape of passed X\n msg = \"Shape of the passed X data is not correct. Expected {0} columns, got {1}.\"\n if X.shape[1] != n_transformed_features:\n raise ValueError(msg.format(n_transformed_features, X.shape[1]))\n\n # create resulting array of appropriate dtype\n dt = np.find_common_type([cat.dtype for cat in self.categories_], [])\n X_tr = np.empty((n_samples, n_features), dtype=dt)\n j = 0\n found_unknown = {}\n\n for i in range(n_features):\n if self.drop is None:\n cats = self.categories_[i]\n else:\n cats = np.delete(self.categories_[i], self.drop_idx_[i])\n n_categories = len(cats)\n\n # Only happens if there was a column with a unique\n # category. In this case we just fill the column with this\n # unique category value.\n if n_categories == 0:\n X_tr[:, i] = self.categories_[i][self.drop_idx_[i]]\n j += n_categories\n continue\n sub = X[:, j : j + n_categories] # for sparse X argmax returns 2D matrix, ensure 1D array\n labels = np.asarray(sub.argmax(axis=1)).flatten()\n X_tr[:, i] = cats[labels]\n if self.handle_unknown == \"ignore\":\n unknown = np.asarray(sub.sum(axis=1) == 0).flatten()\n # ignored unknown categories: we have a row of all zero\n if unknown.any():\n found_unknown[i] = unknown\n # drop will either be None or handle_unknown will be error. If\n # self.drop is not None, then we can safely assume that all of\n # the nulls in each column are the dropped value\n elif self.drop is not None:\n dropped = np.asarray(sub.sum(axis=1) == 0).flatten()\n if dropped.any():\n X_tr[dropped, i] = self.categories_[i][self.drop_idx_[i]]\n\n j += n_categories\n\n # if ignored are found: potentially need to upcast result to\n # insert None values\n if found_unknown:\n if X_tr.dtype != object:\n X_tr = X_tr.astype(object)\n\n for idx, mask in found_unknown.items():\n X_tr[mask, idx] = None\n\n return X_tr\n\n def get_feature_names(self, input_features=None):\n \"\"\"Return feature names for output features.\n\n Parameters\n ----------\n input_features : list of string, length n_features, optional\n String names for input features if available. By default,\n \"x0\", \"x1\", ... \"xn_features\" is used.\n\n Returns\n -------\n output_feature_names : array of string, length n_output_features\n\n \"\"\"\n check_is_fitted(self, \"categories_\")\n cats = self.categories_\n if input_features is None:\n input_features = [\"x%d\" % i for i in range(len(cats))]\n elif len(input_features) != len(self.categories_):\n raise ValueError(\n \"input_features should have length equal to number of features ({}), got {}\".format(\n len(self.categories_), len(input_features)\n )\n )\n\n feature_names = []\n for i in range(len(cats)):\n names = [input_features[i] + \"_\" + str(t) for t in cats[i]]\n if self.drop is not None:\n names.pop(self.drop_idx_[i])\n feature_names.extend(names)\n\n return np.array(feature_names, dtype=object)\n\n\nclass OrdinalMergeRaresHandleUnknownEncoder(_BaseEncoder):\n \"\"\"Encode categorical features as an integer array.\n\n The input to this transformer should be an array-like of integers or\n strings, denoting the values taken on by categorical (discrete) features.\n The features are converted to ordinal integers. This results in\n a single column of integers (0 to n_categories - 1) per feature.\n\n Read more in the :ref:`User Guide `.\n\n Parameters\n ----------\n categories : 'auto' or a list of lists/arrays of values.\n Categories (unique values) per feature:\n\n - 'auto' : Determine categories automatically from the training data.\n - list : ``categories[i]`` holds the categories expected in the ith\n column. The passed categories should not mix strings and numeric\n values, and should be sorted in case of numeric values.\n\n The used categories can be found in the ``categories_`` attribute.\n\n dtype : number type, default np.float64\n Desired dtype of output.\n\n max_levels : int, default=None\n One less than the maximum number of categories to keep (max_levels = 2 means we keep 3 distinct categories).\n Infrequent categories are grouped together and mapped to the highest int\n Unknown categories encountered at test time are mapped to another extra category. Embedding layers should be able to take in max_levels + 1 categories!\n\n Attributes\n ----------\n categories_ : list of arrays\n The categories of each feature determined during fitting\n (in order of the features in X and corresponding with the output\n of ``transform``).\n\n infrequent_indices_: list of arrays of shape(n_infrequent_categories)\n ``infrequent_indices_[i]`` contains a list of indices in\n ``categories_[i]`` corresponding to the infrequent categories.\n\n \"\"\"\n\n def __init__(self, categories=\"auto\", dtype=np.float64, max_levels=None):\n self.categories = categories\n self.dtype = dtype\n self.max_levels = max_levels\n self._label_encoder = None\n\n def fit(self, X, y=None):\n \"\"\"Fit the OrdinalEncoder to X.\n\n Parameters\n ----------\n X : array-like, shape [n_samples, n_features]\n The data to determine the categories of each feature.\n\n Returns\n -------\n self\n\n \"\"\"\n self._label_encoder = LabelEncoderFeatureGenerator(verbosity=0)\n self._label_encoder.fit(X=X)\n X = self._label_encoder.transform(X)\n X = np.array(X).tolist() # converts all elements in X to the same type (i.e. cannot mix floats, ints, and str)\n self._fit(X, handle_unknown=\"ignore\")\n\n self.categories_as_sets_ = [np.array(list(set(categories))) for categories in self.categories_]\n # new level introduced to account for unknown categories, always = 1 + total number of categories seen during training\n self.categories_unknown_level_ = [min(len(categories), self.max_levels) for categories in self.categories_]\n self.categories_len_ = [len(categories) for categories in self.categories_]\n return self\n\n def transform(self, X):\n \"\"\"Transform X to ordinal codes.\n\n Parameters\n ----------\n X : array-like, shape [n_samples, n_features]\n The data to encode.\n\n Returns\n -------\n X_out : sparse matrix or a 2-d array\n Transformed input.\n\n \"\"\"\n X = self._label_encoder.transform(X)\n X_og_array = np.array(X) # original X array before transform\n X_int, _ = self._transform(\n X, handle_unknown=\"ignore\"\n ) # will contain zeros for 0th category as well as unknown values.\n\n for i in range(X_int.shape[1]):\n X_col_data = X_og_array[:, i]\n cat_set = self.categories_as_sets_[i]\n unknown_elements = np.isin(X_col_data, cat_set, invert=True)\n X_int[unknown_elements, i] = self.categories_unknown_level_[\n i\n ] # replace entries with unknown categories with feature_i_numlevels + 1 value. Do NOT modify self.categories_\n\n return X_int.astype(self.dtype, copy=False)\n\n def inverse_transform(self, X):\n \"\"\"Convert the data back to the original representation.\n In case unknown categories are encountered (all zeros in the one-hot encoding), ``None`` is used to represent this category.\n\n Parameters\n ----------\n X : array-like or sparse matrix, shape [n_samples, n_encoded_features]\n The transformed data.\n\n Returns\n -------\n X_tr : array-like, shape [n_samples, n_features]\n Inverse transformed array.\n\n \"\"\"\n check_is_fitted(self, \"categories_\")\n X = check_array(X, accept_sparse=\"csr\")\n\n n_samples, _ = X.shape\n n_features = len(self.categories_)\n\n # validate shape of passed X\n msg = \"Shape of the passed X data is not correct. Expected {0} columns, got {1}.\"\n if X.shape[1] != n_features:\n raise ValueError(msg.format(n_features, X.shape[1]))\n\n # create resulting array of appropriate dtype\n dt = np.find_common_type([cat.dtype for cat in self.categories_], [])\n X_tr = np.empty((n_samples, n_features), dtype=dt)\n\n for i in range(n_features):\n possible_categories = np.append(self.categories_[i], None)\n labels = X[:, i].astype(\"int64\", copy=False)\n X_tr[:, i] = self.categories_[i][labels]\n\n return X_tr\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/utils/data_preprocessor.py",
"content": "\"\"\"Data preprocessing helper functions for tabular neural network models\"\"\"\n\nfrom collections import OrderedDict\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import ( # can also consider: PowerTransformer\n FunctionTransformer,\n QuantileTransformer,\n StandardScaler,\n)\n\nfrom .categorical_encoders import OneHotMergeRaresHandleUnknownEncoder, OrdinalMergeRaresHandleUnknownEncoder\n\n\ndef create_preprocessor(\n impute_strategy,\n max_category_levels,\n unique_category_str,\n continuous_features,\n skewed_features,\n onehot_features,\n embed_features,\n bool_features,\n):\n \"\"\"Creates sklearn ColumnTransformer that can be fit to training data to preprocess it for tabular neural network.\"\"\"\n transformers = [] # order of various column transformers in this list is important!\n if continuous_features:\n continuous_transformer = Pipeline(\n steps=[(\"imputer\", SimpleImputer(strategy=impute_strategy)), (\"scaler\", StandardScaler())]\n )\n transformers.append((\"continuous\", continuous_transformer, continuous_features))\n if skewed_features:\n power_transformer = Pipeline(\n steps=[\n (\"imputer\", SimpleImputer(strategy=impute_strategy)),\n (\"quantile\", QuantileTransformer(output_distribution=\"normal\")),\n ]\n ) # Or output_distribution = 'uniform'\n transformers.append((\"skewed\", power_transformer, skewed_features))\n if onehot_features:\n onehot_transformer = Pipeline(\n steps=[(\"onehot\", OneHotMergeRaresHandleUnknownEncoder(max_levels=max_category_levels, sparse=False))]\n ) # test-time unknown values will be encoded as all zeros vector\n transformers.append((\"onehot\", onehot_transformer, onehot_features))\n if embed_features: # Ordinal transformer applied to convert to-be-embedded categorical features to integer levels\n ordinal_transformer = Pipeline(\n steps=[(\"ordinal\", OrdinalMergeRaresHandleUnknownEncoder(max_levels=max_category_levels))]\n ) # returns 0-n when max_category_levels = n-1. category n is reserved for unknown test-time categories.\n transformers.append((\"ordinal\", ordinal_transformer, embed_features))\n try:\n out = ColumnTransformer(\n transformers=transformers,\n remainder=\"passthrough\",\n force_int_remainder_cols=False,\n ) # numeric features are processed in the same order as in numeric_features vector, so feature-names remain the same.\n except:\n # TODO: Avoid try/except once scikit-learn 1.5 is minimum\n # Needed for scikit-learn 1.4 and 1.9+, force_int_remainder_cols is deprecated in 1.7 and introduced in 1.5\n # ref: https://github.com/autogluon/autogluon/issues/5289\n out = ColumnTransformer(\n transformers=transformers,\n remainder=\"passthrough\",\n ) # numeric features are processed in the same order as in numeric_features vector, so feature-names remain the same.\n return out\n\n\ndef convert_df_dtype_to_str(df):\n return df.astype(str)\n\n\ndef get_feature_arraycol_map(processor, max_category_levels):\n \"\"\"Returns OrderedDict of feature-name -> list of column-indices in processed data array corresponding to this feature\"\"\"\n feature_preserving_transforms = set(\n [\"continuous\", \"skewed\", \"ordinal\", \"bool\", \"remainder\"]\n ) # these transforms do not alter dimensionality of feature\n feature_arraycol_map = {} # unordered version\n current_colindex = 0\n for transformer in processor.transformers_:\n transformer_name = transformer[0]\n transformed_features = transformer[2]\n if transformer_name in feature_preserving_transforms:\n for feature in transformed_features:\n if feature in feature_arraycol_map:\n raise ValueError(\"same feature is processed by two different column transformers: %s\" % feature)\n feature_arraycol_map[feature] = [current_colindex]\n current_colindex += 1\n elif transformer_name == \"onehot\":\n oh_encoder = [step for (name, step) in transformer[1].steps if name == \"onehot\"][0]\n for i in range(len(transformed_features)):\n feature = transformed_features[i]\n if feature in feature_arraycol_map:\n raise ValueError(\"same feature is processed by two different column transformers: %s\" % feature)\n oh_dimensionality = min(len(oh_encoder.categories_[i]), max_category_levels + 1)\n feature_arraycol_map[feature] = list(range(current_colindex, current_colindex + oh_dimensionality))\n current_colindex += oh_dimensionality\n else:\n raise ValueError(\"unknown transformer encountered: %s\" % transformer_name)\n return OrderedDict([(key, feature_arraycol_map[key]) for key in feature_arraycol_map])\n\n\ndef get_feature_type_map(feature_arraycol_map, types_of_features):\n \"\"\"Returns OrderedDict of feature-name -> feature_type string (options: 'vector', 'embed').\"\"\"\n if feature_arraycol_map is None:\n raise ValueError(\n \"Must first call get_feature_arraycol_map() to set feature_arraycol_map before calling get_feature_type_map()\"\n )\n vector_features = (\n types_of_features[\"continuous\"]\n + types_of_features[\"skewed\"]\n + types_of_features[\"onehot\"]\n + types_of_features[\"bool\"]\n )\n feature_type_map = OrderedDict()\n for feature_name in feature_arraycol_map:\n if feature_name in vector_features:\n feature_type_map[feature_name] = \"vector\"\n elif feature_name in types_of_features[\"embed\"]:\n feature_type_map[feature_name] = \"embed\"\n else:\n feature_type_map[feature_name] = \"vector\"\n return feature_type_map\n"
},
{
"path": "tabular/src/autogluon/tabular/models/tabular_nn/utils/nn_architecture_utils.py",
"content": "\"\"\"Helper functions related to NN architectures\"\"\"\n\nimport numpy as np\n\nfrom autogluon.core.constants import REGRESSION\n\n\ndef get_embed_sizes(train_dataset, params, num_categs_per_feature):\n \"\"\"Returns list of embedding sizes for each categorical variable.\n Selects this adaptively based on training_dataset.\n Note: Assumes there is at least one embed feature.\n \"\"\"\n max_embedding_dim = params[\"max_embedding_dim\"]\n embed_exponent = params[\"embed_exponent\"]\n size_factor = params[\"embedding_size_factor\"]\n embed_dims = [\n int(size_factor * max(2, min(max_embedding_dim, 1.6 * num_categs_per_feature[i] ** embed_exponent)))\n for i in range(len(num_categs_per_feature))\n ]\n return embed_dims\n\n\ndef infer_y_range(y_vals, y_range_extend):\n \"\"\"Infers good output range for neural network when used for regression.\"\"\"\n min_y = float(y_vals.min())\n max_y = float(y_vals.max())\n std_y = np.std(y_vals)\n y_ext = y_range_extend * std_y\n if min_y >= 0: # infer y must be nonnegative\n min_y = max(0, min_y - y_ext)\n else:\n min_y = min_y - y_ext\n if max_y <= 0: # infer y must be non-positive\n max_y = min(0, max_y + y_ext)\n else:\n max_y = max_y + y_ext\n return (min_y, max_y)\n\n\ndef get_default_layers(problem_type, num_net_outputs, max_layer_width):\n \"\"\"Default sizes for NN layers.\"\"\"\n if problem_type == REGRESSION:\n default_layer_sizes = [\n 256,\n 128,\n ] # overall network will have 4 layers. Input layer, 256-unit hidden layer, 128-unit hidden layer, output layer.\n else:\n default_sizes = [256, 128] # will be scaled adaptively\n # base_size = max(1, min(num_net_outputs, 20)/2.0) # scale layer width based on number of classes\n base_size = max(\n 1, min(num_net_outputs, 100) / 50\n ) # TODO: Updated because it improved model quality and made training far faster\n default_layer_sizes = [defaultsize * base_size for defaultsize in default_sizes]\n layer_expansion_factor = 1 # TODO: consider scaling based on num_rows, eg: layer_expansion_factor = 2-np.exp(-max(0,train_dataset.num_examples-10000))\n return [int(min(max_layer_width, layer_expansion_factor * defaultsize)) for defaultsize in default_layer_sizes]\n\n\ndef default_numeric_embed_dim(train_dataset, max_layer_width, first_layer_width):\n \"\"\"Default embedding dimensionality for numeric features.\"\"\"\n vector_dim = train_dataset.dataset._data[train_dataset.vectordata_index].shape[\n 1\n ] # total dimensionality of vector features\n prop_vector_features = train_dataset.num_vector_features() / float(\n train_dataset.num_features\n ) # Fraction of features that are numeric\n min_numeric_embed_dim = 32\n max_numeric_embed_dim = max_layer_width\n return int(\n min(\n max_numeric_embed_dim,\n max(min_numeric_embed_dim, first_layer_width * prop_vector_features * np.log10(vector_dim + 10)),\n )\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/models/text_prediction/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/text_prediction/text_prediction_v1_model.py",
"content": "from __future__ import annotations\n\nimport logging\n\nfrom autogluon.common.features.types import (\n R_CATEGORY,\n R_FLOAT,\n R_INT,\n R_OBJECT,\n S_IMAGE_PATH,\n S_TEXT_AS_CATEGORY,\n S_TEXT_NGRAM,\n S_TEXT_SPECIAL,\n)\n\nfrom ..automm.automm_model import MultiModalPredictorModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass TextPredictorModel(MultiModalPredictorModel):\n \"\"\"MultimodalPredictor that doesn't use image features\"\"\"\n\n ag_key = \"AG_TEXT_NN\"\n ag_name = \"TextPredictor\"\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_INT, R_FLOAT, R_CATEGORY, R_OBJECT],\n ignored_type_group_special=[S_TEXT_NGRAM, S_TEXT_AS_CATEGORY, S_TEXT_SPECIAL, S_IMAGE_PATH],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\"]\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/callbacks.py",
"content": "import logging\nimport time\nfrom collections import OrderedDict\n\nfrom xgboost import DMatrix\nfrom xgboost.callback import EarlyStopping, TrainingCallback\n\nfrom autogluon.common.utils.lite import disable_if_lite_mode\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.utils.early_stopping import SimpleES\n\nfrom .xgboost_utils import learning_curve_func_generator\n\nlogger = logging.getLogger(__name__)\n\n\nclass CustomMetricCallback(TrainingCallback):\n \"\"\"Calculating additional custom metrics during training.\n\n Custom metrics can be found by calling evals_result() on the associated XGBClassifier or XGBRegressor\n object used for training. i.e.\n\n model.evals_result() = {\n \"validation_0\" : {\n \"metric_1\": [...],\n \"metric_2\": [...],\n \"metric_3\": [...],\n ...\n },\n \"validation_1\": {\n \"metric_1\": [...],\n \"metric_2\": [...],\n \"metric_3\": [...],\n ...\n },\n ...\n }\n\n Parameters\n ----------\n scorers : list(Scorer)\n List of all metrics, represented as Scorer objects, to be computed at each iteration.\n eval_sets : dict(str: tuple)\n Dict of {name: eval_set} pairs, where eval_set = (X, y), containing the datasets used to train the model.\n X is the output of AbstractModel's preprocess method, and y is the truth values fed into an AbstractModel's _fit() method\n e.g. {\"train\": (X, y), \"val\": (X_val, y_val)}\n problem_type : str\n Autogluon constant communicating the current problem_type (i.e. BINARY or REGRESSION)\n use_error : bool\n Whether the scorers specified should calculate metrics in score or error format\n \"\"\"\n\n def __init__(self, scorers, eval_sets, problem_type, use_error=True):\n self.metrics = [\n learning_curve_func_generator(scorer, problem_type=problem_type, use_error=use_error) for scorer in scorers\n ]\n self.eval_sets = [\n (name, DMatrix(eval_set[0], label=eval_set[1]), eval_set[1]) for name, eval_set in eval_sets.items()\n ]\n\n def after_iteration(self, model, epoch, evals_log):\n y_preds = [model.predict(eval_set[1]) for eval_set in self.eval_sets]\n\n if epoch == 0:\n for eval_name, _, _ in self.eval_sets:\n if eval_name not in evals_log:\n evals_log[eval_name] = OrderedDict()\n for metric in self.metrics:\n if metric.__name__ not in evals_log[eval_name]:\n evals_log[eval_name][metric.__name__] = []\n\n for i, (eval_name, _, y_true) in enumerate(self.eval_sets):\n for metric in self.metrics:\n evals_log[eval_name][metric.__name__].append(metric(y_true, y_preds[i]))\n\n return False\n\n\nclass EarlyStoppingCustom(EarlyStopping):\n \"\"\"\n Augments early stopping in XGBoost to also consider time_limit, memory usage, and usage of adaptive early stopping methods.\n\n Parameters\n ----------\n rounds : int or tuple\n If int, The possible number of rounds without the trend occurrence.\n If tuple, contains early stopping class as first element and class init kwargs as second element.\n \"\"\"\n\n def __init__(self, rounds, time_limit=None, start_time=None, verbose=False, min_delta=2e-6, **kwargs):\n if rounds is None:\n # Disable early stopping via rounds\n rounds = 999999\n # Add a tiny min_delta so training doesn't go on for extremely long if only tiny improvements are being made\n # (can occur when validation error is almost 0, such as val log_loss <0.00005)\n super().__init__(rounds=999999, min_delta=min_delta, **kwargs)\n if isinstance(rounds, int):\n self.es = SimpleES(patience=rounds)\n else:\n self.es = rounds[0](**rounds[1])\n self.time_limit = time_limit\n self.start_time = start_time\n self.verbose = verbose\n self._mem_status = None\n self._mem_init_rss = None\n\n @disable_if_lite_mode(ret=lambda self, model: super().before_training(model=model))\n def before_training(self, model):\n model = super().before_training(model=model)\n if self.start_time is None:\n self.start_time = time.time()\n self._mem_status = ResourceManager.get_process()\n self._mem_init_rss = self._mem_status.memory_info().rss\n return model\n\n def after_iteration(self, model, epoch, evals_log):\n should_stop = super().after_iteration(model, epoch, evals_log)\n if should_stop:\n return should_stop\n is_best_iter = self.current_rounds == 0\n should_stop = self.es.update(cur_round=epoch, is_best=is_best_iter)\n if should_stop:\n return should_stop\n if self._time_check(model=model, epoch=epoch):\n return True\n if epoch % 10 == 0 and self._memory_check(model=model):\n return True\n return should_stop\n\n def _time_check(self, model, epoch):\n if self.time_limit is not None:\n time_elapsed = time.time() - self.start_time\n time_left = self.time_limit - time_elapsed\n if time_left <= 0:\n if self.verbose:\n logger.log(\n 20,\n f\"Ran out of time, early stopping on iteration {epoch}. Best iteration is: \\t[{model.attr('best_iteration')}]\\t{model.attr('best_score')}\",\n )\n return True\n return False\n\n @disable_if_lite_mode(ret=False)\n def _memory_check(self, model):\n available = ResourceManager.get_available_virtual_mem()\n cur_rss = self._mem_status.memory_info().rss\n if cur_rss < self._mem_init_rss:\n self._mem_init_rss = cur_rss\n estimated_model_size_mb = (cur_rss - self._mem_init_rss) >> 20\n available_mb = available >> 20\n\n model_size_memory_ratio = estimated_model_size_mb / available_mb\n\n if (model_size_memory_ratio > 0.75) or (available_mb < 512):\n logger.warning(\"Warning: Large XGB model size may cause OOM error if training continues\")\n logger.warning(f\"Available Memory: {available_mb} MB\")\n logger.warning(f\"Estimated XGB model size: {estimated_model_size_mb} MB\")\n if self.verbose:\n logger.warning(\n f\"Warning: Early stopped XGB model prior to optimal result to avoid OOM error. Please increase available memory to avoid subpar model quality.\\n\"\n )\n logger.warning(\n f\"Early stopping. Best iteration is: \\t[{model.attr('best_iteration')}]\\t{model.attr('best_score')}\"\n )\n return True\n elif self.verbose and (model_size_memory_ratio > 0.25):\n logger.log(15, f\"Available Memory: {available_mb} MB\")\n logger.log(15, f\"Estimated XGB model size: {estimated_model_size_mb} MB\")\n return False\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/hyperparameters/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/hyperparameters/parameters.py",
"content": "from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS\n\nDEFAULT_NUM_BOOST_ROUND = 10000\nMAX_CATEGORY_LEVELS = 100 # maximum number of allowed levels per categorical feature\n# Options: [10, 100, 200, 300, 400, 500, 1000, 10000]\n\n\ndef get_param_baseline(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_param_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_param_multiclass_baseline(num_classes=num_classes)\n elif problem_type == SOFTCLASS:\n return get_param_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_param_regression_baseline()\n else:\n return get_param_binary_baseline()\n\n\ndef get_base_params():\n base_params = {\n \"n_estimators\": DEFAULT_NUM_BOOST_ROUND,\n \"learning_rate\": 0.1,\n \"n_jobs\": -1,\n \"proc.max_category_levels\": MAX_CATEGORY_LEVELS,\n }\n return base_params\n\n\ndef get_param_binary_baseline():\n params = get_base_params()\n baseline_params = {\n \"objective\": \"binary:logistic\",\n \"booster\": \"gbtree\",\n }\n params.update(baseline_params)\n return params\n\n\ndef get_param_multiclass_baseline(num_classes):\n params = get_base_params()\n baseline_params = {\n \"objective\": \"multi:softprob\",\n \"booster\": \"gbtree\",\n \"num_class\": num_classes,\n }\n params.update(baseline_params)\n return params\n\n\ndef get_param_regression_baseline():\n params = get_base_params()\n baseline_params = {\n \"objective\": \"reg:squarederror\",\n \"booster\": \"gbtree\",\n }\n params.update(baseline_params)\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/hyperparameters/searchspaces.py",
"content": "\"\"\"Default hyperparameter search spaces used in XGBoost Boosting model\"\"\"\n\nfrom autogluon.common import space\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\n\nDEFAULT_NUM_BOOST_ROUND = 10000 # default for HPO\n\n\ndef get_default_searchspace(problem_type, num_classes=None):\n if problem_type == BINARY:\n return get_searchspace_binary_baseline()\n elif problem_type == MULTICLASS:\n return get_searchspace_multiclass_baseline(num_classes=num_classes)\n elif problem_type == REGRESSION:\n return get_searchspace_regression_baseline()\n else:\n return get_searchspace_binary_baseline()\n\n\ndef get_base_searchspace():\n base_params = {\n \"n_estimators\": DEFAULT_NUM_BOOST_ROUND,\n \"booster\": \"gbtree\",\n \"n_jobs\": -1,\n \"learning_rate\": space.Real(lower=5e-3, upper=0.2, default=0.1, log=True),\n \"max_depth\": space.Int(lower=3, upper=10, default=6),\n \"min_child_weight\": space.Int(lower=1, upper=5, default=1),\n \"colsample_bytree\": space.Real(lower=0.5, upper=1.0, default=1.0),\n # Below lines are commented out as they made search worse. Refine ranges before considering reintroducing these hyperparameters.\n # 'gamma': Real(lower=0, upper=5, default=0),\n # 'subsample': Real(lower=0.5, upper=1.0, default=1.0),\n # 'reg_alpha': Real(lower=0.0, upper=10.0, default=0.0),\n # 'reg_lambda': Real(lower=0.0, upper=10.0, default=1.0),\n }\n return base_params\n\n\ndef get_searchspace_multiclass_baseline(num_classes):\n params = get_base_searchspace()\n baseline_params = {\n \"objective\": \"multi:softmax\",\n \"num_class\": num_classes,\n }\n params.update(baseline_params)\n return params\n\n\ndef get_searchspace_binary_baseline():\n params = get_base_searchspace()\n baseline_params = {\n \"objective\": \"binary:logistic\",\n }\n params.update(baseline_params)\n return params\n\n\ndef get_searchspace_regression_baseline():\n params = get_base_searchspace()\n baseline_params = {\n \"objective\": \"reg:squarederror\",\n }\n params.update(baseline_params)\n return params\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/xgboost_model.py",
"content": "from __future__ import annotations\n\nimport logging\nimport math\nimport os\nimport time\n\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT\nfrom autogluon.common.utils.lite import disable_if_lite_mode\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.common.utils.try_import import try_import_xgboost\nfrom autogluon.core.constants import MULTICLASS, PROBLEM_TYPES_CLASSIFICATION, REGRESSION, SOFTCLASS\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.models._utils import get_early_stopping_rounds\n\nfrom . import xgboost_utils\nfrom .hyperparameters.parameters import get_param_baseline\nfrom .hyperparameters.searchspaces import get_default_searchspace\n\nlogger = logging.getLogger(__name__)\n\n\nclass XGBoostModel(AbstractModel):\n \"\"\"\n XGBoost model: https://xgboost.readthedocs.io/en/latest/\n\n Hyperparameter options: https://xgboost.readthedocs.io/en/latest/parameter.html\n \"\"\"\n\n ag_key = \"XGB\"\n ag_name = \"XGBoost\"\n ag_priority = 40\n seed_name = \"seed\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n self._ohe: bool = True\n self._ohe_generator = None\n self._xgb_model_type = None\n\n def _set_default_params(self):\n default_params = get_param_baseline(problem_type=self.problem_type, num_classes=self.num_classes)\n for param, val in default_params.items():\n self._set_default_param_value(param, val)\n\n def _get_default_searchspace(self):\n return get_default_searchspace(problem_type=self.problem_type, num_classes=self.num_classes)\n\n def _get_default_auxiliary_params(self) -> dict:\n default_auxiliary_params = super()._get_default_auxiliary_params()\n extra_auxiliary_params = dict(\n valid_raw_types=[R_BOOL, R_INT, R_FLOAT, R_CATEGORY],\n )\n default_auxiliary_params.update(extra_auxiliary_params)\n return default_auxiliary_params\n\n # Use specialized XGBoost metric if available (fast), otherwise use custom func generator\n def get_eval_metric(self):\n eval_metric = xgboost_utils.convert_ag_metric_to_xgbm(\n ag_metric_name=self.stopping_metric.name, problem_type=self.problem_type\n )\n if eval_metric is None:\n eval_metric = xgboost_utils.func_generator(metric=self.stopping_metric, problem_type=self.problem_type)\n return eval_metric\n\n def _preprocess(self, X, is_train=False, max_category_levels=None, **kwargs):\n X = super()._preprocess(X=X, **kwargs)\n\n if is_train:\n if self._ohe:\n self._ohe_generator = xgboost_utils.OheFeatureGenerator(max_levels=max_category_levels)\n self._ohe_generator.fit(X)\n\n if self._ohe:\n X = self._ohe_generator.transform(X)\n\n return X\n\n def _fit(\n self,\n X,\n y,\n X_val=None,\n y_val=None,\n time_limit=None,\n num_gpus=0,\n num_cpus=None,\n sample_weight=None,\n sample_weight_val=None,\n verbosity=2,\n **kwargs,\n ):\n # TODO: utilize sample_weight_val in early-stopping if provided\n start_time = time.time()\n ag_params = self._get_ag_params()\n params = self._get_model_params()\n generate_curves = ag_params.get(\"generate_curves\", False)\n\n if generate_curves:\n X_test = kwargs.get(\"X_test\", None)\n y_test = kwargs.get(\"y_test\", None)\n else:\n X_test = None\n y_test = None\n\n if num_cpus:\n params[\"n_jobs\"] = num_cpus\n max_category_levels = params.pop(\"proc.max_category_levels\", 100)\n enable_categorical = params.get(\"enable_categorical\", False)\n if enable_categorical:\n \"\"\"Skip one-hot-encoding and pass categoricals directly to XGBoost\"\"\"\n self._ohe = False\n else:\n \"\"\"One-hot-encode categorical features\"\"\"\n self._ohe = True\n\n if verbosity <= 2:\n verbose = False\n log_period = None\n elif verbosity == 3:\n verbose = True\n log_period = 50\n else:\n verbose = True\n log_period = 1\n\n eval_set = {}\n X = self.preprocess(X, y=y, is_train=True, max_category_levels=max_category_levels)\n num_rows_train = X.shape[0]\n\n # NOTE: xgb eval_metric param supports: default xgb metric(s), str or list(str) OR custom_metric generated by func_generator() in xgboost_utils\n # xgb does not support list(custom_metrics). Instead, use the CustomMetricCallback defined in xgb callbacks file\n if \"eval_metric\" not in params:\n eval_metric = self.get_eval_metric()\n if eval_metric is not None:\n params[\"eval_metric\"] = eval_metric\n eval_metric_name = eval_metric.__name__ if not isinstance(eval_metric, str) else eval_metric\n else:\n eval_metric_name = (\n params[\"eval_metric\"].__name__ if not isinstance(params[\"eval_metric\"], str) else params[\"eval_metric\"]\n )\n\n if X_val is None:\n early_stopping_rounds = None\n eval_set = None\n else:\n X_val = self.preprocess(X_val, is_train=False)\n eval_set[\"val\"] = (X_val, y_val)\n early_stopping_rounds = ag_params.get(\"early_stop\", \"adaptive\")\n if isinstance(early_stopping_rounds, (str, tuple, list)):\n early_stopping_rounds = self._get_early_stopping_rounds(\n num_rows_train=num_rows_train, strategy=early_stopping_rounds\n )\n\n if generate_curves and eval_set is not None:\n scorers = ag_params.get(\"curve_metrics\", [self.eval_metric])\n use_curve_metric_error = ag_params.get(\"use_error_for_curve_metrics\", False)\n metric_names = [scorer.name for scorer in scorers]\n\n eval_set[\"train\"] = (X, y)\n if X_test is not None:\n X_test = self.preprocess(X_test, is_train=False)\n eval_set[\"test\"] = (X_test, y_test)\n\n if num_gpus != 0:\n if \"device\" not in params:\n # FIXME: figure out which GPUs are available to this model instead of hardcoding GPU 0.\n # Need to update BaggedEnsembleModel\n params[\"device\"] = \"cuda:0\"\n if \"tree_method\" not in params:\n params[\"tree_method\"] = \"hist\"\n\n try_import_xgboost()\n from xgboost.callback import EvaluationMonitor\n\n from .callbacks import CustomMetricCallback, EarlyStoppingCustom\n\n if eval_set is not None and \"callbacks\" not in params:\n callbacks = []\n if generate_curves:\n callbacks.append(\n CustomMetricCallback(\n scorers=scorers,\n eval_sets=eval_set,\n problem_type=self.problem_type,\n use_error=use_curve_metric_error,\n )\n )\n if log_period is not None:\n callbacks.append(EvaluationMonitor(period=log_period))\n\n callbacks.append(\n EarlyStoppingCustom(\n early_stopping_rounds,\n start_time=start_time,\n time_limit=time_limit,\n verbose=verbose,\n metric_name=eval_metric_name, # forces stopping_metric rather than arbitrary last metric\n data_name=\"validation_0\", # forces val dataset rather than arbitrary last dataset\n )\n )\n params[\"callbacks\"] = callbacks\n\n if eval_set is not None:\n # important that val dataset is listed first\n # (since validation_0 is specified in EarlyStoppingCustom callback)\n eval_set = [eval_set[\"val\"]]\n\n from xgboost import XGBClassifier, XGBRegressor\n\n model_type = XGBClassifier if self.problem_type in PROBLEM_TYPES_CLASSIFICATION else XGBRegressor\n\n import warnings\n\n with warnings.catch_warnings():\n # FIXME: v1.1: Upgrade XGBoost to 2.0.1+ to avoid deprecation warnings from Pandas 2.1+ during XGBoost fit.\n warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n if params.get(\"device\", \"cpu\") == \"cuda:0\":\n # verbosity=0 to hide UserWarning: Falling back to prediction using DMatrix due to mismatched devices.\n # TODO: Find a way to hide this warning without setting verbosity=0\n # ref: https://github.com/dmlc/xgboost/issues/9791\n params[\"verbosity\"] = 0\n self.model = model_type(**params)\n self.model.fit(X=X, y=y, eval_set=eval_set, verbose=False, sample_weight=sample_weight)\n\n if generate_curves:\n\n def filter(d, keys):\n # ensure only specified curve metrics are included\n return {key: d[key] for key in keys if key in d}\n\n eval_results = self.model.evals_result().copy()\n del eval_results[\"validation_0\"]\n\n curves = {name: filter(metrics, metric_names) for name, metrics in eval_results.items()}\n self.save_learning_curves(metrics=metric_names, curves=curves)\n\n bst = self.model.get_booster()\n # TODO: Investigate speed-ups from GPU inference\n # bst.set_param({\"predictor\": \"gpu_predictor\"})\n\n if eval_set is not None:\n self.params_trained[\"n_estimators\"] = bst.best_iteration + 1\n # Don't save the callback or eval_metric objects\n self.model.set_params(callbacks=None, eval_metric=None)\n\n def _predict_proba(self, X, num_cpus=-1, **kwargs):\n X = self.preprocess(X, **kwargs)\n if self.problem_type in [MULTICLASS, SOFTCLASS]:\n # Bug fix for \"xgboost>=2,<2.0.3\" : https://github.com/dmlc/xgboost/issues/9807\n self.model.set_params(n_jobs=num_cpus, objective=\"multi:softprob\")\n else:\n self.model.set_params(n_jobs=num_cpus)\n\n if self.problem_type == REGRESSION:\n return self.model.predict(X)\n\n y_pred_proba = self.model.predict_proba(X)\n return self._convert_proba_to_unified_form(y_pred_proba)\n\n def _get_early_stopping_rounds(self, num_rows_train, strategy=\"auto\"):\n return get_early_stopping_rounds(num_rows_train=num_rows_train, strategy=strategy)\n\n def _get_num_classes(self, y):\n if self.problem_type == MULTICLASS:\n if self.num_classes is not None:\n num_classes = self.num_classes\n else:\n num_classes = 10 # Guess if not given, can do better by looking at y\n elif self.problem_type == SOFTCLASS: # TODO: delete this elif if it's unnecessary.\n num_classes = y.shape[1]\n else:\n num_classes = 1\n return num_classes\n\n def _ag_params(self) -> set:\n return {\"early_stop\", \"generate_curves\", \"curve_metrics\", \"use_error_for_curve_metrics\"}\n\n def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int:\n hyperparameters = self._get_model_params()\n return self.estimate_memory_usage_static(\n X=X,\n problem_type=self.problem_type,\n num_classes=self.num_classes,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n\n @classmethod\n def _estimate_memory_usage_static(\n cls,\n *,\n X: pd.DataFrame,\n hyperparameters: dict = None,\n num_classes: int = 1,\n **kwargs,\n ) -> int:\n if hyperparameters is None:\n hyperparameters = {}\n num_classes = (\n num_classes if num_classes else 1\n ) # self.num_classes could be None after initialization if it's a regression problem\n data_mem_usage = get_approximate_df_mem_usage(X).sum()\n data_mem_usage_bytes = (\n data_mem_usage * 7 + data_mem_usage / 4 * num_classes\n ) # TODO: Extremely crude approximation, can be vastly improved\n\n max_bin = hyperparameters.get(\"max_bin\", 256)\n max_depth = hyperparameters.get(\"max_depth\", 6)\n max_leaves = hyperparameters.get(\"max_leaves\", 0)\n if max_leaves is None:\n max_leaves = 0\n\n if max_depth > 12 or max_depth == 0: # 0 = uncapped\n max_depth = 12 # Try our best if the value is very large, only treat it as 12.\n\n if max_leaves != 0: # if capped max_leaves\n # make the effective max_depth for calculations be the lesser of the two constraints\n max_depth = min(max_depth, math.ceil(math.log2(max_leaves)))\n\n # Formula based on manual testing, aligns with LightGBM histogram sizes\n # This approximation is less accurate than it is for LightGBM and CatBoost.\n # Note that max_depth didn't appear to reduce memory usage below 6, and it was unclear if it increased memory usage above 6.\n if max_depth < 7:\n depth_modifier = math.pow(2, 6)\n elif max_depth < 9:\n depth_modifier = math.pow(2, max_depth)\n else:\n depth_modifier = math.pow(2, max_depth - 1)\n histogram_mem_usage_bytes = 20 * depth_modifier * len(X.columns) * max_bin\n histogram_mem_usage_bytes *= 1.2 # Add a 20% buffer\n\n mem_size_per_estimator = num_classes * max_depth * 500 # very rough estimate\n n_estimators = hyperparameters.get(\"n_estimators\", 10000)\n n_estimators_min = min(n_estimators, 1000)\n mem_size_estimators = (\n n_estimators_min * mem_size_per_estimator\n ) # memory estimate after fitting up to 1000 estimators\n\n approx_mem_size_req = data_mem_usage_bytes + histogram_mem_usage_bytes + mem_size_estimators\n return approx_mem_size_req\n\n def _validate_fit_memory_usage(\n self,\n mem_error_threshold: float = 1.0,\n mem_warning_threshold: float = 0.75,\n mem_size_threshold: int = 1e9,\n **kwargs,\n ):\n return super()._validate_fit_memory_usage(\n mem_error_threshold=mem_error_threshold,\n mem_warning_threshold=mem_warning_threshold,\n mem_size_threshold=mem_size_threshold,\n **kwargs,\n )\n\n def get_minimum_resources(self, is_gpu_available=False):\n minimum_resources = {\n \"num_cpus\": 1,\n }\n if is_gpu_available:\n minimum_resources[\"num_gpus\"] = 0.5\n return minimum_resources\n\n @disable_if_lite_mode(ret=(1, 0))\n def _get_default_resources(self):\n # only_physical_cores=True is faster in training\n num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True)\n num_gpus = 0\n return num_cpus, num_gpus\n\n def save(self, path: str = None, verbose=True) -> str:\n _model = self.model\n self.model = None\n if _model is not None:\n self._xgb_model_type = _model.__class__\n path = super().save(path=path, verbose=verbose)\n if _model is not None:\n # Halves disk usage compared to .json / .pkl\n _model.save_model(os.path.join(path, \"xgb.ubj\"))\n self.model = _model\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths=True, verbose=True):\n model = super().load(path=path, reset_paths=reset_paths, verbose=verbose)\n if model._xgb_model_type is not None:\n model.model = model._xgb_model_type()\n # Much faster to load using .ubj than .json (10x+ speedup)\n model.model.load_model(os.path.join(path, \"xgb.ubj\"))\n model._xgb_model_type = None\n return model\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"softclass\"]\n\n @classmethod\n def _class_tags(cls):\n return {\n \"can_estimate_memory_usage_static\": True,\n \"supports_learning_curves\": True,\n }\n\n def _more_tags(self):\n # `can_refit_full=True` because n_estimators is communicated at end of `_fit`:\n # self.params_trained['n_estimators'] = bst.best_ntree_limit\n return {\"can_refit_full\": True}\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xgboost/xgboost_utils.py",
"content": "from collections import OrderedDict\n\nimport numpy as np\nfrom scipy.sparse import csr_matrix, hstack\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS\nfrom autogluon.core.metrics import Scorer\n\nfrom ..tabular_nn.utils.categorical_encoders import OneHotMergeRaresHandleUnknownEncoder\n\n_ag_to_xgbm_metric_dict = {\n BINARY: dict(accuracy=\"error\", log_loss=\"logloss\", roc_auc=\"auc\"),\n MULTICLASS: dict(\n accuracy=\"merror\",\n log_loss=\"mlogloss\",\n ),\n REGRESSION: dict(\n mean_absolute_error=\"mae\",\n mean_squared_error=\"rmse\",\n root_mean_squared_error=\"rmse\",\n ),\n}\n\n\ndef convert_ag_metric_to_xgbm(ag_metric_name, problem_type):\n return _ag_to_xgbm_metric_dict.get(problem_type, dict()).get(ag_metric_name, None)\n\n\ndef func_generator(metric: Scorer, problem_type: str):\n \"\"\"\n Create a custom metric compatible with XGBoost, based on the XGBoost 1.6+ API.\n Note that XGBoost needs lower is better metrics.\n\n Params:\n -------\n metric : Scorer\n The autogluon Scorer object to be converted into an XGBoost custom metric.\n problem_type: str\n The current problem type.\n\n Returns:\n --------\n Callable[y_true, y_hat]\n XGBoost custom metric wrapper function.\n \"\"\"\n needs_pred_proba = not metric.needs_pred\n sign = -1 if metric.greater_is_better else 1\n\n if needs_pred_proba:\n\n def custom_metric(y_true, y_hat):\n return sign * metric(y_true, y_hat)\n\n else:\n if problem_type in [MULTICLASS, SOFTCLASS]:\n\n def custom_metric(y_true, y_hat):\n y_hat = y_hat.argmax(axis=1)\n return sign * metric(y_true, y_hat)\n\n elif problem_type == BINARY:\n\n def custom_metric(y_true, y_hat):\n y_hat = np.round(y_hat)\n return sign * metric(y_true, y_hat)\n\n else:\n\n def custom_metric(y_true, y_hat):\n return sign * metric(y_true, y_hat)\n\n # Note: Must include \"_\" prefix due to xgboost internal metric naming conflicts (e.g. precision)\n custom_metric.__name__ = f\"_{metric.name}\"\n\n return custom_metric\n\n\ndef learning_curve_func_generator(metric: Scorer, problem_type: str, use_error: bool = False):\n \"\"\"\n Create a custom metric compatible with XGBoost inputs (but in greater is better format).\n NOTE: Do not use these custom metrics with XGBoost internally.\n\n Documentation of XGBoost support for Custom Metrics:\n Trying to use Multiple Custom Metrics:\n https://stackoverflow.com/questions/44527485/how-to-pass-multiple-custom-metrics-eval-metric-in-python-xgboost\n Multiple Custom Not possible: https://github.com/dmlc/xgboost/issues/2408\n Possible Workaround: https://github.com/dmlc/xgboost/issues/1125 -> Didn't work\n Resolution: Instead, use custom metrics by passing in list of AutoGluon Scorers into custom metric callback\n\n Params:\n -------\n metric : Scorer\n The autogluon Scorer object to be converted into an XGBoost custom metric.\n problem_type: str\n The current problem type.\n use_error: bool\n Whether the custom metric should be computed in error or score format.\n\n Returns:\n --------\n Callable[y_true, y_hat]\n XGBoost custom metric wrapper function.\n \"\"\"\n sign = -1 if metric.greater_is_better else 1\n func = func_generator(metric=metric, problem_type=problem_type)\n\n def custom_metric(y_true, y_hat):\n result = sign * func(y_true, y_hat)\n if use_error:\n return metric.convert_score_to_error(result)\n return result\n\n # Set custom metric name to scorer metric name\n # Note: no need for _ prefix because these metrics aren't\n # to be used by xgboost internally\n custom_metric.__name__ = metric.name\n\n return custom_metric\n\n\nclass OheFeatureGenerator(BaseEstimator, TransformerMixin):\n def __init__(self, max_levels=None):\n self._feature_map = OrderedDict() # key: feature_name, value: feature_type\n self.labels = OrderedDict()\n self.cat_cols = []\n self.other_cols = []\n self.ohe_encs = None\n self.max_levels = max_levels\n\n def fit(self, X, y=None):\n self.cat_cols = list(X.select_dtypes(include=\"category\").columns)\n self.other_cols = list(X.select_dtypes(exclude=\"category\").columns)\n self.ohe_encs = OneHotMergeRaresHandleUnknownEncoder(max_levels=self.max_levels)\n\n if self.cat_cols:\n self.ohe_encs.fit(X[self.cat_cols])\n assert len(self.cat_cols) == len(self.ohe_encs.categories_)\n\n for cat_col, categories in zip(self.cat_cols, self.ohe_encs.categories_):\n categories_ = categories.tolist()\n self.labels[cat_col] = categories_\n # Update feature map ({name: type})\n for category in categories_:\n self._feature_map[f\"{cat_col}_{category}\"] = \"i\" # one-hot encoding data type is boolean\n\n if self.other_cols:\n for c in self.other_cols:\n self._feature_map[c] = \"int\" if X[c].dtypes == int else \"float\"\n return self\n\n def transform(self, X, y=None):\n X_list = []\n if self.cat_cols:\n X_list.append(self.ohe_encs.transform(X[self.cat_cols]))\n if self.other_cols:\n X_list.append(csr_matrix(X[self.other_cols]))\n return hstack(X_list, format=\"csr\")\n\n def get_feature_names(self):\n return list(self._feature_map.keys())\n\n def get_feature_types(self):\n return list(self._feature_map.values())\n\n def get_original_feature_names(self):\n return self.cat_cols + self.other_cols\n"
},
{
"path": "tabular/src/autogluon/tabular/models/xt/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/models/xt/xt_model.py",
"content": "from __future__ import annotations\n\nfrom autogluon.core.constants import QUANTILE, REGRESSION\n\nfrom ..rf.rf_model import RFModel\n\n\nclass XTModel(RFModel):\n \"\"\"\n Extra Trees model (scikit-learn): https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier\n \"\"\"\n\n ag_key = \"XT\"\n ag_name = \"ExtraTrees\"\n ag_priority = 60\n\n def _get_model_type(self):\n if self.problem_type == REGRESSION:\n from sklearn.ensemble import ExtraTreesRegressor\n\n return ExtraTreesRegressor\n elif self.problem_type == QUANTILE:\n from ..rf.rf_quantile import ExtraTreesQuantileRegressor\n\n return ExtraTreesQuantileRegressor\n else:\n from sklearn.ensemble import ExtraTreesClassifier\n\n return ExtraTreesClassifier\n\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\"]\n"
},
{
"path": "tabular/src/autogluon/tabular/predictor/__init__.py",
"content": "from .interpretable_predictor import InterpretableTabularPredictor\nfrom .predictor import TabularPredictor\n"
},
{
"path": "tabular/src/autogluon/tabular/predictor/interpretable_predictor.py",
"content": "import logging\n\nimport numpy as np\nimport pandas as pd\n\nfrom .predictor import TabularPredictor\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Consider removing this unless improved. I suspect it will simply confuse users more than help them.\nclass InterpretableTabularPredictor(TabularPredictor):\n \"\"\"\n EXPERIMENTAL\n\n AutoGluon InterpretableTabularPredictor predicts values in a column of a tabular dataset (classification or regression).\n InterpretableTabularPredictor shares the same functionality as TabularPredictor, but is limited to simple models\n that are easier to interpret visually via simple rules.\n\n InterpretableTabularPredictor should be used when accuracy is not important,\n and instead interpretability is the key requirement.\n\n Categorical features are one-hot-encoded to preserve interpretability.\n\n Stacking and bagging are not available in this predictor to preserve interpretability.\n \"\"\"\n\n def fit(self, train_data, tuning_data=None, time_limit=None, *, presets=\"interpretable\", **kwargs):\n logger.log(\n 30,\n f\"EXPERIMENTAL WARNING: Fitting {self.__class__.__name__}\\n\"\n f\"\\tThis class is experimental and could be removed without warning in a future release.\\n\"\n f\"\\tTo avoid confusing results, please only provide categorical and numeric features.\\n\"\n f\"\\tText and datetime features will result in confusing rules that are hard to interpret.\",\n )\n\n return super().fit(\n train_data=train_data,\n tuning_data=tuning_data,\n time_limit=time_limit,\n presets=presets,\n **kwargs,\n )\n\n def _validate_fit_extra_kwargs(self, kwargs, extra_valid_keys=None) -> dict:\n kwargs = super()._validate_fit_extra_kwargs(kwargs=kwargs, extra_valid_keys=extra_valid_keys)\n print(kwargs)\n if \"num_bag_folds\" in kwargs and kwargs[\"num_bag_folds\"] is not None and kwargs[\"num_bag_folds\"] > 1:\n raise ValueError(f\"{self.__class__.__name__} does not support `num_bag_folds`.\")\n if \"num_bag_sets\" in kwargs and kwargs[\"num_bag_sets\"] is not None and kwargs[\"num_bag_sets\"] > 1:\n raise ValueError(f\"{self.__class__.__name__} does not support `num_bag_sets`.\")\n if \"num_stack_levels\" in kwargs and kwargs[\"num_stack_levels\"] is not None and kwargs[\"num_stack_levels\"] >= 1:\n raise ValueError(f\"{self.__class__.__name__} does not support `num_stack_levels`.\")\n if \"auto_stack\" in kwargs and kwargs[\"auto_stack\"]:\n raise ValueError(f\"{self.__class__.__name__} does not support `auto_stack`.\")\n return kwargs\n\n def leaderboard_interpretable(self, verbose: bool = False, **kwargs) -> pd.DataFrame:\n \"\"\"\n Leaderboard of fitted interpretable models along with their corresponding complexities.\n Identical to `.leaderboard`, but with an additional 'complexity' column indicating\n the number of rules used in the model.\n\n Models which do not support calculating 'complexity' will be filtered from this result.\n \"\"\"\n silent = kwargs.pop(\"silent\", None)\n if silent is not None:\n verbose = not silent\n leaderboard = self.leaderboard(**kwargs)\n\n complexities = []\n info = self.info()\n for i in range(leaderboard.shape[0]):\n model_name = leaderboard.iloc[i][\"model\"]\n complexities.append(info[\"model_info\"][model_name].get(\"complexity\", np.nan))\n leaderboard.insert(2, \"complexity\", complexities) # insert directly after score_test/score_val\n leaderboard = leaderboard[~pd.isna(leaderboard.complexity)] # remove non-interpretable models\n score_col = \"score_test\" if \"score_test\" in leaderboard.columns else \"score_val\"\n leaderboard = leaderboard.sort_values(by=[score_col, \"complexity\"], ascending=[False, True], ignore_index=True)\n if verbose:\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n print(leaderboard)\n return leaderboard\n\n def print_interpretable_rules(self, complexity_threshold: int = 10, model_name: str = None):\n \"\"\"\n Print the rules of the highest performing model below the complexity threshold.\n\n Parameters\n ----------\n complexity_threshold : int, default=10\n Threshold for complexity (number of rules) of fitted models to show.\n If not model complexity is below this threshold, prints the model with the lowest complexity.\n model_name : str, default=None\n Optionally print rules for a particular model, ignoring the complexity threshold.\n \"\"\"\n if model_name is None:\n summaries = self.leaderboard_interpretable()\n summaries_filtered = summaries[summaries.complexity <= complexity_threshold]\n if summaries_filtered.shape[0] == 0:\n summaries_filtered = summaries\n model_name = summaries_filtered.iloc[0][\"model\"] # best model is at top\n agmodel = self._trainer.load_model(model_name)\n imodel = agmodel.model\n print(imodel)\n\n # TODO: I have not been able to extract any insight from the output of this method.\n # I don't see how it is useful.\n def explain_classification_errors(self, data, model=None, print_rules: bool = True):\n \"\"\"Explain classification errors by fitting a rule-based model to them\n\n Parameters\n ----------\n data : str or :class:`pd.DataFrame`\n The data to make predictions for. Should contain same column names as training Dataset and follow same format\n (may contain extra columns that won't be used by Predictor, including the label-column itself).\n If str is passed, `data` will be loaded using the str value as the file path.\n model : str (optional)\n The name of the model to get predictions from. Defaults to None, which uses the highest scoring model on the validation set.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`\n print_rules : bool, optional\n Whether to print the learned rules\n\n Returns\n -------\n cls : imodels.classifier\n Interpretable rule-based classifier with fit/predict methods\n \"\"\"\n import imodels\n\n if model is None:\n model = self.model_best\n data = self._get_dataset(data)\n predictions = self.predict(data=data, model=model, as_pandas=True)\n labels = data[self.label]\n data_transformed = self.transform_features(data=data, model=model)\n labels_transformed = self.transform_labels(labels=labels)\n cls, columns = imodels.explain_classification_errors(\n data_transformed, predictions, labels_transformed, print_rules=print_rules\n )\n return cls\n"
},
{
"path": "tabular/src/autogluon/tabular/predictor/predictor.py",
"content": "from __future__ import annotations\n\nimport copy\nimport inspect\nimport logging\nimport math\nimport os\nimport pprint\nimport shutil\nimport time\nimport warnings\nfrom typing import Any, Literal, Optional, Union, overload\n\nimport networkx as nx\nimport numpy as np\nimport pandas as pd\nfrom packaging import version\n\nfrom autogluon.common import FeatureMetadata, TabularDataset\nfrom autogluon.common.loaders import load_json\nfrom autogluon.common.savers import save_json\nfrom autogluon.common.utils.cv_splitter import CVSplitter\nfrom autogluon.common.utils.decorators import apply_presets\nfrom autogluon.common.utils.file_utils import get_directory_size, get_directory_size_per_file\nfrom autogluon.common.utils.hyperparameter_utils import (\n get_hyperparameter_str_deprecation_msg,\n is_advanced_hyperparameter_format,\n)\nfrom autogluon.common.utils.log_utils import (\n add_log_to_file,\n set_logger_verbosity,\n warn_if_mlflow_autologging_is_enabled,\n)\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager, get_resource_manager\nfrom autogluon.common.utils.system_info import get_ag_system_info\nfrom autogluon.common.utils.try_import import try_import_ray\nfrom autogluon.common.utils.utils import (\n check_saved_predictor_version,\n compare_autogluon_metadata,\n get_autogluon_metadata,\n setup_outputdir,\n)\nfrom autogluon.core.callbacks import AbstractCallback\nfrom autogluon.core.constants import (\n AUTO_WEIGHT,\n BALANCE_WEIGHT,\n BINARY,\n MULTICLASS,\n PROBLEM_TYPES_CLASSIFICATION,\n PSEUDO_MODEL_SUFFIX,\n QUANTILE,\n REGRESSION,\n)\nfrom autogluon.core.data.label_cleaner import LabelCleanerMulticlassToBinary\nfrom autogluon.core.metrics import Scorer, get_metric\nfrom autogluon.core.problem_type import problem_type_info\nfrom autogluon.core.pseudolabeling.pseudolabeling import filter_ensemble_pseudo, filter_pseudo\nfrom autogluon.core.scheduler.scheduler_factory import scheduler_factory\nfrom autogluon.core.stacked_overfitting.utils import check_stacked_overfitting_from_leaderboard\nfrom autogluon.core.utils import (\n get_pred_from_proba_df,\n plot_performance_vs_trials,\n plot_summary_of_models,\n plot_tabular_models,\n)\nfrom autogluon.core.utils.loaders import load_pkl, load_str\nfrom autogluon.core.utils.savers import save_pkl, save_str\nfrom autogluon.core.utils.utils import generate_train_test_split_combined\n\nfrom ..configs.feature_generator_presets import get_default_feature_generator\nfrom ..configs.hyperparameter_configs import get_hyperparameter_config\nfrom ..configs.pipeline_presets import (\n USE_BAG_HOLDOUT_AUTO_THRESHOLD,\n get_validation_and_stacking_method,\n)\nfrom ..configs.presets_configs import tabular_presets_alias, tabular_presets_dict\nfrom ..learner import AbstractTabularLearner, DefaultLearner\nfrom ..registry import ag_model_registry\nfrom ..trainer.abstract_trainer import AbstractTabularTrainer\nfrom ..version import __version__\n\nlogger = logging.getLogger(__name__) # return autogluon root logger\n\n\n# Extra TODOs (Stretch): Can occur post v1.0\n# TODO: make core_kwargs a kwargs argument to predictor.fit\n# TODO: add aux_kwargs to predictor.fit\n# TODO: consider adding kwarg option for data which has already been preprocessed by feature generator to skip feature generation.\n# TODO: Resolve raw text feature usage in default feature generator\n# TODO: num_bag_sets -> ag_args\nclass TabularPredictor:\n \"\"\"\n AutoGluon TabularPredictor predicts values in a column of a tabular dataset (classification or regression).\n\n Parameters\n ----------\n label : str\n Name of the column that contains the target variable to predict.\n problem_type : str, default = None\n Type of prediction problem, i.e. is this a binary/multiclass classification or regression problem (options: 'binary', 'multiclass', 'regression', 'quantile').\n If `problem_type = None`, the prediction problem type is inferred based on the label-values in provided dataset.\n eval_metric : str or Scorer, default = None\n Metric by which predictions will be ultimately evaluated on test data.\n AutoGluon tunes factors such as hyperparameters, early-stopping, ensemble-weights, etc. in order to improve this metric on validation data.\n\n If `eval_metric = None`, it is automatically chosen based on `problem_type`.\n Defaults to 'accuracy' for binary and multiclass classification, 'root_mean_squared_error' for regression, and 'pinball_loss' for quantile.\n\n Otherwise, options for classification:\n ['accuracy', 'balanced_accuracy', 'log_loss', 'f1', 'f1_macro', 'f1_micro', 'f1_weighted',\n 'roc_auc', 'roc_auc_ovo', 'roc_auc_ovo_macro', 'roc_auc_ovo_weighted', 'roc_auc_ovr', 'roc_auc_ovr_macro', 'roc_auc_ovr_micro',\n 'roc_auc_ovr_weighted', 'average_precision', 'precision', 'precision_macro', 'precision_micro', 'precision_weighted',\n 'recall', 'recall_macro', 'recall_micro', 'recall_weighted', 'mcc', 'pac_score']\n Options for regression:\n ['root_mean_squared_error', 'mean_squared_error', 'mean_absolute_error', 'median_absolute_error', 'mean_absolute_percentage_error', 'r2', 'symmetric_mean_absolute_percentage_error']\n For more information on these options, see `sklearn.metrics`: https://scikit-learn.org/stable/modules/classes.html#sklearn-metrics-metrics\n For metric source code, see `autogluon.core.metrics`.\n\n You can also pass your own evaluation function here as long as it follows formatting of the functions defined in folder `autogluon.core.metrics`.\n For detailed instructions on creating and using a custom metric, refer to https://auto.gluon.ai/stable/tutorials/tabular/advanced/tabular-custom-metric.html\n path : Union[str, pathlib.Path], default = None\n Path to directory where models and intermediate outputs should be saved.\n If unspecified, a time-stamped folder called \"AutogluonModels/ag-[TIMESTAMP]\" will be created in the working directory to store all models.\n Note: To call `fit()` twice and save all results of each fit, you must specify different `path` locations or don't specify `path` at all.\n Otherwise files from first `fit()` will be overwritten by second `fit()`.\n verbosity : int, default = 2\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n Verbosity levels:\n 0: Only log exceptions\n 1: Only log warnings + exceptions\n 2: Standard logging\n 3: Verbose logging (ex: log validation score every 50 iterations)\n 4: Maximally verbose logging (ex: log validation score every iteration)\n log_to_file: bool, default = False\n Whether to save the logs into a file for later reference\n log_file_path: str, default = \"auto\"\n File path to save the logs.\n If auto, logs will be saved under `predictor_path/logs/predictor_log.txt`.\n Will be ignored if `log_to_file` is set to False\n sample_weight : str, default = None\n If specified, this column-name indicates which column of the data should be treated as sample weights. This column will NOT be considered as a predictive feature.\n Sample weights should be non-negative (and cannot be nan), with larger values indicating which rows are more important than others.\n If you want your usage of sample weights to match results obtained outside of this Predictor, then ensure sample weights for your training (or tuning) data sum to the number of rows in the training (or tuning) data.\n You may also specify two special strings: 'auto_weight' (automatically choose a weighting strategy based on the data) or 'balance_weight' (equally weight classes in classification, no effect in regression). If specifying your own sample_weight column, make sure its name does not match these special strings.\n weight_evaluation : bool, default = False\n Only considered when `sample_weight` column is not None. Determines whether sample weights should be taken into account when computing evaluation metrics on validation/test data.\n If True, then weighted metrics will be reported based on the sample weights provided in the specified `sample_weight` (in which case `sample_weight` column must also be present in test data).\n In this case, the 'best' model used by default for prediction will also be decided based on a weighted version of evaluation metric.\n Note: we do not recommend specifying `weight_evaluation` when `sample_weight` is 'auto_weight' or 'balance_weight', instead specify appropriate `eval_metric`.\n groups : str, default = None\n [Experimental] If specified, AutoGluon will use the column named the value of groups in `train_data` during `.fit` as the data splitting indices for the purposes of bagging.\n This column will not be used as a feature during model training.\n This parameter is ignored if bagging is not enabled. To instead specify a custom validation set with bagging disabled, specify `tuning_data` in `.fit`.\n The data will be split via `sklearn.model_selection.LeaveOneGroupOut`.\n Use this option to control the exact split indices AutoGluon uses.\n It is not recommended to use this option unless it is required for very specific situations.\n Bugs may arise from edge cases if the provided groups are not valid to properly train models, such as if not all classes are present during training in multiclass classification. It is up to the user to sanitize their groups.\n\n As an example, if you want your data folds to preserve adjacent rows in the table without shuffling, then for 3 fold bagging with 6 rows of data, the groups column values should be [0, 0, 1, 1, 2, 2].\n positive_class : str or int, default = None\n Used to determine the positive class in binary classification.\n This is used for certain metrics such as 'f1' which produce different scores depending on which class is considered the positive class.\n If not set, will be inferred as the second element of the existing unique classes after sorting them.\n If classes are [0, 1], then 1 will be selected as the positive class.\n If classes are ['def', 'abc'], then 'def' will be selected as the positive class.\n If classes are [True, False], then True will be selected as the positive class.\n **kwargs :\n learner_type : AbstractLearner, default = DefaultLearner\n A class which inherits from `AbstractLearner`. This dictates the inner logic of predictor.\n If you don't know what this is, keep it as the default.\n learner_kwargs : dict, default = None\n Kwargs to send to the learner. Options include:\n\n ignored_columns : list, default = None\n Banned subset of column names that predictor may not use as predictive features (e.g. unique identifier to a row or user-ID).\n These columns are ignored during `fit()`.\n label_count_threshold : int, default = 10\n For multi-class classification problems, this is the minimum number of times a label must appear in dataset in order to be considered an output class.\n AutoGluon will ignore any classes whose labels do not appear at least this many times in the dataset (i.e. will never predict them).\n cache_data : bool, default = True\n When enabled, the training and validation data are saved to disk for future reuse.\n Enables advanced functionality in predictor such as `fit_extra()` and feature importance calculation on the original data.\n trainer_type : AbstractTabularTrainer, default = AutoTrainer\n A class inheriting from `AbstractTabularTrainer` that controls training/ensembling of many models.\n If you don't know what this is, keep it as the default.\n default_base_path : str | Path | None, default = None\n A default base path to use for the time-stamped folder if `path` is None.\n If None, defaults to `AutogluonModels`. Only used if `path` is None, and thus\n only used for local paths, not s3 paths.\n \"\"\"\n\n Dataset = TabularDataset\n predictor_file_name = \"predictor.pkl\"\n _predictor_version_file_name = \"version.txt\"\n _predictor_metadata_file_name = \"metadata.json\"\n _predictor_log_file_name = \"predictor_log.txt\"\n\n def __init__(\n self,\n label: str,\n problem_type: str = None,\n eval_metric: str | Scorer = None,\n path: str = None,\n verbosity: int = 2,\n log_to_file: bool = False,\n log_file_path: str = \"auto\",\n sample_weight: str = None,\n weight_evaluation: bool = False,\n groups: str = None,\n positive_class: int | str | None = None,\n **kwargs,\n ):\n self.verbosity = verbosity\n set_logger_verbosity(self.verbosity)\n if sample_weight == AUTO_WEIGHT: # TODO: update auto_weight strategy and make it the default\n sample_weight = None\n logger.log(15, f\"{AUTO_WEIGHT} currently does not use any sample weights.\")\n self.sample_weight = sample_weight\n self.weight_evaluation = weight_evaluation # TODO: sample_weight and weight_evaluation can both be properties that link to self._learner.sample_weight, self._learner.weight_evaluation\n self._decision_threshold = (\n None # TODO: Each model should have its own decision threshold instead of one global threshold\n )\n if self.sample_weight in [AUTO_WEIGHT, BALANCE_WEIGHT] and self.weight_evaluation:\n logger.warning(\n f\"We do not recommend specifying weight_evaluation when sample_weight='{self.sample_weight}', instead specify appropriate eval_metric.\"\n )\n self._validate_init_kwargs(kwargs)\n path = setup_outputdir(path=path, default_base_path=kwargs.get(\"default_base_path\"))\n\n learner_type = kwargs.get(\"learner_type\", DefaultLearner)\n learner_kwargs = kwargs.get(\"learner_kwargs\", dict())\n quantile_levels = kwargs.get(\"quantile_levels\", None)\n if positive_class is not None:\n learner_kwargs[\"positive_class\"] = positive_class\n\n self._learner: AbstractTabularLearner = learner_type(\n path_context=path,\n label=label,\n feature_generator=None,\n eval_metric=eval_metric,\n problem_type=problem_type,\n quantile_levels=quantile_levels,\n sample_weight=self.sample_weight,\n weight_evaluation=self.weight_evaluation,\n groups=groups,\n **learner_kwargs,\n )\n self._learner_type = type(self._learner)\n self._trainer: AbstractTabularTrainer = None\n self._sub_fits: list[str] = []\n self._stacked_overfitting_occurred: bool | None = None\n self._fit_strategy = None\n\n if log_to_file:\n self._setup_log_to_file(log_file_path=log_file_path)\n\n @property\n def classes_(self) -> list:\n \"\"\"\n For multiclass problems, this list contains the class labels in sorted order of `predict_proba()` output.\n For binary problems, this list contains the class labels in sorted order of `predict_proba(as_multiclass=True)` output.\n `classes_[0]` corresponds to internal label = 0 (negative class), `classes_[1]` corresponds to internal label = 1 (positive class).\n This is relevant for certain metrics such as F1 where True and False labels impact the metric score differently.\n For other problem types, will equal None.\n For example if `pred = predict_proba(x, as_multiclass=True)`, then ith index of `pred` provides predicted probability that `x` belongs to class given by `classes_[i]`.\n \"\"\"\n return self._learner.class_labels\n\n @property\n def class_labels(self) -> list:\n \"\"\"Alias to self.classes_\"\"\"\n return self.classes_\n\n @property\n def class_labels_internal(self) -> list:\n \"\"\"\n For multiclass problems, this list contains the internal class labels in sorted order of internal `predict_proba()` output.\n For binary problems, this list contains the internal class labels in sorted order of internal `predict_proba(as_multiclass=True)` output.\n The value will always be `class_labels_internal=[0, 1]` for binary problems, with 0 as the negative class, and 1 as the positive class.\n For other problem types, will equal None.\n \"\"\"\n return self._learner.label_cleaner.ordered_class_labels_transformed\n\n @property\n def class_labels_internal_map(self) -> dict:\n \"\"\"\n For binary and multiclass classification problems, this dictionary contains the mapping of the original labels to the internal labels.\n For example, in binary classification, label values of 'True' and 'False' will be mapped to the internal representation `1` and `0`.\n Therefore, class_labels_internal_map would equal {'True': 1, 'False': 0}\n For other problem types, will equal None.\n For multiclass, it is possible for not all of the label values to have a mapping.\n This indicates that the internal models will never predict those missing labels, and training rows associated with the missing labels were dropped.\n \"\"\"\n return self._learner.label_cleaner.inv_map\n\n @property\n def quantile_levels(self) -> list[float]:\n return self._learner.quantile_levels\n\n @property\n def eval_metric(self) -> Scorer:\n \"\"\"The metric used to evaluate predictive performance\"\"\"\n return self._learner.eval_metric\n\n @property\n def original_features(self) -> list[str]:\n \"\"\"Original features user passed in to fit before processing\"\"\"\n self._assert_is_fit()\n return self._learner.original_features\n\n @property\n def problem_type(self) -> str:\n \"\"\"What type of prediction problem this Predictor has been trained for\"\"\"\n return self._learner.problem_type\n\n @property\n def decision_threshold(self) -> float | None:\n \"\"\"\n The decision threshold used to convert prediction probabilities to predictions.\n Only relevant for binary classification, otherwise the value will be None.\n Valid values are in the range [0.0, 1.0]\n You can obtain an optimized `decision_threshold` by first calling `predictor.calibrate_decision_threshold()`.\n Useful to set for metrics such as `balanced_accuracy` and `f1` as `0.5` is often not an optimal threshold.\n Predictions are calculated via the following logic on the positive class: `1 if pred > decision_threshold else 0`\n \"\"\"\n if self._decision_threshold is not None:\n return self._decision_threshold\n elif self.problem_type == BINARY:\n return 0.5\n else:\n return None\n\n def set_decision_threshold(self, decision_threshold: float):\n \"\"\"\n Set `predictor.decision_threshold`. Problem type must be 'binary', and the value must be between 0 and 1.\n \"\"\"\n assert self.problem_type == BINARY\n assert decision_threshold >= 0\n assert decision_threshold <= 1\n if decision_threshold != self.decision_threshold:\n logger.log(\n 20,\n f\"Updating predictor.decision_threshold from {self.decision_threshold} -> {decision_threshold}\\n\"\n f\"\\tThis will impact how prediction probabilities are converted to predictions in binary classification.\\n\"\n f\"\\tPrediction probabilities of the positive class >{decision_threshold} \"\n f\"will be predicted as the positive class ({self.positive_class}). \"\n f\"This can significantly impact metric scores.\\n\"\n f\"\\tYou can update this value via `predictor.set_decision_threshold`.\\n\"\n f\"\\tYou can calculate an optimal decision threshold on the validation data via `predictor.calibrate_decision_threshold()`.\",\n )\n self._decision_threshold = decision_threshold\n\n def features(self, feature_stage: str = \"original\") -> list:\n \"\"\"\n Returns a list of feature names dependent on the value of feature_stage.\n\n Parameters\n ----------\n feature_stage : str, default = 'original'\n If 'original', returns the list of features specified in the original training data. This feature set is required in input data when making predictions.\n If 'transformed', returns the list of features after pre-processing by the feature generator.\n\n Returns\n -------\n Returns a list of feature names\n \"\"\"\n if feature_stage == \"original\":\n return self.feature_metadata_in.get_features()\n elif feature_stage == \"transformed\":\n return self.feature_metadata.get_features()\n else:\n raise ValueError(f\"Unknown feature_stage: '{feature_stage}'. Must be one of {['original', 'transformed']}\")\n\n @property\n def has_val(self) -> bool:\n \"\"\"\n Return True if holdout validation data was used during fit, else return False.\n \"\"\"\n self._assert_is_fit(\"has_val\")\n return self._trainer.has_val\n\n @property\n def feature_metadata(self) -> FeatureMetadata:\n \"\"\"\n Returns the internal FeatureMetadata.\n\n Inferred data type of each predictive variable after preprocessing transformation (i.e. column of training data table used to predict `label`).\n Contains both raw dtype and special dtype information. Each feature has exactly 1 raw dtype (such as 'int', 'float', 'category') and zero to many special dtypes (such as 'datetime_as_int', 'text', 'text_ngram').\n Special dtypes are AutoGluon specific feature types that are used to identify features with meaning beyond what the raw dtype can convey.\n `feature_metadata.type_map_raw`: Dictionary of feature name -> raw dtype mappings.\n `feature_metadata.type_group_map_special`: Dictionary of lists of special feature names, grouped by special feature dtype.\n \"\"\"\n return self._trainer.feature_metadata\n\n @property\n def feature_metadata_in(self) -> FeatureMetadata:\n \"\"\"\n Returns the input FeatureMetadata.\n\n Inferred data type of each predictive variable before preprocessing transformation.\n Contains both raw dtype and special dtype information. Each feature has exactly 1 raw dtype (such as 'int', 'float', 'category') and zero to many special dtypes (such as 'datetime_as_int', 'text', 'text_ngram').\n Special dtypes are AutoGluon specific feature types that are used to identify features with meaning beyond what the raw dtype can convey.\n `feature_metadata.type_map_raw`: Dictionary of feature name -> raw dtype mappings.\n `feature_metadata.type_group_map_special`: Dictionary of lists of special feature names, grouped by special feature dtype.\n \"\"\"\n return self._learner.feature_generator.feature_metadata_in\n\n @property\n def label(self) -> str | int:\n \"\"\"\n Name of table column that contains data from the variable to predict (often referred to as: labels, response variable, target variable, dependent variable, y, etc).\n \"\"\"\n return self._learner.label\n\n @property\n def path(self) -> str:\n \"\"\"Path to directory where all models used by this Predictor are stored\"\"\"\n return self._learner.path\n\n @apply_presets(tabular_presets_dict, tabular_presets_alias)\n def fit(\n self,\n train_data: pd.DataFrame | str,\n tuning_data: pd.DataFrame | str = None,\n time_limit: float = None,\n presets: list[str] | str = None,\n hyperparameters: dict | str = None,\n feature_metadata: str | FeatureMetadata = \"infer\",\n infer_limit: float = None,\n infer_limit_batch_size: int = None,\n fit_weighted_ensemble: bool = True,\n fit_full_last_level_weighted_ensemble: bool = True,\n full_weighted_ensemble_additionally: bool = False,\n dynamic_stacking: bool | str = False,\n calibrate_decision_threshold: bool | str = \"auto\",\n num_cpus: int | str = \"auto\",\n num_gpus: int | str = \"auto\",\n fit_strategy: Literal[\"sequential\", \"parallel\"] = \"sequential\",\n memory_limit: float | str = \"auto\",\n callbacks: list[AbstractCallback | list | tuple] = None,\n **kwargs,\n ) -> \"TabularPredictor\":\n \"\"\"\n Fit models to predict a column of a data table (label) based on the other columns (features).\n\n Parameters\n ----------\n train_data : :class:`pd.DataFrame` or str\n Table of the training data as a pandas DataFrame.\n If str is passed, `train_data` will be loaded using the str value as the file path.\n tuning_data : :class:`pd.DataFrame` or str, optional\n Another dataset containing validation data reserved for tuning processes such as early stopping, hyperparameter tuning, and ensembling.\n This dataset should be in the same format as `train_data`.\n If str is passed, `tuning_data` will be loaded using the str value as the file path.\n Note: If `refit_full=True` is specified, the final model may be fit on `tuning_data` as well as `train_data`.\n Note: Because `tuning_data` is used to determine which model is the 'best' model, as well as to determine the ensemble weights,\n it should not be considered a fully unseen dataset. It is possible that AutoGluon will be overfit to the `tuning_data`.\n To ensure an unbiased evaluation, use separate unseen test data to evaluate the final model using `predictor.leaderboard(test_data, display=True)`.\n Do not provide your evaluation test data as `tuning_data`!\n If bagging is not enabled and `tuning_data = None`: `fit()` will automatically hold out some random validation samples from `train_data`.\n If bagging is enabled and `tuning_data = None`: no tuning data will be used. Instead, AutoGluon will perform cross-validation.\n If bagging is enabled: `use_bag_holdout=True` must be specified in order to provide tuning data. If specified, AutoGluon will still perform cross-validation for model fits, but will use `tuning_data` for optimizing the weighted ensemble weights and model calibration.\n time_limit : float, default = None\n Approximately how long `fit()` should run for (wallclock time in seconds).\n If not specified, `fit()` will run until all models have completed training, but will not repeatedly bag models unless `num_bag_sets` is specified.\n presets : list or str or dict, default = ['medium_quality']\n List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`.\n It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon.\n As an example, to get the most accurate overall predictor (regardless of its efficiency), set `presets='best_quality'` (or `extreme_quality` if a GPU is available).\n To get good quality with minimal disk usage, set `presets=['good_quality', 'optimize_for_deployment']`\n Any user-specified arguments in `fit()` will override the values used by presets.\n If specifying a list of presets, later presets will override earlier presets if they alter the same argument.\n For precise definitions of the provided presets, see file: `autogluon/tabular/configs/presets_configs.py`.\n Users can specify custom presets by passing in a dictionary of argument values as an element to the list.\n\n Available Presets: ['extreme_quality', 'best_quality', 'high_quality', 'good_quality', 'medium_quality', 'experimental_quality', 'optimize_for_deployment', 'interpretable', 'ignore_text']\n\n It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other.\n\n In-depth Preset Info:\n extreme_quality={...}\n New in v1.5: The state-of-the-art for tabular machine learning.\n Requires `pip install autogluon.tabular[tabarena]` to install TabPFN, TabICL, and TabDPT.\n Significantly more accurate than `best_quality` on datasets <= 100000 samples. Requires a GPU.\n Will use recent tabular foundation models TabPFNv2, TabICL, TabDPT, and Mitra to maximize performance.\n Recommended for applications that benefit from the best possible model accuracy.\n\n best_quality_v150={...}\n New in v1.5: Better quality than 'best_quality' and 5x+ faster to train. Give it a try!\n\n best_quality={'auto_stack': True, 'dynamic_stacking': 'auto', 'hyperparameters': 'zeroshot'}\n Best predictive accuracy with little consideration to inference time or disk usage. Achieve even better results by specifying a large time_limit value.\n Recommended for applications that benefit from the best possible model accuracy.\n\n high_quality_v150={...}\n New in v1.5: Better quality than 'high_quality' and 5x+ faster to train. Give it a try!\n\n high_quality={'auto_stack': True, 'dynamic_stacking': 'auto', 'hyperparameters': 'zeroshot', 'refit_full': True, 'set_best_to_refit_full': True, 'save_bag_folds': False}\n High predictive accuracy with fast inference. ~8x faster inference and ~8x lower disk usage than `best_quality`.\n Recommended for applications that require reasonable inference speed and/or model size.\n\n good_quality={'auto_stack': True, 'dynamic_stacking': 'auto', 'hyperparameters': 'light', 'refit_full': True, 'set_best_to_refit_full': True, 'save_bag_folds': False}\n Good predictive accuracy with very fast inference. ~4x faster inference and ~4x lower disk usage than `high_quality`.\n Recommended for applications that require fast inference speed.\n\n medium_quality={'auto_stack': False}\n Medium predictive accuracy with very fast inference and very fast training time. ~20x faster training than `good_quality`.\n This is the default preset in AutoGluon, but should generally only be used for quick prototyping, as `good_quality` results in significantly better predictive accuracy and faster inference time.\n\n experimental_quality={'auto_stack': True, 'dynamic_stacking': 'auto', 'hyperparameters': 'experimental', 'fit_strategy': 'parallel', 'num_gpus': 0}\n This preset acts as a testing ground for cutting edge features and models which could later be added to the `best_quality` preset in future releases.\n Recommended when `best_quality` was already being used and the user wants to push performance even further.\n\n optimize_for_deployment={'keep_only_best': True, 'save_space': True}\n Optimizes result immediately for deployment by deleting unused models and removing training artifacts.\n Often can reduce disk usage by ~2-4x with no negatives to model accuracy or inference speed.\n This will disable numerous advanced functionality, but has no impact on inference.\n This will make certain functionality less informative, such as `predictor.leaderboard()` and `predictor.fit_summary()`.\n Because unused models will be deleted under this preset, methods like `predictor.leaderboard()` and `predictor.fit_summary()` will no longer show the full set of models that were trained during `fit()`.\n Recommended for applications where the inner details of AutoGluon's training is not important and there is no intention of manually choosing between the final models.\n This preset pairs well with the other presets such as `good_quality` to make a very compact final model.\n Identical to calling `predictor.delete_models(models_to_keep='best')` and `predictor.save_space()` directly after `fit()`.\n\n interpretable={'auto_stack': False, 'hyperparameters': 'interpretable'}\n Fits only interpretable rule-based models from the imodels package.\n Trades off predictive accuracy for conciseness.\n\n ignore_text={'_feature_generator_kwargs': {'enable_text_ngram_features': False, 'enable_text_special_features': False, 'enable_raw_text_features': False}}\n Disables automated feature generation when text features are detected.\n This is useful to determine how beneficial text features are to the end result, as well as to ensure features are not mistaken for text when they are not.\n Ignored if `feature_generator` was also specified.\n\n hyperparameters : str or dict, default = 'default'\n Determines the hyperparameters used by the models.\n If `str` is passed, will use a preset hyperparameter configuration.\n Valid `str` options: ['default', 'zeroshot', 'zeroshot_2025_tabfm', 'light', 'very_light', 'toy', 'multimodal']\n 'default': Default AutoGluon hyperparameters intended to get strong accuracy with reasonable disk usage and inference time. Used in the 'medium_quality' preset.\n 'zeroshot': A powerful model portfolio learned from TabRepo's ensemble simulation on 200 datasets. Contains ~100 models and is used in 'best_quality' and 'high_quality' presets.\n 'zeroshot_2025_tabfm': Absolute cutting edge portfolio learned from TabArena's ensemble simulation that leverages tabular foundation models. Contains 22 models and is used in the `extreme_quality` preset.\n 'light': Results in smaller models. Generally will make inference speed much faster and disk usage much lower, but with worse accuracy. Used in the 'good_quality' preset.\n 'very_light': Results in much smaller models. Behaves similarly to 'light', but in many cases with over 10x less disk usage and a further reduction in accuracy.\n 'toy': Results in extremely small models. Only use this when prototyping, as the model quality will be severely reduced.\n 'multimodal': [EXPERIMENTAL] Trains a multimodal transformer model alongside tabular models. Requires that some text columns appear in the data and GPU.\n When combined with 'best_quality' `presets` option, this can achieve extremely strong results in multimodal data tables that contain columns with text in addition to numeric/categorical columns.\n Reference `autogluon/tabular/configs/hyperparameter_configs.py` for information on the hyperparameters associated with each preset.\n Keys are strings that indicate which model types to train.\n Stable model options include:\n 'GBM' (LightGBM)\n 'CAT' (CatBoost)\n 'XGB' (XGBoost)\n 'EBM' (Explainable Boosting Machine)\n 'REALMLP' (RealMLP)\n 'TABM' (TabM)\n 'MITRA' (Mitra)\n 'TABICL' (TabICL)\n 'TABPFNV2' (TabPFNv2)\n 'RF' (random forest)\n 'XT' (extremely randomized trees)\n 'KNN' (k-nearest neighbors)\n 'LR' (linear regression)\n 'NN_TORCH' (neural network implemented in Pytorch)\n 'FASTAI' (neural network with FastAI backend)\n 'AG_AUTOMM' (`MultimodalPredictor` from `autogluon.multimodal`. Supports Tabular, Text, and Image modalities. GPU is required.)\n Experimental model options include:\n 'FT_TRANSFORMER' (Tabular Transformer, GPU is recommended. Does not scale well to >100 features. Recommended to use TabM instead.)\n 'AG_TEXT_NN' (Multimodal Text+Tabular model, GPU is required. Recommended to instead use its successor, 'AG_AUTOMM'.)\n 'AG_IMAGE_NN' (Image model, GPU is required. Recommended to instead use its successor, 'AG_AUTOMM'.)\n If a certain key is missing from hyperparameters, then `fit()` will not train any models of that type. Omitting a model key from hyperparameters is equivalent to including this model key in `excluded_model_types`.\n For example, set `hyperparameters = { 'NN_TORCH':{...} }` if say you only want to train (PyTorch) neural networks and no other types of models.\n Values = dict of hyperparameter settings for each model type, or list of dicts.\n Each hyperparameter can either be a single fixed value or a search space containing many possible values.\n Unspecified hyperparameters will be set to default values (or default search spaces if `hyperparameter_tune_kwargs='auto'`).\n Caution: Any provided search spaces will error if `hyperparameter_tune_kwargs=None` (Default).\n To train multiple models of a given type, set the value to a list of hyperparameter dictionaries.\n For example, `hyperparameters = {'RF': [{'criterion': 'gini'}, {'criterion': 'entropy'}]}` will result in 2 random forest models being trained with separate hyperparameters.\n Advanced functionality: Bring your own model / Custom model support\n AutoGluon fully supports custom models. For a detailed tutorial on creating and using custom models with AutoGluon, refer to https://auto.gluon.ai/stable/tutorials/tabular/advanced/tabular-custom-model.html\n Advanced functionality: Custom stack levels\n By default, AutoGluon reuses the same models and model hyperparameters at each level during stack ensembling.\n To customize this behaviour, create a hyperparameters dictionary separately for each stack level, and then add them as values to a new dictionary, with keys equal to the stack level.\n Example: `hyperparameters = {1: {'RF': rf_params1}, 2: {'CAT': [cat_params1, cat_params2], 'NN_TORCH': {}}}`\n This will result in a stack ensemble that has one custom random forest in level 1 followed by two CatBoost models with custom hyperparameters and a default neural network in level 2, for a total of 4 models.\n If a level is not specified in `hyperparameters`, it will default to using the highest specified level to train models. This can also be explicitly controlled by adding a 'default' key.\n\n Default:\n hyperparameters = {\n 'NN_TORCH': {},\n 'GBM': [\n {'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}},\n {},\n {\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ],\n 'CAT': {},\n 'XGB': {},\n 'FASTAI': {},\n 'RF': [\n {'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}},\n {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}},\n {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression']}},\n ],\n 'XT': [\n {'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}},\n {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}},\n {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression']}},\n ],\n 'KNN': [\n {'weights': 'uniform', 'ag_args': {'name_suffix': 'Unif'}},\n {'weights': 'distance', 'ag_args': {'name_suffix': 'Dist'}},\n ],\n }\n\n Details regarding the hyperparameters you can specify for each model are provided in the following files:\n NN: `autogluon.tabular.models.tabular_nn.hyperparameters.parameters`\n Note: certain hyperparameter settings may cause these neural networks to train much slower.\n GBM: `autogluon.tabular.models.lgb.hyperparameters.parameters`\n See also the lightGBM docs: https://lightgbm.readthedocs.io/en/latest/Parameters.html\n CAT: `autogluon.tabular.models.catboost.hyperparameters.parameters`\n See also the CatBoost docs: https://catboost.ai/docs/concepts/parameter-tuning.html\n XGB: `autogluon.tabular.models.xgboost.hyperparameters.parameters`\n See also the XGBoost docs: https://xgboost.readthedocs.io/en/latest/parameter.html\n FASTAI: `autogluon.tabular.models.fastainn.hyperparameters.parameters`\n See also the FastAI docs: https://docs.fast.ai/tabular.learner.html\n RF: See sklearn documentation: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html\n Note: Hyperparameter tuning is disabled for this model.\n XT: See sklearn documentation: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html\n Note: Hyperparameter tuning is disabled for this model.\n KNN: See sklearn documentation: https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html\n Note: Hyperparameter tuning is disabled for this model.\n LR: `autogluon.tabular.models.lr.hyperparameters.parameters`\n Note: Hyperparameter tuning is disabled for this model.\n Note: 'penalty' parameter can be used for regression to specify regularization method: 'L1' and 'L2' values are supported.\n Advanced functionality: Custom AutoGluon model arguments\n These arguments are optional and can be specified in any model's hyperparameters.\n Example: `hyperparameters = {'RF': {..., 'ag_args': {'name_suffix': 'CustomModelSuffix', 'disable_in_hpo': True}}`\n Individual arguments can be passed for ag_args_fit by adding the prefix `ag.`: `hyperparameters = {'RF': {..., 'ag.num_cpus': 1}}`\n Individual arguments can be passed for ag_args_ensemble by adding the prefix `ag.ens`: `hyperparameters = {'RF': {..., 'ag.ens.fold_fitting_strategy': 'sequential_local'}}`\n ag_args: Dictionary of customization options related to meta properties of the model such as its name, the order it is trained, the problem types it is valid for, and the type of HPO it utilizes.\n Valid keys:\n name: (str) The name of the model. This overrides AutoGluon's naming logic and all other name arguments if present.\n name_main: (str) The main name of the model. Example: 'RandomForest'.\n name_prefix: (str) Add a custom prefix to the model name. Unused by default.\n name_suffix: (str) Add a custom suffix to the model name. Unused by default.\n priority: (int) Determines the order in which the model is trained. Larger values result in the model being trained earlier. Default values range from 100 (KNN) to 0 (custom), dictated by model type. If you want this model to be trained first, set priority = 999.\n problem_types: (list) List of valid problem types for the model. `problem_types=['binary']` will result in the model only being trained if `problem_type` is 'binary'.\n disable_in_hpo: (bool) If True, the model will only be trained if `hyperparameter_tune_kwargs=None`.\n valid_stacker: (bool) If False, the model will not be trained as a level 2 or higher stacker model.\n valid_base: (bool) If False, the model will not be trained as a level 1 (base) model.\n hyperparameter_tune_kwargs: (dict) Refer to :meth:`TabularPredictor.fit` hyperparameter_tune_kwargs argument. If specified here, will override global HPO settings for this model.\n Reference the default hyperparameters for example usage of these options.\n ag_args_fit: Dictionary of model fit customization options related to how and with what constraints the model is trained. These parameters affect stacker fold models, but not stacker models themselves.\n Clarification: `time_limit` is the internal time in seconds given to a particular model to train, which is dictated in part by the `time_limit` argument given during `predictor.fit()` but is not the same.\n Valid keys:\n stopping_metric: (str or :class:`autogluon.core.metrics.Scorer`, default=None) The metric to use for early stopping of the model. If None, model will decide.\n max_memory_usage_ratio: (float, default=1.0) The ratio of memory usage relative to the default to allow before early stopping or killing the model. Values greater than 1.0 will be increasingly prone to out-of-memory errors.\n max_time_limit_ratio: (float, default=1.0) The ratio of the provided time_limit to use during model `fit()`. If `time_limit=10` and `max_time_limit_ratio=0.3`, time_limit would be changed to 3. Does not alter max_time_limit or min_time_limit values.\n max_time_limit: (float, default=None) Maximum amount of time to allow this model to train for (in sec). If the provided time_limit is greater than this value, it will be replaced by max_time_limit.\n min_time_limit: (float, default=0) Allow this model to train for at least this long (in sec), regardless of the time limit it would otherwise be granted.\n If `min_time_limit >= max_time_limit`, time_limit will be set to min_time_limit.\n If `min_time_limit=None`, time_limit will be set to None and the model will have no training time restriction.\n num_cpus : (int or str, default='auto')\n How many CPUs to use during model fit.\n If 'auto', model will decide.\n num_gpus : (int or str, default='auto')\n How many GPUs to use during model fit.\n If 'auto', model will decide. Some models can use GPUs but don't by default due to differences in model quality.\n Set to 0 to disable usage of GPUs.\n max_rows : (int, default=None)\n If train_data has more rows than `max_rows`, the model will raise an AssertionError at the start of fit.\n max_features : (int, default=None)\n If train_data has more features than `max_features`, the model will raise an AssertionError at the start of fit.\n max_classes : (int, default==None)\n If train_data has more classes than `max_classes`, the model will raise an AssertionError at the start of fit.\n problem_types : (list[str], default=None)\n If the task is not a problem_type in `problem_types`, the model will raise an AssertionError at the start of fit.\n ignore_constraints : (bool, default=False)\n If True, will ignore the values of `max_rows`, `max_features`, `max_classes`, and `problem_type`, treating them as None.\n ag_args_ensemble: Dictionary of hyperparameters shared by all models that control how they are ensembled, if bag mode is enabled.\n Valid keys:\n use_orig_features: [True, False, \"never\"], default True\n Whether a stack model will use the original features along with the stack features to train (akin to skip-connections).\n If True, will use the original data features.\n If False, will discard the original data features and only use stack features, except when no stack features exist (such as in layer 1).\n If \"never\", will always discard the original data features. Will be skipped in layer 1.\n valid_stacker : bool, default True\n If True, will be marked as valid to include as a stacker model.\n If False, will only be fit as a base model (layer 1) and will not be fit in stack layers (layer 2+).\n max_base_models : int, default 0\n Maximum number of base models whose predictions form the features input to this stacker model.\n If more than `max_base_models` base models are available, only the top `max_base_models` models with highest validation score are used.\n If 0, the logic is skipped.\n max_base_models_per_type : int | str, default \"auto\"\n Similar to `max_base_models`. If more than `max_base_models_per_type` of any particular model type are available,\n only the top `max_base_models_per_type` of that type are used. This occurs before the `max_base_models` filter.\n If \"auto\", the value will be adaptively set based on the number of training samples.\n More samples will lead to larger values, starting at 1 with <1000 samples, increasing up to 12 at >=50000 samples.\n If 0, the logic is skipped.\n num_folds: (int, default=None) If specified, the number of folds to fit in the bagged model.\n If specified, overrides any other value used to determine the number of folds such as predictor.fit `num_bag_folds` argument.\n max_sets: (int, default=None) If specified, the maximum sets to fit in the bagged model.\n The lesser of `max_sets` and the predictor.fit `num_bag_sets` argument will be used for the given model.\n Useful if a particular model is expensive relative to others and you want to avoid repeated bagging of the expensive model while still repeated bagging the cheaper models.\n save_bag_folds: (bool, default=True)\n If True, bagged models will save their fold models (the models from each individual fold of bagging). This is required to use bagged models for prediction.\n If False, bagged models will not save their fold models. This means that bagged models will not be valid models during inference.\n This should only be set to False when planning to call `predictor.refit_full()` or when `refit_full` is set and `set_best_to_refit_full=True`.\n Particularly useful if disk usage is a concern. By not saving the fold models, bagged models will use only very small amounts of disk space during training.\n In many training runs, this will reduce peak disk usage by >10x.\n fold_fitting_strategy: (AbstractFoldFittingStrategy default=auto) Whether to fit folds in parallel or in sequential order.\n If parallel_local, folds will be trained in parallel with evenly distributed computing resources. This could bring 2-4x speedup compared to SequentialLocalFoldFittingStrategy, but could consume much more memory.\n If sequential_local, folds will be trained in sequential.\n If auto, strategy will be determined by OS and whether ray is installed or not. MacOS support for parallel_local is unstable, and may crash if enabled.\n num_folds_parallel: (int or str, default='auto') Number of folds to be trained in parallel if using ParallelLocalFoldFittingStrategy. Consider lowering this value if you encounter either out of memory issue or CUDA out of memory issue(when trained on gpu).\n if 'auto', will try to train all folds in parallel.\n\n feature_metadata : :class:`autogluon.common.FeatureMetadata` or str, default = 'infer'\n The feature metadata used in various inner logic in feature preprocessing.\n If 'infer', will automatically construct a FeatureMetadata object based on the properties of `train_data`.\n In this case, `train_data` is input into :meth:`autogluon.common.FeatureMetadata.from_df` to infer `feature_metadata`.\n If 'infer' incorrectly assumes the dtypes of features, consider explicitly specifying `feature_metadata`.\n infer_limit : float, default = None\n The inference time limit in seconds per row to adhere to during fit.\n If infer_limit=0.05 and infer_limit_batch_size=1000, AutoGluon will avoid training models that take longer than 50 ms/row to predict when given a batch of 1000 rows to predict (must predict 1000 rows in no more than 50 seconds).\n If bagging is enabled, the inference time limit will be respected based on estimated inference speed of `_FULL` models after refit_full is called, NOT on the inference speed of the bagged ensembles.\n The inference times calculated for models are assuming `predictor.persist('all')` is called after fit.\n If None, no limit is enforced.\n If it is impossible to satisfy the constraint, an exception will be raised.\n infer_limit_batch_size : int, default = None\n The batch size to use when predicting in bulk to estimate per-row inference time.\n Must be an integer greater than 0.\n If None and `infer_limit` is specified, will default to 10000.\n It is recommended to set to 10000 unless you must satisfy an online-inference scenario.\n Small values, especially `infer_limit_batch_size=1`, will result in much larger per-row inference times and should be avoided if possible.\n Refer to `infer_limit` for more details on how this is used.\n If specified when `infer_limit=None`, the inference time will be logged during training but will not be limited.\n fit_weighted_ensemble : bool, default = True\n If True, a WeightedEnsembleModel will be fit in each stack layer.\n A weighted ensemble will often be stronger than an individual model while being very fast to train.\n It is recommended to keep this value set to True to maximize predictive quality.\n fit_full_last_level_weighted_ensemble : bool, default = True\n If True, the WeightedEnsembleModel of the last stacking level will be fit with all (successful) models from all previous layers as base models.\n If stacking is disabled, settings this to True or False makes no difference because the WeightedEnsembleModel L2 always uses all models from L1.\n It is recommended to keep this value set to True to maximize predictive quality.\n full_weighted_ensemble_additionally : bool, default = False\n If True, AutoGluon will fit two WeightedEnsembleModels after training all stacking levels. Setting this to True, simulates calling\n `fit_weighted_ensemble()` after calling `fit()`. Has no affect if `fit_full_last_level_weighted_ensemble` is False and does not fit an additional\n WeightedEnsembleModel if stacking is disabled.\n dynamic_stacking: bool | str, default = False\n If True and `num_stack_levels` > 0, AutoGluon will dynamically determine whether to use stacking or not by first validating AutoGluon's stacking\n behavior. This is done to avoid so-called stacked overfitting that can make traditional multi-layer stacking, as used in AutoGluon, fail drastically\n and produce unreliable validation scores.\n It is recommended to keep this value set to True or \"auto\" when using stacking,\n as long as it is unknown whether the data is affected by stacked overfitting.\n If it is known that the data is unaffected by stacked overfitting, then setting this value to False is expected to maximize predictive quality.\n If enabled, by default, AutoGluon performs dynamic stacking by spending 25% of the provided time limit for detection and all remaining\n time for fitting AutoGluon. This can be adjusted by specifying `ds_args` with different parameters to `fit()`.\n If \"auto\", will be set to `not use_bag_holdout`.\n See the documentation of `ds_args` for more information.\n calibrate_decision_threshold : bool | str, default = \"auto\"\n If True, will automatically calibrate the decision threshold at the end of fit for calls to `.predict` based on the evaluation metric.\n If \"auto\", will be set to True if `eval_metric.needs_class=True` and `problem_type=\"binary\"`.\n By default, the decision threshold is `0.5`, however for some metrics such as `f1` and `balanced_accuracy`,\n scores can be significantly improved by choosing a threshold other than `0.5`.\n Only valid for `problem_type='binary'`. Ignored for all other problem types.\n num_cpus: int | str, default = \"auto\"\n The total amount of cpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of cpus available and the model requirement for best performance.\n Users generally don't need to set this value\n num_gpus: int | str, default = \"auto\"\n The total amount of gpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of gpus available and the model requirement for best performance.\n Users generally don't need to set this value\n fit_strategy: Literal[\"sequential\", \"parallel\"], default = \"sequential\"\n The strategy used to fit models.\n If \"sequential\", models will be fit sequentially. This is the most stable option with the most readable logging.\n If \"parallel\", models will be fit in parallel with ray, splitting available compute between them.\n Note: \"parallel\" is experimental and may run into issues. It was first added in version 1.2.0.\n Note: \"parallel\" does not yet support running with GPUs.\n For machines with 16 or more CPU cores, it is likely that \"parallel\" will be faster than \"sequential\".\n\n .. versionadded:: 1.2.0\n\n memory_limit: float | str, default = \"auto\"\n The total amount of memory in GB you want AutoGluon predictor to use. \"auto\" means AutoGluon will use all available memory on the system\n (that is detectable by psutil).\n Note that this is only a soft limit! AutoGluon uses this limit to skip training models that are expected to require too much memory or stop\n training a model that would exceed the memory limit. AutoGluon does not guarantee the enforcement of this limit (yet). Nevertheless, we expect\n AutoGluon to abide by the limit in most cases or, at most, go over the limit by a small margin.\n For most virtualized systems (e.g., in the cloud) and local usage on a server or laptop, \"auto\" is ideal for this parameter. We recommend manually\n setting the memory limit (and any other resources) on systems with shared resources that are controlled by the operating system (e.g., SLURM and\n cgroups). Otherwise, AutoGluon might wrongly assume more resources are available for fitting a model than the operating system allows,\n which can result in model training failing or being very inefficient.\n callbacks : list[AbstractCallback], default = None\n :::{warning}\n Callbacks are an experimental feature and may change in future releases without warning.\n Callback support is preliminary and targeted towards developers.\n :::\n A list of callback objects inheriting from `autogluon.core.callbacks.AbstractCallback`.\n These objects will be called before and after each model fit within trainer.\n They have the ability to skip models or early stop the training process.\n They can also theoretically change the entire logical flow of the trainer code by interacting with the passed `trainer` object.\n For more details, refer to `AbstractCallback` source code.\n If None, no callback objects will be used.\n\n [Note] Callback objects can be mutated in-place by the fit call if they are stateful.\n Ensure that you avoid reusing a mutated callback object between multiple fit calls.\n\n [Note] Callback objects are deleted from trainer at the end of the fit call. They will not impact operations such as `refit_full` or `fit_extra`.\n **kwargs :\n auto_stack : bool, default = False\n Whether AutoGluon should automatically utilize bagging and multi-layer stack ensembling to boost predictive accuracy.\n Set this = True if you are willing to tolerate longer training times in order to maximize predictive accuracy!\n Automatically sets `num_bag_folds` and `num_stack_levels` arguments based on dataset properties.\n Note: Setting `num_bag_folds` and `num_stack_levels` arguments will override `auto_stack`.\n Note: This can increase training time (and inference time) by up to 20x, but can greatly improve predictive performance.\n num_bag_folds : int, default = None\n Number of folds used for bagging of models. When `num_bag_folds = k`, training time is roughly increased by a factor of `k` (set = 0 to disable bagging).\n Disabled by default (0), but we recommend values between 5-10 to maximize predictive performance.\n Increasing num_bag_folds will result in models with lower bias but that are more prone to overfitting.\n `num_bag_folds = 1` is an invalid value, and will raise a ValueError.\n Values > 10 may produce diminishing returns, and can even harm overall results due to overfitting.\n To further improve predictions, avoid increasing `num_bag_folds` much beyond 10 and instead increase `num_bag_sets`.\n num_bag_sets : int, default = None\n Number of repeats of kfold bagging to perform (values must be >= 1). Total number of models trained during bagging = `num_bag_folds * num_bag_sets`.\n Defaults to 1 when unspecified. Value is ignored if `num_bag_folds<=2`.\n Values greater than 1 will result in superior predictive performance, especially on smaller problems and with stacking enabled (reduces overall variance).\n Be warned: This will drastically increase overall runtime, and if using a time limit, can very commonly lead to worse performance.\n It is recommended to increase this value only as a last resort, as it is the least computationally efficient method to improve performance.\n num_stack_levels : int, default = None\n Number of stacking levels to use in stack ensemble. Roughly increases model training time by factor of `num_stack_levels+1` (set = 0 to disable stack ensembling).\n Disabled by default (0), but we recommend `num_stack_levels=1` to maximize predictive performance.\n To prevent overfitting, `num_bag_folds >= 2` must also be set or else a ValueError will be raised.\n delay_bag_sets : bool, default = False\n Controls when repeats of kfold bagging are executed in AutoGluon when under a time limit.\n We suggest sticking to `False` to avoid overfitting.\n If True, AutoGluon delays repeating kfold bagging until after evaluating all models\n from `hyperparameters`, if there is enough time. This allows AutoGluon to explore\n more hyperparameters to obtain a better final performance but it may lead to\n more overfitting.\n If False, AutoGluon repeats kfold bagging immediately after evaluating each model.\n Thus, AutoGluon might evaluate fewer models with less overfitting.\n holdout_frac : float, default = None\n Fraction of train_data to holdout as tuning data for optimizing hyperparameters (ignored unless `tuning_data = None`, ignored if `num_bag_folds != 0` unless `use_bag_holdout == True`).\n Default value (if None) is selected based on the number of rows in the training data. Default values range from 0.2 at 2,500 rows to 0.01 at 250,000 rows.\n Default value is doubled if `hyperparameter_tune_kwargs` is set, up to a maximum of 0.2.\n Disabled if `num_bag_folds >= 2` unless `use_bag_holdout == True`.\n use_bag_holdout : bool | str, default = False\n If True, a `holdout_frac` portion of the data is held-out from model bagging.\n This held-out data is only used to score models and determine weighted ensemble weights.\n Enable this if there is a large gap between score_val and score_test in stack models.\n Note: If `tuning_data` was specified, `tuning_data` is used as the holdout data.\n Disabled if not bagging.\n If \"auto\", will be set to True if the training data has >= 1000000 rows, else it will be set to False.\n hyperparameter_tune_kwargs : str or dict, default = None\n Hyperparameter tuning strategy and kwargs (for example, how many HPO trials to run).\n If None, then hyperparameter tuning will not be performed.\n You can either choose to provide a preset\n Valid preset values:\n 'auto': Performs HPO via bayesian optimization search on NN_TORCH and FASTAI models, and random search on other models using local scheduler.\n 'random': Performs HPO via random search using local scheduler.\n Or provide a dict to specify searchers and schedulers\n Valid keys:\n 'num_trials': How many HPO trials to run\n 'scheduler': Which scheduler to use\n Valid values:\n 'local': Local scheduler that schedule trials FIFO\n 'searcher': Which searching algorithm to use\n 'local_random': Uses the 'random' searcher\n 'random': Perform random search\n 'auto': Perform bayesian optimization search on NN_TORCH and FASTAI models. Perform random search on other models.\n The 'scheduler' and 'searcher' key are required when providing a dict.\n An example of a valid dict:\n hyperparameter_tune_kwargs = {\n 'num_trials': 5,\n 'scheduler' : 'local',\n 'searcher': 'auto',\n }\n feature_prune_kwargs: dict, default = None\n Performs layer-wise feature pruning via recursive feature elimination with permutation feature importance.\n This fits all models in a stack layer once, discovers a pruned set of features, fits all models in the stack layer\n again with the pruned set of features, and updates input feature lists for models whose validation score improved.\n If None, do not perform feature pruning. If empty dictionary, perform feature pruning with default configurations.\n For valid dictionary keys, refer to :class:`autogluon.core.utils.feature_selection.FeatureSelector` and\n `autogluon.core.trainer.abstract_trainer.AbstractTabularTrainer._proxy_model_feature_prune` documentation.\n To force all models to work with the pruned set of features, set force_prune=True in the dictionary.\n ag_args : dict, default = None\n Keyword arguments to pass to all models (i.e. common hyperparameters shared by all AutoGluon models).\n See the `ag_args` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args` parameter for all models in `hyperparameters`.\n If a key in `ag_args` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n ag_args_fit : dict, default = None\n Keyword arguments to pass to all models.\n See the `ag_args_fit` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args_fit` parameter for all models in `hyperparameters`.\n If a key in `ag_args_fit` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n ag_args_ensemble : dict, default = None\n Keyword arguments to pass to all models.\n See the `ag_args_ensemble` argument from \"Advanced functionality: Custom AutoGluon model arguments\" in the `hyperparameters` argument documentation for valid values.\n Identical to specifying `ag_args_ensemble` parameter for all models in `hyperparameters`.\n If a key in `ag_args_ensemble` is already specified for a model in `hyperparameters`, it will not be altered through this argument.\n ds_args : dict, see below for default\n Keyword arguments for dynamic stacking, only used if `dynamic_stacking=True`. These keyword arguments control the behavior of dynamic stacking\n and determine how AutoGluon tries to detect stacked overfitting. To detect stacked overfitting, AutoGluon will fit itself (so called sub-fits)\n on a subset (for holdout) or multiple subsets (for repeated cross-validation) and use the predictions of AutoGluon on the validation data to\n detect stacked overfitting. The sub-fits stop and stacking will be disabled if any sub-fit shows stacked overfitting.\n Allowed keys and values are:\n `detection_time_frac` : float in (0,1), default = 1/4\n Determines how much of the original training time is used for detecting stacked overfitting.\n When using (repeated) cross-validation, each sub-fit will be fit for `1/n_splits * detection_time_frac * time_limit`.\n If no time limit is given to AutoGluon, this parameter is ignored and AutoGluon is fit without a time limit in the sub-fit.\n `validation_procedure`: str, default = 'holdout'\n Determines the validation procedure used to detect stacked overfitting. Can be either `cv` or `holdout`.\n If `validation_procedure='holdout'` and `holdout_data` is not specified (default), then `holdout_frac` determines the holdout data.\n If `validation_procedure='holdout'` and `holdout_data` is specified, then the provided `holdout_data` is used for validation.\n If `validation_procedure='cv'`, `n_folds` and `n_repeats` determine the kind cross-validation procedure.\n `holdout_frac` : float in (0,1), default = 1/9\n Determines how much of the original training data is used for the holdout data during holdout validation.\n Ignored if `holdout_data` is not None.\n `n_folds` : int in [2, +inf), default = 2\n Number of folds to use for cross-validation.\n `n_repeats` : int [1, +inf), default = 1\n Number of repeats to use for repeated cross-validation. If set to 1, performs 1-repeated cross-validation which is equivalent to\n cross-validation without repeats.\n `memory_safe_fits` : bool, default = True\n If True, AutoGluon runs each sub-fit in a ray-based subprocess to avoid memory leakage that exist due to Python's lackluster\n garbage collector.\n `clean_up_fits` : bool, default = True\n If True, AutoGluon will remove all saved information from sub-fits from disk.\n If False, the sub-fits are kept on disk and `self._sub_fits` will store paths to the sub-fits, which can be loaded just like any other\n predictor from disk using `TabularPredictor.load()`.\n `enable_ray_logging` : bool, default = True\n If True, will log the dynamic stacking sub-fit when ray is used (`memory_safe_fits=True`).\n Note that because of how ray works, this may cause extra unwanted logging in the main fit process after dynamic stacking completes.\n `enable_callbacks` : bool, default = False\n If True, will perform a deepcopy on the specified user callbacks and enable them during the DyStack call.\n If False, will not include callbacks in the DyStack call.\n `holdout_data`: str or :class:`pd.DataFrame`, default = None\n Another dataset containing validation data reserved for detecting stacked overfitting. This dataset should be in the same format as\n `train_data`. If str is passed, `holdout_data` will be loaded using the str value as the file path.\n If `holdout_data` is not None, the sub-fit is fit on all of `train_data` and the full fit is fit on all of `train_data` and\n `holdout_data` combined.\n included_model_types : list, default = None\n To only include listed model types for training during `fit()`.\n Models that are listed in `included_model_types` but not in `hyperparameters` will be ignored.\n Reference `hyperparameters` documentation for what models correspond to each value.\n Useful when only a subset of model needs to be trained and the `hyperparameters` dictionary is difficult or time-consuming.\n Example: To include both 'GBM' and 'FASTAI' models, specify `included_model_types=['GBM', 'FASTAI']`.\n excluded_model_types : list, default = None\n Banned subset of model types to avoid training during `fit()`, even if present in `hyperparameters`.\n Reference `hyperparameters` documentation for what models correspond to each value.\n Useful when a particular model type such as 'KNN' or 'custom' is not desired but altering the `hyperparameters` dictionary is difficult or time-consuming.\n Example: To exclude both 'KNN' and 'custom' models, specify `excluded_model_types=['KNN', 'custom']`.\n refit_full : bool or str, default = False\n Whether to retrain all models on all of the data (training + validation) after the normal training procedure.\n This is equivalent to calling `predictor.refit_full(model=refit_full)` after fit.\n If `refit_full=True`, it will be treated as `refit_full='all'`.\n If `refit_full=False`, refitting will not occur.\n Valid str values:\n `all`: refits all models.\n `best`: refits only the best model (and its ancestors if it is a stacker model).\n `{model_name}`: refits only the specified model (and its ancestors if it is a stacker model).\n For bagged models:\n Reduces a model's inference time by collapsing bagged ensembles into a single model fit on all of the training data.\n This process will typically result in a slight accuracy reduction and a large inference speedup.\n The inference speedup will generally be between 10-200x faster than the original bagged ensemble model.\n The inference speedup factor is equivalent to (k * n), where k is the number of folds (`num_bag_folds`) and n is the number of finished repeats (`num_bag_sets`) in the bagged ensemble.\n The runtime is generally 10% or less of the original fit runtime.\n The runtime can be roughly estimated as 1 / (k * n) of the original fit runtime, with k and n defined above.\n For non-bagged models:\n Optimizes a model's accuracy by retraining on 100% of the data without using a validation set.\n Will typically result in a slight accuracy increase and no change to inference time.\n The runtime will be approximately equal to the original fit runtime.\n This process does not alter the original models, but instead adds additional models.\n If stacker models are refit by this process, they will use the refit_full versions of the ancestor models during inference.\n Models produced by this process will not have validation scores, as they use all of the data for training.\n Therefore, it is up to the user to determine if the models are of sufficient quality by including test data in `predictor.leaderboard(test_data)`.\n If the user does not have additional test data, they should reference the original model's score for an estimate of the performance of the refit_full model.\n Warning: Be aware that utilizing refit_full models without separately verifying on test data means that the model is untested, and has no guarantee of being consistent with the original model.\n The time taken by this process is not enforced by `time_limit`.\n save_bag_folds : bool, default = True\n If True, will save the bagged fold models to disk.\n If False, will not save the bagged fold models, only keeping their metadata and out-of-fold predictions.\n Note: The bagged models will not be available for prediction, only use this if you intend to call `refit_full`.\n The purpose of setting it to False is that it greatly decreases the peak disk usage of the predictor during the fit call when bagging.\n Note that this makes refit_full slightly more likely to crash in scenarios where the dataset is large relative to available system memory.\n This is because by default, refit_full will fall back to cloning the first fold of the bagged model in case it lacks memory to refit.\n However, if `save_bag_folds=False`, this fallback isn't possible, as there is not fold model to clone because it wasn't saved.\n In this scenario, refit will raise an exception for `save_bag_folds=False`, but will succeed if `save_bag_folds=True`.\n Final disk usage of predictor will be identical regardless of the setting after `predictor.delete_models(models_to_keep=\"best\")` is called post-fit.\n set_best_to_refit_full : bool, default = False\n If True, will change the default model that Predictor uses for prediction when model is not specified to the refit_full version of the model that exhibited the highest validation score.\n Only valid if `refit_full` is set.\n keep_only_best : bool, default = False\n If True, only the best model and its ancestor models are saved in the outputted `predictor`. All other models are deleted.\n If you only care about deploying the most accurate predictor with the smallest file-size and no longer need any of the other trained models or functionality beyond prediction on new data, then set: `keep_only_best=True`, `save_space=True`.\n This is equivalent to calling `predictor.delete_models(models_to_keep='best')` directly after `fit()`.\n If used with `refit_full` and `set_best_to_refit_full`, the best model will be the refit_full model, and the original bagged best model will be deleted.\n `refit_full` will be automatically set to 'best' in this case to avoid training models which will be later deleted.\n save_space : bool, default = False\n If True, reduces the memory and disk size of predictor by deleting auxiliary model files that aren't needed for prediction on new data.\n This is equivalent to calling `predictor.save_space()` directly after `fit()`.\n This has NO impact on inference accuracy.\n It is recommended if the only goal is to use the trained model for prediction.\n Certain advanced functionality may no longer be available if `save_space=True`. Refer to `predictor.save_space()` documentation for more details.\n feature_generator : :class:`autogluon.features.generators.AbstractFeatureGenerator`, default = :class:`autogluon.features.generators.AutoMLPipelineFeatureGenerator`\n The feature generator used by AutoGluon to process the input data to the form sent to the models. This often includes automated feature generation and data cleaning.\n It is generally recommended to keep the default feature generator unless handling an advanced use-case.\n To control aspects of the default feature generation process, you can pass in an :class:`AutoMLPipelineFeatureGenerator` object constructed using some of these kwargs:\n enable_numeric_features : bool, default True\n Whether to keep features of 'int' and 'float' raw types.\n These features are passed without alteration to the models.\n Appends IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=['int', 'float']))) to the generator group.\n enable_categorical_features : bool, default True\n Whether to keep features of 'object' and 'category' raw types.\n These features are processed into memory optimized 'category' features.\n Appends CategoryFeatureGenerator() to the generator group.\n enable_datetime_features : bool, default True\n Whether to keep features of 'datetime' raw type and 'object' features identified as 'datetime_as_object' features.\n These features will be converted to 'int' features representing milliseconds since epoch.\n Appends DatetimeFeatureGenerator() to the generator group.\n enable_text_special_features : bool, default True\n Whether to use 'object' features identified as 'text' features to generate 'text_special' features such as word count, capital letter ratio, and symbol counts.\n Appends TextSpecialFeatureGenerator() to the generator group.\n enable_text_ngram_features : bool, default True\n Whether to use 'object' features identified as 'text' features to generate 'text_ngram' features.\n Appends TextNgramFeatureGenerator(vectorizer=vectorizer) to the generator group.\n enable_raw_text_features : bool, default False\n Whether to keep the raw text features.\n Appends IdentityFeatureGenerator(infer_features_in_args=dict(required_special_types=['text'])) to the generator group.\n vectorizer : CountVectorizer, default CountVectorizer(min_df=30, ngram_range=(1, 3), max_features=10000, dtype=np.uint8)\n sklearn CountVectorizer object to use in TextNgramFeatureGenerator.\n Only used if `enable_text_ngram_features=True`.\n unlabeled_data : pd.DataFrame, default = None\n [Experimental Parameter] UNUSED.\n Collection of data without labels that we can use to pretrain on.\n This is the same schema as train_data, except without the labels.\n Currently, unlabeled_data is not used by any model.\n verbosity : int\n If specified, overrides the existing `predictor.verbosity` value.\n raise_on_model_failure: bool, default = False\n If True, will raise on any exception during model training.\n This is useful when using a debugger during development to identify the cause of model failures.\n This should only be used for debugging.\n If False, will try to skip to the next model if an exception occurred during model training.\n This is the default logic and is a core principle of AutoGluon's design.\n\n .. versionadded:: 1.3.0\n raise_on_no_models_fitted: bool, default = True\n If True, will raise a RuntimeError if no models were successfully fit during `fit()`.\n calibrate: bool or str, default = 'auto'\n Note: It is recommended to use ['auto', False] as the values and avoid True.\n If 'auto' will automatically set to True if the problem_type and eval_metric are suitable for calibration.\n If True and the problem_type is classification, temperature scaling will be used to calibrate the Predictor's estimated class probabilities\n (which may improve metrics like log_loss) and will train a scalar parameter on the validation set.\n If True and the problem_type is quantile regression, conformalization will be used to calibrate the Predictor's estimated quantiles\n (which may improve the prediction interval coverage, and bagging could further improve it) and will compute a set of scalar parameters on the validation set.\n test_data : str or :class:`pd.DataFrame`, default = None\n Table of the test data.\n If str is passed, `test_data` will be loaded using the str value as the file path.\n NOTE: This test_data is NEVER SEEN by the model during training and, if specified, is only used for logging purposes (i.e. for learning curve generation).\n This test_data should be treated the same way test data is used in predictor.leaderboard.\n learning_curves : bool or dict, default = None\n If bool and is True, default learning curve hyperparameter ag_args will be initialized for each of the models included in the ensemble.\n By default, learning curves will include eval_metric scores specified in fit call arguments.\n This can be overwritten as shown below.\n If dict, user can pass learning_curves parameters to be initialized as ag_args in the following format:\n learning_curves = {\n \"metrics\": str or list(str) or Scorer or list(Scorer):\n autogluon metric scorer(s) to be calculated at each iteration, represented as Scorer object(s) or scorer name(s) (str)\n \"use_error\": bool : whether to use error or score format for metrics listed above\n }\n\n Returns\n -------\n :class:`TabularPredictor` object. Returns self.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularDataset, TabularPredictor\n >>> train_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv')\n >>> label = 'class'\n >>> predictor = TabularPredictor(label=label).fit(train_data)\n >>> test_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv')\n >>> leaderboard = predictor.leaderboard(test_data)\n >>> y_test = test_data[label]\n >>> test_data = test_data.drop(columns=[label])\n >>> y_pred = predictor.predict(test_data)\n >>> perf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred)\n\n To maximize predictive performance, use the following:\n\n >>> eval_metric = 'roc_auc' # set this to the metric you ultimately care about\n >>> time_limit = 3600 # set as long as you are willing to wait (in sec)\n >>> predictor = TabularPredictor(label=label, eval_metric=eval_metric).fit(train_data, presets=['best_quality'], time_limit=time_limit)\n \"\"\"\n if self.is_fit:\n raise AssertionError(\n \"Predictor is already fit! To fit additional models, refer to `predictor.fit_extra`, or create a new `Predictor`.\"\n )\n\n verbosity = kwargs.get(\"verbosity\", self.verbosity)\n set_logger_verbosity(verbosity)\n warn_if_mlflow_autologging_is_enabled(logger=logger)\n\n if verbosity >= 2:\n if verbosity == 2:\n logger.log(20, \"Verbosity: 2 (Standard Logging)\")\n elif verbosity == 3:\n logger.log(20, \"Verbosity: 3 (Detailed Logging)\")\n elif verbosity >= 4:\n logger.log(20, f\"Verbosity: {verbosity} (Maximum Logging)\")\n\n resource_manager: ResourceManager = get_resource_manager()\n include_gpu_count = resource_manager.get_gpu_count_torch() or verbosity >= 3\n sys_msg = get_ag_system_info(path=self.path, include_gpu_count=include_gpu_count)\n logger.log(20, sys_msg)\n\n if presets:\n if not isinstance(presets, list):\n presets = [presets]\n logger.log(20, f\"Presets specified: {presets}\")\n else:\n logger.log(\n 20,\n \"No presets specified! To achieve strong results with AutoGluon, it is recommended to use the available presets. Defaulting to `'medium'`...\\n\"\n \"\\tRecommended Presets (For more details refer to https://auto.gluon.ai/stable/tutorials/tabular/tabular-essentials.html#presets):\\n\"\n \"\\tpresets='extreme' : New in v1.5: The state-of-the-art for tabular data. Massively better than 'best' on datasets <100000 samples by using new Tabular Foundation Models (TFMs) meta-learned on https://tabarena.ai: TabPFNv2, TabICL, Mitra, TabDPT, and TabM. Requires a GPU and `pip install autogluon.tabular[tabarena]` to install TabPFN, TabICL, and TabDPT.\\n\"\n \"\\tpresets='best' : Maximize accuracy. Recommended for most users. Use in competitions and benchmarks.\\n\"\n \"\\tpresets='best_v150': New in v1.5: Better quality than 'best' and 5x+ faster to train. Give it a try!\\n\"\n \"\\tpresets='high' : Strong accuracy with fast inference speed.\\n\"\n \"\\tpresets='high_v150': New in v1.5: Better quality than 'high' and 5x+ faster to train. Give it a try!\\n\"\n \"\\tpresets='good' : Good accuracy with very fast inference speed.\\n\"\n \"\\tpresets='medium' : Fast training time, ideal for initial prototyping.\",\n )\n\n kwargs_orig = kwargs.copy()\n\n if verbosity >= 3:\n logger.log(20, \"============ fit kwarg info ============\")\n logger.log(20, \"User Specified kwargs:\")\n logger.log(20, f\"{pprint.pformat(kwargs_orig)}\")\n\n kwargs = self._validate_fit_kwargs(kwargs=kwargs)\n\n if verbosity >= 3:\n logger.log(20, \"Full kwargs:\")\n logger.log(20, f\"{pprint.pformat(kwargs)}\")\n logger.log(20, \"========================================\")\n\n self._validate_num_cpus(num_cpus=num_cpus)\n self._validate_num_gpus(num_gpus=num_gpus)\n self._validate_and_set_memory_limit(memory_limit=memory_limit)\n self._validate_calibrate_decision_threshold(calibrate_decision_threshold=calibrate_decision_threshold)\n self._validate_fit_strategy(fit_strategy=fit_strategy)\n\n auto_stack = kwargs[\"auto_stack\"]\n feature_generator = kwargs[\"feature_generator\"]\n unlabeled_data = kwargs[\"unlabeled_data\"]\n ag_args = kwargs[\"ag_args\"]\n ag_args_fit = kwargs[\"ag_args_fit\"]\n ag_args_ensemble = kwargs[\"ag_args_ensemble\"]\n included_model_types = kwargs[\"included_model_types\"]\n excluded_model_types = kwargs[\"excluded_model_types\"]\n use_bag_holdout = kwargs[\"use_bag_holdout\"]\n ds_args: dict = kwargs[\"ds_args\"]\n delay_bag_sets: bool = kwargs[\"delay_bag_sets\"]\n test_data = kwargs[\"test_data\"]\n learning_curves = kwargs[\"learning_curves\"]\n raise_on_model_failure = kwargs[\"raise_on_model_failure\"]\n\n if ag_args is None:\n ag_args = {}\n ag_args = self._set_hyperparameter_tune_kwargs_in_ag_args(\n kwargs[\"hyperparameter_tune_kwargs\"], ag_args, time_limit=time_limit\n )\n\n feature_generator_init_kwargs = kwargs[\"_feature_generator_kwargs\"]\n if feature_generator_init_kwargs is None:\n feature_generator_init_kwargs = dict()\n\n train_data, tuning_data, test_data, unlabeled_data = self._validate_fit_data(\n train_data=train_data, tuning_data=tuning_data, test_data=test_data, unlabeled_data=unlabeled_data\n )\n infer_limit, infer_limit_batch_size = self._validate_infer_limit(\n infer_limit=infer_limit, infer_limit_batch_size=infer_limit_batch_size\n )\n\n # TODO: Temporary for v1.4. Make this more extensible for v1.5 by letting users make their own dynamic hyperparameters.\n dynamic_hyperparameters = kwargs[\"_experimental_dynamic_hyperparameters\"]\n if dynamic_hyperparameters:\n logger.log(20, \"`extreme_v140` preset uses a dynamic portfolio based on dataset size...\")\n assert hyperparameters is None, (\n \"hyperparameters must be unspecified when `_experimental_dynamic_hyperparameters=True`.\"\n )\n n_samples = len(train_data)\n if n_samples > 30000:\n data_size = \"large\"\n else:\n data_size = \"small\"\n assert data_size in [\"large\", \"small\"]\n if data_size == \"large\":\n logger.log(\n 20,\n \"\\tDetected data size: large (>30000 samples), using `zeroshot` portfolio (identical to 'best_quality' preset).\",\n )\n hyperparameters = \"zeroshot\"\n else:\n if \"num_stack_levels\" not in kwargs_orig:\n # disable stacking for tabfm portfolio\n num_stack_levels = 0\n kwargs[\"num_stack_levels\"] = 0\n logger.log(\n 20,\n \"\\tDetected data size: small (<=30000 samples), using `zeroshot_2025_tabfm` portfolio.\"\n \"\\n\\t\\tNote: `zeroshot_2025_tabfm` portfolio requires a CUDA compatible GPU for best performance.\"\n \"\\n\\t\\tMake sure you have all the relevant dependencies installed: \"\n \"`pip install autogluon.tabular[tabarena]`.\"\n \"\\n\\t\\tIt is strongly recommended to use a machine with 64+ GB memory \"\n \"and a CUDA compatible GPU with 32+ GB vRAM when using this preset. \"\n \"\\n\\t\\tThis portfolio will download foundation model weights from HuggingFace during training. \"\n \"Ensure you have an internet connection or have pre-downloaded the weights to use these models.\"\n \"\\n\\t\\tThis portfolio was meta-learned with TabArena: https://tabarena.ai\",\n )\n hyperparameters = \"zeroshot_2025_tabfm\"\n\n if hyperparameters is None:\n hyperparameters = \"default\"\n if isinstance(hyperparameters, str):\n hyperparameters_str = hyperparameters\n hyperparameters = get_hyperparameter_config(hyperparameters)\n logger.log(\n 20,\n f\"Using hyperparameters preset: hyperparameters='{hyperparameters_str}'\",\n )\n self._validate_hyperparameters(hyperparameters=hyperparameters)\n self.fit_hyperparameters_ = hyperparameters\n\n if \"enable_raw_text_features\" not in feature_generator_init_kwargs:\n if self._check_if_hyperparameters_handle_text(hyperparameters=hyperparameters):\n feature_generator_init_kwargs[\"enable_raw_text_features\"] = True\n\n if feature_metadata is not None and isinstance(feature_metadata, str) and feature_metadata == \"infer\":\n feature_metadata = None\n self._set_feature_generator(\n feature_generator=feature_generator,\n feature_metadata=feature_metadata,\n init_kwargs=feature_generator_init_kwargs,\n )\n\n if self.problem_type is not None:\n inferred_problem_type = self.problem_type\n else:\n self._learner.validate_label(X=train_data)\n inferred_problem_type = self._learner.infer_problem_type(y=train_data[self.label], silent=True)\n\n learning_curves = self._initialize_learning_curve_params(\n learning_curves=learning_curves, problem_type=inferred_problem_type\n )\n if len(learning_curves) == 0:\n test_data = None\n if ag_args_fit is not None:\n ag_args_fit.update(learning_curves)\n else:\n ag_args_fit = learning_curves\n\n use_bag_holdout_was_auto = False\n dynamic_stacking_was_auto = False\n if isinstance(use_bag_holdout, str) and use_bag_holdout == \"auto\":\n use_bag_holdout = None\n use_bag_holdout_was_auto = True\n if isinstance(dynamic_stacking, str) and dynamic_stacking == \"auto\":\n dynamic_stacking = None\n dynamic_stacking_was_auto = True\n\n (\n num_bag_folds,\n num_bag_sets,\n num_stack_levels,\n dynamic_stacking,\n use_bag_holdout,\n holdout_frac,\n refit_full,\n ) = get_validation_and_stacking_method(\n num_bag_folds=kwargs[\"num_bag_folds\"],\n num_bag_sets=kwargs[\"num_bag_sets\"],\n use_bag_holdout=use_bag_holdout,\n holdout_frac=kwargs[\"holdout_frac\"],\n auto_stack=auto_stack,\n num_stack_levels=kwargs[\"num_stack_levels\"],\n dynamic_stacking=dynamic_stacking,\n refit_full=kwargs[\"refit_full\"],\n num_train_rows=len(train_data),\n problem_type=inferred_problem_type,\n hpo_enabled=ag_args.get(\"hyperparameter_tune_kwargs\", None) is not None,\n )\n\n num_bag_folds, num_bag_sets, num_stack_levels, dynamic_stacking, use_bag_holdout = self._sanitize_stack_args(\n num_bag_folds=num_bag_folds,\n num_bag_sets=num_bag_sets,\n num_stack_levels=num_stack_levels,\n num_train_rows=len(train_data),\n dynamic_stacking=dynamic_stacking,\n use_bag_holdout=use_bag_holdout,\n use_bag_holdout_was_auto=use_bag_holdout_was_auto,\n dynamic_stacking_was_auto=dynamic_stacking_was_auto,\n )\n if auto_stack:\n logger.log(\n 20,\n f\"Stack configuration (auto_stack={auto_stack}): \"\n f\"num_stack_levels={num_stack_levels}, num_bag_folds={num_bag_folds}, num_bag_sets={num_bag_sets}\",\n )\n\n if kwargs[\"save_bag_folds\"] is not None and kwargs[\"_save_bag_folds\"] is not None:\n raise ValueError(\n f\"Cannot specify both `save_bag_folds` and `_save_bag_folds` at the same time. \"\n f\"(save_bag_folds={kwargs['save_bag_folds']}, _save_bag_folds={kwargs['_save_bag_folds']}\"\n )\n elif kwargs[\"_save_bag_folds\"] is not None:\n kwargs[\"save_bag_folds\"] = kwargs[\"_save_bag_folds\"]\n\n if kwargs[\"save_bag_folds\"] is not None:\n assert isinstance(kwargs[\"save_bag_folds\"], bool), (\n f\"save_bag_folds must be a bool, found: {type(kwargs['save_bag_folds'])}\"\n )\n if use_bag_holdout and not kwargs[\"save_bag_folds\"]:\n logger.log(\n 30,\n \"WARNING: Attempted to disable saving of bagged fold models when `use_bag_holdout=True`. Forcing `save_bag_folds=True` to avoid errors.\",\n )\n else:\n if num_bag_folds > 0 and not kwargs[\"save_bag_folds\"]:\n logger.log(\n 20,\n f\"Note: `save_bag_folds=False`! This will greatly reduce peak disk usage during fit (by ~{num_bag_folds}x), \"\n f\"but runs the risk of an out-of-memory error during model refit if memory is small relative to the data size.\\n\"\n f\"\\tYou can avoid this risk by setting `save_bag_folds=True`.\",\n )\n if ag_args_ensemble is None:\n ag_args_ensemble = {}\n ag_args_ensemble[\"save_bag_folds\"] = kwargs[\"save_bag_folds\"]\n\n if time_limit is None:\n mb_mem_usage_train_data = get_approximate_df_mem_usage(train_data, sample_ratio=0.2).sum() / 1e6\n num_rows_train = len(train_data)\n if mb_mem_usage_train_data >= 50 or num_rows_train >= 100000:\n logger.log(\n 20,\n f\"Warning: Training may take a very long time because `time_limit` was not specified and `train_data` is large ({num_rows_train} samples, {round(mb_mem_usage_train_data, 2)} MB).\",\n )\n logger.log(\n 20,\n \"\\tConsider setting `time_limit` to ensure training finishes within an expected duration or experiment with a small portion of `train_data` to identify an ideal `presets` and `hyperparameters` configuration.\",\n )\n\n core_kwargs = {\n \"total_resources\": {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n },\n \"ag_args\": ag_args,\n \"ag_args_ensemble\": ag_args_ensemble,\n \"ag_args_fit\": ag_args_fit,\n \"included_model_types\": included_model_types,\n \"excluded_model_types\": excluded_model_types,\n \"feature_prune_kwargs\": kwargs.get(\"feature_prune_kwargs\", None),\n \"delay_bag_sets\": delay_bag_sets,\n \"fit_strategy\": fit_strategy,\n }\n aux_kwargs = {\n \"total_resources\": {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n },\n }\n if fit_weighted_ensemble is False:\n aux_kwargs[\"fit_weighted_ensemble\"] = False\n aux_kwargs[\"fit_full_last_level_weighted_ensemble\"] = fit_full_last_level_weighted_ensemble\n aux_kwargs[\"full_weighted_ensemble_additionally\"] = full_weighted_ensemble_additionally\n\n ag_fit_kwargs = dict(\n X=train_data,\n X_val=tuning_data,\n X_test=test_data,\n X_unlabeled=unlabeled_data,\n holdout_frac=holdout_frac,\n num_bag_folds=num_bag_folds,\n num_bag_sets=num_bag_sets,\n num_stack_levels=num_stack_levels,\n hyperparameters=hyperparameters,\n core_kwargs=core_kwargs,\n aux_kwargs=aux_kwargs,\n time_limit=time_limit,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n verbosity=verbosity,\n use_bag_holdout=use_bag_holdout,\n callbacks=callbacks,\n raise_on_model_failure=raise_on_model_failure,\n )\n ag_post_fit_kwargs = dict(\n keep_only_best=kwargs[\"keep_only_best\"],\n refit_full=refit_full,\n set_best_to_refit_full=kwargs[\"set_best_to_refit_full\"],\n save_space=kwargs[\"save_space\"],\n calibrate=kwargs[\"calibrate\"],\n calibrate_decision_threshold=calibrate_decision_threshold,\n infer_limit=infer_limit,\n num_cpus=num_cpus,\n num_gpus=num_gpus,\n fit_strategy=fit_strategy,\n raise_on_no_models_fitted=kwargs[\"raise_on_no_models_fitted\"],\n )\n if dynamic_stacking:\n logger.log(\n 20,\n f\"DyStack is enabled (dynamic_stacking={dynamic_stacking}). \"\n \"AutoGluon will try to determine whether the input data is affected by stacked overfitting and enable or disable stacking as a consequence.\",\n )\n num_stack_levels, time_limit = self._dynamic_stacking(\n **ds_args, ag_fit_kwargs=ag_fit_kwargs, ag_post_fit_kwargs=ag_post_fit_kwargs\n )\n logger.info(\n f\"Starting main fit with num_stack_levels={num_stack_levels}.\\n\"\n f\"\\tFor future fit calls on this dataset, you can skip DyStack to save time: \"\n f\"`predictor.fit(..., dynamic_stacking=False, num_stack_levels={num_stack_levels})`\"\n )\n\n if (time_limit is not None) and (time_limit <= 0):\n raise AssertionError(\n f\"Not enough time left to train models for the full fit. \"\n f\"Consider specifying a larger time_limit or setting `dynamic_stacking=False`. Time remaining: {time_limit:.2f}s\"\n )\n\n ag_fit_kwargs[\"num_stack_levels\"] = num_stack_levels\n ag_fit_kwargs[\"time_limit\"] = time_limit\n\n # keep track of the fit strategy used for future calls\n self._fit_strategy = fit_strategy\n\n self._fit(ag_fit_kwargs=ag_fit_kwargs, ag_post_fit_kwargs=ag_post_fit_kwargs)\n\n return self\n\n def _fit(self, ag_fit_kwargs: dict, ag_post_fit_kwargs: dict):\n self.save(silent=True) # Save predictor to disk to enable prediction and training after interrupt\n self._learner.fit(**ag_fit_kwargs)\n self._set_post_fit_vars()\n self._post_fit(**ag_post_fit_kwargs)\n self.save()\n\n # TODO: When >2 layers, will only choose between using all layers or using only base models. Would be better to choose the optimal layer.\n def _dynamic_stacking(\n self,\n ag_fit_kwargs: dict,\n ag_post_fit_kwargs: dict,\n validation_procedure: bool,\n detection_time_frac: float,\n holdout_frac: float,\n n_folds: int,\n n_repeats: int,\n memory_safe_fits: bool,\n clean_up_fits: bool,\n enable_ray_logging: bool,\n enable_callbacks: bool,\n holdout_data: Optional[Union[str, pd.DataFrame, None]] = None,\n ):\n \"\"\"Dynamically determines if stacking is used or not by validating the behavior of a sub-fit of AutoGluon that uses stacking on held out data.\n See `ds_args` in the docstring of `fit()` for details on the parameters.\"\"\"\n time_start = time.time()\n time_limit_og = ag_fit_kwargs[\"time_limit\"]\n org_num_stack_levels = ag_fit_kwargs[\"num_stack_levels\"]\n ds_fit_context = os.path.join(self._learner.path_context_og, \"ds_sub_fit\")\n logger.info(\n \"\\tThis is used to identify the optimal `num_stack_levels` value. \"\n \"Copies of AutoGluon will be fit on subsets of the data. \"\n \"Then holdout validation data is used to detect stacked overfitting.\"\n )\n\n if time_limit_og is not None:\n time_limit = int(time_limit_og * detection_time_frac)\n logger.info(\n f\"\\tRunning DyStack for up to {time_limit}s of the {time_limit_og}s of remaining time ({detection_time_frac * 100:.0f}%).\"\n )\n else:\n logger.info(\"\\tWarning: No time limit provided for DyStack. This could take awhile.\")\n time_limit = None\n\n # -- Avoid copying data\n X = ag_fit_kwargs.pop(\"X\")\n X_val = ag_fit_kwargs.pop(\"X_val\")\n X_unlabeled = ag_fit_kwargs.pop(\"X_unlabeled\")\n callbacks = None\n if not enable_callbacks:\n callbacks = ag_fit_kwargs.pop(\"callbacks\")\n inner_ag_fit_kwargs = copy.deepcopy(ag_fit_kwargs)\n if not enable_callbacks:\n ag_fit_kwargs[\"callbacks\"] = callbacks\n inner_ag_fit_kwargs[\"X_val\"] = X_val\n inner_ag_fit_kwargs[\"X_unlabeled\"] = X_unlabeled\n inner_ag_post_fit_kwargs = copy.deepcopy(ag_post_fit_kwargs)\n inner_ag_post_fit_kwargs[\"keep_only_best\"] = (\n False # Do not keep only best, otherwise it eliminates the purpose of the comparison\n )\n inner_ag_post_fit_kwargs[\"calibrate\"] = (\n False # Do not calibrate as calibration is only applied to the model with the best validation score\n )\n # FIXME: Ensure all weighted ensembles have skip connections\n\n # Verify problem type is set\n if self.problem_type is None:\n self._learner.problem_type = self._learner.infer_problem_type(y=X[self.label], silent=True)\n\n ds_fit_kwargs = dict(\n clean_up_fits=clean_up_fits,\n memory_safe_fits=memory_safe_fits,\n enable_ray_logging=enable_ray_logging,\n )\n\n # -- Validation Method\n if validation_procedure == \"holdout\":\n if holdout_data is None:\n ds_fit_kwargs.update(\n dict(holdout_frac=holdout_frac, ds_fit_context=os.path.join(ds_fit_context, \"sub_fit_ho\"))\n )\n else:\n _, holdout_data, _, _ = self._validate_fit_data(train_data=X, tuning_data=holdout_data)\n ds_fit_kwargs[\"ds_fit_context\"] = os.path.join(ds_fit_context, \"sub_fit_custom_ho\")\n\n stacked_overfitting = self._sub_fit_memory_save_wrapper(\n train_data=X,\n time_limit=time_limit,\n time_start=time_start,\n ds_fit_kwargs=ds_fit_kwargs,\n ag_fit_kwargs=inner_ag_fit_kwargs,\n ag_post_fit_kwargs=inner_ag_post_fit_kwargs,\n holdout_data=holdout_data,\n )\n else:\n # Holdout is false, use (repeated) cross-validation\n is_stratified = self.problem_type in [BINARY, MULTICLASS]\n is_binned = self.problem_type in [REGRESSION, QUANTILE]\n self._learner._validate_groups(X=X, X_val=X_val) # Validate splits before splitting\n splits = CVSplitter(\n n_splits=n_folds,\n n_repeats=n_repeats,\n groups=self._learner.groups,\n stratify=is_stratified,\n bin=is_binned,\n random_state=42,\n ).split(X=X.drop(self.label, axis=1), y=X[self.label])\n n_splits = len(splits)\n logger.info(\n f'\\tStarting (repeated-)cross-validation-based sub-fits for dynamic stacking. Context path: \"{ds_fit_context}\"'\n f\"Run at most {n_splits} sub-fits based on {n_repeats}-repeated {n_folds}-fold cross-validation.\"\n )\n np.random.RandomState(42).shuffle(\n splits\n ) # shuffle splits to mix up order such that if only one of the repeats shows leakage we might stop early.\n for split_index, (train_indices, val_indices) in enumerate(splits):\n if time_limit is None:\n sub_fit_time = None\n else:\n time_spend_sub_fits_so_far = int(time.time() - time_start)\n rest_time = time_limit - time_spend_sub_fits_so_far\n sub_fit_time = int(\n 1 / (n_splits - split_index) * rest_time\n ) # if we are faster, give more time to rest of the folds.\n if sub_fit_time <= 0:\n logger.info(\n \"\\tStop cross-validation during dynamic stacking early as no more time left. Consider specifying a larger time_limit.\"\n )\n break\n ds_fit_kwargs.update(\n dict(\n train_indices=train_indices,\n val_indices=val_indices,\n ds_fit_context=os.path.join(ds_fit_context, f\"sub_fit_{split_index}\"),\n )\n )\n logger.info(\n f\"\\tStarting sub-fit {split_index + 1}. Time limit for the sub-fit of this split is: {'unlimited' if sub_fit_time is None else sub_fit_time}.\"\n )\n stacked_overfitting = self._sub_fit_memory_save_wrapper(\n train_data=X,\n time_limit=time_limit,\n time_start=time_start,\n ds_fit_kwargs=ds_fit_kwargs,\n ag_fit_kwargs=inner_ag_fit_kwargs,\n ag_post_fit_kwargs=inner_ag_post_fit_kwargs,\n holdout_data=holdout_data,\n )\n if stacked_overfitting:\n break\n\n del splits\n\n if clean_up_fits:\n try:\n shutil.rmtree(path=ds_fit_context)\n except FileNotFoundError:\n pass\n\n # -- Determine rest time and new num_stack_levels\n time_spend_sub_fits = time.time() - time_start\n num_stack_levels = 0 if stacked_overfitting else org_num_stack_levels\n self._stacked_overfitting_occurred = stacked_overfitting\n\n logger.info(\n f\"\\t{num_stack_levels}\\t = Optimal num_stack_levels (Stacked Overfitting Occurred: {self._stacked_overfitting_occurred})\"\n )\n log_str = f\"\\t{round(time_spend_sub_fits)}s\\t = DyStack runtime\"\n if time_limit_og is None:\n time_limit_fit_full = None\n else:\n time_limit_fit_full = time_limit_og - time_spend_sub_fits\n log_str += f\" |\\t{round(time_limit_fit_full)}s\\t = Remaining runtime\"\n logger.info(log_str)\n\n # -- Revert back\n del inner_ag_fit_kwargs\n if holdout_data is None:\n ag_fit_kwargs[\"X\"] = X\n else:\n logger.log(\n 20,\n \"\\tConcatenating holdout data from dynamic stacking to the training data for the full fit (and reset the index).\",\n )\n ag_fit_kwargs[\"X\"] = pd.concat([X, holdout_data], ignore_index=True)\n\n ag_fit_kwargs[\"X_val\"] = X_val\n ag_fit_kwargs[\"X_unlabeled\"] = X_unlabeled\n\n return num_stack_levels, time_limit_fit_full\n\n def _sub_fit_memory_save_wrapper(\n self,\n train_data: Union[str, pd.DataFrame],\n time_limit: float,\n time_start: float,\n ds_fit_kwargs: dict,\n ag_fit_kwargs: dict,\n ag_post_fit_kwargs: dict,\n holdout_data=Union[str, pd.DataFrame, None],\n ):\n \"\"\"Tries to run the sub-fit in a subprocess (managed by ray). Similar to AutoGluon's parallel fold fitting strategies,\n this code does not shut down ray after usage. Otherwise, we would also kill outer-scope ray usage.\"\"\"\n memory_safe_fits = ds_fit_kwargs.get(\"memory_safe_fits\", True)\n enable_ray_logging = ds_fit_kwargs.get(\"enable_ray_logging\", True)\n normal_fit = False\n total_resources = ag_fit_kwargs[\"core_kwargs\"][\"total_resources\"]\n\n if memory_safe_fits == \"auto\":\n num_gpus = total_resources.get(\"num_gpus\", \"auto\")\n if num_gpus == \"auto\":\n num_gpus = ResourceManager.get_gpu_count_torch()\n if num_gpus > 0:\n logger.log(\n 30,\n \"DyStack: Disabling memory safe fit mode in DyStack \"\n \"because GPUs were detected and num_gpus='auto' (GPUs cannot be used in memory safe fit mode). \"\n \"If you want to use memory safe fit mode, manually set `num_gpus=0`.\",\n )\n if num_gpus > 0:\n memory_safe_fits = False\n else:\n memory_safe_fits = True\n\n if memory_safe_fits:\n try:\n _ds_ray = try_import_ray()\n if not _ds_ray.is_initialized():\n if enable_ray_logging:\n logger.info(\n \"\\tRunning DyStack sub-fit in a ray process to avoid memory leakage. \"\n \"Enabling ray logging (enable_ray_logging=True). Specify `ds_args={'enable_ray_logging': False}` if you experience logging issues.\"\n )\n _ds_ray.init()\n else:\n logger.info(\n \"\\tRunning DyStack sub-fit in a ray process to avoid memory leakage. \"\n \"Logs will not be shown until this process is complete (enable_ray_logging=False). \"\n \"You can experimentally enable logging by specifying `ds_args={'enable_ray_logging': True}`.\"\n )\n _ds_ray.init(\n logging_level=logging.ERROR,\n log_to_driver=False,\n )\n except Exception as e:\n warnings.warn(\n f\"Failed to use ray for memory safe fits. Falling back to normal fit. Error: {repr(e)}\",\n stacklevel=2,\n )\n _ds_ray = None\n\n if time_limit is not None:\n # Subtract time taken to initialize ray\n time_limit -= time.time() - time_start\n if time_limit <= 0:\n logger.log(30, \"Warning: Not enough time to fit DyStack! Skipping...\")\n return False\n\n if holdout_data is None:\n logger.info(f'\\t\\tContext path: \"{ds_fit_kwargs[\"ds_fit_context\"]}\"')\n else:\n logger.info(\n f'\\t\\tRunning DyStack holdout-based sub-fit with custom validation data. Context path: \"{ds_fit_kwargs[\"ds_fit_context\"]}\"'\n )\n\n if _ds_ray is not None:\n # Handle resources\n # FIXME: what about distributed?\n\n num_cpus = total_resources.get(\"num_cpus\", \"auto\")\n\n if num_cpus == \"auto\":\n num_cpus = ResourceManager.get_cpu_count()\n\n # num_gpus is treated oddly in ray, commented out until we find a better solution\n # num_gpus = total_resources.get(\"num_gpus\", \"auto\")\n # if num_gpus == \"auto\":\n # num_gpus = ResourceManager.get_gpu_count()\n\n # Handle expensive data via put\n ag_fit_kwargs_ref = _ds_ray.put(ag_fit_kwargs)\n predictor_ref = _ds_ray.put(self)\n train_data_ref = _ds_ray.put(train_data)\n\n if holdout_data is not None:\n holdout_data_ref = _ds_ray.put(holdout_data)\n else:\n holdout_data_ref = None\n\n # Call sub fit in its own subprocess via ray\n sub_fit_caller = _ds_ray.remote(max_calls=1)(_dystack)\n # FIXME: For some reason ray does not treat `num_cpus` and `num_gpus` the same.\n # For `num_gpus`, the process will reserve the capacity and is unable to share it to child ray processes, causing a deadlock.\n # For `num_cpus`, the value is completely ignored by children, and they can even use more num_cpus than the parent.\n # Because of this, num_gpus is set to 0 here to avoid a deadlock, but num_cpus does not need to be changed.\n # For more info, refer to Ray documentation: https://docs.ray.io/en/latest/ray-core/tasks/nested-tasks.html#yielding-resources-while-blocked\n ref = sub_fit_caller.options(num_cpus=num_cpus, num_gpus=0).remote(\n predictor=predictor_ref,\n train_data=train_data_ref,\n time_limit=time_limit,\n ds_fit_kwargs=ds_fit_kwargs,\n ag_fit_kwargs=ag_fit_kwargs_ref,\n ag_post_fit_kwargs=ag_post_fit_kwargs,\n holdout_data=holdout_data_ref,\n )\n finished, unfinished = _ds_ray.wait([ref], num_returns=1)\n stacked_overfitting, ho_leaderboard, exception = _ds_ray.get(finished[0])\n\n # TODO: This is present to ensure worker logs are properly logged and don't get skipped / printed out of order.\n # Ideally find a faster way to do this that doesn't introduce a 100 ms overhead.\n time.sleep(0.1)\n else:\n normal_fit = True\n else:\n normal_fit = True\n\n if normal_fit:\n stacked_overfitting, ho_leaderboard, exception = _dystack(\n predictor=self,\n train_data=train_data,\n time_limit=time_limit,\n ds_fit_kwargs=ds_fit_kwargs,\n ag_fit_kwargs=ag_fit_kwargs,\n ag_post_fit_kwargs=ag_post_fit_kwargs,\n holdout_data=holdout_data,\n )\n\n if exception is not None:\n logger.log(40, f\"Warning: Exception encountered during DyStack sub-fit:\\n\\t{exception}\")\n if ho_leaderboard is not None:\n logger.log(20, \"Leaderboard on holdout data (DyStack):\")\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n # Rename to avoid confusion for the user\n logger.log(20, ho_leaderboard.rename({\"score_test\": \"score_holdout\"}, axis=1))\n\n if not normal_fit and enable_ray_logging:\n try:\n # Disables ray logging to avoid log spam in main process after DyStack completes\n # This is somewhat of a hack, and needs to use private APIs of ray. It is unclear how to do this in a different way.\n # Note: Once this is done, it cannot be undone,\n # and the only known way to re-enable ray logging in the process is to call `ray.shutdown()` and `ray.init()`\n _ds_ray._private.ray_logging.global_worker_stdstream_dispatcher.remove_handler(\"ray_print_logs\")\n except Exception as e:\n logger.log(\n 40,\n \"WARNING: ray logging verbosity fix raised an exception. Ray might give overly verbose logging output. \"\n \"Please open a GitHub issue to notify the AutoGluon developers of this issue. \"\n \"You can avoid this issue by specifying `ds_args={'enable_ray_logging': False}`. Exception detailed below:\"\n f\"\\n{e}\",\n )\n\n return stacked_overfitting\n\n def _post_fit(\n self,\n keep_only_best=False,\n refit_full=False,\n set_best_to_refit_full=False,\n save_space=False,\n calibrate=False,\n calibrate_decision_threshold=False,\n infer_limit=None,\n num_cpus: int | str = \"auto\",\n num_gpus: int | str = \"auto\",\n refit_full_kwargs: dict = None,\n fit_strategy: Literal[\"auto\", \"sequential\", \"parallel\"] = \"sequential\",\n raise_on_no_models_fitted: bool = True,\n ):\n if refit_full_kwargs is None:\n refit_full_kwargs = {}\n if not self.model_names():\n if raise_on_no_models_fitted:\n raise RuntimeError(\n \"No models were trained successfully during fit().\"\n \" Inspect the log output or increase verbosity to determine why no models were fit.\"\n \" Alternatively, set `raise_on_no_models_fitted` to False during the fit call.\"\n )\n\n logger.log(30, \"Warning: No models found, skipping post_fit logic...\")\n return\n\n if refit_full is True:\n if keep_only_best is True:\n if set_best_to_refit_full is True:\n refit_full = \"best\"\n else:\n logger.warning(\n f\"refit_full was set to {refit_full}, but keep_only_best=True and set_best_to_refit_full=False. Disabling refit_full to avoid training models which would be automatically deleted.\"\n )\n refit_full = False\n else:\n refit_full = \"all\"\n\n if refit_full is not False:\n if infer_limit is not None:\n infer_limit = infer_limit - self._learner.preprocess_1_time\n trainer_model_best = self._trainer.get_model_best(infer_limit=infer_limit, infer_limit_as_child=True)\n logger.log(\n 20, \"Automatically performing refit_full as a post-fit operation (due to `.fit(..., refit_full=True)`\"\n )\n if set_best_to_refit_full:\n _set_best_to_refit_full = trainer_model_best\n else:\n _set_best_to_refit_full = False\n if refit_full == \"best\":\n self.refit_full(\n model=trainer_model_best,\n set_best_to_refit_full=_set_best_to_refit_full,\n num_cpus=num_cpus,\n num_gpus=num_gpus,\n fit_strategy=fit_strategy,\n **refit_full_kwargs,\n )\n else:\n self.refit_full(\n model=refit_full,\n set_best_to_refit_full=_set_best_to_refit_full,\n num_cpus=num_cpus,\n num_gpus=num_gpus,\n fit_strategy=fit_strategy,\n **refit_full_kwargs,\n )\n\n if calibrate == \"auto\":\n if self.problem_type in PROBLEM_TYPES_CLASSIFICATION and self.eval_metric.needs_proba:\n calibrate = True\n elif self.problem_type == QUANTILE:\n calibrate = True\n else:\n calibrate = False\n if calibrate:\n num_rows_val_for_calibration = self._trainer.num_rows_val_for_calibration\n if self.problem_type == BINARY:\n # Tested on \"adult\" dataset\n min_val_rows_for_calibration = 3000\n elif self.problem_type == MULTICLASS:\n # Tested on \"covertype\" dataset\n min_val_rows_for_calibration = 500\n else:\n # problem_type == \"quantile\"\n # TODO: Haven't benchmarked, this is just a guess\n min_val_rows_for_calibration = 1000\n if num_rows_val_for_calibration < min_val_rows_for_calibration:\n calibrate = False\n logger.log(\n 30,\n f\"Disabling calibration for metric `{self.eval_metric.name}` due to having \"\n f\"fewer than {min_val_rows_for_calibration} rows of validation data for calibration, \"\n f\"to avoid overfitting ({num_rows_val_for_calibration} rows). \"\n f\"Force calibration via specifying `calibrate=True`. (calibrate='auto')\",\n )\n\n if calibrate:\n if self.problem_type in PROBLEM_TYPES_CLASSIFICATION:\n self._trainer.calibrate_model()\n elif self.problem_type == QUANTILE:\n self._trainer.calibrate_model()\n else:\n logger.log(\n 30,\n \"WARNING: `calibrate=True` is only applicable to classification or quantile regression problems. Skipping calibration...\",\n )\n\n if isinstance(calibrate_decision_threshold, str) and calibrate_decision_threshold == \"auto\":\n calibrate_decision_threshold = self._can_calibrate_decision_threshold()\n if calibrate_decision_threshold and self.eval_metric.name == \"precision\":\n # precision becomes undefined when no true positives exist.\n # This interacts weirdly with threshold calibration where val score will be 1.0, but test score can be 0.0 due to being undefined.\n calibrate_decision_threshold = False\n logger.log(\n 30,\n \"Disabling decision threshold calibration for metric `precision` to avoid undefined results. \"\n \"Force calibration via specifying `calibrate_decision_threshold=True`.\",\n )\n elif calibrate_decision_threshold and self.eval_metric.name == \"accuracy\":\n num_rows_val_for_calibration = self._trainer.num_rows_val_for_calibration\n min_val_rows_for_calibration = 10000\n if num_rows_val_for_calibration < min_val_rows_for_calibration:\n calibrate_decision_threshold = False\n logger.log(\n 20,\n f\"Disabling decision threshold calibration for metric `accuracy` due to having \"\n f\"fewer than {min_val_rows_for_calibration} rows of validation data for calibration, \"\n f\"to avoid overfitting ({num_rows_val_for_calibration} rows).\"\n f\"\\n\\t`accuracy` is generally not improved through threshold calibration. \"\n f\"Force calibration via specifying `calibrate_decision_threshold=True`.\",\n )\n elif calibrate_decision_threshold:\n num_rows_val_for_calibration = self._trainer.num_rows_val_for_calibration\n min_val_rows_for_calibration = 50\n if num_rows_val_for_calibration < min_val_rows_for_calibration:\n calibrate_decision_threshold = False\n logger.log(\n 30,\n f\"Disabling decision threshold calibration for metric `{self.eval_metric.name}` due to having \"\n f\"fewer than {min_val_rows_for_calibration} rows of validation data for calibration \"\n f\"to avoid overfitting ({num_rows_val_for_calibration} rows). \"\n f\"Force calibration via specifying `calibrate_decision_threshold=True`.\",\n )\n if calibrate_decision_threshold:\n logger.log(\n 20,\n \"Enabling decision threshold calibration (calibrate_decision_threshold='auto', metric is valid, problem_type is 'binary')\",\n )\n if calibrate_decision_threshold:\n if self.problem_type != BINARY:\n logger.log(\n 30,\n \"WARNING: `calibrate_decision_threshold=True` is only applicable to binary classification. Skipping calibration...\",\n )\n else:\n best_threshold = self.calibrate_decision_threshold()\n self.set_decision_threshold(decision_threshold=best_threshold)\n\n if keep_only_best:\n self.delete_models(models_to_keep=\"best\", dry_run=False)\n\n if save_space:\n self.save_space()\n\n def _can_calibrate_decision_threshold(self) -> bool:\n return self.eval_metric.needs_class and self.problem_type == BINARY\n\n # TODO: Consider adding infer_limit to fit_extra\n def fit_extra(\n self,\n hyperparameters: str | dict[str, Any],\n time_limit: float = None,\n base_model_names: list[str] = None,\n fit_weighted_ensemble: bool = True,\n fit_full_last_level_weighted_ensemble: bool = True,\n full_weighted_ensemble_additionally: bool = False,\n num_cpus: str | int = \"auto\",\n num_gpus: str | int = \"auto\",\n fit_strategy: Literal[\"auto\", \"sequential\", \"parallel\"] = \"auto\",\n memory_limit: float | str = \"auto\",\n **kwargs,\n ) -> \"TabularPredictor\":\n \"\"\"\n Fits additional models after the original :meth:`TabularPredictor.fit` call.\n The original train_data and tuning_data will be used to train the models.\n\n Parameters\n ----------\n hyperparameters : str or dict\n Refer to argument documentation in :meth:`TabularPredictor.fit`.\n If `base_model_names` is specified and hyperparameters is using the level-based key notation,\n the key of the level which directly uses the base models should be 1. The level in the hyperparameters\n dictionary is relative, not absolute.\n time_limit : int, default = None\n Refer to argument documentation in :meth:`TabularPredictor.fit`.\n base_model_names : list[str], default = None\n The names of the models to use as base models for this fit call.\n Base models will provide their out-of-fold predictions as additional features to the models in `hyperparameters`.\n If specified, all models trained will be stack ensembles.\n If None, models will be trained as if they were specified in :meth:`TabularPredictor.fit`, without depending on existing models.\n Only valid if bagging is enabled.\n fit_weighted_ensemble : bool, default = True\n If True, a WeightedEnsembleModel will be fit in each stack layer.\n A weighted ensemble will often be stronger than an individual model while being very fast to train.\n It is recommended to keep this value set to True to maximize predictive quality.\n fit_full_last_level_weighted_ensemble : bool, default = True\n If True, the WeightedEnsembleModel of the last stacking level will be fit with all (successful) models from all previous layers as base models.\n If stacking is disabled, settings this to True or False makes no difference because the WeightedEnsembleModel L2 always uses all models from L1.\n It is recommended to keep this value set to True to maximize predictive quality.\n full_weighted_ensemble_additionally : bool, default = False\n If True, AutoGluon will fit two WeightedEnsembleModels after training all stacking levels. Setting this to True, simulates calling\n `fit_weighted_ensemble()` after calling `fit()`. Has no affect if `fit_full_last_level_weighted_ensemble` is False and does not fit an additional\n WeightedEnsembleModel if stacking is disabled.\n num_cpus: int, default = \"auto\"\n The total amount of cpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of cpus available and the model requirement for best performance.\n Users generally don't need to set this value\n num_gpus: int, default = \"auto\"\n The total amount of gpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of gpus available and the model requirement for best performance.\n Users generally don't need to set this value\n fit_strategy: Literal[\"auto\", \"sequential\", \"parallel\"], default = \"auto\"\n The strategy used to fit models.\n If \"auto\", uses the same fit_strategy as used in the original :meth:`TabularPredictor.fit` call.\n If \"sequential\", models will be fit sequentially. This is the most stable option with the most readable logging.\n If \"parallel\", models will be fit in parallel with ray, splitting available compute between them.\n Note: \"parallel\" is experimental and may run into issues. It was first added in version 1.2.0.\n For machines with 16 or more CPU cores, it is likely that \"parallel\" will be faster than \"sequential\".\n\n .. versionadded:: 1.2.0\n\n memory_limit: float | str, default = \"auto\"\n The total amount of memory in GB you want AutoGluon predictor to use. \"auto\" means AutoGluon will use all available memory on the system\n (that is detectable by psutil).\n Note that this is only a soft limit! AutoGluon uses this limit to skip training models that are expected to require too much memory or stop\n training a model that would exceed the memory limit. AutoGluon does not guarantee the enforcement of this limit (yet). Nevertheless, we expect\n AutoGluon to abide by the limit in most cases or, at most, go over the limit by a small margin.\n For most virtualized systems (e.g., in the cloud) and local usage on a server or laptop, \"auto\" is ideal for this parameter. We recommend manually\n setting the memory limit (and any other resources) on systems with shared resources that are controlled by the operating system (e.g., SLURM and\n cgroups). Otherwise, AutoGluon might wrongly assume more resources are available for fitting a model than the operating system allows,\n which can result in model training failing or being very inefficient.\n **kwargs :\n Refer to kwargs documentation in :meth:`TabularPredictor.fit`.\n Note that the following kwargs are not available in `fit_extra` as they cannot be changed from their values set in `fit()`:\n [`holdout_frac`, `num_bag_folds`, `auto_stack`, `feature_generator`, `unlabeled_data`]\n Moreover, `dynamic_stacking` is also not available in `fit_extra` as the detection of stacked overfitting is only supported at the first fit time.\n pseudo_data : pd.DataFrame, default = None\n Data that has been self labeled by Autogluon model and will be incorporated into training during 'fit_extra'\n \"\"\"\n self._assert_is_fit(\"fit_extra\")\n time_start = time.time()\n\n kwargs_orig = kwargs.copy()\n kwargs = self._validate_fit_extra_kwargs(kwargs)\n\n verbosity = kwargs.get(\"verbosity\", self.verbosity)\n set_logger_verbosity(verbosity)\n\n if verbosity >= 3:\n logger.log(20, \"============ fit kwarg info ============\")\n logger.log(20, \"User Specified kwargs:\")\n logger.log(20, f\"{pprint.pformat(kwargs_orig)}\")\n logger.log(20, \"Full kwargs:\")\n logger.log(20, f\"{pprint.pformat(kwargs)}\")\n logger.log(20, \"========================================\")\n\n self._validate_num_cpus(num_cpus=num_cpus)\n self._validate_num_gpus(num_gpus=num_gpus)\n self._validate_and_set_memory_limit(memory_limit=memory_limit)\n\n if fit_strategy == \"auto\":\n fit_strategy = self._fit_strategy\n self._validate_fit_strategy(fit_strategy=fit_strategy)\n\n # TODO: Allow disable aux (default to disabled)\n # TODO: num_bag_sets\n # num_bag_sets = kwargs['num_bag_sets']\n num_stack_levels = kwargs[\"num_stack_levels\"]\n # save_bag_folds = kwargs['save_bag_folds'] # TODO: Enable\n\n ag_args = kwargs[\"ag_args\"]\n ag_args_fit = kwargs[\"ag_args_fit\"]\n ag_args_ensemble = kwargs[\"ag_args_ensemble\"]\n excluded_model_types = kwargs[\"excluded_model_types\"]\n pseudo_data = kwargs.get(\"pseudo_data\", None)\n\n # TODO: Since data preprocessor is fitted on original train_data it cannot account for if\n # labeled pseudo data has new labels unseen in the original train. Probably need to refit\n # data preprocessor if this is the case.\n if pseudo_data is not None:\n X_pseudo, y_pseudo, y_pseudo_og = self._sanitize_pseudo_data(pseudo_data=pseudo_data)\n else:\n X_pseudo = None\n y_pseudo = None\n y_pseudo_og = None\n\n if ag_args is None:\n ag_args = {}\n ag_args = self._set_hyperparameter_tune_kwargs_in_ag_args(\n kwargs[\"hyperparameter_tune_kwargs\"], ag_args, time_limit=time_limit\n )\n\n fit_new_weighted_ensemble = False # TODO: Add as option\n aux_kwargs = {\n \"total_resources\": {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n },\n }\n if fit_weighted_ensemble is False:\n aux_kwargs = {\"fit_weighted_ensemble\": False}\n aux_kwargs[\"fit_full_last_level_weighted_ensemble\"] = fit_full_last_level_weighted_ensemble\n aux_kwargs[\"full_weighted_ensemble_additionally\"] = full_weighted_ensemble_additionally\n\n if isinstance(hyperparameters, str):\n hyperparameters = get_hyperparameter_config(hyperparameters)\n\n if num_stack_levels is None:\n hyperparameter_keys = list(hyperparameters.keys())\n highest_level = 1\n for key in hyperparameter_keys:\n if isinstance(key, int):\n highest_level = max(key, highest_level)\n num_stack_levels = highest_level - 1\n\n # TODO: make core_kwargs a kwargs argument to predictor.fit, add aux_kwargs to predictor.fit\n core_kwargs = {\n \"total_resources\": {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n },\n \"ag_args\": ag_args,\n \"ag_args_ensemble\": ag_args_ensemble,\n \"ag_args_fit\": ag_args_fit,\n \"excluded_model_types\": excluded_model_types,\n \"fit_strategy\": fit_strategy,\n }\n\n # FIXME: v1.2 pseudo_data can be passed in `fit()` but it is ignored!\n if X_pseudo is not None and y_pseudo is not None:\n core_kwargs[\"X_pseudo\"] = X_pseudo\n core_kwargs[\"y_pseudo\"] = y_pseudo\n\n # TODO: Add special error message if called and training/val data was not cached.\n X, y, X_val, y_val = self._trainer.load_data()\n\n if y_pseudo is not None and self.problem_type in PROBLEM_TYPES_CLASSIFICATION:\n y_og = self._learner.label_cleaner.inverse_transform(y)\n y_og_classes = y_og.unique()\n y_pseudo_classes = y_pseudo_og.unique()\n matching_classes = np.isin(y_pseudo_classes, y_og_classes)\n\n if not matching_classes.all():\n raise Exception(\n f\"Pseudo training data contains classes not in original train data: {y_pseudo_classes[~matching_classes]}\"\n )\n\n name_suffix = kwargs.get(\"name_suffix\", \"\")\n\n fit_models = self._trainer.train_multi_levels(\n X=X,\n y=y,\n hyperparameters=hyperparameters,\n X_val=X_val,\n y_val=y_val,\n base_model_names=base_model_names,\n time_limit=time_limit,\n relative_stack=True,\n level_end=num_stack_levels + 1,\n core_kwargs=core_kwargs,\n aux_kwargs=aux_kwargs,\n name_suffix=name_suffix,\n )\n\n if time_limit is not None:\n time_limit = time_limit - (time.time() - time_start)\n\n if fit_new_weighted_ensemble:\n if time_limit is not None:\n time_limit_weighted = max(time_limit, 60)\n else:\n time_limit_weighted = None\n fit_models += self.fit_weighted_ensemble(time_limit=time_limit_weighted)\n\n refit_full_kwargs = dict(\n X_pseudo=X_pseudo,\n y_pseudo=y_pseudo,\n )\n\n self._post_fit(\n keep_only_best=kwargs[\"keep_only_best\"],\n refit_full=kwargs[\"refit_full\"],\n set_best_to_refit_full=kwargs[\"set_best_to_refit_full\"],\n save_space=kwargs[\"save_space\"],\n calibrate=kwargs[\"calibrate\"],\n refit_full_kwargs=refit_full_kwargs,\n )\n self.save()\n return self\n\n def _get_all_fit_extra_args(self):\n ret = list(self._fit_extra_kwargs_dict().keys()) + list(inspect.signature(self.fit_extra).parameters.keys())\n ret.remove(\"kwargs\")\n\n return ret\n\n def _fit_weighted_ensemble_pseudo(self):\n \"\"\"\n Fits weighted ensemble on top models trained with pseudo labeling, then if new\n weighted ensemble model is best model then sets `model_best` in trainer to\n weighted ensemble model.\n \"\"\"\n logger.log(15, \"Fitting weighted ensemble using top models\")\n weighted_ensemble_model_name = self.fit_weighted_ensemble()[0]\n\n # TODO: This is a hack! self.predict_prob does not update to use weighted ensemble\n # if it's the best model.\n # TODO: There should also be PL added to weighted ensemble model name to notify\n # users it is a model trained with PL models if they are indeed ensembled\n model_best_name = self._trainer.leaderboard().iloc[0][\"model\"]\n if model_best_name == weighted_ensemble_model_name:\n self._trainer.model_best = model_best_name\n self._trainer.save()\n logger.log(15, \"Weighted ensemble was the best model for current iteration of pseudo labeling\")\n else:\n logger.log(15, \"Weighted ensemble was not the best model for current iteration of pseudo labeling\")\n\n def _predict_pseudo(self, X_test: pd.DataFrame, use_ensemble: bool):\n if use_ensemble:\n if self.problem_type in PROBLEM_TYPES_CLASSIFICATION:\n test_pseudo_idxes_true, y_pred_proba, y_pred = filter_ensemble_pseudo(\n predictor=self, unlabeled_data=X_test\n )\n else:\n test_pseudo_idxes_true, y_pred = filter_ensemble_pseudo(predictor=self, unlabeled_data=X_test)\n y_pred_proba = y_pred.copy()\n else:\n if self.can_predict_proba:\n y_pred_proba = self.predict_proba(data=X_test, as_multiclass=True)\n y_pred = get_pred_from_proba_df(y_pred_proba, problem_type=self.problem_type)\n else:\n y_pred = self.predict(data=X_test)\n y_pred_proba = y_pred\n test_pseudo_idxes_true = filter_pseudo(y_pred_proba_og=y_pred_proba, problem_type=self.problem_type)\n return y_pred, y_pred_proba, test_pseudo_idxes_true\n\n def _run_pseudolabeling(\n self,\n unlabeled_data: pd.DataFrame,\n max_iter: int,\n return_pred_prob: bool = False,\n use_ensemble: bool = False,\n fit_ensemble: bool = False,\n fit_ensemble_every_iter: bool = False,\n **kwargs,\n ):\n \"\"\"\n Runs pseudolabeling algorithm using the same hyperparameters and model and fit settings\n used in original model unless specified by the user. This is an internal function that iteratively\n self labels unlabeled test data then incorporates all self labeled data above a threshold into training.\n Will keep incorporating self labeled data into training until validation score does not improve\n\n Parameters:\n -----------\n unlabeled_data: pd.DataFrame\n Extra unlabeled data (could be the test data) to assign pseudolabels to and incorporate as extra training data.\n max_iter: int\n Maximum allowed number of iterations, where in each iteration, the data are pseudolabeled\n by the current predictor and the predictor is refit including the pseudolabled data in its training set.\n return_pred_proba: bool, default = False\n Transductive learning setting, will return predictive probabilities of unlabeled_data\n use_ensemble: bool, default = False\n If True will use ensemble pseudo labeling algorithm if False will use best model\n pseudo labeling method\n fit_ensemble: bool, default = False\n If True will fit weighted ensemble on final best models. Fitting weighted ensemble will be done after fitting\n of models is completed unless otherwise specified. If False will not fit weighted ensemble on final best\n models.\n fit_ensemble_every_iter: bool, default = False\n If True will fit weighted ensemble model using combination of best models\n for every iteration of pseudo label algorithm. If False and fit_ensemble\n is True, will just do it at the very end of training pseudo labeled models.\n\n Returns:\n --------\n self: TabularPredictor\n \"\"\"\n previous_score = (\n self.leaderboard(set_refit_score_to_parent=True)\n .set_index(\"model\", drop=True)\n .loc[self.model_best][\"score_val\"]\n )\n y_pseudo_og = pd.Series()\n X_test = unlabeled_data.copy()\n\n y_pred, y_pred_proba, test_pseudo_idxes_true = self._predict_pseudo(X_test=X_test, use_ensemble=use_ensemble)\n y_pred_proba_og = y_pred_proba\n\n for i in range(max_iter):\n if len(X_test) == 0:\n logger.log(20, \"No more unlabeled data to pseudolabel. Done with pseudolabeling...\")\n break\n\n iter_print = str(i + 1)\n logger.log(20, f\"Beginning iteration {iter_print} of pseudolabeling out of max {max_iter}\")\n\n if len(test_pseudo_idxes_true) < 1:\n logger.log(\n 20,\n f\"Could not confidently assign pseudolabels for any of the provided rows in iteration {iter_print}. Done with pseudolabeling...\",\n )\n break\n else:\n logger.log(\n 20,\n f\"Pseudolabeling algorithm confidently assigned pseudolabels to {len(test_pseudo_idxes_true)} rows of data \"\n f\"on iteration {iter_print}. Adding to train data\",\n )\n\n test_pseudo_idxes = pd.Series(data=False, index=y_pred_proba.index)\n test_pseudo_idxes_false = test_pseudo_idxes[~test_pseudo_idxes.index.isin(test_pseudo_idxes_true.index)]\n test_pseudo_idxes[test_pseudo_idxes_true.index] = True\n\n if len(test_pseudo_idxes_true) != 0:\n if len(y_pseudo_og) == 0:\n y_pseudo_og = y_pred.loc[test_pseudo_idxes_true.index].copy()\n else:\n y_pseudo_og = pd.concat(\n [y_pseudo_og, y_pred.loc[test_pseudo_idxes_true.index]], verify_integrity=True\n )\n\n pseudo_data = unlabeled_data.loc[y_pseudo_og.index]\n pseudo_data[self.label] = y_pseudo_og\n self.fit_extra(pseudo_data=pseudo_data, name_suffix=PSEUDO_MODEL_SUFFIX.format(iter=(i + 1)), **kwargs)\n\n if fit_ensemble and fit_ensemble_every_iter:\n self._fit_weighted_ensemble_pseudo()\n\n current_score = (\n self.leaderboard(set_refit_score_to_parent=True)\n .set_index(\"model\", drop=True)\n .loc[self.model_best][\"score_val\"]\n )\n logger.log(\n 20,\n f\"Pseudolabeling algorithm changed validation score from: {previous_score}, to: {current_score}\"\n f\" using evaluation metric: {self.eval_metric.name}\",\n )\n\n if previous_score >= current_score:\n # No improvement from pseudo labelling this iteration, stop iterating\n break\n else:\n # Cut down X_test to not include pseudo labeled data\n X_test = X_test.loc[test_pseudo_idxes[~test_pseudo_idxes].index]\n previous_score = current_score\n\n # Update y_pred_proba and test_pseudo_idxes_true based on the latest pseudolabelled iteration\n y_pred, y_pred_proba, test_pseudo_idxes_true = self._predict_pseudo(\n X_test=X_test, use_ensemble=use_ensemble\n )\n # Update the y_pred_proba_og variable if an improvement was achieved\n if return_pred_prob and test_pseudo_idxes_false is not None:\n y_pred_proba_og.loc[test_pseudo_idxes_false.index] = y_pred_proba.loc[\n test_pseudo_idxes_false.index\n ]\n\n if fit_ensemble and not fit_ensemble_every_iter:\n self._fit_weighted_ensemble_pseudo()\n if return_pred_prob:\n if self.can_predict_proba:\n y_pred_proba_og = self.predict_proba(unlabeled_data)\n else:\n y_pred_proba_og = self.predict(unlabeled_data)\n\n if return_pred_prob:\n return self, y_pred_proba_og\n else:\n return self\n\n # TODO: `fit_ensemble` and `use_ensemble` seem redundant, and don't use calibration, making them worse than when they are disabled.\n # TODO: Supporting L2+ models is very complicated. It requires predicting with the original models via `predictor.predict_proba_multi` on `pseudo_data`,\n # then keeping track of these pred_proba and passing them to the appropriate models at fit time.\n @apply_presets(tabular_presets_dict, tabular_presets_alias)\n def fit_pseudolabel(\n self,\n pseudo_data: pd.DataFrame,\n max_iter: int = 3,\n return_pred_prob: bool = False,\n use_ensemble: bool = False,\n fit_ensemble: bool = False,\n fit_ensemble_every_iter: bool = False,\n **kwargs,\n ):\n \"\"\"\n [Advanced] Uses additional data (`pseudo_data`) to try to achieve better model quality.\n Pseudo data can come either with or without the `label` column.\n\n If `pseudo_data` is labeled, then models will be refit using the `pseudo_data` as additional training data.\n If bagging, each fold of the bagged ensemble will use all the `pseudo_data` as additional training data.\n `pseudo_data` will never be used for validation/scoring.\n\n If the data is unlabeled, such as providing the batched test data without ground truth available, then transductive learning is leveraged.\n In transductive learning, the existing predictor will predict on `pseudo_data`\n to identify the most confident rows (For example all rows with predictive probability above 95%).\n These rows will then be pseudo-labelled, given the label of the most confident class.\n The pseudo-labelled rows will then be used as additional training data when fitting the models.\n Then, if `max_iter > 1`, this process can repeat itself, using the new models to predict on the unused `pseudo_data` rows\n to see if any new rows should be used in the next iteration as training data.\n We recommend specifying `return_pred_prob=True` if the data is unlabeled to get the correct prediction probabilities on the `pseudo_data`,\n rather than calling `predictor.predict_proba(pseudo_data)`.\n\n For example:\n Original fit: 10000 `train_data` rows with 10-fold bagging\n Bagged fold models will use 9000 `train_data` rows for training, and 1000 for validation.\n `fit_pseudolabel` is called with 5000 row labelled `pseudo_data`.\n Bagged fold models are then fit again with `_PSEUDO` suffix.\n 10000 train_data rows with 10-fold bagging + 5000 `pseudo_data` rows.\n Bagged fold models will use 9000 `train_data` rows + 5000 `pseudo_data` rows = 14000 rows for training, and 1000 for validation.\n Note: The same validation rows will be used as was done in the original fit, so that validation scores are directly comparable.\n Alternatively, `fit_pseduolabel` is called with 5000 rows unlabelled `pseudo_data`.\n Predictor predicts on the `pseudo_data`, finds 965 rows with confident predictions.\n Set the ground truth of those 965 rows as the most confident prediction.\n Bagged fold models are then fit with `_PSEUDO` suffix.\n 10000 train_data rows with 10-fold bagging + 965 labelled `pseudo_data` rows.\n Bagged fold models will use 9000 `train_data` rows + 965 `pseudo_data` rows = 9965 rows for training, and 1000 for validation.\n Note: The same validation rows will be used as was done in the original fit, so that validation scores are directly comparable.\n Repeat the process using the new pseudo-labelled predictor on the remaining `pseudo_data`.\n In the example, lets assume 188 new `pseudo_data` rows have confident predictions.\n Now the total labelled `pseudo_data` rows is 965 + 188 = 1153.\n Then repeat the process, up to `max_iter` times: ex 10000 train_data rows with 10-fold bagging + 1153 `pseudo_data` rows.\n Early stopping will trigger if validation score improvement is not observed.\n\n Note: pseudo_data is only used for L1 models. Support for L2+ models is not yet implemented. L2+ models will only use the original train_data.\n\n Parameters\n ----------\n pseudo_data : :class:`pd.DataFrame`\n Extra data to incorporate into training. Pre-labeled test data allowed. If no labels\n then pseudo-labeling algorithm will predict and filter out which rows to incorporate into training\n max_iter: int, default = 3\n Maximum iterations of pseudo-labeling allowed\n return_pred_prob: bool, default = False\n Returns held-out predictive probabilities from pseudo-labeling. If test_data is labeled then\n returns model's predictive probabilities.\n use_ensemble: bool, default = False\n If True will use ensemble pseudo labeling algorithm. If False will just use best model\n for pseudo labeling algorithm.\n fit_ensemble: bool, default = False\n If True with fit weighted ensemble model using combination of best models.\n Fitting weighted ensemble will be done after fitting has\n been completed unless otherwise specified. If False will not fit weighted ensemble\n over models trained with pseudo labeling and models trained without it.\n fit_ensemble_every_iter: bool, default = False\n If True fits weighted ensemble model for every iteration of pseudo labeling algorithm. If False\n and fit_ensemble is True will fit after all pseudo labeling training is done.\n **kwargs:\n If predictor is not already fit, then kwargs are for the functions 'fit' and 'fit_extra':\n Refer to parameters documentation in :meth:`TabularPredictor.fit`.\n Refer to parameters documentation in :meth:`TabularPredictor.fit_extra`.\n If predictor is fit kwargs are for 'fit_extra':\n Refer to parameters documentation in :meth:`TabularPredictor.fit_extra`.\n\n Returns\n -------\n self : TabularPredictor\n Returns self, which is a Python class of TabularPredictor\n \"\"\"\n if len(pseudo_data) < 1:\n raise Exception(\"No pseudo data given\")\n\n self._validate_unique_indices(pseudo_data, \"pseudo_data\")\n\n was_fit = self.is_fit\n if not was_fit:\n if \"train_data\" not in kwargs.keys():\n Exception(\n \"Autogluon is required to be fit or given 'train_data' in order to run 'fit_pseudolabel'.\"\n \" Autogluon is not fit and 'train_data' was not given\"\n )\n\n logger.log(20, \"Predictor not fit prior to pseudolabeling. Fitting now...\")\n self.fit(**kwargs)\n\n if self.problem_type is MULTICLASS and self.eval_metric.name != \"accuracy\":\n logger.warning(\n \"AutoGluon has detected the problem type as 'multiclass' and \"\n f\"eval_metric is {self.eval_metric.name}, we recommend using\"\n f\"fit_pseudolabeling when eval metric is 'accuracy'\"\n )\n\n is_labeled = self.label in pseudo_data.columns\n\n hyperparameters = kwargs.get(\"hyperparameters\", None)\n if hyperparameters is None:\n if self.is_fit:\n hyperparameters = self.fit_hyperparameters_\n elif isinstance(hyperparameters, str):\n hyperparameters = get_hyperparameter_config(hyperparameters)\n\n kwargs[\"hyperparameters\"] = hyperparameters\n fit_extra_args = self._get_all_fit_extra_args()\n fit_extra_kwargs = {key: value for key, value in kwargs.items() if key in fit_extra_args}\n\n # If first fit was in this method call and `num_stack_levels` wasn't specified, reuse the number of stack levels used in the first fit.\n # TODO: Consider making calculating this information easier, such as keeping track of meta-info from the latest/original fit call.\n # Currently we use `stack_name == core` to figure out the number of stack levels, but this is somewhat brittle.\n if \"num_stack_levels\" not in fit_extra_kwargs and not was_fit:\n models_core: list[str] = [\n m\n for m, stack_name in self._trainer.get_models_attribute_dict(attribute=\"stack_name\").items()\n if stack_name == \"core\"\n ]\n num_stack_levels = (\n max(self._trainer.get_models_attribute_dict(attribute=\"level\", models=models_core).values()) - 1\n )\n fit_extra_kwargs[\"num_stack_levels\"] = num_stack_levels\n if is_labeled:\n logger.log(20, \"Fitting predictor using the provided pseudolabeled examples as extra training data...\")\n self.fit_extra(\n pseudo_data=pseudo_data, name_suffix=PSEUDO_MODEL_SUFFIX.format(iter=\"\")[:-1], **fit_extra_kwargs\n )\n\n if fit_ensemble:\n logger.log(15, \"Fitting weighted ensemble model using best models\")\n self.fit_weighted_ensemble()\n\n if return_pred_prob:\n y_pred_proba = self.predict_proba(pseudo_data) if self.can_predict_proba else self.predict(pseudo_data)\n return self, y_pred_proba\n else:\n return self\n else:\n logger.log(\n 20,\n \"Given test_data for pseudo labeling did not contain labels. \"\n \"AutoGluon will assign pseudo labels to data and use it for extra training data...\",\n )\n return self._run_pseudolabeling(\n unlabeled_data=pseudo_data,\n max_iter=max_iter,\n return_pred_prob=return_pred_prob,\n use_ensemble=use_ensemble,\n fit_ensemble=fit_ensemble,\n fit_ensemble_every_iter=fit_ensemble_every_iter,\n **fit_extra_kwargs,\n )\n\n def predict(\n self,\n data: pd.DataFrame | str,\n model: str | None = None,\n as_pandas: bool = True,\n transform_features: bool = True,\n *,\n decision_threshold: float | None = None,\n ) -> pd.Series | np.ndarray:\n \"\"\"\n Use trained models to produce predictions of `label` column values for new data.\n\n Parameters\n ----------\n data : :class:`pd.DataFrame` or str\n The data to make predictions for. Should contain same column names as training data and follow same format\n (may contain extra columns that won't be used by Predictor, including the label-column itself).\n If str is passed, `data` will be loaded using the str value as the file path.\n model : str (optional)\n The name of the model to get predictions from. Defaults to None, which uses the highest scoring model on the validation set.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`\n as_pandas : bool, default = True\n Whether to return the output as a :class:`pd.Series` (True) or :class:`np.ndarray` (False).\n transform_features : bool, default = True\n If True, preprocesses data before predicting with models.\n If False, skips global feature preprocessing.\n This is useful to save on inference time if you have already called `data = predictor.transform_features(data)`.\n decision_threshold : float, default = None\n The decision threshold used to convert prediction probabilities to predictions.\n Only relevant for binary classification, otherwise ignored.\n If None, defaults to `predictor.decision_threshold`.\n Valid values are in the range [0.0, 1.0]\n You can obtain an optimized `decision_threshold` by first calling `predictor.calibrate_decision_threshold()`.\n Useful to set for metrics such as `balanced_accuracy` and `f1` as `0.5` is often not an optimal threshold.\n Predictions are calculated via the following logic on the positive class: `1 if pred > decision_threshold else 0`\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset. Either :class:`np.ndarray` or :class:`pd.Series` depending on `as_pandas` argument.\n \"\"\"\n self._assert_is_fit(\"predict\")\n data = self._get_dataset(data)\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n return self._learner.predict(\n X=data,\n model=model,\n as_pandas=as_pandas,\n transform_features=transform_features,\n decision_threshold=decision_threshold,\n )\n\n def predict_proba(\n self,\n data: pd.DataFrame | str,\n model: str | None = None,\n as_pandas: bool = True,\n as_multiclass: bool = True,\n transform_features: bool = True,\n ) -> pd.DataFrame | pd.Series | np.ndarray:\n \"\"\"\n Use trained models to produce predicted class probabilities rather than class-labels (if task is classification).\n If `predictor.problem_type` is regression or quantile, this will raise an AssertionError.\n\n Parameters\n ----------\n data : :class:`pd.DataFrame` or str\n The data to make predictions for. Should contain same column names as training dataset and follow same format\n (may contain extra columns that won't be used by Predictor, including the label-column itself).\n If str is passed, `data` will be loaded using the str value as the file path.\n model : str (optional)\n The name of the model to get prediction probabilities from. Defaults to None, which uses the highest scoring model on the validation set.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n as_pandas : bool, default = True\n Whether to return the output as a pandas object (True) or numpy array (False).\n Pandas object is a DataFrame if this is a multiclass problem or `as_multiclass=True`, otherwise it is a Series.\n If the output is a DataFrame, the column order will be equivalent to `predictor.class_labels`.\n as_multiclass : bool, default = True\n Whether to return binary classification probabilities as if they were for multiclass classification.\n Output will contain two columns, and if `as_pandas=True`, the column names will correspond to the binary class labels.\n The columns will be the same order as `predictor.class_labels`.\n If False, output will contain only 1 column for the positive class (get positive_class name via `predictor.positive_class`).\n Only impacts output for binary classification problems.\n transform_features : bool, default = True\n If True, preprocesses data before predicting with models.\n If False, skips global feature preprocessing.\n This is useful to save on inference time if you have already called `data = predictor.transform_features(data)`.\n\n Returns\n -------\n Array of predicted class-probabilities, corresponding to each row in the given data.\n May be a :class:`np.ndarray` or :class:`pd.DataFrame` / :class:`pd.Series` depending on `as_pandas` and `as_multiclass` arguments and the type of prediction problem.\n For binary classification problems, the output contains for each datapoint the predicted probabilities of the negative and positive classes, unless you specify `as_multiclass=False`.\n \"\"\"\n self._assert_is_fit(\"predict_proba\")\n if not self.can_predict_proba:\n raise AssertionError(\n f'`predictor.predict_proba` is not supported when problem_type=\"{self.problem_type}\". '\n f\"Please call `predictor.predict` instead. \"\n f\"You can check the value of `predictor.can_predict_proba` to tell if predict_proba is valid.\"\n )\n data = self._get_dataset(data)\n return self._learner.predict_proba(\n X=data,\n model=model,\n as_pandas=as_pandas,\n as_multiclass=as_multiclass,\n transform_features=transform_features,\n )\n\n def predict_from_proba(\n self, y_pred_proba: pd.DataFrame | np.ndarray, decision_threshold: float | None = None\n ) -> pd.Series | np.array:\n \"\"\"\n Given prediction probabilities, convert to predictions.\n\n Parameters\n ----------\n y_pred_proba : :class:`pd.DataFrame` or :class:`np.ndarray`\n The prediction probabilities to convert to predictions.\n Obtainable via the output of `predictor.predict_proba`.\n decision_threshold : float, default = None\n The decision threshold used to convert prediction probabilities to predictions.\n Only relevant for binary classification, otherwise ignored.\n If None, defaults to `predictor.decision_threshold`.\n Valid values are in the range [0.0, 1.0]\n You can obtain an optimized `decision_threshold` by first calling `predictor.calibrate_decision_threshold()`.\n Useful to set for metrics such as `balanced_accuracy` and `f1` as `0.5` is often not an optimal threshold.\n Predictions are calculated via the following logic on the positive class: `1 if pred > decision_threshold else 0`\n\n Returns\n -------\n Array of predictions, one corresponding to each row in given dataset. Either :class:`np.ndarray` or :class:`pd.Series` depending on `y_pred_proba` dtype.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularPredictor\n >>> predictor = TabularPredictor(label='class').fit('train.csv', label='class')\n >>> y_pred_proba = predictor.predict_proba('test.csv')\n >>>\n >>> # y_pred and y_pred_from_proba are identical\n >>> y_pred = predictor.predict('test.csv')\n >>> y_pred_from_proba = predictor.predict_from_proba(y_pred_proba=y_pred_proba)\n \"\"\"\n if not self.can_predict_proba:\n raise AssertionError(\n f'`predictor.predict_from_proba` is not supported when problem_type=\"{self.problem_type}\".'\n )\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n return self._learner.get_pred_from_proba(y_pred_proba=y_pred_proba, decision_threshold=decision_threshold)\n\n @property\n def can_predict_proba(self) -> bool:\n \"\"\"\n Return True if predictor can return prediction probabilities via `.predict_proba`, otherwise return False.\n Raises an AssertionError if called before fitting.\n \"\"\"\n self._assert_is_fit(\"can_predict_proba\")\n return problem_type_info.can_predict_proba(problem_type=self.problem_type)\n\n @property\n def is_fit(self) -> bool:\n \"\"\"\n Return True if `predictor.fit` has been called, otherwise return False.\n \"\"\"\n return self._learner.is_fit\n\n def evaluate(\n self,\n data: pd.DataFrame | str,\n model: str = None,\n decision_threshold: float = None,\n display: bool = False,\n auxiliary_metrics: bool = True,\n detailed_report: bool = False,\n **kwargs,\n ) -> dict:\n \"\"\"\n Report the predictive performance evaluated over a given dataset.\n This is basically a shortcut for: `pred_proba = predict_proba(data); evaluate_predictions(data[label], pred_proba)`.\n\n Parameters\n ----------\n data : str or :class:`pd.DataFrame`\n This dataset must also contain the `label` with the same column-name as previously specified.\n If str is passed, `data` will be loaded using the str value as the file path.\n If `self.sample_weight` is set and `self.weight_evaluation==True`, then a column with the sample weight name is checked and used for weighted metric evaluation if it exists.\n model : str (optional)\n The name of the model to get prediction probabilities from. Defaults to None, which uses the highest scoring model on the validation set.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n decision_threshold : float, default = None\n The decision threshold to use when converting prediction probabilities to predictions.\n This will impact the scores of metrics such as `f1` and `accuracy`.\n If None, defaults to `predictor.decision_threshold`. Ignored unless `problem_type='binary'`.\n Refer to the `predictor.decision_threshold` docstring for more information.\n display : bool, default = False\n If True, performance results are printed.\n auxiliary_metrics: bool, default = True\n Should we compute other (`problem_type` specific) metrics in addition to the default metric?\n detailed_report : bool, default = False\n Should we computed more detailed versions of the `auxiliary_metrics`? (requires `auxiliary_metrics = True`)\n\n Returns\n -------\n Returns dict where keys = metrics, values = performance along each metric. To get the `eval_metric` score, do `output[predictor.eval_metric.name]`\n NOTE: Metrics scores always show in higher is better form.\n This means that metrics such as log_loss and root_mean_squared_error will have their signs FLIPPED, and values will be negative.\n \"\"\"\n if \"silent\" in kwargs:\n # keep `silent` logic for backwards compatibility\n assert isinstance(kwargs[\"silent\"], bool)\n display = not kwargs.pop(\"silent\")\n if len(kwargs) > 0:\n for key in kwargs:\n raise TypeError(f\"TabularPredictor.evaluate() got an unexpected keyword argument '{key}'\")\n self._assert_is_fit(\"evaluate\")\n data = self._get_dataset(data)\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n if self.can_predict_proba:\n y_pred = self.predict_proba(data=data, model=model)\n else:\n y_pred = self.predict(data=data, model=model)\n if self.sample_weight is not None and self.weight_evaluation and self.sample_weight in data:\n sample_weight = data[self.sample_weight]\n else:\n sample_weight = None\n return self.evaluate_predictions(\n y_true=data[self.label],\n y_pred=y_pred,\n sample_weight=sample_weight,\n decision_threshold=decision_threshold,\n display=display,\n auxiliary_metrics=auxiliary_metrics,\n detailed_report=detailed_report,\n )\n\n def evaluate_predictions(\n self,\n y_true,\n y_pred,\n sample_weight=None,\n decision_threshold=None,\n display: bool = False,\n auxiliary_metrics=True,\n detailed_report=False,\n **kwargs,\n ) -> dict:\n \"\"\"\n Evaluate the provided prediction probabilities against ground truth labels.\n Evaluation is based on the `eval_metric` previously specified in init, or default metrics if none was specified.\n\n Parameters\n ----------\n y_true : :class:`np.array` or :class:`pd.Series`\n The ordered collection of ground-truth labels.\n y_pred : :class:`pd.Series` or :class:`pd.DataFrame`\n The ordered collection of prediction probabilities or predictions.\n Obtainable via the output of `predictor.predict_proba`.\n Caution: For certain types of `eval_metric` (such as 'roc_auc'), `y_pred` must be predicted-probabilities rather than predicted labels.\n sample_weight : :class:`pd.Series`, default = None\n Sample weight for each row of data. If None, uniform sample weights are used.\n decision_threshold : float, default = None\n The decision threshold to use when converting prediction probabilities to predictions.\n This will impact the scores of metrics such as `f1` and `accuracy`.\n If None, defaults to `predictor.decision_threshold`. Ignored unless `problem_type='binary'`.\n Refer to the `predictor.decision_threshold` docstring for more information.\n display : bool, default = False\n If True, performance results are printed.\n auxiliary_metrics: bool, default = True\n Should we compute other (`problem_type` specific) metrics in addition to the default metric?\n detailed_report : bool, default = False\n Should we computed more detailed versions of the `auxiliary_metrics`? (requires `auxiliary_metrics = True`)\n\n Returns\n -------\n Returns dict where keys = metrics, values = performance along each metric.\n NOTE: Metrics scores always show in higher is better form.\n This means that metrics such as log_loss and root_mean_squared_error will have their signs FLIPPED, and values will be negative.\n \"\"\"\n if \"silent\" in kwargs:\n # keep `silent` logic for backwards compatibility\n assert isinstance(kwargs[\"silent\"], bool)\n display = not kwargs.pop(\"silent\")\n if len(kwargs) > 0:\n for key in kwargs:\n raise TypeError(f\"TabularPredictor.evaluate_predictions() got an unexpected keyword argument '{key}'\")\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n return self._learner.evaluate_predictions(\n y_true=y_true,\n y_pred=y_pred,\n sample_weight=sample_weight,\n decision_threshold=decision_threshold,\n display=display,\n auxiliary_metrics=auxiliary_metrics,\n detailed_report=detailed_report,\n )\n\n def leaderboard(\n self,\n data: pd.DataFrame | str | None = None,\n extra_info: bool = False,\n extra_metrics: list | None = None,\n decision_threshold: float | None = None,\n score_format: str = \"score\",\n only_pareto_frontier: bool = False,\n skip_score: bool = False,\n refit_full: bool | None = None,\n set_refit_score_to_parent: bool = False,\n display: bool = False,\n **kwargs,\n ) -> pd.DataFrame:\n \"\"\"\n Output summary of information about models produced during `fit()` as a :class:`pd.DataFrame`.\n Includes information on test and validation scores for all models, model training times, inference times, and stack levels.\n Output DataFrame columns include:\n 'model': The name of the model.\n\n 'score_val': The validation score of the model on the 'eval_metric'.\n NOTE: Metrics scores always show in higher is better form.\n This means that metrics such as log_loss and root_mean_squared_error will have their signs FLIPPED, and values will be negative.\n This is necessary to avoid the user needing to know the metric to understand if higher is better when looking at leaderboard.\n 'eval_metric': The evaluation metric name used to calculate the scores.\n This should be identical to `predictor.eval_metric.name`.\n 'pred_time_val': The inference time required to compute predictions on the validation data end-to-end.\n Equivalent to the sum of all 'pred_time_val_marginal' values for the model and all of its base models.\n 'fit_time': The fit time required to train the model end-to-end (Including base models if the model is a stack ensemble).\n Equivalent to the sum of all 'fit_time_marginal' values for the model and all of its base models.\n 'pred_time_val_marginal': The inference time required to compute predictions on the validation data (Ignoring inference times for base models).\n Note that this ignores the time required to load the model into memory when bagging is disabled.\n 'fit_time_marginal': The fit time required to train the model (Ignoring base models).\n 'stack_level': The stack level of the model.\n A model with stack level N can take any set of models with stack level less than N as input, with stack level 1 models having no model inputs.\n 'can_infer': If model is able to perform inference on new data. If False, then the model either was not saved, was deleted, or an ancestor of the model cannot infer.\n `can_infer` is often False when `save_bag_folds=False` was specified in initial `fit()`.\n 'fit_order': The order in which models were fit. The first model fit has `fit_order=1`, and the Nth model fit has `fit_order=N`. The order corresponds to the first child model fit in the case of bagged ensembles.\n\n Parameters\n ----------\n data : str or :class:`pd.DataFrame` (optional)\n This dataset must also contain the label-column with the same column-name as specified during fit().\n If extra_metrics=None and skip_score=True, then the label column is not required.\n If specified, then the leaderboard returned will contain additional columns 'score_test', 'pred_time_test', and 'pred_time_test_marginal'.\n 'score_test': The score of the model on the 'eval_metric' for the data provided.\n NOTE: Metrics scores always show in higher is better form.\n This means that metrics such as log_loss and root_mean_squared_error will have their signs FLIPPED, and values will be negative.\n This is necessary to avoid the user needing to know the metric to understand if higher is better when looking at leaderboard.\n 'pred_time_test': The true end-to-end wall-clock inference time of the model for the data provided.\n Equivalent to the sum of all 'pred_time_test_marginal' values for the model and all of its base models.\n 'pred_time_test_marginal': The inference time of the model for the data provided, minus the inference time for the model's base models, if it has any.\n Note that this ignores the time required to load the model into memory when bagging is disabled.\n If str is passed, `data` will be loaded using the str value as the file path.\n extra_info : bool, default = False\n If `True`, will return extra columns with advanced info.\n This requires additional computation as advanced info data is calculated on demand.\n Additional output columns when `extra_info=True` include:\n 'num_features': Number of input features used by the model.\n Some models may ignore certain features in the preprocessed data.\n 'num_models': Number of models that actually make up this \"model\" object.\n For non-bagged models, this is 1. For bagged models, this is equal to the number of child models (models trained on bagged folds) the bagged ensemble contains.\n 'num_models_w_ancestors': Equivalent to the sum of 'num_models' values for the model and its' ancestors (see below).\n 'memory_size': The amount of memory in bytes the model requires when persisted in memory. This is not equivalent to the amount of memory the model may use during inference.\n For bagged models, this is the sum of the 'memory_size' of all child models.\n 'memory_size_w_ancestors': Equivalent to the sum of 'memory_size' values for the model and its' ancestors.\n This is the amount of memory required to avoid loading any models in-between inference calls to get predictions from this model.\n For online-inference, this is critical. It is important that the machine performing online inference has memory more than twice this value to avoid loading models for every call to inference by persisting models in memory.\n 'memory_size_min': The amount of memory in bytes the model minimally requires to perform inference.\n For non-bagged models, this is equivalent to 'memory_size'.\n For bagged models, this is equivalent to the largest child model's 'memory_size_min'.\n To minimize memory usage, child models can be loaded and un-persisted one by one to infer. This is the default behavior if a bagged model was not already persisted in memory prior to inference.\n 'memory_size_min_w_ancestors': Equivalent to the max of the 'memory_size_min' values for the model and its' ancestors.\n This is the minimum required memory to infer with the model by only loading one model at a time, as each of its ancestors will also have to be loaded into memory.\n For offline-inference where latency is not a concern, this should be used to determine the required memory for a machine if 'memory_size_w_ancestors' is too large.\n 'num_ancestors': Number of ancestor models for the given model.\n\n 'num_descendants': Number of descendant models for the given model.\n\n 'model_type': The type of the given model.\n If the model is an ensemble type, 'child_model_type' will indicate the inner model type. A stack ensemble of bagged LightGBM models would have 'StackerEnsembleModel' as its model type.\n 'child_model_type': The child model type. None if the model is not an ensemble. A stack ensemble of bagged LightGBM models would have 'LGBModel' as its child type.\n child models are models which are used as a group to generate a given bagged ensemble model's predictions. These are the models trained on each fold of a bagged ensemble.\n For 10-fold bagging, the bagged ensemble model would have 10 child models.\n For 10-fold bagging with 3 repeats, the bagged ensemble model would have 30 child models.\n Note that child models are distinct from ancestors and descendants.\n 'hyperparameters': The hyperparameter values specified for the model.\n All hyperparameters that do not appear in this dict remained at their default values.\n 'hyperparameters_fit': The hyperparameters set by the model during fit.\n This overrides the 'hyperparameters' value for a particular key if present in 'hyperparameters_fit' to determine the fit model's final hyperparameters.\n This is most commonly set for hyperparameters that indicate model training iterations or epochs, as early stopping can find a different value from what 'hyperparameters' indicated.\n In these cases, the provided hyperparameter in 'hyperparameters' is used as a maximum for the model, but the model is still able to early stop at a smaller value during training to achieve a better validation score or to satisfy time constraints.\n For example, if a NN model was given `epochs=500` as a hyperparameter, but found during training that `epochs=60` resulted in optimal validation score, it would use `epoch=60` and `hyperparameters_fit={'epoch': 60}` would be set.\n 'ag_args_fit': Special AutoGluon arguments that influence model fit.\n See the documentation of the `hyperparameters` argument in `TabularPredictor.fit()` for more information.\n 'features': List of feature names used by the model.\n\n 'child_hyperparameters': Equivalent to 'hyperparameters', but for the model's children.\n\n 'child_hyperparameters_fit': Equivalent to 'hyperparameters_fit', but for the model's children.\n\n 'child_ag_args_fit': Equivalent to 'ag_args_fit', but for the model's children.\n\n 'ancestors': The model's ancestors. Ancestor models are the models which are required to make predictions during the construction of the model's input features.\n If A is an ancestor of B, then B is a descendant of A.\n If a model's ancestor is deleted, the model is no longer able to infer on new data, and its 'can_infer' value will be False.\n A model can only have ancestor models whose 'stack_level' are lower than itself.\n 'stack_level'=1 models have no ancestors.\n 'descendants': The model's descendants. Descendant models are the models which require this model to make predictions during the construction of their input features.\n If A is a descendant of B, then B is an ancestor of A.\n If this model is deleted, then all descendant models will no longer be able to infer on new data, and their 'can_infer' values will be False.\n A model can only have descendant models whose 'stack_level' are higher than itself.\n extra_metrics : list, default = None\n A list of metrics to calculate scores for and include in the output DataFrame.\n Only valid when `data` is specified. The scores refer to the scores on `data` (same data as used to calculate the `score_test` column).\n This list can contain any values which would also be valid for `eval_metric` in predictor init.\n For example, `extra_metrics=['accuracy', 'roc_auc', 'log_loss']` would be valid in binary classification.\n This example would return 3 additional columns in the output DataFrame, whose column names match the names of the metrics.\n Passing `extra_metrics=[predictor.eval_metric]` would return an extra column in the name of the eval metric that has identical values to `score_test`.\n This also works with custom metrics. If passing an object instead of a string, the column name will be equal to the `.name` attribute of the object.\n NOTE: Metrics scores always show in higher is better form.\n This means that metrics such as log_loss and root_mean_squared_error will have their signs FLIPPED, and values will be negative.\n This is necessary to avoid the user needing to know the metric to understand if higher is better when looking at leaderboard.\n decision_threshold : float, default = None\n The decision threshold to use when converting prediction probabilities to predictions.\n This will impact the scores of metrics such as `f1` and `accuracy`.\n If None, defaults to `predictor.decision_threshold`. Ignored unless `problem_type='binary'`.\n Refer to the `predictor.decision_threshold` docstring for more information.\n NOTE: `score_val` will not be impacted by this value in v0.8.\n `score_val` will always show the validation scores achieved with a decision threshold of `0.5`.\n Only test scores will be properly updated.\n score_format : {'score', 'error'}\n If \"score\", leaderboard is returned as normal.\n If \"error\", the column \"score_val\" is converted to \"metric_error_val\", and \"score_test\" is converted to \"metric_error_test\".\n \"metric_error\" is calculated by taking `predictor.eval_metric.convert_score_to_error(score)`.\n This will result in errors where 0 is perfect and lower is better.\n only_pareto_frontier : bool, default = False\n If `True`, only return model information of models in the Pareto frontier of the accuracy/latency trade-off (models which achieve the highest score within their end-to-end inference time).\n At minimum this will include the model with the highest score and the model with the lowest inference time.\n This is useful when deciding which model to use during inference if inference time is a consideration.\n Models filtered out by this process would never be optimal choices for a user that only cares about model inference time and score.\n skip_score : bool, default = False\n [Advanced, primarily for developers]\n If `True`, will skip computing `score_test` if `data` is specified. `score_test` will be set to NaN for all models.\n `pred_time_test` and related columns will still be computed.\n refit_full : bool, default = None\n If True, will return only models that have been refit (ex: have `_FULL` in the name).\n If False, will return only models that have not been refit.\n If None, will return all models.\n set_refit_score_to_parent : bool, default = False\n If True, the `score_val` of refit models will be set to the `score_val` of their parent.\n While this does not represent the genuine validation score of the refit model, it is a reasonable proxy.\n display : bool, default = False\n If True, the output DataFrame is printed to stdout.\n\n Returns\n -------\n :class:`pd.DataFrame` of model performance summary information.\n \"\"\"\n if \"silent\" in kwargs:\n # keep `silent` logic for backwards compatibility\n assert isinstance(kwargs[\"silent\"], bool)\n display = not kwargs.pop(\"silent\")\n if len(kwargs) > 0:\n for key in kwargs:\n raise TypeError(f\"TabularPredictor.leaderboard() got an unexpected keyword argument '{key}'\")\n self._assert_is_fit(\"leaderboard\")\n data = self._get_dataset(data, allow_nan=True)\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n return self._learner.leaderboard(\n X=data,\n extra_info=extra_info,\n extra_metrics=extra_metrics,\n decision_threshold=decision_threshold,\n only_pareto_frontier=only_pareto_frontier,\n score_format=score_format,\n skip_score=skip_score,\n refit_full=refit_full,\n set_refit_score_to_parent=set_refit_score_to_parent,\n display=display,\n )\n\n def learning_curves(self) -> tuple[dict, dict]:\n \"\"\"\n Retrieves learning curves generated during predictor.fit().\n Will not work if the learning_curves flag was not set during training.\n Note that learning curves are only generated for iterative learners with\n learning curve support.\n\n Parameters\n ----------\n None\n\n Returns\n -------\n metadata: dict\n A dictionary containing metadata related to the training process.\n\n model_data: dict\n A dictionary containing the learning curves across all models.\n To see curve_data format, refer to AbstractModel's save_learning_curves() method.\n {\n \"model\": curve_data,\n \"model\": curve_data,\n \"model\": curve_data,\n \"model\": curve_data,\n }\n \"\"\"\n metadata = self.info()\n metadata = {\n \"problem_type\": metadata[\"problem_type\"],\n \"eval_metric\": metadata[\"eval_metric\"],\n \"num_classes\": metadata[\"num_classes\"],\n \"num_rows_train\": metadata[\"num_rows_train\"],\n \"num_rows_val\": metadata[\"num_rows_val\"],\n \"num_rows_test\": metadata[\"num_rows_test\"],\n \"models\": {\n model: {\n \"model_name\": info[\"name\"],\n \"model_type\": info[\"model_type\"],\n \"stopping_metric\": info[\"stopping_metric\"],\n \"hyperparameters\": info[\"hyperparameters\"],\n \"hyperparameters_fit\": info[\"hyperparameters_fit\"],\n \"ag_args_fit\": info[\"ag_args_fit\"],\n \"predict_time\": info[\"predict_time\"],\n \"fit_time\": info[\"fit_time\"],\n \"val_score\": info[\"val_score\"],\n }\n for model, info in metadata[\"model_info\"].items()\n if info.get(\"has_learning_curves\", False)\n },\n }\n\n model_data = {}\n for model in metadata[\"models\"].values():\n model_name = model[\"model_name\"]\n model_data[model_name] = self._trainer.get_model_learning_curves(model=model_name)\n\n return metadata, model_data\n\n def model_failures(self, verbose: bool = False) -> pd.DataFrame:\n \"\"\"\n [Advanced] Get the model failures that occurred during the fitting of this model, in the form of a pandas DataFrame.\n\n This is useful for in-depth debugging of model failures and identifying bugs.\n\n For more information on model failures, refer to `predictor.info()['model_info_failures']`\n\n Parameters\n ----------\n verbose: bool, default = False\n If True, the output DataFrame is printed to stdout.\n\n Returns\n -------\n model_failures_df: pd.DataFrame\n A DataFrame of model failures. Each row corresponds to a model failure, and columns correspond to meta information about that model.\n\n Included Columns:\n \"model\": The name of the model that failed\n \"exc_type\": The class name of the exception raised\n \"total_time\": The total time in seconds taken by the model prior to the exception (lost time due to the failure)\n \"model_type\": The class name of the model\n \"child_model_type\": The child class name of the model\n \"is_initialized\"\n \"is_fit\"\n \"is_valid\"\n \"can_infer\"\n \"num_features\"\n \"num_models\"\n \"memory_size\"\n \"hyperparameters\"\n \"hyperparameters_fit\"\n \"child_hyperparameters\"\n \"child_hyperparameters_fit\"\n \"exc_str\": The string message contained in the raised exception\n \"exc_traceback\": The full traceback message of the exception as a string\n \"exc_order\": The order of the model failure (starting from 1)\n \"\"\"\n self._assert_is_fit(\"model_failures\")\n model_failures_df = self._trainer.model_failures()\n if verbose:\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n print(model_failures_df)\n return model_failures_df\n\n def predict_proba_multi(\n self,\n data: pd.DataFrame = None,\n models: list[str] = None,\n as_pandas: bool = True,\n as_multiclass: bool = True,\n transform_features: bool = True,\n inverse_transform: bool = True,\n ) -> dict[str, pd.DataFrame] | dict[str, pd.Series] | dict[str, np.ndarray]:\n \"\"\"\n Returns a dictionary of prediction probabilities where the key is\n the model name and the value is the model's prediction probabilities on the data.\n\n Equivalent output to:\n ```\n predict_proba_dict = {}\n for m in models:\n predict_proba_dict[m] = predictor.predict_proba(data, model=m)\n ```\n\n Note that this will generally be much faster than calling :meth:`TabularPredictor.predict_proba` separately for each model\n because this method leverages the model dependency graph to avoid redundant computation.\n\n Parameters\n ----------\n data : str or DataFrame, default = None\n The data to predict on.\n If None:\n If self.has_val, the validation data is used.\n Else, prediction is skipped and the out-of-fold (OOF) prediction probabilities are returned, equivalent to:\n ```\n predict_proba_dict = {}\n for m in models:\n predict_proba_dict[m] = predictor.predict_proba_oof(model=m)\n ```\n models : list[str], default = None\n The list of models to get predictions for.\n If None, all models that can infer are used.\n as_pandas : bool, default = True\n Whether to return the output of each model as a pandas object (True) or numpy array (False).\n Pandas object is a :class:`pd.DataFrame` if this is a multiclass problem or `as_multiclass=True`, otherwise it is a :class:`pd.Series`.\n If the output is a :class:`pd.DataFrame`, the column order will be equivalent to `predictor.classes_`.\n as_multiclass : bool, default = True\n Whether to return binary classification probabilities as if they were for multiclass classification.\n Output will contain two columns, and if `as_pandas=True`, the column names will correspond to the binary class labels.\n The columns will be the same order as `predictor.class_labels`.\n If False, output will contain only 1 column for the positive class (get positive_class name via `predictor.positive_class`).\n Only impacts output for binary classification problems.\n transform_features : bool, default = True\n If True, preprocesses data before predicting with models.\n If False, skips global feature preprocessing.\n This is useful to save on inference time if you have already called `data = predictor.transform_features(data)`.\n inverse_transform : bool, default = True\n If True, will return prediction probabilities in the original format.\n If False (advanced), will return prediction probabilities in AutoGluon's internal format.\n\n Returns\n -------\n dict\n Dictionary with model names as keys and model prediction probabilities as values.\n \"\"\"\n self._assert_is_fit(\"predict_proba_multi\")\n if not self.can_predict_proba:\n raise AssertionError(\n f'`predictor.predict_proba_multi` is not supported when problem_type=\"{self.problem_type}\". '\n f\"Please call `predictor.predict_multi` instead. \"\n f\"You can check the value of `predictor.can_predict_proba` to tell if predict_proba_multi is valid.\"\n )\n data = self._get_dataset(data, allow_nan=True)\n return self._learner.predict_proba_multi(\n X=data,\n models=models,\n as_pandas=as_pandas,\n as_multiclass=as_multiclass,\n transform_features=transform_features,\n inverse_transform=inverse_transform,\n use_refit_parent_oof=True,\n )\n\n @overload\n def predict_multi(\n self,\n data: pd.DataFrame = None,\n models: list[str] = None,\n as_pandas: Literal[True] = True,\n transform_features: bool = True,\n inverse_transform: bool = True,\n decision_threshold: float = None,\n ) -> dict[str, pd.Series]: ...\n\n @overload\n def predict_multi(\n self,\n data: pd.DataFrame = None,\n models: list[str] = None,\n *,\n as_pandas: Literal[False],\n transform_features: bool = True,\n inverse_transform: bool = True,\n decision_threshold: float = None,\n ) -> dict[str, np.ndarray]: ...\n\n def predict_multi(\n self,\n data: pd.DataFrame = None,\n models: list[str] = None,\n as_pandas: bool = True,\n transform_features: bool = True,\n inverse_transform: bool = True,\n *,\n decision_threshold: float = None,\n ) -> dict[str, pd.Series] | dict[str, np.ndarray]:\n \"\"\"\n Returns a dictionary of predictions where the key is\n the model name and the value is the model's prediction probabilities on the data.\n\n Equivalent output to:\n ```\n predict_dict = {}\n for m in models:\n predict_dict[m] = predictor.predict(data, model=m)\n ```\n\n Note that this will generally be much faster than calling :meth:`TabularPredictor.predict` separately for each model\n because this method leverages the model dependency graph to avoid redundant computation.\n\n Parameters\n ----------\n data : DataFrame, default = None\n The data to predict on.\n If None:\n If self.has_val, the validation data is used.\n Else, prediction is skipped and the out-of-fold (OOF) predictions are returned, equivalent to:\n ```\n predict_dict = {}\n for m in models:\n predict_dict[m] = predictor.predict_oof(model=m)\n ```\n models : list[str], default = None\n The list of models to get predictions for.\n If None, all models that can infer are used.\n as_pandas : bool, default = True\n Whether to return the output of each model as a :class:`pd.Series` (True) or :class:`np.ndarray` (False).\n transform_features : bool, default = True\n If True, preprocesses data before predicting with models.\n If False, skips global feature preprocessing.\n This is useful to save on inference time if you have already called `data = predictor.transform_features(data)`.\n inverse_transform : bool, default = True\n If True, will return predictions in the original format.\n If False (advanced), will return predictions in AutoGluon's internal format.\n decision_threshold : float, default = None\n The decision threshold used to convert prediction probabilities to predictions.\n Only relevant for binary classification, otherwise ignored.\n If None, defaults to `0.5`.\n Valid values are in the range [0.0, 1.0]\n You can obtain an optimized `decision_threshold` by first calling :meth:`TabularPredictor.calibrate_decision_threshold`.\n Useful to set for metrics such as `balanced_accuracy` and `f1` as `0.5` is often not an optimal threshold.\n Predictions are calculated via the following logic on the positive class: `1 if pred > decision_threshold else 0`\n\n Returns\n -------\n dict[str, pd.Series] | dict[str, np.ndarray]\n Dictionary with model names as keys and model predictions as values.\n \"\"\"\n self._assert_is_fit(\"predict_multi\")\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n data = self._get_dataset(data, allow_nan=True)\n return self._learner.predict_multi(\n X=data,\n models=models,\n as_pandas=as_pandas,\n transform_features=transform_features,\n inverse_transform=inverse_transform,\n decision_threshold=decision_threshold,\n )\n\n def fit_summary(self, verbosity: int = 3, show_plot: bool = False) -> dict:\n \"\"\"\n Output summary of information about models produced during `fit()`.\n May create various generated summary plots and store them in folder: `predictor.path`.\n\n Parameters\n ----------\n verbosity : int, default = 3\n Controls how detailed of a summary to output.\n Set <= 0 for no output printing, 1 to print just high-level summary,\n 2 to print summary and create plots, >= 3 to print all information produced during `fit()`.\n show_plot : bool, default = False\n If True, shows the model summary plot in browser when verbosity > 1.\n\n Returns\n -------\n Dict containing various detailed information. We do not recommend directly printing this dict as it may be very large.\n \"\"\"\n self._assert_is_fit(\"fit_summary\")\n # hpo_used = len(self._trainer.hpo_results) > 0\n hpo_used = False # Disabled until a more memory efficient hpo_results object is implemented.\n model_types = self._trainer.get_models_attribute_dict(attribute=\"type\")\n model_inner_types = self._trainer.get_models_attribute_dict(attribute=\"type_inner\")\n model_typenames = {key: model_types[key].__name__ for key in model_types}\n model_innertypenames = {\n key: model_inner_types[key].__name__ for key in model_types if key in model_inner_types\n }\n MODEL_STR = \"Model\"\n ENSEMBLE_STR = \"Ensemble\"\n for model in model_typenames:\n if (\n (model in model_innertypenames)\n and (ENSEMBLE_STR not in model_innertypenames[model])\n and (ENSEMBLE_STR in model_typenames[model])\n ):\n new_model_typename = model_typenames[model] + \"_\" + model_innertypenames[model]\n if new_model_typename.endswith(MODEL_STR):\n new_model_typename = new_model_typename[: -len(MODEL_STR)]\n model_typenames[model] = new_model_typename\n\n unique_model_types = set(model_typenames.values()) # no more class info\n # all fit() information that is returned:\n results = {\n \"model_types\": model_typenames, # dict with key = model-name, value = type of model (class-name)\n \"model_performance\": self._trainer.get_models_attribute_dict(\"val_score\"),\n # dict with key = model-name, value = validation performance\n \"model_best\": self._trainer.model_best, # the name of the best model (on validation data)\n \"model_paths\": self._trainer.get_models_attribute_dict(\"path\"),\n # dict with key = model-name, value = path to model file\n \"model_fit_times\": self._trainer.get_models_attribute_dict(\"fit_time\"),\n \"model_pred_times\": self._trainer.get_models_attribute_dict(\"predict_time\"),\n \"num_bag_folds\": self._trainer.k_fold,\n \"max_stack_level\": self._trainer.get_max_level(),\n }\n if self.problem_type == QUANTILE:\n results[\"num_quantiles\"] = len(self.quantile_levels)\n elif self.problem_type != REGRESSION:\n results[\"num_classes\"] = self._trainer.num_classes\n # if hpo_used:\n # results['hpo_results'] = self._trainer.hpo_results\n # get dict mapping model name to final hyperparameter values for each model:\n model_hyperparams = {}\n for model_name in self._trainer.get_model_names():\n model_obj = self._trainer.load_model(model_name)\n model_hyperparams[model_name] = model_obj.params\n results[\"model_hyperparams\"] = model_hyperparams\n\n if verbosity > 0: # print stuff\n print(\"*** Summary of fit() ***\")\n print(\"Estimated performance of each model:\")\n results[\"leaderboard\"] = self._learner.leaderboard(display=True)\n # self._summarize('model_performance', 'Validation performance of individual models', results)\n # self._summarize('model_best', 'Best model (based on validation performance)', results)\n # self._summarize('hyperparameter_tune', 'Hyperparameter-tuning used', results)\n print(\"Number of models trained: %s\" % len(results[\"model_performance\"]))\n print(\"Types of models trained:\")\n print(unique_model_types)\n num_fold_str = \"\"\n bagging_used = results[\"num_bag_folds\"] > 0\n if bagging_used:\n num_fold_str = f\" (with {results['num_bag_folds']} folds)\"\n print(\"Bagging used: %s %s\" % (bagging_used, num_fold_str))\n num_stack_str = \"\"\n stacking_used = results[\"max_stack_level\"] > 2\n if stacking_used:\n num_stack_str = f\" (with {results['max_stack_level']} levels)\"\n print(\"Multi-layer stack-ensembling used: %s %s\" % (stacking_used, num_stack_str))\n hpo_str = \"\"\n # if hpo_used and verbosity <= 2:\n # hpo_str = \" (call fit_summary() with verbosity >= 3 to see detailed HPO info)\"\n # print(\"Hyperparameter-tuning used: %s %s\" % (hpo_used, hpo_str))\n # TODO: uncomment once feature_prune is functional: self._summarize('feature_prune', 'feature-selection used', results)\n print(\"Feature Metadata (Processed):\")\n print(\"(raw dtype, special dtypes):\")\n print(self.feature_metadata)\n if verbosity > 1: # create plots\n plot_tabular_models(\n results,\n output_directory=self.path,\n save_file=\"SummaryOfModels.html\",\n plot_title=\"Models produced during fit()\",\n show_plot=show_plot,\n )\n if hpo_used:\n for model_type in results[\"hpo_results\"]:\n if \"trial_info\" in results[\"hpo_results\"][model_type]:\n plot_summary_of_models(\n results[\"hpo_results\"][model_type],\n output_directory=self.path,\n save_file=model_type + \"_HPOmodelsummary.html\",\n plot_title=f\"Models produced during {model_type} HPO\",\n show_plot=show_plot,\n )\n plot_performance_vs_trials(\n results[\"hpo_results\"][model_type],\n output_directory=self.path,\n save_file=model_type + \"_HPOperformanceVStrials.png\",\n plot_title=f\"HPO trials for {model_type} models\",\n show_plot=show_plot,\n )\n if verbosity > 2: # print detailed information\n if hpo_used:\n hpo_results = results[\"hpo_results\"]\n print(\"*** Details of Hyperparameter optimization ***\")\n for model_type in hpo_results:\n hpo_model = hpo_results[model_type]\n if \"trial_info\" in hpo_model:\n print(\n f\"HPO for {model_type} model: Num. configurations tried = {len(hpo_model['trial_info'])}, Time spent = {hpo_model['total_time']}s, Search strategy = {hpo_model['search_strategy']}\"\n )\n print(\n f\"Best hyperparameter-configuration (validation-performance: {self.eval_metric} = {hpo_model['validation_performance']}):\"\n )\n print(hpo_model[\"best_config\"])\n \"\"\"\n if bagging_used:\n pass # TODO: print detailed bagging info\n if stacking_used:\n pass # TODO: print detailed stacking info, like how much it improves validation performance\n if results['feature_prune']:\n pass # TODO: print detailed feature-selection info once feature-selection is functional.\n \"\"\"\n if verbosity > 0:\n print(\"*** End of fit() summary ***\")\n return results\n\n def transform_features(\n self,\n data: pd.DataFrame | str = None,\n model: str = None,\n base_models: list[str] = None,\n return_original_features: bool = True,\n ) -> pd.DataFrame:\n \"\"\"\n Transforms data features through the AutoGluon feature generator.\n This is useful to gain an understanding of how AutoGluon interprets the data features.\n The output of this function can be used to train further models, even outside of AutoGluon.\n This can be useful for training your own models on the same data representation as AutoGluon.\n Individual AutoGluon models like the neural network may apply additional feature transformations that are not reflected in this method.\n This method only applies universal transforms employed by all AutoGluon models.\n When `data=None`, `base_models=[{best_model}], and bagging was enabled during fit():\n This returns the out-of-fold predictions of the best model, which can be used as training input to a custom user stacker model.\n\n Parameters\n ----------\n data: :class:`pd.DataFrame` or str (optional)\n The data to apply feature transformation to.\n This data does not require the label column.\n If str is passed, `data` will be loaded using the str value as the file path.\n If not specified, the original data used during fit() will be used if fit() was previously called with `cache_data=True`. Otherwise, an exception will be raised.\n For non-bagged mode predictors:\n The data used when not specified is the validation set.\n This can either be an automatically generated validation set or the user-defined `tuning_data` if passed during fit().\n If all parameters are unspecified, then the output is equivalent to `predictor.load_data_internal(data='val', return_X=True, return_y=False)[0]`.\n To get the label values of the output, call `predictor.load_data_internal(data='val', return_X=False, return_y=True)[1]`.\n If the original training set is desired, it can be passed in through `data`.\n Warning: Do not pass the original training set if `model` or `base_models` are set. This will result in overfit feature transformation.\n For bagged mode predictors:\n The data used when not specified is the full training set.\n If all parameters are unspecified, then the output is equivalent to `predictor.load_data_internal(data='train', return_X=True, return_y=False)[0]`.\n To get the label values of the output, call `predictor.load_data_internal(data='train', return_X=False, return_y=True)[1]`.\n `base_model` features generated in this instance will be from out-of-fold predictions.\n Note that the training set may differ from the training set originally passed during fit(), as AutoGluon may choose to drop or duplicate rows during training.\n Warning: Do not pass the original training set through `data` if `model` or `base_models` are set. This will result in overfit feature transformation. Instead set `data=None`.\n model: str, default = None\n Model to generate input features for.\n The output data will be equivalent to the input data that would be sent into `model.predict_proba(data)`.\n Note: This only applies to cases where `data` is not the training data.\n If `None`, then only return generically preprocessed features prior to any model fitting.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n Specifying a `refit_full` model will cause an exception if `data=None`.\n `base_models=None` is a requirement when specifying `model`.\n base_models: list[str], default = None\n List of model names to use as base_models for a hypothetical stacker model when generating input features.\n If `None`, then only return generically preprocessed features prior to any model fitting.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n If a stacker model S exists with `base_models=M`, then setting `base_models=M` is equivalent to setting `model=S`.\n `model=None` is a requirement when specifying `base_models`.\n return_original_features: bool, default = True\n Whether to return the original features.\n If False, only returns the additional output columns from specifying `model` or `base_models`.\n This is useful to set to False if the intent is to use the output as input to further stacker models without the original features.\n\n Returns\n -------\n :class:`pd.DataFrame` of the provided `data` after feature transformation has been applied.\n This output does not include the label column, and will remove it if present in the supplied `data`.\n If a transformed label column is desired, use `predictor.transform_labels`.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularPredictor\n >>> predictor = TabularPredictor(label='class').fit('train.csv', label='class', auto_stack=True) # predictor is in bagged mode.\n >>> model = 'WeightedEnsemble_L2'\n >>> train_data_transformed = predictor.transform_features(model=model) # Internal training DataFrame used as input to `model.fit()` for each model trained in predictor.fit()`\n >>> test_data_transformed = predictor.transform_features('test.csv', model=model) # Internal test DataFrame used as input to `model.predict_proba()` during `predictor.predict_proba(test_data, model=model)`\n\n \"\"\"\n self._assert_is_fit(\"transform_features\")\n data = self._get_dataset(data, allow_nan=True)\n return self._learner.get_inputs_to_stacker(\n dataset=data, model=model, base_models=base_models, use_orig_features=return_original_features\n )\n\n def transform_labels(\n self, labels: np.ndarray | pd.Series, inverse: bool = False, proba: bool = False\n ) -> pd.Series | pd.DataFrame:\n \"\"\"\n Transforms data labels to the internal label representation.\n This can be useful for training your own models on the same data label representation as AutoGluon.\n Regression problems do not differ between original and internal representation, and thus this method will return the provided labels.\n Warning: When `inverse=False`, it is possible for the output to contain NaN label values in multiclass problems if the provided label was dropped during training.\n\n Parameters\n ----------\n labels: :class:`np.ndarray` or :class:`pd.Series`\n Labels to transform.\n If `proba=False`, an example input would be the output of `predictor.predict(test_data)`.\n If `proba=True`, an example input would be the output of `predictor.predict_proba(test_data, as_multiclass=False)`.\n inverse: bool, default = False\n When `True`, the input labels are treated as being in the internal representation and the original representation is outputted.\n proba: bool, default = False\n When `True`, the input labels are treated as probabilities and the output will be the internal representation of probabilities.\n In this case, it is expected that `labels` be a :class:`pd.DataFrame` or :class:`np.ndarray`.\n If the `problem_type` is multiclass:\n The input column order must be equal to `predictor.class_labels`.\n The output column order will be equal to `predictor.class_labels_internal`.\n if `inverse=True`, the same logic applies, but with input and output columns interchanged.\n When `False`, the input labels are treated as actual labels and the output will be the internal representation of the labels.\n In this case, it is expected that `labels` be a :class:`pd.Series` or :class:`np.ndarray`.\n\n Returns\n -------\n :class:`pd.Series` of labels if `proba=False` or :class:`pd.DataFrame` of label probabilities if `proba=True`.\n\n \"\"\"\n self._assert_is_fit(\"transform_labels\")\n return self._learner.transform_labels(y=labels, inverse=inverse, proba=proba)\n\n def feature_importance(\n self,\n data=None,\n model: str = None,\n features: list = None,\n feature_stage: str = \"original\",\n subsample_size: int = 5000,\n time_limit: float = None,\n num_shuffle_sets: int = None,\n include_confidence_band: bool = True,\n confidence_level: float = 0.99,\n silent: bool = False,\n ):\n \"\"\"\n Calculates feature importance scores for the given model via permutation importance. Refer to https://explained.ai/rf-importance/ for an explanation of permutation importance.\n A feature's importance score represents the performance drop that results when the model makes predictions on a perturbed copy of the data where this feature's values have been randomly shuffled across rows.\n A feature score of 0.01 would indicate that the predictive performance dropped by 0.01 when the feature was randomly shuffled.\n The higher the score a feature has, the more important it is to the model's performance.\n If a feature has a negative score, this means that the feature is likely harmful to the final model, and a model trained with the feature removed would be expected to achieve a better predictive performance.\n Note that calculating feature importance can be a very computationally expensive process, particularly if the model uses hundreds or thousands of features. In many cases, this can take longer than the original model training.\n To estimate how long `feature_importance(model, data, features)` will take, it is roughly the time taken by `predict_proba(data, model)` multiplied by the number of features.\n\n Note: For highly accurate importance and p_value estimates, it is recommended to set `subsample_size` to at least 5000 if possible and `num_shuffle_sets` to at least 10.\n\n Parameters\n ----------\n data : str or :class:`pd.DataFrame` (optional)\n This data must also contain the label-column with the same column-name as specified during `fit()`.\n If specified, then the data is used to calculate the feature importance scores.\n If str is passed, `data` will be loaded using the str value as the file path.\n If not specified, the original data used during `fit()` will be used if `cache_data=True`. Otherwise, an exception will be raised.\n Do not pass the training data through this argument, as the feature importance scores calculated will be biased due to overfitting.\n More accurate feature importances will be obtained from new data that was held-out during `fit()`.\n model : str, default = None\n Model to get feature importances for, if None the best model is chosen.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`\n features : list, default = None\n List of str feature names that feature importances are calculated for and returned, specify None to get all feature importances.\n If you only want to compute feature importances for some of the features, you can pass their names in as a list of str.\n Valid feature names change depending on the `feature_stage`.\n To get the list of feature names for `feature_stage='original'`, call `predictor.feature_metadata_in.get_features()`.\n To get the list of feature names for `feature_stage='transformed'`, call `list(predictor.transform_features().columns)`.\n To get the list of feature names for `feature_stage=`transformed_model`, call `list(predictor.transform_features(model={model_name}).columns)`.\n [Advanced] Can also contain tuples as elements of (feature_name, feature_list) form.\n feature_name can be any string so long as it is unique with all other feature names / features in the list.\n feature_list can be any list of valid features in the data.\n This will compute importance of the combination of features in feature_list, naming the set of features in the returned DataFrame feature_name.\n This importance will differ from adding the individual importances of each feature in feature_list, and will be more accurate to the overall group importance.\n Example: ['featA', 'featB', 'featC', ('featBC', ['featB', 'featC'])]\n In this example, the importance of 'featBC' will be calculated by jointly permuting 'featB' and 'featC' together as if they were a single two-dimensional feature.\n feature_stage : str, default = 'original'\n What stage of feature-processing should importances be computed for.\n Options:\n 'original':\n Compute importances of the original features.\n Warning: `data` must be specified with this option, otherwise an exception will be raised.\n 'transformed':\n Compute importances of the post-internal-transformation features (after automated feature engineering). These features may be missing some original features, or add new features entirely.\n An example of new features would be ngram features generated from a text column.\n Warning: For bagged models, feature importance calculation is not yet supported with this option when `data=None`. Doing so will raise an exception.\n 'transformed_model':\n Compute importances of the post-model-transformation features. These features are the internal features used by the requested model. They may differ greatly from the original features.\n If the model is a stack ensemble, this will include stack ensemble features such as the prediction probability features of the stack ensemble's base (ancestor) models.\n subsample_size : int, default = 5000\n The number of rows to sample from `data` when computing feature importance.\n If `subsample_size=None` or `data` contains fewer than `subsample_size` rows, all rows will be used during computation.\n Larger values increase the accuracy of the feature importance scores.\n Runtime linearly scales with `subsample_size`.\n time_limit : float, default = None\n Time in seconds to limit the calculation of feature importance.\n If None, feature importance will calculate without early stopping.\n A minimum of 1 full shuffle set will always be evaluated. If a shuffle set evaluation takes longer than `time_limit`, the method will take the length of a shuffle set evaluation to return regardless of the `time_limit`.\n num_shuffle_sets : int, default = None\n The number of different permutation shuffles of the data that are evaluated.\n Larger values will increase the quality of the importance evaluation.\n It is generally recommended to increase `subsample_size` before increasing `num_shuffle_sets`.\n Defaults to 5 if `time_limit` is None or 10 if `time_limit` is specified.\n Runtime linearly scales with `num_shuffle_sets`.\n include_confidence_band: bool, default = True\n If True, returned DataFrame will include two additional columns specifying confidence interval for the true underlying importance value of each feature.\n Increasing `subsample_size` and `num_shuffle_sets` will tighten the confidence interval.\n confidence_level: float, default = 0.99\n This argument is only considered when `include_confidence_band` is True, and can be used to specify the confidence level used for constructing confidence intervals.\n For example, if `confidence_level` is set to 0.99, then the returned DataFrame will include columns 'p99_high' and 'p99_low' which indicates that the true feature importance will be between 'p99_high' and 'p99_low' 99% of the time (99% confidence interval).\n More generally, if `confidence_level` = 0.XX, then the columns containing the XX% confidence interval will be named 'pXX_high' and 'pXX_low'.\n silent : bool, default = False\n Whether to suppress logging output.\n\n Returns\n -------\n :class:`pd.DataFrame` of feature importance scores with 6 columns:\n index: The feature name.\n 'importance': The estimated feature importance score.\n 'stddev': The standard deviation of the feature importance score. If NaN, then not enough num_shuffle_sets were used to calculate a variance.\n 'p_value': P-value for a statistical t-test of the null hypothesis: importance = 0, vs the (one-sided) alternative: importance > 0.\n Features with low p-value appear confidently useful to the predictor, while the other features may be useless to the predictor (or even harmful to include in its training data).\n A p-value of 0.01 indicates that there is a 1% chance that the feature is useless or harmful, and a 99% chance that the feature is useful.\n A p-value of 0.99 indicates that there is a 99% chance that the feature is useless or harmful, and a 1% chance that the feature is useful.\n 'n': The number of shuffles performed to estimate importance score (corresponds to sample-size used to determine confidence interval for true score).\n 'pXX_high': Upper end of XX% confidence interval for true feature importance score (where XX=99 by default).\n 'pXX_low': Lower end of XX% confidence interval for true feature importance score.\n \"\"\"\n self._assert_is_fit(\"feature_importance\")\n data = self._get_dataset(data, allow_nan=True)\n if (data is None) and (not self._trainer.is_data_saved):\n raise AssertionError(\n \"No data was provided and there is no cached data to load for feature importance calculation. `cache_data=True` must be set in the `TabularPredictor` init `learner_kwargs` argument call to enable this functionality when data is not specified.\"\n )\n if data is not None:\n self._validate_unique_indices(data, \"data\")\n\n if num_shuffle_sets is None:\n num_shuffle_sets = 10 if time_limit else 5\n\n fi_df = self._learner.get_feature_importance(\n model=model,\n X=data,\n features=features,\n feature_stage=feature_stage,\n subsample_size=subsample_size,\n time_limit=time_limit,\n num_shuffle_sets=num_shuffle_sets,\n silent=silent,\n )\n\n if include_confidence_band:\n if confidence_level <= 0.5 or confidence_level >= 1.0:\n raise ValueError(\"confidence_level must lie between 0.5 and 1.0\")\n ci_str = \"{:0.0f}\".format(confidence_level * 100)\n import scipy.stats\n\n num_features = len(fi_df)\n ci_low_dict = dict()\n ci_high_dict = dict()\n for i in range(num_features):\n fi = fi_df.iloc[i]\n mean = fi[\"importance\"]\n stddev = fi[\"stddev\"]\n n = fi[\"n\"]\n if np.isnan(stddev) or np.isnan(n) or np.isnan(mean) or n == 1:\n ci_high = np.nan\n ci_low = np.nan\n else:\n t_val = scipy.stats.t.ppf(1 - (1 - confidence_level) / 2, n - 1)\n ci_high = mean + t_val * stddev / math.sqrt(n)\n ci_low = mean - t_val * stddev / math.sqrt(n)\n ci_high_dict[fi.name] = ci_high\n ci_low_dict[fi.name] = ci_low\n high_str = \"p\" + ci_str + \"_high\"\n low_str = \"p\" + ci_str + \"_low\"\n fi_df[high_str] = pd.Series(ci_high_dict)\n fi_df[low_str] = pd.Series(ci_low_dict)\n return fi_df\n\n def compile(self, models=\"best\", with_ancestors=True, compiler_configs=\"auto\"):\n \"\"\"\n Compile models for accelerated prediction.\n This can be helpful to reduce prediction latency and improve throughput.\n\n Note that this is currently an experimental feature, the supported compilers can be ['native', 'onnx'].\n\n In order to compile with a specific compiler, that compiler must be installed in the Python environment.\n\n Parameters\n ----------\n models : list of str or str, default = 'best'\n Model names of models to compile.\n If 'best' then the model with the highest validation score is compiled (this is the model used for prediction by default).\n If 'all' then all models are compiled.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n with_ancestors : bool, default = True\n If True, all ancestor models of the provided models will also be compiled.\n compiler_configs : dict or str, default = \"auto\"\n If \"auto\", defaults to the following:\n compiler_configs = {\n \"RF\": {\"compiler\": \"onnx\"},\n \"XT\": {\"compiler\": \"onnx\"},\n \"NN_TORCH\": {\"compiler\": \"onnx\"},\n }\n Otherwise, specify a compiler_configs dictionary manually. Keys can be exact model names or model types.\n Exact model names take priority over types if both are valid for a model.\n Types can be either the true type such as RandomForestModel or the shorthand \"RF\".\n The dictionary key logic for types is identical to the logic in the hyperparameters argument of `predictor.fit`\n\n Example values within the configs:\n compiler : str, default = None\n The compiler that is used for model compilation.\n batch_size : int, default = None\n The batch size that is optimized for model prediction.\n By default, the batch size is None. This means the compiler would try to leverage dynamic shape for prediction.\n Using batch_size=1 would be more suitable for online prediction, which expects a result from one data point.\n However, it can be slow for batch processing, because of the overhead of multiple kernel execution.\n Increasing batch size to a number that is larger than 1 would help increase the prediction throughput.\n This comes with an expense of utilizing larger memory for prediction.\n \"\"\"\n self._assert_is_fit(\"compile\")\n if isinstance(compiler_configs, str):\n if compiler_configs == \"auto\":\n compiler_configs = {\n \"RF\": {\"compiler\": \"onnx\"},\n \"XT\": {\"compiler\": \"onnx\"},\n \"NN_TORCH\": {\"compiler\": \"onnx\"},\n }\n else:\n raise ValueError(f'Unknown compiler_configs preset: \"{compiler_configs}\"')\n self._trainer.compile(model_names=models, with_ancestors=with_ancestors, compiler_configs=compiler_configs)\n\n def persist(self, models=\"best\", with_ancestors=True, max_memory=0.4) -> list[str]:\n \"\"\"\n Persist models in memory for reduced inference latency. This is particularly important if the models are being used for online-inference where low latency is critical.\n If models are not persisted in memory, they are loaded from disk every time they are asked to make predictions.\n\n Parameters\n ----------\n models : list of str or str, default = 'best'\n Model names of models to persist.\n If 'best' then the model with the highest validation score is persisted (this is the model used for prediction by default).\n If 'all' then all models are persisted.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n with_ancestors : bool, default = True\n If True, all ancestor models of the provided models will also be persisted.\n If False, stacker models will not have the models they depend on persisted unless those models were specified in `models`. This will slow down inference as the ancestor models will still need to be loaded from disk for each predict call.\n Only relevant for stacker models.\n max_memory : float, default = 0.4\n Proportion of total available memory to allow for the persisted models to use.\n If the models' summed memory usage requires a larger proportion of memory than max_memory, they are not persisted. In this case, the output will be an empty list.\n If None, then models are persisted regardless of estimated memory usage. This can cause out-of-memory errors.\n\n Returns\n -------\n List of persisted model names.\n \"\"\"\n self._assert_is_fit(\"persist\")\n try:\n return self._learner.persist_trainer(\n low_memory=False, models=models, with_ancestors=with_ancestors, max_memory=max_memory\n )\n except Exception as e:\n valid_models = self.model_names()\n if isinstance(models, list):\n invalid_models = [m for m in models if m not in valid_models]\n if invalid_models:\n raise ValueError(\n f\"Invalid models specified. The following models do not exist:\\n\\t{invalid_models}\\nValid models:\\n\\t{valid_models}\"\n )\n raise e\n\n def unpersist(self, models=\"all\") -> list[str]:\n \"\"\"\n Unpersist models in memory for reduced memory usage.\n If models are not persisted in memory, they are loaded from disk every time they are asked to make predictions.\n Note: Another way to reset the predictor and unpersist models is to reload the predictor from disk via `predictor = TabularPredictor.load(predictor.path)`.\n\n Parameters\n ----------\n models : list of str or str, default = 'all'\n Model names of models to unpersist.\n If 'all' then all models are unpersisted.\n Valid models are listed in this `predictor` by calling `predictor.model_names(persisted=True)`.\n\n Returns\n -------\n List of unpersisted model names.\n \"\"\"\n self._assert_is_fit(\"unpersist\")\n return self._learner.load_trainer().unpersist(model_names=models)\n\n # TODO: `total_resources = None` during refit, fix this.\n # refit_full doesn't account for user-specified resources at fit time, nor does it allow them to specify for refit.\n def refit_full(\n self,\n model: str | list[str] = \"all\",\n set_best_to_refit_full: bool = True,\n train_data_extra: pd.DataFrame = None,\n num_cpus: int | str = \"auto\",\n num_gpus: int | str = \"auto\",\n fit_strategy: Literal[\"auto\", \"sequential\", \"parallel\"] = \"auto\",\n **kwargs,\n ) -> dict[str, str]:\n \"\"\"\n Retrain model on all of the data (training + validation).\n For bagged models:\n Optimizes a model's inference time by collapsing bagged ensembles into a single model fit on all of the training data.\n This process will typically result in a slight accuracy reduction and a large inference speedup.\n The inference speedup will generally be between 10-200x faster than the original bagged ensemble model.\n The inference speedup factor is equivalent to (k * n), where k is the number of folds (`num_bag_folds`) and n is the number of finished repeats (`num_bag_sets`) in the bagged ensemble.\n The runtime is generally 10% or less of the original fit runtime.\n The runtime can be roughly estimated as 1 / (k * n) of the original fit runtime, with k and n defined above.\n For non-bagged models:\n Optimizes a model's accuracy by retraining on 100% of the data without using a validation set.\n Will typically result in a slight accuracy increase and no change to inference time.\n The runtime will be approximately equal to the original fit runtime.\n This process does not alter the original models, but instead adds additional models.\n If stacker models are refit by this process, they will use the refit_full versions of the ancestor models during inference.\n Models produced by this process will not have validation scores, as they use all of the data for training.\n Therefore, it is up to the user to determine if the models are of sufficient quality by including test data in `predictor.leaderboard(test_data)`.\n If the user does not have additional test data, they should reference the original model's score for an estimate of the performance of the refit_full model.\n Warning: Be aware that utilizing refit_full models without separately verifying on test data means that the model is untested, and has no guarantee of being consistent with the original model.\n `cache_data` must have been set to `True` during the original training to enable this functionality.\n\n Parameters\n ----------\n model : str | list[str], default = 'all'\n Model name of model(s) to refit.\n If 'all' then all models are refitted.\n If 'best' then the model with the highest validation score is refit.\n All ancestor models will also be refit in the case that the selected model is a weighted or stacker ensemble.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`.\n set_best_to_refit_full : bool | str, default = True\n If True, sets best model to the refit_full version of the prior best model.\n This means the model used when `predictor.predict(data)` is called will be the refit_full version instead of the original version of the model.\n Ignored if `model` is not the best model.\n If str, interprets as a model name and sets best model to the refit_full version of the model `set_best_to_refit_full`.\n train_data_extra : pd.DataFrame, default = None\n If specified, will be used as additional rows of training data when refitting models.\n Requires label column. Will only be used for L1 models.\n num_cpus: int | str, default = \"auto\"\n The total amount of cpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of cpus available and the model requirement for best performance.\n Users generally don't need to set this value\n num_gpus: int | str, default = \"auto\"\n The total amount of gpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of gpus available and the model requirement for best performance.\n Users generally don't need to set this value\n fit_strategy: Literal[\"auto\", \"sequential\", \"parallel\"], default = \"auto\"\n The strategy used to fit models.\n If \"auto\", uses the same fit_strategy as used in the original :meth:`TabularPredictor.fit` call.\n If \"sequential\", models will be fit sequentially. This is the most stable option with the most readable logging.\n If \"parallel\", models will be fit in parallel with ray, splitting available compute between them.\n Note: \"parallel\" is experimental and may run into issues. It was first added in version 1.2.0.\n For machines with 16 or more CPU cores, it is likely that \"parallel\" will be faster than \"sequential\".\n\n .. versionadded:: 1.2.0\n\n **kwargs\n [Advanced] Developer debugging arguments.\n\n Returns\n -------\n Dictionary of original model names -> refit_full model names.\n \"\"\"\n self._assert_is_fit(\"refit_full\")\n ts = time.time()\n model_best = self._model_best(can_infer=None)\n if model == \"best\":\n model = model_best\n logger.log(\n 20,\n \"Refitting models via `predictor.refit_full` using all of the data (combined train and validation)...\\n\"\n '\\tModels trained in this way will have the suffix \"_FULL\" and have NaN validation score.\\n'\n \"\\tThis process is not bound by time_limit, but should take less time than the original `predictor.fit` call.\\n\"\n '\\tTo learn more, refer to the `.refit_full` method docstring which explains how \"_FULL\" models differ from normal models.',\n )\n\n self._validate_num_cpus(num_cpus=num_cpus)\n self._validate_num_gpus(num_gpus=num_gpus)\n total_resources = {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n }\n\n if fit_strategy == \"auto\":\n fit_strategy = self._fit_strategy\n self._validate_fit_strategy(fit_strategy=fit_strategy)\n\n if train_data_extra is not None:\n assert kwargs.get(\"X_pseudo\", None) is None, \"Cannot pass both train_data_extra and X_pseudo arguments\"\n assert kwargs.get(\"y_pseudo\", None) is None, \"Cannot pass both train_data_extra and y_pseudo arguments\"\n X_pseudo, y_pseudo, _ = self._sanitize_pseudo_data(pseudo_data=train_data_extra, name=\"train_data_extra\")\n kwargs[\"X_pseudo\"] = X_pseudo\n kwargs[\"y_pseudo\"] = y_pseudo\n refit_full_dict = self._learner.refit_ensemble_full(\n model=model, total_resources=total_resources, fit_strategy=fit_strategy, **kwargs\n )\n\n if set_best_to_refit_full:\n if isinstance(set_best_to_refit_full, str):\n model_to_set_best = set_best_to_refit_full\n else:\n model_to_set_best = model_best\n model_refit_map = self._trainer.model_refit_map()\n if model_to_set_best in model_refit_map:\n self._trainer.model_best = model_refit_map[model_to_set_best]\n # Note: model_best will be overwritten if additional training is done with new models,\n # since model_best will have validation score of None and any new model will have a better validation score.\n # This has the side effect of having the possibility of model_best being overwritten by a worse model than the original model_best.\n self._trainer.save()\n logger.log(\n 20,\n f'Updated best model to \"{self._trainer.model_best}\" (Previously \"{model_best}\"). '\n f'AutoGluon will default to using \"{self._trainer.model_best}\" for predict() and predict_proba().',\n )\n elif model_to_set_best in model_refit_map.values():\n # Model best is already a refit full model\n prev_best = self._trainer.model_best\n self._trainer.model_best = model_to_set_best\n self._trainer.save()\n logger.log(\n 20,\n f'Updated best model to \"{self._trainer.model_best}\" (Previously \"{prev_best}\"). '\n f'AutoGluon will default to using \"{self._trainer.model_best}\" for predict() and predict_proba().',\n )\n else:\n logger.warning(\n f'Best model (\"{model_to_set_best}\") is not present in refit_full dictionary. '\n f'Training may have failed on the refit model. AutoGluon will default to using \"{model_best}\" for predict() and predict_proba().'\n )\n\n te = time.time()\n logger.log(\n 20, f'Refit complete, total runtime = {round(te - ts, 2)}s ... Best model: \"{self._trainer.model_best}\"'\n )\n return refit_full_dict\n\n @property\n def model_best(self) -> str:\n \"\"\"\n Returns the string model name of the best model by validation score that can infer.\n This is the same model used during inference when `predictor.predict` is called without specifying a model.\n This can be updated to be a model other than the model with best validation score by methods such as refit_full and set_model_best.\n\n Returns\n -------\n String model name of the best model\n \"\"\"\n return self._model_best(can_infer=True)\n\n def _model_best(self, can_infer=None) -> str:\n self._assert_is_fit(\"model_best\")\n # TODO: Set self._trainer.model_best to the best model at end of fit instead of best WeightedEnsemble.\n if self._trainer.model_best is not None:\n models = self._trainer.get_model_names(can_infer=can_infer)\n if self._trainer.model_best in models:\n return self._trainer.model_best\n return self._trainer.get_model_best(can_infer=can_infer)\n\n def set_model_best(self, model: str, save_trainer: bool = False):\n \"\"\"\n Sets the model to be used by default when calling `predictor.predict(data)`.\n By default, this is the model with the best validation score, but this is not always the case.\n If manually set, this can be overwritten internally if further training occurs, such as through fit_extra, refit_full, or distill.\n\n Parameters\n ----------\n model : str\n Name of model to set to best. If model does not exist or cannot infer, raises an AssertionError.\n save_trainer : bool, default = False\n If True, self._trainer is saved with the new model_best value, such that it is reflected when predictor is loaded in future from disk.\n \"\"\"\n self._assert_is_fit(\"set_model_best\")\n models = self._trainer.get_model_names(can_infer=True)\n if model in models:\n self._trainer.model_best = model\n else:\n raise AssertionError(f'Model \"{model}\" is not a valid model to specify as best! Valid models: {models}')\n if save_trainer:\n self._trainer.save()\n\n def model_refit_map(self, inverse=False) -> dict[str, str]:\n \"\"\"\n Returns a dictionary of original model name -> refit full model name.\n Empty unless `refit_full=True` was set during fit or `predictor.refit_full()` was called.\n This can be useful when determining the best model based off of `predictor.leaderboard()`, then getting the _FULL version of the model by passing its name as the key to this dictionary.\n\n Parameters\n ----------\n inverse : bool, default = False\n If True, instead returns a dictionary of refit full model name -> original model name (Swap keys with values)\n\n Returns\n -------\n Dictionary of original model name -> refit full model name.\n \"\"\"\n self._assert_is_fit(\"model_refit_map\")\n return self._trainer.model_refit_map(inverse=inverse)\n\n def info(self):\n \"\"\"\n [EXPERIMENTAL] Returns a dictionary of `predictor` metadata.\n Warning: This functionality is currently in preview mode.\n The metadata information returned may change in structure in future versions without warning.\n The definitions of various metadata values are not yet documented.\n The output of this function should not be used for programmatic decisions.\n Contains information such as row count, column count, model training time, validation scores, hyperparameters, and much more.\n\n Returns\n -------\n Dictionary of `predictor` metadata.\n \"\"\"\n self._assert_is_fit(\"info\")\n return self._learner.get_info(include_model_info=True, include_model_failures=True)\n\n def model_info(self, model: str) -> dict:\n \"\"\"\n Returns metadata information about the given model.\n Equivalent output to `predictor.info()[\"model_info\"][model]`\n\n Parameters\n ----------\n model: str\n The name of the model to get info for.\n\n Returns\n -------\n model_info: dict\n Model info dictionary\n\n \"\"\"\n return self._trainer.get_model_info(model=model)\n\n # TODO: Add entire `hyperparameters` dict method for multiple models (including stack ensemble)\n # TODO: Add unit test\n def model_hyperparameters(\n self,\n model: str,\n include_ag_args_ensemble: bool = True,\n output_format: Literal[\"user\", \"all\"] = \"user\",\n ) -> dict:\n \"\"\"\n Returns the hyperparameters of a given model.\n\n Parameters\n ----------\n model: str\n The name of the model to get hyperparameters for.\n include_ag_args_ensemble: bool, default True\n If True, includes the ag_args_ensemble parameters if they exist (for example, when bagging is enabled).\n output_format: {\"user\", \"all\"}, default \"user\"\n If \"user\", returns the same hyperparameters specified by the user (only non-defaults).\n If \"all\", returns all hyperparameters used by the model (including default hyperparameters not specified by the user)\n Regardless of the output_format, they both are functionally equivalent if passed to AutoGluon.\n\n Returns\n -------\n model_hyperparameters: dict\n Dictionary of model hyperparameters.\n Equivalent to the model_hyperparameters specified by the user for this model in:\n `predictor.fit(..., hyperparameters={..., model_key: [..., model_hyperparameters]})`\n\n \"\"\"\n # TODO: Move logic into trainer?\n info_model = self.model_info(model=model)\n if output_format == \"user\":\n if \"bagged_info\" in info_model:\n hyperparameters = info_model[\"bagged_info\"][\"child_hyperparameters_user\"].copy()\n if include_ag_args_ensemble and info_model[\"hyperparameters_user\"]:\n hyperparameters[\"ag_args_ensemble\"] = info_model[\"hyperparameters_user\"]\n else:\n hyperparameters = info_model[\"hyperparameters_user\"]\n elif output_format == \"all\":\n if \"bagged_info\" in info_model:\n hyperparameters = info_model[\"bagged_info\"][\"child_hyperparameters\"].copy()\n if info_model[\"bagged_info\"][\"child_ag_args_fit\"]:\n hyperparameters[\"ag_args_fit\"] = info_model[\"bagged_info\"][\"child_ag_args_fit\"]\n if include_ag_args_ensemble:\n bag_hyperparameters = info_model[\"hyperparameters\"].copy()\n if info_model[\"ag_args_fit\"]:\n bag_hyperparameters[\"ag_args_fit\"] = info_model[\"ag_args_fit\"]\n if bag_hyperparameters:\n hyperparameters[\"ag_args_ensemble\"] = bag_hyperparameters\n else:\n hyperparameters = info_model[\"hyperparameters\"]\n else:\n raise ValueError(f\"output_format={output_format} is unknown!\")\n return hyperparameters\n\n # TODO: Add data argument\n # TODO: Add option to disable OOF generation of newly fitted models\n # TODO: Move code logic to learner/trainer\n # TODO: Add fit() arg to perform this automatically at end of training\n # TODO: Consider adding cutoff arguments such as top-k models\n def fit_weighted_ensemble(\n self,\n base_models: list = None,\n name_suffix: str = \"Best\",\n expand_pareto_frontier: bool = False,\n time_limit: float = None,\n refit_full: bool = False,\n num_cpus: int | str = \"auto\",\n num_gpus: int | str = \"auto\",\n ):\n \"\"\"\n Fits new weighted ensemble models to combine predictions of previously-trained models.\n `cache_data` must have been set to `True` during the original training to enable this functionality.\n\n Parameters\n ----------\n base_models: list, default = None\n List of model names the weighted ensemble can consider as candidates.\n If None, all previously trained models are considered except for weighted ensemble models.\n As an example, to train a weighted ensemble that can only have weights assigned to the models 'model_a' and 'model_b', set `base_models=['model_a', 'model_b']`\n name_suffix: str, default = 'Best'\n Name suffix to add to the name of the newly fitted ensemble model.\n expand_pareto_frontier: bool, default = False\n If True, will train N-1 weighted ensemble models instead of 1, where `N=len(base_models)`.\n The final model trained when True is equivalent to the model trained when False.\n These weighted ensemble models will attempt to expand the pareto frontier.\n This will create many different weighted ensembles which have different accuracy/memory/inference-speed trade-offs.\n This is particularly useful when inference speed is an important consideration.\n time_limit: float, default = None\n Time in seconds each weighted ensemble model is allowed to train for. If `expand_pareto_frontier=True`, the `time_limit` value is applied to each model.\n If None, the ensemble models train without time restriction.\n refit_full : bool, default = False\n If True, will apply refit_full to all weighted ensembles created during this call.\n Identical to calling `predictor.refit_full(model=predictor.fit_weighted_ensemble(...))`\n num_cpus: int | str, default = \"auto\"\n The total amount of cpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of cpus available and the model requirement for best performance.\n Users generally don't need to set this value\n num_gpus: int | str, default = \"auto\"\n The total amount of gpus you want AutoGluon predictor to use.\n Auto means AutoGluon will make the decision based on the total number of gpus available and the model requirement for best performance.\n Users generally don't need to set this value\n\n Returns\n -------\n List of newly trained weighted ensemble model names.\n If an exception is encountered while training an ensemble model, that model's name will be absent from the list.\n \"\"\"\n self._assert_is_fit(\"fit_weighted_ensemble\")\n trainer = self._learner.load_trainer()\n\n if trainer.bagged_mode:\n X = trainer.load_X()\n y = trainer.load_y()\n fit = True\n else:\n X = trainer.load_X_val()\n y = trainer.load_y_val()\n fit = False\n\n total_resources = {\n \"num_cpus\": num_cpus,\n \"num_gpus\": num_gpus,\n }\n\n stack_name = \"aux1\"\n if base_models is None:\n base_models = trainer.get_model_names(stack_name=\"core\")\n\n X_stack_preds = trainer.get_inputs_to_stacker(\n X=X, base_models=base_models, fit=fit, use_orig_features=False, use_val_cache=True\n )\n\n models = []\n\n if expand_pareto_frontier:\n leaderboard = self.leaderboard()\n leaderboard = leaderboard[leaderboard[\"model\"].isin(base_models)]\n leaderboard = leaderboard.sort_values(by=\"pred_time_val\")\n models_to_check = leaderboard[\"model\"].tolist()\n for i in range(1, len(models_to_check) - 1):\n models_to_check_now = models_to_check[: i + 1]\n max_base_model_level = max([trainer.get_model_level(base_model) for base_model in models_to_check_now])\n weighted_ensemble_level = max_base_model_level + 1\n models += trainer.generate_weighted_ensemble(\n X=X_stack_preds,\n y=y,\n level=weighted_ensemble_level,\n stack_name=stack_name,\n base_model_names=models_to_check_now,\n name_suffix=name_suffix + \"_Pareto\" + str(i),\n time_limit=time_limit,\n total_resources=total_resources,\n )\n\n max_base_model_level = max([trainer.get_model_level(base_model) for base_model in base_models])\n weighted_ensemble_level = max_base_model_level + 1\n models += trainer.generate_weighted_ensemble(\n X=X_stack_preds,\n y=y,\n level=weighted_ensemble_level,\n stack_name=stack_name,\n base_model_names=base_models,\n name_suffix=name_suffix,\n time_limit=time_limit,\n total_resources=total_resources,\n )\n\n if refit_full:\n refit_models_dict = self.refit_full(model=models, num_cpus=num_cpus, num_gpus=num_gpus)\n models += [refit_models_dict[m] for m in models]\n\n return models\n\n def calibrate_decision_threshold(\n self,\n data: pd.DataFrame | str | None = None,\n metric: str | Scorer | None = None,\n model: str = \"best\",\n decision_thresholds: int | list[float] = 25,\n secondary_decision_thresholds: int | None = 19,\n subsample_size: int | None = 1000000,\n verbose: bool = True,\n ) -> float:\n \"\"\"\n Calibrate the decision threshold in binary classification to optimize a given metric.\n You can pass the output of this method as input to `predictor.set_decision_threshold` to update the predictor.\n Will raise an AssertionError if `predictor.problem_type != 'binary'`.\n\n Note that while calibrating the decision threshold can help to improve a given metric,\n other metrics may end up having worse scores.\n For example, calibrating on `balanced_accuracy` will often harm `accuracy`.\n Users should keep this in mind while leveraging decision threshold calibration.\n\n Parameters\n ----------\n data : pd.DataFrame or str, optional\n The data to use for calibration. Must contain the label column.\n We recommend to keep this value as None unless you are an advanced user and understand the implications.\n If None, will use internal data such as the holdout validation data or out-of-fold predictions.\n metric : autogluon.core.metrics.Scorer or str, default = None\n The metric to optimize during calibration.\n If None, uses `predictor.eval_metric`.\n model : str, default = 'best'\n The model to use prediction probabilities of when calibrating the threshold.\n If 'best', will use `predictor.model_best`.\n decision_thresholds : int | list[float], default = 25\n The number of decision thresholds on either side of `0.5` to search.\n The default of 25 will result in 51 searched thresholds: [0.00, 0.02, 0.04, ..., 0.48, 0.50, 0.52, ..., 0.96, 0.98, 1.00]\n Alternatively, a list of decision thresholds can be passed and only the thresholds in the list will be searched.\n secondary_decision_thresholds : int | None, default = 19\n The number of secondary decision thresholds to check on either side of the threshold identified in the first phase.\n Skipped if None.\n For example, if decision_thresholds=50 and 0.14 was identified as the optimal threshold, while secondary_decision_threshold=9,\n Then the following additional thresholds are checked:\n [0.131, 0.132, 0.133, 0.134, 0.135, 0.136, 0.137, 0.138, 0.139, 0.141, 0.142, 0.143, 0.144, 0.145, 0.146, 0.147, 0.148, 0.149]\n subsample_size : int | None, default = 1000000\n When `subsample_size` is not None and `data` contains more rows than `subsample_size`, samples to `subsample_size` rows to speed up calibration.\n Usually it is not necessary to use more than 1 million rows for calibration.\n verbose : bool, default = True\n If True, will log information about the calibration process.\n\n Returns\n -------\n Decision Threshold: A float between 0 and 1 defining the decision boundary for predictions that\n maximizes the `metric` score on the `data` for the `model`.\n \"\"\"\n # TODO: v1.2\n # Calculate optimal threshold for each model separately when deciding best model\n # time limit\n # update validation scores of models based on threshold\n # speed up the logic / search for optimal threshold more efficiently\n # make threshold calibration part of internal optimization, such as during fit_weighted_ensemble.\n # precision has strange edge-cases where it flips from 1.0 to 0.0 score due to becoming undefined\n # consider warning users who pass this metric,\n # or edit this metric so they do not flip value when undefined.\n # UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n # Use `zero_division` parameter to control this behavior.\n\n self._assert_is_fit(\"calibrate_decision_threshold\")\n assert self.problem_type == BINARY, (\n f'calibrate_decision_threshold is only available for `problem_type=\"{BINARY}\"`'\n )\n data = self._get_dataset(data, allow_nan=True)\n\n if metric is None:\n metric = self.eval_metric\n if model == \"best\":\n model = self.model_best\n\n return self._learner.calibrate_decision_threshold(\n data=data,\n metric=metric,\n model=model,\n decision_thresholds=decision_thresholds,\n secondary_decision_thresholds=secondary_decision_thresholds,\n subsample_size=subsample_size,\n verbose=verbose,\n )\n\n def predict_oof(\n self,\n model: str = None,\n *,\n transformed=False,\n train_data=None,\n internal_oof=False,\n decision_threshold=None,\n can_infer=None,\n ) -> pd.Series:\n \"\"\"\n Note: This is advanced functionality not intended for normal usage.\n\n Returns the out-of-fold (OOF) predictions for every row in the training data.\n\n For a similar method, refer to :meth:`TabularPredictor.predict_multi` with `data=None`.\n For more information, refer to `predict_proba_oof()` documentation.\n\n Parameters\n ----------\n model : str (optional)\n Refer to `predict_proba_oof()` documentation.\n transformed : bool, default = False\n Refer to `predict_proba_oof()` documentation.\n train_data : pd.DataFrame, default = None\n Refer to `predict_proba_oof()` documentation.\n internal_oof : bool, default = False\n Refer to `predict_proba_oof()` documentation.\n decision_threshold : float, default = None\n Refer to `predict_multi` documentation.\n can_infer : bool, default = None\n Refer to `predict_proba_oof()` documentation.\n\n Returns\n -------\n :class:`pd.Series` object of the out-of-fold training predictions of the model.\n \"\"\"\n self._assert_is_fit(\"predict_oof\")\n if decision_threshold is None:\n decision_threshold = self.decision_threshold\n y_pred_proba_oof = self.predict_proba_oof(\n model=model,\n transformed=transformed,\n as_multiclass=True,\n train_data=train_data,\n internal_oof=internal_oof,\n can_infer=can_infer,\n )\n y_pred_oof = get_pred_from_proba_df(\n y_pred_proba_oof, problem_type=self.problem_type, decision_threshold=decision_threshold\n )\n if transformed:\n return self._learner.label_cleaner.to_transformed_dtype(y_pred_oof)\n return y_pred_oof\n\n # TODO: Remove train_data argument once we start caching the raw original data: Can just load that instead.\n def predict_proba_oof(\n self,\n model: str = None,\n *,\n transformed=False,\n as_multiclass=True,\n train_data=None,\n internal_oof=False,\n can_infer=None,\n ) -> pd.DataFrame | pd.Series:\n \"\"\"\n Note: This is advanced functionality not intended for normal usage.\n\n Returns the out-of-fold (OOF) predicted class probabilities for every row in the training data.\n OOF prediction probabilities may provide unbiased estimates of generalization accuracy (reflecting how predictions will behave on new data)\n Predictions for each row are only made using models that were fit to a subset of data where this row was held-out.\n\n For a similar method, refer to :meth:`TabularPredictor.predict_proba_multi` with `data=None`.\n\n Warning: This method will raise an exception if called on a model that is not a bagged ensemble. Only bagged models (such a stacker models) can produce OOF predictions.\n This also means that refit_full models and distilled models will raise an exception.\n Warning: If intending to join the output of this method with the original training data, be aware that a rare edge-case issue exists:\n Multiclass problems with rare classes combined with the use of the 'log_loss' eval_metric may have forced AutoGluon to duplicate rows in the training data to satisfy minimum class counts in the data.\n If this has occurred, then the indices and row counts of the returned :class:`pd.Series` in this method may not align with the training data.\n In this case, consider fetching the processed training data using `predictor.load_data_internal()` instead of using the original training data.\n A more benign version of this issue occurs when 'log_loss' wasn't specified as the eval_metric but rare classes were dropped by AutoGluon.\n In this case, not all original training data rows will have an OOF prediction. It is recommended to either drop these rows during the join or to get direct predictions on the missing rows via :meth:`TabularPredictor.predict_proba`.\n\n Parameters\n ----------\n model : str (optional)\n The name of the model to get out-of-fold predictions from. Defaults to None, which uses the highest scoring model on the validation set.\n Valid models are listed in this `predictor` by calling `predictor.model_names()`\n transformed : bool, default = False\n Whether the output values should be of the original label representation (False) or the internal label representation (True).\n The internal representation for binary and multiclass classification are integers numbering the k possible classes from 0 to k-1, while the original representation is identical to the label classes provided during fit.\n Generally, most users will want the original representation and keep `transformed=False`.\n as_multiclass : bool, default = True\n Whether to return binary classification probabilities as if they were for multiclass classification.\n Output will contain two columns, and if `transformed=False`, the column names will correspond to the binary class labels.\n The columns will be the same order as `predictor.class_labels`.\n If False, output will contain only 1 column for the positive class (get positive_class name via `predictor.positive_class`).\n Only impacts output for binary classification problems.\n train_data : pd.DataFrame, default = None\n Specify the original `train_data` to ensure that any training rows that were originally dropped internally are properly handled.\n If None, then output will not contain all rows if training rows were dropped internally during fit.\n If `train_data` is specified and `model` is unable to predict and rows were dropped internally, an exception will be raised.\n internal_oof : bool, default = False\n [Advanced Option] Return the internal OOF preds rather than the externally facing OOF preds.\n Internal OOF preds may have more/fewer rows than was provided in train_data, and are incompatible with external data.\n If you don't know what this does, keep it as False.\n can_infer : bool, default = None\n Only used if `model` is not specified.\n This is used to determine if the best model must be one that is able to predict on new data (True).\n If None, the best model does not need to be able to infer on new data.\n\n Returns\n -------\n :class:`pd.Series` or :class:`pd.DataFrame` object of the out-of-fold training prediction probabilities of the model.\n \"\"\"\n self._assert_is_fit(\"predict_proba_oof\")\n if model is None:\n model = self._model_best(can_infer=can_infer)\n if not self._trainer.bagged_mode:\n raise AssertionError(\"Predictor must be in bagged mode to get out-of-fold predictions.\")\n if self._trainer.get_model_attribute(model=model, attribute=\"refit_full\", default=False):\n model_to_get_oof = self._trainer.get_model_attribute(model=model, attribute=\"refit_full_parent\")\n # TODO: bagged-with-holdout refit to bagged-no-holdout should still be able to return out-of-fold predictions\n else:\n model_to_get_oof = model\n if model != model_to_get_oof:\n logger.log(20, f'Using OOF from \"{model_to_get_oof}\" as a proxy for \"{model}\".')\n if self._trainer.get_model_attribute_full(model=model_to_get_oof, attribute=\"val_in_fit\", func=max):\n raise AssertionError(\n f\"Model {model_to_get_oof} does not have out-of-fold predictions because it used a validation set during training.\"\n )\n y_pred_proba_oof_transformed = self.transform_features(\n base_models=[model_to_get_oof], return_original_features=False\n )\n if not internal_oof:\n is_duplicate_index = y_pred_proba_oof_transformed.index.duplicated(keep=\"first\")\n if is_duplicate_index.any():\n logger.log(\n 20,\n \"Detected duplicate indices... This means that data rows may have been duplicated during training. \"\n \"Removing all duplicates except for the first instance.\",\n )\n y_pred_proba_oof_transformed = y_pred_proba_oof_transformed[is_duplicate_index == False]\n if self._learner._pre_X_rows is not None and len(y_pred_proba_oof_transformed) < self._learner._pre_X_rows:\n len_diff = self._learner._pre_X_rows - len(y_pred_proba_oof_transformed)\n if train_data is None:\n logger.warning(\n f\"WARNING: {len_diff} rows of training data were dropped internally during fit. \"\n f\"The output will not contain all original training rows.\\n\"\n f\"If attempting to get `oof_pred_proba`, DO NOT pass `train_data` into `predictor.predict_proba` or `predictor.transform_features`!\\n\"\n f\"Instead this can be done by the following \"\n f\"(Ensure `train_data` is identical to when it was used in fit):\\n\"\n f\"oof_pred_proba = predictor.predict_proba_oof(train_data=train_data)\\n\"\n f\"oof_pred = predictor.predict_oof(train_data=train_data)\\n\"\n )\n else:\n missing_idx = list(train_data.index.difference(y_pred_proba_oof_transformed.index))\n if len(missing_idx) > 0:\n missing_idx_data = train_data.loc[missing_idx]\n missing_pred_proba = self.transform_features(\n data=missing_idx_data, base_models=[model], return_original_features=False\n )\n y_pred_proba_oof_transformed = pd.concat([y_pred_proba_oof_transformed, missing_pred_proba])\n y_pred_proba_oof_transformed = y_pred_proba_oof_transformed.reindex(list(train_data.index))\n\n if self.problem_type == MULTICLASS and self._learner.label_cleaner.problem_type_transform == MULTICLASS:\n y_pred_proba_oof_transformed.columns = copy.deepcopy(\n self._learner.label_cleaner.ordered_class_labels_transformed\n )\n elif self.problem_type == QUANTILE:\n y_pred_proba_oof_transformed.columns = self.quantile_levels\n else:\n y_pred_proba_oof_transformed.columns = [self.label]\n y_pred_proba_oof_transformed = y_pred_proba_oof_transformed[self.label]\n if as_multiclass and self.problem_type == BINARY:\n y_pred_proba_oof_transformed = LabelCleanerMulticlassToBinary.convert_binary_proba_to_multiclass_proba(\n y_pred_proba_oof_transformed, as_pandas=True\n )\n elif self.problem_type == MULTICLASS:\n if transformed:\n y_pred_proba_oof_transformed = (\n LabelCleanerMulticlassToBinary.convert_binary_proba_to_multiclass_proba(\n y_pred_proba_oof_transformed, as_pandas=True\n )\n )\n y_pred_proba_oof_transformed.columns = copy.deepcopy(\n self._learner.label_cleaner.ordered_class_labels_transformed\n )\n if transformed:\n return y_pred_proba_oof_transformed\n else:\n return self.transform_labels(labels=y_pred_proba_oof_transformed, inverse=True, proba=True)\n\n @property\n def positive_class(self) -> int | str:\n \"\"\"\n Returns the positive class name in binary classification. Useful for computing metrics such as F1 which require a positive and negative class.\n In binary classification, :class:`TabularPredictor.predict_proba(as_multiclass=False)` returns the estimated probability that each row belongs to the positive class.\n Will print a warning and return None if called when `predictor.problem_type != 'binary'`.\n\n Returns\n -------\n The positive class name in binary classification or None if the problem is not binary classification.\n \"\"\"\n return self._learner.positive_class\n\n def load_data_internal(self, data=\"train\", return_X=True, return_y=True):\n \"\"\"\n Loads the internal data representation used during model training.\n Individual AutoGluon models like the neural network may apply additional feature transformations that are not reflected in this method.\n This method only applies universal transforms employed by all AutoGluon models.\n Warning, the internal representation may:\n Have different features compared to the original data.\n Have different row counts compared to the original data.\n Have indices which do not align with the original data.\n Have label values which differ from those in the original data.\n Internal data representations should NOT be combined with the original data, in most cases this is not possible.\n\n Parameters\n ----------\n data : str, default = 'train'\n The data to load.\n Valid values are:\n 'train':\n Load the training data used during model training.\n This is a transformed and augmented version of the `train_data` passed in `fit()`.\n 'val':\n Load the validation data used during model training.\n This is a transformed and augmented version of the `tuning_data` passed in `fit()`.\n If `tuning_data=None` was set in `fit()`, then `tuning_data` is an automatically generated validation set created by splitting `train_data`.\n Warning: Will raise an exception if called by a bagged predictor, as bagged predictors have no validation data.\n return_X : bool, default = True\n Whether to return the internal data features\n If set to `False`, then the first element in the returned tuple will be None.\n return_y : bool, default = True\n Whether to return the internal data labels\n If set to `False`, then the second element in the returned tuple will be None.\n\n Returns\n -------\n Tuple of (:class:`pd.DataFrame`, :class:`pd.Series`) corresponding to the internal data features and internal data labels, respectively.\n\n \"\"\"\n self._assert_is_fit(\"load_data_internal\")\n if data == \"train\":\n load_X = self._trainer.load_X\n load_y = self._trainer.load_y\n elif data == \"val\":\n load_X = self._trainer.load_X_val\n load_y = self._trainer.load_y_val\n else:\n raise ValueError(f\"data must be one of: ['train', 'val'], but was '{data}'.\")\n X = load_X() if return_X else None\n y = load_y() if return_y else None\n return X, y\n\n def save_space(self, remove_data=True, remove_fit_stack=True, requires_save=True, reduce_children=False):\n \"\"\"\n Reduces the memory and disk size of predictor by deleting auxiliary model files that aren't needed for prediction on new data.\n This function has NO impact on inference accuracy.\n It is recommended to invoke this method if the only goal is to use the trained model for prediction.\n However, certain advanced functionality may no longer be available after `save_space()` has been called.\n\n Parameters\n ----------\n remove_data : bool, default = True\n Whether to remove cached files of the original training and validation data.\n Only reduces disk usage, it has no impact on memory usage.\n This is especially useful when the original data was large.\n This is equivalent to setting `cache_data=False` during the original `fit()`.\n Will disable all advanced functionality that requires `cache_data=True`.\n remove_fit_stack : bool, default = True\n Whether to remove information required to fit new stacking models and continue fitting bagged models with new folds.\n Only reduces disk usage, it has no impact on memory usage.\n This includes:\n out-of-fold (OOF) predictions\n This is useful for multiclass problems with many classes, as OOF predictions can become very large on disk. (1 GB per model in extreme cases)\n This disables `predictor.refit_full()` for stacker models.\n requires_save : bool, default = True\n Whether to remove information that requires the model to be saved again to disk.\n Typically this only includes flag variables that don't have significant impact on memory or disk usage, but should technically be updated due to the removal of more important information.\n An example is the `is_data_saved` boolean variable in `trainer`, which should be updated to `False` if `remove_data=True` was set.\n reduce_children : bool, default = False\n Whether to apply the reduction rules to bagged ensemble children models. These are the models trained for each fold of the bagged ensemble.\n This should generally be kept as `False` since the most important memory and disk reduction techniques are automatically applied to these models during the original `fit()` call.\n\n \"\"\"\n self._assert_is_fit(\"save_space\")\n self._trainer.reduce_memory_size(\n remove_data=remove_data,\n remove_fit_stack=remove_fit_stack,\n remove_fit=True,\n remove_info=False,\n requires_save=requires_save,\n reduce_children=reduce_children,\n )\n\n def delete_models(\n self,\n models_to_keep: str | list[str] | None = None,\n models_to_delete: str | list[str] | None = None,\n allow_delete_cascade: bool = False,\n delete_from_disk: bool = True,\n dry_run: bool = False,\n ):\n \"\"\"\n Deletes models from `predictor`.\n This can be helpful to minimize memory usage and disk usage, particularly for model deployment.\n This will remove all references to the models in `predictor`.\n For example, removed models will not appear in `predictor.leaderboard()`.\n WARNING: If `delete_from_disk=True`, this will DELETE ALL FILES in the deleted model directories, regardless if they were created by AutoGluon or not.\n DO NOT STORE FILES INSIDE OF THE MODEL DIRECTORY THAT ARE UNRELATED TO AUTOGLUON.\n\n Parameters\n ----------\n models_to_keep : str or list[str], default = None\n Name of model or models to not delete.\n All models that are not specified and are also not required as a dependency of any model in `models_to_keep` will be deleted.\n Specify `models_to_keep='best'` to keep only the best model and its model dependencies.\n `models_to_delete` must be None if `models_to_keep` is set.\n To see the list of possible model names, use: `predictor.model_names()` or `predictor.leaderboard()`.\n models_to_delete : str or list[str], default = None\n Name of model or models to delete.\n All models that are not specified but depend on a model in `models_to_delete` will also be deleted.\n `models_to_keep` must be None if `models_to_delete` is set.\n allow_delete_cascade : bool, default = False\n If `False`, if unspecified dependent models of models in `models_to_delete` exist an exception will be raised instead of deletion occurring.\n An example of a dependent model is m1 if m2 is a stacker model and takes predictions from m1 as inputs. In this case, m1 would be a dependent model of m2.\n If `True`, all dependent models of models in `models_to_delete` will be deleted.\n Has no effect if `models_to_delete=None`.\n delete_from_disk : bool, default = True\n If `True`, deletes the models from disk if they were persisted.\n WARNING: This deletes the entire directory for the deleted models, and ALL FILES located there.\n It is highly recommended to first run with `dry_run=True` to understand which directories will be deleted.\n dry_run : bool, default = False\n If `True`, then deletions don't occur, and logging statements are printed describing what would have occurred.\n Set `dry_run=False` to perform the deletions.\n\n \"\"\"\n self._assert_is_fit(\"delete_models\")\n if models_to_keep == \"best\":\n models_to_keep = self.model_best\n self._trainer.delete_models(\n models_to_keep=models_to_keep,\n models_to_delete=models_to_delete,\n allow_delete_cascade=allow_delete_cascade,\n delete_from_disk=delete_from_disk,\n dry_run=dry_run,\n )\n\n def disk_usage(self) -> int:\n \"\"\"\n Returns the combined size of all files under the `predictor.path` directory in bytes.\n \"\"\"\n return get_directory_size(self.path)\n\n def disk_usage_per_file(self, *, sort_by: str = \"size\", include_path_in_name: bool = False) -> pd.Series:\n \"\"\"\n Returns the size of each file under the `predictor.path` directory in bytes.\n\n Parameters\n ----------\n sort_by : str, default = \"size\"\n If None, output files will be ordered based on order of search in os.walk(path).\n If \"size\", output files will be ordered in descending order of file size.\n If \"name\", output files will be ordered by name in ascending alphabetical order.\n include_path_in_name : bool, default = False\n If True, includes the full path of the file including the input `path` as part of the index in the output pd.Series.\n If False, removes the `path` prefix of the file path in the index of the output pd.Series.\n\n For example, for a file located at `foo/bar/model.pkl`, with path='foo/'\n If True, index will be `foo/bar/model.pkl`\n If False, index will be `bar/model.pkl`\n\n Returns\n -------\n pd.Series with index file path and value file size in bytes.\n \"\"\"\n return get_directory_size_per_file(self.path, sort_by=sort_by, include_path_in_name=include_path_in_name)\n\n def model_names(\n self,\n stack_name: str = None,\n level: int = None,\n can_infer: bool = None,\n models: list[str] = None,\n persisted: bool = None,\n ) -> list[str]:\n \"\"\"\n Returns the list of model names trained in this `predictor` object.\n\n Parameters\n ----------\n stack_name: str, default = None\n If specified, returns only models under a given stack name.\n level: int, default = None\n If specified, returns only models at the given stack level.\n can_infer: bool, default = None\n If specified, returns only models that can/cannot infer on new data.\n models: list[str], default = None\n The list of model names to consider.\n If None, considers all models.\n persisted: bool, default = None\n If None: no filtering will occur based on persisted status\n If True: will return only the models that are persisted in memory via `predictor.persist()`\n If False: will return only the models that are not persisted in memory via `predictor.persist()`\n\n Returns\n -------\n List of model names\n \"\"\"\n self._assert_is_fit(\"model_names\")\n model_names = self._trainer.get_model_names(\n stack_name=stack_name, level=level, can_infer=can_infer, models=models\n )\n if persisted is not None:\n persisted_model_names = list(self._trainer.models.keys())\n if persisted:\n model_names = [m for m in model_names if m in persisted_model_names]\n else:\n model_names = [m for m in model_names if m not in persisted_model_names]\n return model_names\n\n def distill(\n self,\n train_data: pd.DataFrame | str = None,\n tuning_data: pd.DataFrame | str = None,\n augmentation_data: pd.DataFrame = None,\n time_limit: float = None,\n hyperparameters: dict | str = None,\n holdout_frac: float = None,\n teacher_preds: str = \"soft\",\n augment_method: str = \"spunge\",\n augment_args: dict = {\"size_factor\": 5, \"max_size\": int(1e5)},\n models_name_suffix: str = None,\n verbosity: int = None,\n ):\n \"\"\"\n [EXPERIMENTAL]\n Distill AutoGluon's most accurate ensemble-predictor into single models which are simpler/faster and require less memory/compute.\n Distillation can produce a model that is more accurate than the same model fit directly on the original training data.\n After calling `distill()`, there will be more models available in this Predictor, which can be evaluated using `predictor.leaderboard(test_data)` and deployed with: `predictor.predict(test_data, model=MODEL_NAME)`.\n This will raise an exception if `cache_data=False` was previously set in `fit()`.\n\n NOTE: Until catboost v0.24 is released, `distill()` with CatBoost students in multiclass classification requires you to first install catboost-dev: `pip install catboost-dev`\n\n Parameters\n ----------\n train_data : str or :class:`pd.DataFrame`, default = None\n Same as `train_data` argument of `fit()`.\n If None, the same training data will be loaded from `fit()` call used to produce this Predictor.\n tuning_data : str or :class:`pd.DataFrame`, default = None\n Same as `tuning_data` argument of `fit()`.\n If `tuning_data = None` and `train_data = None`: the same training/validation splits will be loaded from `fit()` call used to produce this Predictor,\n unless bagging/stacking was previously used in which case a new training/validation split is performed.\n augmentation_data : :class:`pd.DataFrame`, default = None\n An optional extra dataset of unlabeled rows that can be used for augmenting the dataset used to fit student models during distillation (ignored if None).\n time_limit : int, default = None\n Approximately how long (in seconds) the distillation process should run for.\n If None, no time-constraint will be enforced allowing the distilled models to fully train.\n hyperparameters : dict or str, default = None\n Specifies which models to use as students and what hyperparameter-values to use for them.\n Same as `hyperparameters` argument of `fit()`.\n If = None, then student models will use the same hyperparameters from `fit()` used to produce this Predictor.\n Note: distillation is currently only supported for ['GBM','NN_TORCH','RF','CAT'] student models, other models and their hyperparameters are ignored here.\n holdout_frac : float\n Same as `holdout_frac` argument of :meth:`TabularPredictor.fit`.\n teacher_preds : str, default = 'soft'\n What form of teacher predictions to distill from (teacher refers to the most accurate AutoGluon ensemble-predictor).\n If None, we only train with original labels (no data augmentation).\n If 'hard', labels are hard teacher predictions given by: `teacher.predict()`\n If 'soft', labels are soft teacher predictions given by: `teacher.predict_proba()`\n Note: 'hard' and 'soft' are equivalent for regression problems.\n If `augment_method` is not None, teacher predictions are only used to label augmented data (training data keeps original labels).\n To apply label-smoothing: `teacher_preds='onehot'` will use original training data labels converted to one-hot vectors for multiclass problems (no data augmentation).\n augment_method : str, default='spunge'\n Specifies method to use for generating augmented data for distilling student models.\n Options include:\n None : no data augmentation performed.\n 'munge' : The MUNGE algorithm (https://www.cs.cornell.edu/~caruana/compression.kdd06.pdf).\n 'spunge' : A simpler, more efficient variant of the MUNGE algorithm.\n augment_args : dict, default = {'size_factor':5, 'max_size': int(1e5)}\n Contains the following kwargs that control the chosen `augment_method` (these are ignored if `augment_method=None`):\n 'num_augmented_samples': int, number of augmented datapoints used during distillation. Overrides 'size_factor', 'max_size' if specified.\n 'max_size': float, the maximum number of augmented datapoints to add (ignored if 'num_augmented_samples' specified).\n 'size_factor': float, if n = training data sample-size, we add int(n * size_factor) augmented datapoints, up to 'max_size'.\n Larger values in `augment_args` will slow down the runtime of distill(), and may produce worse results if provided time_limit are too small.\n You can also pass in kwargs for the `spunge_augment`, `munge_augment` functions in `autogluon.tabular.augmentation.distill_utils`.\n models_name_suffix : str, default = None\n Optional suffix that can be appended at the end of all distilled student models' names.\n Note: all distilled models will contain '_DSTL' substring in their name by default.\n verbosity : int, default = None\n Controls amount of printed output during distillation (4 = highest, 0 = lowest).\n Same as `verbosity` parameter of :class:`TabularPredictor`.\n If None, the same `verbosity` used in previous fit is employed again.\n\n Returns\n -------\n List of names (str) corresponding to the distilled models.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularDataset, TabularPredictor\n >>> train_data = TabularDataset('train.csv')\n >>> predictor = TabularPredictor(label='class').fit(train_data, auto_stack=True)\n >>> distilled_model_names = predictor.distill()\n >>> test_data = TabularDataset('test.csv')\n >>> ldr = predictor.leaderboard(test_data)\n >>> model_to_deploy = distilled_model_names[0]\n >>> predictor.predict(test_data, model=model_to_deploy)\n\n \"\"\"\n self._assert_is_fit(\"distill\")\n if isinstance(hyperparameters, str):\n hyperparameters = get_hyperparameter_config(hyperparameters)\n return self._learner.distill(\n X=train_data,\n X_val=tuning_data,\n time_limit=time_limit,\n hyperparameters=hyperparameters,\n holdout_frac=holdout_frac,\n verbosity=verbosity,\n models_name_suffix=models_name_suffix,\n teacher_preds=teacher_preds,\n augmentation_data=augmentation_data,\n augment_method=augment_method,\n augment_args=augment_args,\n )\n\n # TODO: v1.0 Move core logic to `trainer` level.\n # TODO: v1.0 Make it use leaderboard directly, allow to specify columns to include in the plot.\n # TODO: See if we can incorporate into tutorials (without causing crashes for users who try them)\n # Might require using a different tool than pygraphviz to avoid the apt-get commands\n # TODO: v1.0 Rename to `plot_model_graph`\n # TODO: v1.0 Maybe add ensemble weights to the edges.\n def plot_ensemble_model(\n self, model: str = \"best\", *, prune_unused_nodes: bool = True, filename: str = \"ensemble_model.png\"\n ) -> str:\n \"\"\"\n Output the visualized stack ensemble architecture of a model trained by `fit()`.\n The plot is stored to a file, `ensemble_model.png` in folder `predictor.path` (or by the name specified in `filename`)\n\n This function requires `graphviz` and `pygraphviz` to be installed because this visualization depends on those package.\n Unless this function will raise `ImportError` without being able to generate the visual of the ensemble model.\n\n To install the required package, run the below commands (for Ubuntu linux):\n\n $ sudo apt-get install graphviz graphviz-dev\n $ pip install pygraphviz\n\n For other platforms, refer to https://graphviz.org/ for Graphviz install, and https://pygraphviz.github.io/ for PyGraphviz.\n\n Parameters\n ----------\n model : str, default 'best'\n The model to highlight in golden orange, with all component models highlighted in yellow.\n If 'best', will default to the best model returned from `self.model_best`\n prune_unused_nodes : bool, default True\n If True, only plot the models that are components of the specified `model`.\n If False, will plot all models.\n filename : str, default 'ensemble_model.png'\n The filename to save the plot as. Will be located under the `self.path` folder.\n\n Returns\n -------\n The file name with the full path to the saved graphic on disk.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularDataset, TabularPredictor\n >>> train_data = TabularDataset('train.csv')\n >>> predictor = TabularPredictor(label='class').fit(train_data)\n >>> path_to_png = predictor.plot_ensemble_model()\n >>>\n >>> # To view the plot inside a Jupyter Notebook, use the below code:\n >>> from IPython.display import Image, display\n >>> display(Image(filename=path_to_png))\n\n \"\"\"\n self._assert_is_fit(\"plot_ensemble_model\")\n try:\n import pygraphviz # noqa: F401\n except ImportError:\n raise ImportError(\n \"Visualizing ensemble network architecture requires the `pygraphviz` library. \"\n \"Try `sudo apt-get install graphviz graphviz-dev` followed by `pip install pygraphviz` to install on Linux, \"\n \"or refer to the method docstring for detailed installation instructions for other operating systems.\"\n )\n\n G = self._trainer.model_graph.copy()\n\n primary_model = model\n if primary_model == \"best\":\n primary_model = self.model_best\n all_models = self.model_names()\n assert primary_model in all_models, f'Unknown model \"{primary_model}\"! Valid models: {all_models}'\n if prune_unused_nodes == True:\n models_to_keep = self._trainer.get_minimum_model_set(model=primary_model)\n G = nx.subgraph(G, models_to_keep)\n\n models = list(G.nodes)\n fit_times = self._trainer.get_models_attribute_full(models=models, attribute=\"fit_time\")\n predict_times = self._trainer.get_models_attribute_full(models=models, attribute=\"predict_time\")\n\n A = nx.nx_agraph.to_agraph(G)\n\n for node in A.iternodes():\n node_name = node.name\n fit_time = fit_times[node_name]\n predict_time = predict_times[node_name]\n if fit_time is None:\n fit_time_str = \"NaN\"\n else:\n fit_time_str = f\"{fit_time:.1f}s\"\n if predict_time is None:\n predict_time_str = \"NaN\"\n else:\n predict_time_str = f\"{predict_time:.2f}s\"\n\n node_val_score = node.attr[\"val_score\"]\n if node_val_score is None or (isinstance(node_val_score, str) and node_val_score == \"None\"):\n node_val_score_str = \"NaN\"\n else:\n node_val_score_str = f\"{float(node.attr['val_score']):.4f}\"\n label = (\n f\"{node.name}\"\n f\"\\nscore_val: {node_val_score_str}\"\n f\"\\nfit_time: {fit_time_str}\"\n f\"\\npred_time_val: {predict_time_str}\"\n )\n # Remove unnecessary attributes\n node.attr.clear()\n node.attr[\"label\"] = label\n\n A.graph_attr.update(rankdir=\"BT\")\n A.node_attr.update(fontsize=10)\n A.node_attr.update(shape=\"rectangle\")\n\n for node in A.iternodes():\n if node.name == primary_model:\n # Golden Orange\n node.attr[\"style\"] = \"filled\"\n node.attr[\"fillcolor\"] = \"#ff9900\"\n node.attr[\"shape\"] = \"box3d\"\n elif nx.has_path(G, node.name, primary_model):\n # Yellow\n node.attr[\"style\"] = \"filled\"\n node.attr[\"fillcolor\"] = \"#ffcc00\"\n # Else: White\n\n model_image_fname = os.path.join(self.path, filename)\n A.draw(model_image_fname, format=\"png\", prog=\"dot\")\n return model_image_fname\n\n @staticmethod\n def _summarize(key, msg, results):\n if key in results:\n print(msg + \": \" + str(results[key]))\n\n @staticmethod\n def _get_dataset(data, allow_nan: bool = False) -> pd.DataFrame | None:\n if data is None:\n if allow_nan:\n return data\n else:\n raise TypeError(\"data=None is invalid. data must be a pd.DataFrame or str file path to data\")\n elif isinstance(data, pd.DataFrame):\n return data\n elif isinstance(data, str):\n return TabularDataset(data)\n elif isinstance(data, pd.Series):\n raise TypeError(\n \"data must be a pd.DataFrame, not pd.Series. \\\n To predict on just single example (ith row of table), use data.iloc[[i]] rather than data.iloc[i]\"\n )\n else:\n raise TypeError(\"data must be a pd.DataFrame or str file path to data\")\n\n def _validate_hyperparameter_tune_kwargs(self, hyperparameter_tune_kwargs, time_limit=None):\n \"\"\"\n Returns True if hyperparameter_tune_kwargs is None or can construct a valid scheduler.\n Returns False if hyperparameter_tune_kwargs results in an invalid scheduler.\n \"\"\"\n if hyperparameter_tune_kwargs is None:\n return True\n\n scheduler_cls, scheduler_params = scheduler_factory(\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n time_out=time_limit,\n nthreads_per_trial=\"auto\",\n ngpus_per_trial=\"auto\",\n )\n\n if scheduler_params.get(\"dist_ip_addrs\", None):\n logger.warning(\n \"Warning: dist_ip_addrs does not currently work for Tabular. Distributed instances will not be utilized.\"\n )\n\n if scheduler_params[\"num_trials\"] == 1:\n logger.warning(\n \"Warning: Specified num_trials == 1 for hyperparameter tuning, disabling HPO. This can occur if time_limit was not specified in `fit()`.\"\n )\n return False\n\n scheduler_ngpus = scheduler_params[\"resource\"].get(\"num_gpus\", 0)\n if scheduler_ngpus is not None and isinstance(scheduler_ngpus, int) and scheduler_ngpus > 1:\n logger.warning(\n f\"Warning: TabularPredictor currently doesn't use >1 GPU per training run. Detected {scheduler_ngpus} GPUs.\"\n )\n\n return True\n\n def _set_hyperparameter_tune_kwargs_in_ag_args(self, hyperparameter_tune_kwargs, ag_args, time_limit):\n if hyperparameter_tune_kwargs is not None and \"hyperparameter_tune_kwargs\" not in ag_args:\n if \"hyperparameter_tune_kwargs\" in ag_args:\n AssertionError(\n \"hyperparameter_tune_kwargs was specified in both ag_args and in kwargs. Please only specify once.\"\n )\n else:\n ag_args[\"hyperparameter_tune_kwargs\"] = hyperparameter_tune_kwargs\n if ag_args.get(\"hyperparameter_tune_kwargs\", None) is not None:\n logger.log(\n 30, \"Warning: hyperparameter tuning is currently experimental and may cause the process to hang.\"\n )\n return ag_args\n\n def _set_post_fit_vars(self, learner: AbstractTabularLearner = None):\n if learner is not None:\n self._learner: AbstractTabularLearner = learner\n self._learner_type = type(self._learner)\n if self._learner.trainer_path is not None:\n self._learner.persist_trainer(low_memory=True)\n self._trainer: AbstractTabularTrainer = self._learner.load_trainer() # Trainer object\n\n @classmethod\n def _load_version_file(cls, path: str) -> str:\n \"\"\"\n Loads the version file that is part of the saved predictor artifact.\n The version file contains a string matching `predictor._learner.version`.\n\n Parameters\n ----------\n path: str\n The path that would be used to load the predictor via `predictor.load(path)`\n\n Returns\n -------\n The version of AutoGluon used to fit the predictor, as a string.\n\n \"\"\"\n version_file_path = os.path.join(path, cls._predictor_version_file_name)\n try:\n version = load_str.load(path=version_file_path)\n except:\n # Loads the old version file used in `autogluon.tabular<=1.1.0`, named `__version__`.\n # This file name was changed because Kaggle does not allow uploading files named `__version__`.\n version_file_path = os.path.join(path, \"__version__\")\n version = load_str.load(path=version_file_path)\n return version\n\n @classmethod\n def _load_metadata_file(cls, path: str, silent: bool = True):\n metadata_file_path = os.path.join(path, cls._predictor_metadata_file_name)\n return load_json.load(path=metadata_file_path, verbose=not silent)\n\n def _save_version_file(self, silent: bool = False):\n version_file_contents = f\"{__version__}\"\n version_file_path = os.path.join(self.path, self._predictor_version_file_name)\n save_str.save(path=version_file_path, data=version_file_contents, verbose=not silent)\n\n def _save_metadata_file(self, silent: bool = False):\n \"\"\"\n Save metadata json file to disk containing information such as\n python version, autogluon version, installed packages, operating system, etc.\n \"\"\"\n metadata_file_path = os.path.join(self.path, self._predictor_metadata_file_name)\n\n metadata = get_autogluon_metadata()\n\n save_json.save(path=metadata_file_path, obj=metadata)\n if not silent:\n logger.log(15, f\"Saving {metadata_file_path}\")\n\n def save(self, silent: bool = False):\n \"\"\"\n Save this Predictor to file in directory specified by this Predictor's `path`.\n Note that :meth:`TabularPredictor.fit` already saves the predictor object automatically\n (we do not recommend modifying the Predictor object yourself as it tracks many trained models).\n\n Parameters\n ----------\n silent : bool, default = False\n Whether to save without logging a message.\n \"\"\"\n path = self.path\n tmp_learner = self._learner\n tmp_trainer = self._trainer\n self._learner.save()\n self._learner = None\n self._trainer = None\n save_pkl.save(path=os.path.join(path, self.predictor_file_name), object=self)\n self._learner = tmp_learner\n self._trainer = tmp_trainer\n self._save_version_file(silent=silent)\n try:\n self._save_metadata_file(silent=silent)\n except Exception as e:\n logger.log(30, f\"Failed to save metadata file due to exception {e}, skipping...\")\n if not silent:\n logger.log(20, f'TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"{self.path}\")')\n\n @classmethod\n def _load(cls, path: str) -> \"TabularPredictor\":\n \"\"\"\n Inner load method, called in `load`.\n \"\"\"\n predictor: TabularPredictor = load_pkl.load(path=os.path.join(path, cls.predictor_file_name))\n learner = predictor._learner_type.load(path)\n predictor._set_post_fit_vars(learner=learner)\n return predictor\n\n @classmethod\n def load(\n cls,\n path: str,\n verbosity: int = None,\n require_version_match: bool = True,\n require_py_version_match: bool = True,\n check_packages: bool = False,\n ) -> \"TabularPredictor\":\n \"\"\"\n Load a TabularPredictor object previously produced by `fit()` from file and returns this object. It is highly recommended the predictor be loaded with the exact AutoGluon version it was fit with.\n\n .. warning::\n\n :meth:`autogluon.tabular.TabularPredictor.load` uses `pickle` module implicitly, which is known to\n be insecure. It is possible to construct malicious pickle data which will execute arbitrary code during\n unpickling. Never load data that could have come from an untrusted source, or that could have been tampered\n with. **Only load data you trust.**\n\n Parameters\n ----------\n path : str\n The path to directory in which this Predictor was previously saved.\n verbosity : int, default = None\n Sets the verbosity level of this Predictor after it is loaded.\n Valid values range from 0 (least verbose) to 4 (most verbose).\n If None, logging verbosity is not changed from existing values.\n Specify larger values to see more information printed when using Predictor during inference, smaller values to see less information.\n Refer to TabularPredictor init for more information.\n require_version_match : bool, default = True\n If True, will raise an AssertionError if the `autogluon.tabular` version of the loaded predictor does not match the installed version of `autogluon.tabular`.\n If False, will allow loading of models trained on incompatible versions, but is NOT recommended. Users may run into numerous issues if attempting this.\n require_py_version_match : bool, default = True\n If True, will raise an AssertionError if the Python version of the loaded predictor does not match the installed Python version.\n Micro version differences such as 3.9.2 and 3.9.7 will log a warning but will not raise an exception.\n If False, will allow loading of models trained on incompatible python versions, but is NOT recommended. Users may run into numerous issues if attempting this.\n check_packages : bool, default = False\n If True, checks package versions of the loaded predictor against the package versions of the current environment.\n Warnings will be logged for each mismatch of package version.\n\n Returns\n -------\n predictor : TabularPredictor\n\n Examples\n --------\n >>> predictor = TabularPredictor.load(path_to_predictor)\n\n \"\"\"\n if verbosity is not None:\n set_logger_verbosity(verbosity) # Reset logging after load (could be in new Python session)\n if path is None:\n raise ValueError(\"path cannot be None in load()\")\n\n try:\n from ..version import __version__\n\n version_current = __version__\n except:\n version_current = None\n\n path = setup_outputdir(path, warn_if_exist=False) # replace ~ with absolute path if it exists\n try:\n version_saved = cls._load_version_file(path=path)\n except:\n logger.warning(\n f'WARNING: Could not find version file at \"{os.path.join(path, cls._predictor_version_file_name)}\".\\n'\n f\"This means that the predictor was fit in an AutoGluon version `<=0.3.1`.\"\n )\n version_saved = None\n\n if version_saved is None:\n predictor = cls._load(path=path)\n try:\n version_saved = predictor._learner.version\n except:\n version_saved = None\n else:\n predictor = None\n if version_saved is None:\n version_saved = \"Unknown (Likely <=0.0.11)\"\n\n check_saved_predictor_version(\n version_current=version_current,\n version_saved=version_saved,\n require_version_match=require_version_match,\n logger=logger,\n )\n\n try:\n metadata_init = cls._load_metadata_file(path=path)\n except:\n logger.warning(\n f'WARNING: Could not find metadata file at \"{os.path.join(path, cls._predictor_metadata_file_name)}\".\\n'\n f\"This could mean that the predictor was fit in a version `<=0.5.2`.\"\n )\n metadata_init = None\n\n metadata_load = get_autogluon_metadata()\n\n if metadata_init is not None:\n try:\n compare_autogluon_metadata(\n original=metadata_init, current=metadata_load, check_packages=check_packages\n )\n except:\n logger.log(30, \"WARNING: Exception raised while comparing metadata files, skipping comparison...\")\n if require_py_version_match:\n if metadata_init[\"py_version\"] != metadata_load[\"py_version\"]:\n raise AssertionError(\n f\"Predictor was created on Python version {metadata_init['py_version']} \"\n f\"but is being loaded with Python version {metadata_load['py_version']}. \"\n f\"Please ensure the versions match to avoid instability. While it is NOT recommended, \"\n f\"this error can be bypassed by specifying `require_py_version_match=False`.\"\n )\n\n if predictor is None:\n predictor = cls._load(path=path)\n\n return predictor\n\n @classmethod\n def load_log(cls, predictor_path: str = None, log_file_path: Optional[str] = None) -> list[str]:\n \"\"\"\n Load log files of a predictor\n\n Parameters\n ----------\n predictor_path: Optional[str], default = None\n Path to the predictor to load the log.\n This can be used when the predictor was initialized with `log_file_path=\"auto\"` to fetch the log file automatically\n log_file_path: Optional[str], default = None\n Path to the log file.\n If you specified a `log_file_path` while initializing the predictor, you should use `log_file_path` to load the log file instead.\n At least one of `predictor_path` or `log_file_path` must to be specified\n\n Returns\n -------\n list[str]\n A list containing lines of the log file\n \"\"\"\n file_path = log_file_path\n if file_path is None:\n assert predictor_path is not None, (\n \"Please either provide `predictor_path` or `log_file_path` to load the log file\"\n )\n file_path = os.path.join(predictor_path, \"logs\", cls._predictor_log_file_name)\n assert os.path.isfile(file_path), f\"Log file does not exist at {file_path}\"\n lines = []\n with open(file_path, \"r\") as f:\n lines = f.readlines()\n return lines\n\n def _setup_log_to_file(self, log_file_path: str):\n if log_file_path == \"auto\":\n log_file_path = os.path.join(self.path, \"logs\", self._predictor_log_file_name)\n log_file_path = os.path.abspath(os.path.normpath(log_file_path))\n os.makedirs(os.path.dirname(log_file_path), exist_ok=True)\n add_log_to_file(log_file_path)\n\n @staticmethod\n def _validate_init_kwargs(kwargs):\n valid_kwargs = {\n \"learner_type\",\n \"learner_kwargs\",\n \"quantile_levels\",\n \"default_base_path\",\n }\n invalid_keys = []\n for key in kwargs:\n if key not in valid_kwargs:\n invalid_keys.append(key)\n if invalid_keys:\n raise ValueError(f\"Invalid kwargs passed: {invalid_keys}\\nValid kwargs: {list(valid_kwargs)}\")\n\n def _validate_fit_kwargs(self, *, kwargs: dict) -> dict:\n # TODO:\n # Valid core_kwargs values:\n # ag_args, ag_args_fit, ag_args_ensemble, stack_name, ensemble_type, name_suffix, time_limit\n # Valid aux_kwargs values:\n # name_suffix, time_limit, stack_name, aux_hyperparameters, ag_args, ag_args_ensemble, fit_weighted_ensemble\n\n # TODO: Remove features from models option for fit_extra\n # TODO: Constructor?\n fit_kwargs_default = dict(\n # data split / ensemble architecture kwargs -> Don't nest but have nested documentation -> Actually do nesting\n holdout_frac=None, # TODO: Potentially error if num_bag_folds is also specified\n num_bag_folds=None,\n # TODO: Potentially move to fit_extra, raise exception if value too large / invalid in fit_extra.\n auto_stack=False,\n use_bag_holdout=False,\n # other\n feature_generator=\"auto\",\n unlabeled_data=None,\n _feature_generator_kwargs=None,\n # learning curves and test data (for logging purposes only)\n learning_curves=False,\n test_data=None,\n raise_on_model_failure=False,\n # experimental\n _experimental_dynamic_hyperparameters=False,\n )\n kwargs, ds_valid_keys = self._sanitize_dynamic_stacking_kwargs(kwargs)\n kwargs = self._validate_fit_extra_kwargs(\n kwargs, extra_valid_keys=list(fit_kwargs_default.keys()) + ds_valid_keys\n )\n kwargs_sanitized = fit_kwargs_default.copy()\n kwargs_sanitized.update(kwargs)\n\n return kwargs_sanitized\n\n def _validate_calibrate_decision_threshold(self, calibrate_decision_threshold: bool | str):\n valid_calibrate_decision_threshold_options = [True, False, \"auto\"]\n if calibrate_decision_threshold not in valid_calibrate_decision_threshold_options:\n raise ValueError(\n f\"`calibrate_decision_threshold` must be a value in \"\n f\"{valid_calibrate_decision_threshold_options}, but is: {calibrate_decision_threshold}\"\n )\n\n def _validate_num_cpus(self, num_cpus: int | str):\n if num_cpus is None:\n raise ValueError(f\"`num_cpus` must be an int or 'auto'. Value: {num_cpus}\")\n if isinstance(num_cpus, str):\n if num_cpus != \"auto\":\n raise ValueError(f\"`num_cpus` must be an int or 'auto'. Value: {num_cpus}\")\n elif not isinstance(num_cpus, int):\n raise TypeError(f\"`num_cpus` must be an int or 'auto'. Found: {type(num_cpus)} | Value: {num_cpus}\")\n else:\n if num_cpus < 1:\n raise ValueError(f\"`num_cpus` must be greater than or equal to 1. (num_cpus={num_cpus})\")\n\n def _validate_num_gpus(self, num_gpus: int | float | str):\n if num_gpus is None:\n raise ValueError(f\"`num_gpus` must be an int, float, or 'auto'. Value: {num_gpus}\")\n if isinstance(num_gpus, str):\n if num_gpus != \"auto\":\n raise ValueError(f\"`num_gpus` must be an int, float, or 'auto'. Value: {num_gpus}\")\n elif not isinstance(num_gpus, (int, float)):\n raise TypeError(\n f\"`num_gpus` must be an int, float, or 'auto'. Found: {type(num_gpus)} | Value: {num_gpus}\"\n )\n else:\n if num_gpus < 0:\n raise ValueError(f\"`num_gpus` must be greater than or equal to 0. (num_gpus={num_gpus})\")\n\n def _validate_and_set_memory_limit(self, memory_limit: float | str):\n if memory_limit is None:\n raise ValueError(f\"`memory_limit` must be an int, float, or 'auto'. Value: {memory_limit}\")\n if isinstance(memory_limit, str):\n if memory_limit != \"auto\":\n raise ValueError(f\"`memory_limit` must be an int, float, or 'auto'. Value: {memory_limit}\")\n elif not isinstance(memory_limit, (int, float)):\n raise TypeError(\n f\"`memory_limit` must be an int, float, or 'auto'. Found: {type(memory_limit)} | Value: {memory_limit}\"\n )\n else:\n if memory_limit <= 0:\n raise ValueError(f\"`memory_limit` must be greater than 0. (memory_limit={memory_limit})\")\n\n if memory_limit != \"auto\":\n logger.log(20, f\"Enforcing custom memory (soft) limit of {memory_limit} GB!\")\n os.environ[\"AG_MEMORY_LIMIT_IN_GB\"] = str(memory_limit)\n\n def _validate_fit_strategy(self, fit_strategy: str):\n valid_values = [\"sequential\", \"parallel\"]\n if fit_strategy not in valid_values:\n raise ValueError(f\"fit_strategy must be one of {valid_values}. Value: {fit_strategy}\")\n\n def _fit_extra_kwargs_dict(self) -> dict:\n \"\"\"\n Returns:\n --------\n dict of fit_extra args:\n verbosity: Which levels of logger should be printed\n pseudo_data: pseudo labeled data to be incorporated into train\n but not used in validation\n name_suffix: A suffix string to be added to the individual model names\n \"\"\"\n return dict(\n # data split / ensemble architecture kwargs -> Don't nest but have nested documentation -> Actually do nesting\n num_bag_sets=None,\n delay_bag_sets=False,\n num_stack_levels=None,\n hyperparameter_tune_kwargs=None,\n # core_kwargs -> +1 nest\n ag_args=None,\n ag_args_fit=None,\n ag_args_ensemble=None,\n included_model_types=None,\n excluded_model_types=None,\n # aux_kwargs -> +1 nest\n # post_fit_kwargs -> +1 nest\n set_best_to_refit_full=False,\n keep_only_best=False,\n save_space=False,\n refit_full=False,\n save_bag_folds=None,\n # other\n verbosity=self.verbosity,\n feature_prune_kwargs=None,\n raise_on_no_models_fitted=True,\n # private\n _save_bag_folds=None,\n calibrate=\"auto\",\n # pseudo label\n pseudo_data=None,\n name_suffix=None,\n )\n\n @staticmethod\n def _sanitize_dynamic_stacking_kwargs(kwargs: dict) -> tuple[dict, list[str]]:\n ds_kwargs_key = \"ds_args\"\n ds_args = dict(\n validation_procedure=\"holdout\",\n detection_time_frac=1 / 4,\n holdout_frac=1 / 9,\n n_folds=2,\n n_repeats=1,\n memory_safe_fits=\"auto\",\n clean_up_fits=True,\n holdout_data=None,\n enable_ray_logging=True,\n enable_callbacks=True,\n )\n allowed_kes = set(ds_args.keys())\n\n if ds_kwargs_key in kwargs:\n kwargs[ds_kwargs_key] = copy.deepcopy(kwargs[ds_kwargs_key])\n ds_args.update(kwargs[ds_kwargs_key])\n\n key_mismatch = set(ds_args.keys()) - allowed_kes\n if key_mismatch:\n raise ValueError(f\"Got invalid keys for `ds_args`. Allowed: {allowed_kes}. Got: {key_mismatch}\")\n if (\"validation_procedure\" in ds_args) and (\n (not isinstance(ds_args[\"validation_procedure\"], str))\n or (ds_args[\"validation_procedure\"] not in [\"holdout\", \"cv\"])\n ):\n raise ValueError(\n \"`validation_procedure` in `ds_args` must be str in {'holdout','cv'}. \"\n + f\"Got: {ds_args['validation_procedure']}\"\n )\n for arg_name in [\"clean_up_fits\", \"enable_ray_logging\"]:\n if (arg_name in ds_args) and (not isinstance(ds_args[arg_name], bool)):\n raise ValueError(f\"`{arg_name}` in `ds_args` must be bool. Got: {type(ds_args[arg_name])}\")\n if \"memory_safe_fits\" in ds_args and not isinstance(ds_args[\"memory_safe_fits\"], (bool, str)):\n raise ValueError(\n f\"`memory_safe_fits` in `ds_args` must be bool or 'auto'. Got: {type(ds_args['memory_safe_fits'])}\"\n )\n for arg_name in [\"detection_time_frac\", \"holdout_frac\"]:\n if (arg_name in ds_args) and (\n (not isinstance(ds_args[arg_name], float)) or (ds_args[arg_name] >= 1) or (ds_args[arg_name] <= 0)\n ):\n raise ValueError(\n f\"`{arg_name}` in `ds_args` must be float in (0,1). Got: {type(ds_args[arg_name])}, {ds_args[arg_name]}\"\n )\n if (\"n_folds\" in ds_args) and ((not isinstance(ds_args[\"n_folds\"], int)) or (ds_args[\"n_folds\"] < 2)):\n raise ValueError(\n f\"`n_folds` in `ds_args` must be int in [2, +inf). Got: {type(ds_args['n_folds'])}, {ds_args['n_folds']}\"\n )\n if (\"n_repeats\" in ds_args) and ((not isinstance(ds_args[\"n_repeats\"], int)) or (ds_args[\"n_repeats\"] < 1)):\n raise ValueError(\n f\"`n_repeats` in `ds_args` must be int in [1, +inf). Got: {type(ds_args['n_repeats'])}, {ds_args['n_repeats']}\"\n )\n if (\n (\"holdout_data\" in ds_args)\n and (not isinstance(ds_args[\"holdout_data\"], (str, pd.DataFrame)))\n and (ds_args[\"holdout_data\"] is not None)\n ):\n raise ValueError(\n f\"`holdout_data` in `ds_args` must be None, str, or pd.DataFrame. Got: {type(ds_args['holdout_data'])}\"\n )\n if (ds_args[\"validation_procedure\"] == \"cv\") and (ds_args[\"holdout_data\"] is not None):\n raise ValueError(\n \"`validation_procedure` in `ds_args` is 'cv' but `holdout_data` in `ds_args` is specified.\"\n \"You must decide for either (repeated) cross-validation or holdout validation.\"\n )\n kwargs[ds_kwargs_key] = ds_args\n return kwargs, [ds_kwargs_key]\n\n def _validate_fit_extra_kwargs(self, kwargs: dict, extra_valid_keys: list[str] | None = None):\n fit_extra_kwargs_default = self._fit_extra_kwargs_dict()\n\n allowed_kwarg_names = list(fit_extra_kwargs_default.keys())\n if extra_valid_keys is not None:\n allowed_kwarg_names += extra_valid_keys\n for kwarg_name in kwargs.keys():\n if kwarg_name not in allowed_kwarg_names:\n public_kwarg_options = [kwarg for kwarg in allowed_kwarg_names if kwarg[0] != \"_\"]\n public_kwarg_options.sort()\n raise ValueError(\n f\"Unknown `.fit` keyword argument specified: '{kwarg_name}'\\nValid kwargs: {public_kwarg_options}\"\n )\n\n kwargs_sanitized = fit_extra_kwargs_default.copy()\n kwargs_sanitized.update(kwargs)\n\n # Deepcopy args to avoid altering outer context\n deepcopy_args = [\"ag_args\", \"ag_args_fit\", \"ag_args_ensemble\", \"included_model_types\", \"excluded_model_types\"]\n for deepcopy_arg in deepcopy_args:\n kwargs_sanitized[deepcopy_arg] = copy.deepcopy(kwargs_sanitized[deepcopy_arg])\n\n refit_full = kwargs_sanitized[\"refit_full\"]\n set_best_to_refit_full = kwargs_sanitized[\"set_best_to_refit_full\"]\n if refit_full and not self._learner.cache_data:\n raise ValueError(\n \"`refit_full=True` is only available when `cache_data=True`. Set `cache_data=True` to utilize `refit_full`.\"\n )\n if set_best_to_refit_full and not refit_full:\n raise ValueError(\n \"`set_best_to_refit_full=True` is only available when `refit_full=True`. Set `refit_full=True` to utilize `set_best_to_refit_full`.\"\n )\n valid_calibrate_options = [True, False, \"auto\"]\n calibrate = kwargs_sanitized[\"calibrate\"]\n if calibrate not in valid_calibrate_options:\n raise ValueError(f\"`calibrate` must be a value in {valid_calibrate_options}, but is: {calibrate}\")\n\n return kwargs_sanitized\n\n def _prune_data_features(self, train_features: list, other_features: list, is_labeled: bool) -> tuple[list, list]:\n \"\"\"\n Removes certain columns from the provided datasets that do not contain predictive features.\n\n Parameters\n ----------\n train_features : list\n The features/columns for the incoming training data\n other_features : list\n Features of other auxiliary data that contains the same covariates as the training data.\n Examples of this could be: tuning data, pseudo data\n is_labeled: bool\n Is other_features dataframe labeled or not\n \"\"\"\n if self.sample_weight is not None:\n if self.sample_weight in train_features:\n train_features.remove(self.sample_weight)\n if self.sample_weight in other_features:\n other_features.remove(self.sample_weight)\n if self._learner.groups is not None and is_labeled:\n train_features.remove(self._learner.groups)\n\n return train_features, other_features\n\n def _validate_fit_data(\n self,\n train_data: str | pd.DataFrame,\n tuning_data: str | pd.DataFrame | None = None,\n test_data: str | pd.DataFrame | None = None,\n unlabeled_data: str | pd.DataFrame | None = None,\n ) -> tuple[pd.DataFrame, pd.DataFrame | None, pd.DataFrame | None]:\n if isinstance(train_data, str):\n train_data = TabularDataset(train_data)\n if tuning_data is not None and isinstance(tuning_data, str):\n tuning_data = TabularDataset(tuning_data)\n if test_data is not None and isinstance(test_data, str):\n test_data = TabularDataset(test_data)\n if unlabeled_data is not None and isinstance(unlabeled_data, str):\n unlabeled_data = TabularDataset(unlabeled_data)\n\n if not isinstance(train_data, pd.DataFrame):\n raise AssertionError(\n f\"train_data is required to be a pandas DataFrame, but was instead: {type(train_data)}\"\n )\n\n if len(set(train_data.columns)) < len(train_data.columns):\n raise ValueError(\n \"Column names are not unique, please change duplicated column names (in pandas: train_data.rename(columns={'current_name':'new_name'})\"\n )\n\n self._validate_single_fit_dataset(\n train_data=train_data, other_data=tuning_data, name=\"tuning_data\", is_labeled=True\n )\n self._validate_single_fit_dataset(\n train_data=train_data, other_data=test_data, name=\"test_data\", is_labeled=True\n )\n self._validate_single_fit_dataset(\n train_data=train_data, other_data=unlabeled_data, name=\"unlabeled_data\", is_labeled=False\n )\n\n return train_data, tuning_data, test_data, unlabeled_data\n\n def _validate_single_fit_dataset(\n self, train_data: pd.DataFrame, other_data: pd.DataFrame, name: str, is_labeled: bool = True\n ):\n \"\"\"\n Validates additional dataset, ensuring format is consistent with train dataset.\n\n Parameters:\n -----------\n train_data : DataFrame\n training set dataframe\n other_data : DataFrame\n additional data set\n name : str\n name of additional data set\n is_labeled : bool\n whether the other_data is labeled\n\n Returns:\n --------\n None\n \"\"\"\n if other_data is not None:\n if not isinstance(other_data, pd.DataFrame):\n raise AssertionError(\n f\"{name} is required to be a pandas DataFrame, but was instead: {type(other_data)}\"\n )\n self._validate_unique_indices(data=other_data, name=name)\n train_features = [column for column in train_data.columns if column != self.label]\n other_features = [column for column in other_data.columns if column != self.label]\n train_features, other_features = self._prune_data_features(\n train_features=train_features, other_features=other_features, is_labeled=is_labeled\n )\n train_features = np.array(train_features)\n other_features = np.array(other_features)\n if np.any(train_features != other_features):\n raise ValueError(f\"Column names must match between train_data and {name}\")\n\n if self.label in other_data:\n train_label_type = train_data[self.label].dtype\n other_label_type = other_data[self.label].dtype\n\n if train_label_type != other_label_type:\n logger.warning(\n f\"WARNING: train_data and {name} have mismatched label column dtypes! \"\n f\"train_label_type={train_label_type}, {name}_type={other_label_type}.\\n\"\n f\"\\tYou should ensure the dtypes match to avoid bugs or instability.\\n\"\n f\"\\tAutoGluon will attempt to convert the dtypes to align.\"\n )\n\n def _initialize_learning_curve_params(\n self, learning_curves: dict | bool | None = None, problem_type: str | None = None\n ) -> dict:\n \"\"\"\n Convert users learning_curve dict parameters into ag_param format.\n Also, converts all metrics into list of autogluon Scorer objects.\n\n Parameters:\n -----------\n learning_curves : bool | dict | None\n If bool, whether to generate learning curves.\n If dict, the dictionary of learning_curves parameters passed into predictor from the user.\n If None, will not generate curves.\n problem_type : str | None\n The current problem type.\n\n Returns:\n --------\n params : dict\n The learning curves parameters in ag_params format.\n \"\"\"\n if learning_curves is None or learning_curves == False:\n return {}\n elif type(learning_curves) != dict and type(learning_curves) != bool:\n raise ValueError(\"Learning curves parameter must be a boolean or dict!\")\n\n # metrics defaults to self.eval_metric if not specified\n metrics = None\n use_error = False\n\n if isinstance(learning_curves, dict):\n if \"metrics\" in learning_curves:\n metrics = learning_curves[\"metrics\"]\n if not isinstance(metrics, list):\n metrics = [metrics]\n\n names, scorers = [], []\n for metric in metrics:\n if isinstance(metric, str):\n names.append(metric)\n elif isinstance(metric, Scorer):\n scorers.append(metric)\n\n names = [get_metric(name, problem_type, \"eval_metric\") for name in names]\n metrics = names + scorers\n\n # check for duplicate metrics / aliases\n all_metric_names = [metric.name for metric in metrics]\n if len(set(all_metric_names)) != len(all_metric_names):\n from collections import Counter\n\n counts = {metric: count for metric, count in Counter(all_metric_names).items() if count > 1}\n raise ValueError(f\"The following learning curve metrics appeared more than once: {counts}\")\n\n if \"use_error\" in learning_curves:\n use_error = learning_curves[\"use_error\"]\n\n params = {\n \"ag.generate_curves\": True,\n \"ag.use_error_for_curve_metrics\": use_error,\n }\n\n if metrics:\n params[\"ag.curve_metrics\"] = metrics\n\n return params\n\n @staticmethod\n def _validate_unique_indices(data: pd.DataFrame, name: str):\n is_duplicate_index = data.index.duplicated(keep=False)\n if is_duplicate_index.any():\n duplicate_count = is_duplicate_index.sum()\n raise AssertionError(\n f\"{name} contains {duplicate_count} duplicated indices. \"\n \"Please ensure DataFrame indices are unique.\\n\"\n f\"\\tYou can identify the indices which are duplicated via `{name}.index.duplicated(keep=False)`\"\n )\n\n @staticmethod\n def _validate_infer_limit(infer_limit: float, infer_limit_batch_size: int) -> tuple[float, int]:\n if infer_limit_batch_size is not None:\n if not isinstance(infer_limit_batch_size, int):\n raise ValueError(\n f\"infer_limit_batch_size must be type int, but was instead type {type(infer_limit_batch_size)}\"\n )\n elif infer_limit_batch_size < 1:\n raise AssertionError(f\"infer_limit_batch_size must be >=1, value: {infer_limit_batch_size}\")\n if infer_limit is not None:\n if not isinstance(infer_limit, (int, float)):\n raise ValueError(f\"infer_limit must be type int or float, but was instead type {type(infer_limit)}\")\n if infer_limit <= 0:\n raise AssertionError(f\"infer_limit must be greater than zero! (infer_limit={infer_limit})\")\n if infer_limit is not None and infer_limit_batch_size is None:\n infer_limit_batch_size = 10000\n logger.log(\n 20,\n f\"infer_limit specified, but infer_limit_batch_size was not specified. Setting infer_limit_batch_size={infer_limit_batch_size}\",\n )\n return infer_limit, infer_limit_batch_size\n\n def _set_feature_generator(self, feature_generator=\"auto\", feature_metadata=None, init_kwargs=None):\n if self._learner.feature_generator is not None:\n if isinstance(feature_generator, str) and feature_generator == \"auto\":\n feature_generator = self._learner.feature_generator\n else:\n raise AssertionError(\"FeatureGenerator already exists!\")\n self._learner.feature_generator = get_default_feature_generator(\n feature_generator=feature_generator, feature_metadata=feature_metadata, init_kwargs=init_kwargs\n )\n\n def _sanitize_stack_args(\n self,\n num_bag_folds: int,\n num_bag_sets: int,\n num_stack_levels: int,\n num_train_rows: int,\n dynamic_stacking: bool | str,\n use_bag_holdout: bool | str,\n use_bag_holdout_was_auto: bool,\n dynamic_stacking_was_auto: bool,\n ):\n if not isinstance(num_bag_folds, int):\n raise ValueError(f\"num_bag_folds must be an integer. (num_bag_folds={num_bag_folds})\")\n if not isinstance(num_stack_levels, int):\n raise ValueError(f\"num_stack_levels must be an integer. (num_stack_levels={num_stack_levels})\")\n if num_bag_folds < 2 and num_bag_folds != 0:\n raise ValueError(f\"num_bag_folds must be equal to 0 or >=2. (num_bag_folds={num_bag_folds})\")\n if num_stack_levels != 0 and num_bag_folds == 0:\n raise ValueError(\n f\"num_stack_levels must be 0 if num_bag_folds is 0. (num_stack_levels={num_stack_levels}, num_bag_folds={num_bag_folds})\"\n )\n if not isinstance(num_bag_sets, int):\n raise ValueError(f\"num_bag_sets must be an integer. (num_bag_sets={num_bag_sets})\")\n if not isinstance(dynamic_stacking, bool):\n raise ValueError(f\"dynamic_stacking must be a bool. (dynamic_stacking={dynamic_stacking})\")\n if not isinstance(use_bag_holdout, bool):\n raise ValueError(f\"use_bag_holdout must be a bool. (use_bag_holdout={use_bag_holdout})\")\n\n if use_bag_holdout_was_auto and num_bag_folds != 0:\n if use_bag_holdout:\n log_extra = (\n f\"Reason: num_train_rows >= {USE_BAG_HOLDOUT_AUTO_THRESHOLD}. (num_train_rows={num_train_rows})\"\n )\n else:\n log_extra = (\n f\"Reason: num_train_rows < {USE_BAG_HOLDOUT_AUTO_THRESHOLD}. (num_train_rows={num_train_rows})\"\n )\n logger.log(20, f\"Setting use_bag_holdout from 'auto' to {use_bag_holdout}. {log_extra}\")\n\n if dynamic_stacking and num_stack_levels < 1:\n log_extra_ds = f\"Reason: Stacking is not enabled. (num_stack_levels={num_stack_levels})\"\n if not dynamic_stacking_was_auto:\n logger.log(20, f\"Forcing dynamic_stacking to False. {log_extra_ds}\")\n dynamic_stacking = False\n elif dynamic_stacking_was_auto:\n if dynamic_stacking:\n log_extra_ds = f\"Reason: Enable dynamic_stacking when use_bag_holdout is disabled. (use_bag_holdout={use_bag_holdout})\"\n else:\n log_extra_ds = f\"Reason: Skip dynamic_stacking when use_bag_holdout is enabled. (use_bag_holdout={use_bag_holdout})\"\n logger.log(20, f\"Setting dynamic_stacking from 'auto' to {dynamic_stacking}. {log_extra_ds}\")\n\n return num_bag_folds, num_bag_sets, num_stack_levels, dynamic_stacking, use_bag_holdout\n\n # TODO: Add .delete() method to easily clean-up clones?\n # Would need to be careful that user doesn't delete important things accidentally.\n # TODO: Add .save_zip() and load_zip() methods to pack and unpack artifacts into a single file to simplify deployment code?\n def clone(self, path: str, *, return_clone: bool = False, dirs_exist_ok: bool = False) -> str | \"TabularPredictor\":\n \"\"\"\n Clone the predictor and all of its artifacts to a new location on local disk.\n This is ideal for use-cases where saving a snapshot of the predictor is desired before performing\n more advanced operations (such as fit_extra and refit_full).\n\n Parameters\n ----------\n path : str\n Directory path the cloned predictor will be saved to.\n return_clone : bool, default = False\n If True, returns the loaded cloned TabularPredictor object.\n If False, returns the local path to the cloned TabularPredictor object.\n dirs_exist_ok : bool, default = False\n If True, will clone the predictor even if the path directory already exists, potentially overwriting unrelated files.\n If False, will raise an exception if the path directory already exists and avoid performing the copy.\n\n Returns\n -------\n If return_clone == True, returns the loaded cloned TabularPredictor object.\n If return_clone == False, returns the local path to the cloned TabularPredictor object.\n\n \"\"\"\n assert path != self.path, f\"Cannot clone into the same directory as the original predictor! (path='{path}')\"\n path_clone = shutil.copytree(src=self.path, dst=path, dirs_exist_ok=dirs_exist_ok)\n logger.log(\n 30,\n f\"Cloned {self.__class__.__name__} located in '{self.path}' to '{path_clone}'.\\n\"\n f'\\tTo load the cloned predictor: predictor_clone = {self.__class__.__name__}.load(path=\"{path_clone}\")',\n )\n return self.__class__.load(path=path_clone) if return_clone else path_clone\n\n def clone_for_deployment(\n self, path: str, *, model: str = \"best\", return_clone: bool = False, dirs_exist_ok: bool = False\n ) -> str | \"TabularPredictor\":\n \"\"\"\n Clone the predictor and all of its artifacts to a new location on local disk,\n then delete the clones artifacts unnecessary during prediction.\n This is ideal for use-cases where saving a snapshot of the predictor is desired before performing\n more advanced operations (such as fit_extra and refit_full).\n\n Note that the clone can no longer fit new models,\n and most functionality except for predict and predict_proba will no longer work.\n\n Identical to performing the following operations in order:\n\n predictor_clone = predictor.clone(path=path, return_clone=True, dirs_exist_ok=dirs_exist_ok)\n predictor_clone.delete_models(models_to_keep=model)\n predictor_clone.set_model_best(model=model, save_trainer=True)\n predictor_clone.save_space()\n\n Parameters\n ----------\n path : str\n Directory path the cloned predictor will be saved to.\n model : str, default = 'best'\n The model to use in the optimized predictor clone.\n All other unrelated models will be deleted to save disk space.\n Refer to the `models_to_keep` argument of `predictor.delete_models` for available options.\n Internally calls `predictor_clone.delete_models(models_to_keep=model)`\n return_clone : bool, default = False\n If True, returns the loaded cloned TabularPredictor object.\n If False, returns the local path to the cloned TabularPredictor object.\n dirs_exist_ok : bool, default = False\n If True, will clone the predictor even if the path directory already exists, potentially overwriting unrelated files.\n If False, will raise an exception if the path directory already exists and avoids performing the copy.\n\n Returns\n -------\n If return_clone == True, returns the loaded cloned TabularPredictor object.\n If return_clone == False, returns the local path to the cloned TabularPredictor object.\n \"\"\"\n predictor_clone = self.clone(path=path, return_clone=True, dirs_exist_ok=dirs_exist_ok)\n if model == \"best\":\n model = predictor_clone.model_best\n logger.log(30, f\"Clone: Keeping minimum set of models required to predict with best model '{model}'...\")\n else:\n logger.log(30, f\"Clone: Keeping minimum set of models required to predict with model '{model}'...\")\n predictor_clone.delete_models(models_to_keep=model, dry_run=False)\n if isinstance(model, str) and model in predictor_clone.model_names(can_infer=True):\n predictor_clone.set_model_best(model=model, save_trainer=True)\n logger.log(\n 30,\n \"Clone: Removing artifacts unnecessary for prediction. \"\n \"NOTE: Clone can no longer fit new models, and most functionality except for predict and predict_proba will no longer work\",\n )\n predictor_clone.save_space()\n return predictor_clone if return_clone else predictor_clone.path\n\n def simulation_artifact(self, test_data: pd.DataFrame = None) -> dict:\n \"\"\"\n [Advanced] Computes and returns the necessary information to perform zeroshot HPO simulation.\n For a usage example, refer to https://github.com/autogluon/tabrepo/blob/main/examples/run_quickstart_from_scratch.py\n\n Parameters\n ----------\n test_data: pd.DataFrame, default = None\n The test data to predict with.\n If None, the keys `pred_proba_dict_test` and `y_test` will not be present in the output.\n\n Returns\n -------\n simulation_dict: dict\n The dictionary of information required for zeroshot HPO simulation.\n Keys are as follows:\n pred_proba_dict_val: Dictionary of model name to prediction probabilities (or predictions if regression) on the validation data\n pred_proba_dict_test: Dictionary of model name to prediction probabilities (or predictions if regression) on the test data\n y_val: Pandas Series of ground truth labels for the validation data (internal representation)\n y_test: Pandas Series of ground truth labels for the test data (internal representation)\n eval_metric: The string name of the evaluation metric (obtained via `predictor.eval_metric.name`)\n problem_type: The problem type (obtained via `predictor.problem_type`)\n problem_type_transform: The transformed (internal) problem type (obtained via `predictor._learner.label_cleaner.problem_type_transform,`)\n ordered_class_labels: The original class labels (`predictor._learner.label_cleaner.ordered_class_labels`)\n ordered_Class_labels_transformed: The transformed (internal) class labels (`predictor._learner.label_cleaner.ordered_class_labels_transformed`)\n num_classes: The number of internal classes (`self._learner.label_cleaner.num_classes`)\n label: The label column name (`predictor.label`)\n \"\"\"\n models = self.model_names(can_infer=True)\n\n pred_proba_dict_test = None\n if self.can_predict_proba:\n pred_proba_dict_val = self.predict_proba_multi(inverse_transform=False, as_multiclass=False, models=models)\n if test_data is not None:\n pred_proba_dict_test = self.predict_proba_multi(\n test_data, inverse_transform=False, as_multiclass=False, models=models\n )\n else:\n pred_proba_dict_val = self.predict_multi(inverse_transform=False, models=models)\n if test_data is not None:\n pred_proba_dict_test = self.predict_multi(test_data, inverse_transform=False, models=models)\n\n val_data_source = \"val\" if self.has_val else \"train\"\n _, y_val = self.load_data_internal(data=val_data_source, return_X=False, return_y=True)\n if test_data is not None:\n y_test = test_data[self.label]\n y_test = self.transform_labels(y_test, inverse=False)\n test_info = dict(\n pred_proba_dict_test=pred_proba_dict_test,\n y_test=y_test,\n )\n else:\n test_info = dict()\n\n simulation_dict = dict(\n **test_info,\n pred_proba_dict_val=pred_proba_dict_val,\n y_val=y_val,\n eval_metric=self.eval_metric.name,\n problem_type=self.problem_type,\n problem_type_transform=self._learner.label_cleaner.problem_type_transform,\n ordered_class_labels=self._learner.label_cleaner.ordered_class_labels,\n ordered_class_labels_transformed=self._learner.label_cleaner.ordered_class_labels_transformed,\n num_classes=self._learner.label_cleaner.num_classes,\n label=self.label,\n )\n\n return simulation_dict\n\n @staticmethod\n def _check_if_hyperparameters_handle_text(hyperparameters: dict) -> bool:\n \"\"\"Check if hyperparameters contain a model that supports raw text features as input\"\"\"\n models_in_hyperparameters = set()\n advanced_hyperparameter_format = is_advanced_hyperparameter_format(hyperparameters=hyperparameters)\n if advanced_hyperparameter_format:\n for key in hyperparameters:\n for m in hyperparameters[key]:\n models_in_hyperparameters.add(m)\n else:\n for key in hyperparameters:\n models_in_hyperparameters.add(key)\n models_in_hyperparameters_raw_text_compatible = []\n model_key_to_cls_map = ag_model_registry.key_to_cls_map()\n for m in models_in_hyperparameters:\n if isinstance(m, str):\n # TODO: Technically the use of MODEL_TYPES here is a hack since we should derive valid types from trainer,\n # but this is required prior to trainer existing.\n if m in model_key_to_cls_map:\n m = model_key_to_cls_map[m]\n else:\n continue\n if m._get_class_tags().get(\"handles_text\", False):\n models_in_hyperparameters_raw_text_compatible.append(m)\n\n if models_in_hyperparameters_raw_text_compatible:\n return True\n else:\n return False\n\n @staticmethod\n def _validate_hyperparameters(hyperparameters: dict):\n \"\"\"\n Verifies correctness of hyperparameters object.\n \"\"\"\n valid_types = (list, dict)\n\n def _validate_hyperparameters_util(params: dict):\n assert isinstance(params, dict), f\"`hyperparameters` must be a dict, but found: {type(params)}\"\n for model_type in params:\n if not isinstance(params[model_type], valid_types):\n extra_msg = \"\"\n if isinstance(params[model_type], str) and params[model_type] == \"GBMLarge\":\n if version.parse(__version__) >= version.parse(\"1.3.0\"):\n extra_msg = \"\\n\" + get_hyperparameter_str_deprecation_msg()\n else:\n # Will log a deprecation warning downstream in trainer\n continue\n raise AssertionError(\n f\"Hyperparameters are incorrectly formatted, refer to the documentation for examples. \"\n f\"`hyperparameters` key '{model_type}' has an unexpected value type.\"\n f\"\\n\\tvalid types: {valid_types}\"\n f\"\\n\\tactual type: {type(params[model_type])}\"\n f\"\\n\\tactual value: {params[model_type]}\"\n f\"{extra_msg}\"\n )\n\n advanced_hyperparameter_format = is_advanced_hyperparameter_format(hyperparameters=hyperparameters)\n if advanced_hyperparameter_format:\n for stack_level in hyperparameters:\n _validate_hyperparameters_util(params=hyperparameters[stack_level])\n else:\n _validate_hyperparameters_util(params=hyperparameters)\n\n def _sanitize_pseudo_data(\n self, pseudo_data: pd.DataFrame, name=\"pseudo_data\"\n ) -> tuple[pd.DataFrame, pd.Series, pd.Series]:\n assert isinstance(pseudo_data, pd.DataFrame)\n if self.label not in pseudo_data.columns:\n raise ValueError(f\"'{name}' does not contain the labeled column.\")\n\n if self.sample_weight is not None:\n raise ValueError(f\"Applying 'sample_weight' with {name} is not supported.\")\n\n X_pseudo = pseudo_data.drop(columns=[self.label])\n y_pseudo_og = pseudo_data[self.label]\n X_pseudo = self._learner.transform_features(X_pseudo)\n y_pseudo = self._learner.label_cleaner.transform(y_pseudo_og)\n\n if np.isnan(y_pseudo.unique()).any():\n raise Exception(\n f\"NaN was found in the label column for {name}.Please ensure no NaN values in target column\"\n )\n return X_pseudo, y_pseudo, y_pseudo_og\n\n def _assert_is_fit(self, message_suffix: str = None):\n if not self.is_fit:\n error_message = \"Predictor is not fit. Call `.fit` before calling\"\n if message_suffix is None:\n error_message = f\"{error_message} this method.\"\n else:\n error_message = f\"{error_message} `.{message_suffix}`.\"\n raise AssertionError(error_message)\n\n\ndef _safe_rmtree(path: str, retries: int = 5, delay: float = 0.5):\n \"\"\"\n shutil.rmtree with retry for Windows.\n On Windows, files opened by subprocesses (e.g. Ray) might be\n temporarily locked, causing PermissionError [WinError 32].\n \"\"\"\n import sys\n import time\n\n for attempt in range(retries):\n try:\n shutil.rmtree(path)\n return\n except PermissionError:\n if sys.platform == \"win32\" and attempt < retries - 1:\n time.sleep(delay)\n else:\n raise\n\n\ndef _dystack(\n predictor: TabularPredictor,\n train_data: Union[str, pd.DataFrame],\n time_limit: float,\n ds_fit_kwargs: dict,\n ag_fit_kwargs: dict,\n ag_post_fit_kwargs: dict,\n holdout_data=Union[str, pd.DataFrame, None],\n):\n \"\"\"Perform a sub-fit of a TabularPredictor on the provided input data and arguments for a TabularPredictor. To perform the sub-fit, the `self._learner` is\n used for fitting and predicting. After the sub-fit, `self._learner` is reset to its original state and the results of the sub-fit is returned. The sub-fit's\n training and validation data depends on the arguments in ds_fit_kwargs and holdout_data specified at `fit()`.\"\"\"\n holdout_frac = ds_fit_kwargs.get(\"holdout_frac\", None)\n train_indices = ds_fit_kwargs.get(\"train_indices\", None)\n if holdout_frac is not None:\n train_data, val_data = generate_train_test_split_combined(\n data=train_data,\n label=predictor.label,\n problem_type=predictor.problem_type,\n test_size=holdout_frac,\n random_state=42,\n )\n elif train_indices is not None:\n val_indices = ds_fit_kwargs.get(\"val_indices\", None)\n val_data = train_data.iloc[val_indices]\n train_data = train_data.iloc[train_indices]\n elif holdout_data is not None:\n train_data = train_data\n val_data = holdout_data\n else:\n raise ValueError(\"Unsupported validation procedure during dynamic stacking!\")\n\n set_logger_verbosity(verbosity=ag_fit_kwargs[\"verbosity\"])\n learner_og = copy.deepcopy(predictor._learner)\n\n ag_fit_kwargs[\"X\"] = train_data\n ag_fit_kwargs[\"time_limit\"] = time_limit\n ds_fit_context = ds_fit_kwargs.get(\"ds_fit_context\")\n clean_up_fits = ds_fit_kwargs.get(\"clean_up_fits\")\n\n predictor._learner.set_contexts(path_context=ds_fit_context)\n logger.log(20, \"Running DyStack sub-fit ...\")\n try:\n predictor._fit(ag_fit_kwargs=ag_fit_kwargs, ag_post_fit_kwargs=ag_post_fit_kwargs)\n except Exception as e:\n return False, None, e\n\n if not predictor.model_names():\n logger.log(\n 20, \"Unable to determine stacked overfitting. AutoGluon's sub-fit did not successfully train any models!\"\n )\n stacked_overfitting = False\n ho_leaderboard = None\n else:\n leaderboard_kwargs = dict()\n if predictor.model_best in predictor.model_refit_map(inverse=True):\n leaderboard_kwargs = dict(refit_full=True, set_refit_score_to_parent=True)\n # Determine stacked overfitting\n ho_leaderboard = predictor.leaderboard(data=val_data, **leaderboard_kwargs).reset_index(drop=True)\n stacked_overfitting = check_stacked_overfitting_from_leaderboard(ho_leaderboard)\n\n del predictor._learner\n predictor._learner = learner_og\n\n if clean_up_fits:\n logger.log(20, f\"Deleting DyStack predictor artifacts (clean_up_fits={clean_up_fits}) ...\")\n _safe_rmtree(path=ds_fit_context)\n else:\n predictor._sub_fits.append(ds_fit_context)\n\n return stacked_overfitting, ho_leaderboard, None\n"
},
{
"path": "tabular/src/autogluon/tabular/registry/__init__.py",
"content": "from ._ag_model_registry import ag_model_registry\nfrom ._model_registry import ModelRegistry\n"
},
{
"path": "tabular/src/autogluon/tabular/registry/_ag_model_registry.py",
"content": "from autogluon.core.models import (\n DummyModel,\n GreedyWeightedEnsembleModel,\n SimpleWeightedEnsembleModel,\n)\n\nfrom ..models import (\n BoostedRulesModel,\n CatBoostModel,\n EBMModel,\n FigsModel,\n FTTransformerModel,\n GreedyTreeModel,\n HSTreeModel,\n ImagePredictorModel,\n KNNModel,\n LGBModel,\n LinearModel,\n MitraModel,\n MultiModalPredictorModel,\n NNFastAiTabularModel,\n PrepLGBModel,\n RealMLPModel,\n RealTabPFNv2Model,\n RealTabPFNv25Model,\n RFModel,\n RuleFitModel,\n TabDPTModel,\n TabICLModel,\n TabMModel,\n TabPFNMixModel,\n TabularNeuralNetTorchModel,\n TextPredictorModel,\n XGBoostModel,\n XTModel,\n)\nfrom ._model_registry import ModelRegistry\n\n# When adding a new model officially to AutoGluon, the model class should be added to the bottom of this list.\nREGISTERED_MODEL_CLS_LST = [\n RFModel,\n XTModel,\n KNNModel,\n LGBModel,\n CatBoostModel,\n XGBoostModel,\n RealMLPModel,\n TabularNeuralNetTorchModel,\n LinearModel,\n NNFastAiTabularModel,\n PrepLGBModel,\n TextPredictorModel,\n ImagePredictorModel,\n MultiModalPredictorModel,\n FTTransformerModel,\n TabDPTModel,\n TabICLModel,\n TabMModel,\n TabPFNMixModel,\n RealTabPFNv2Model,\n RealTabPFNv25Model,\n MitraModel,\n GreedyWeightedEnsembleModel,\n SimpleWeightedEnsembleModel,\n RuleFitModel,\n GreedyTreeModel,\n FigsModel,\n HSTreeModel,\n BoostedRulesModel,\n DummyModel,\n EBMModel,\n]\n\n# TODO: Replace logic in `autogluon.tabular.trainer.model_presets.presets` with `ag_model_registry`\nag_model_registry = ModelRegistry(model_cls_list=REGISTERED_MODEL_CLS_LST)\n"
},
{
"path": "tabular/src/autogluon/tabular/registry/_model_registry.py",
"content": "from __future__ import annotations\n\nfrom typing import Type\n\nimport pandas as pd\n\nfrom autogluon.core.models import AbstractModel\n\n\n# TODO: Move to core? Maybe TimeSeries can reuse?\n# TODO: Use this / refer to this in the custom model tutorial\n# TODO: Add to documentation website\n# TODO: Test register logic in AG\nclass ModelRegistry:\n \"\"\"\n ModelRegistry keeps track of all known model classes to AutoGluon.\n It can provide information such as:\n What model classes and keys are valid to specify in an AutoGluon predictor fit call.\n What a model's name is.\n What a model's key is (such as the key specified by the user in `hyperparameters` to refer to a specific model type).\n What a model's priority is (aka which order to fit a list of models).\n\n Additionally, users can register custom models to AutoGluon so the key is recognized in `hyperparameters` and is treated with the proper priority and name.\n They can register new models via `ModelRegistry.add(model_cls)`.\n\n Therefore, if a user creates a custom model `MyCustomModel` that inherits from `AbstractModel`, they can set the class attributes in `MyCustomModel`:\n ag_key: The string key that can be specified in `hyperparameters`. Example: \"GBM\" for LGBModel\n ag_name: The string name that is used in logging and accessing the model. Example: \"LightGBM\" for LGBModel\n ag_priority: The int priority that is used to order the fitting of models. Higher values will be fit before lower values. Default 0. Example: 90 for LGBModel\n ag_priority_to_problem_type: A dictionary of problem_type to priority that overrides `ag_priority` if specified for a given problem_type. Optional.\n\n Then they can say `ag_model_registry.add(MyCustomModel)`.\n Assuming MyCustomModel.ag_key = \"MY_MODEL\", they can now do:\n ```\n predictor.fit(..., hyperparameters={\"MY_MODEL\": ...})\n ```\n \"\"\"\n\n def __init__(self, model_cls_list: list[Type[AbstractModel]] | None = None):\n if model_cls_list is None:\n model_cls_list = []\n assert isinstance(model_cls_list, list)\n self._model_cls_list = []\n self._key_to_cls_map = dict()\n for model_cls in model_cls_list:\n self.add(model_cls)\n\n def exists(self, model_cls: Type[AbstractModel]) -> bool:\n return model_cls in self._model_cls_list\n\n def add(self, model_cls: Type[AbstractModel]):\n \"\"\"\n Adds `model_cls` to the model registry\n \"\"\"\n assert not self.exists(model_cls), f\"Cannot add model_cls that is already registered: {model_cls}\"\n if model_cls.ag_key is None:\n raise AssertionError(\n f\"Cannot add model_cls with `ag_key=None`. \"\n f\"Ensure you set class attribute `ag_key` to a string for your model_cls: {model_cls}\"\n f'\\n\\tFor example, LightGBModel sets `ag_key = \"GBM\"`'\n )\n if model_cls.ag_name is None:\n raise AssertionError(\n f\"Cannot add model_cls with `ag_name=None`. \"\n f\"Ensure you set class attribute `ag_name` to a string for your model_cls: {model_cls}\"\n f'\\n\\tFor example, LightGBModel sets `ag_name = \"LightGBM\"`'\n )\n assert isinstance(model_cls.ag_key, str)\n assert isinstance(model_cls.ag_name, str)\n assert isinstance(model_cls.ag_priority, int)\n if model_cls.ag_key in self._key_to_cls_map:\n raise AssertionError(\n f\"Cannot register a model class that shares a model key with an already registered model class.\"\n f\"\\n`model_cls.ag_key` must be unique among registered models:\"\n f\"\\n\\t New Class: {model_cls}\"\n f\"\\n\\tConflicting Class: {self._key_to_cls_map[model_cls.ag_key]}\"\n f\"\\n\\tConflicting ag_key: {model_cls.ag_key}\"\n )\n self._model_cls_list.append(model_cls)\n self._key_to_cls_map[model_cls.ag_key] = model_cls\n\n def remove(self, model_cls: Type[AbstractModel]):\n \"\"\"\n Removes `model_cls` from the model registry\n \"\"\"\n assert self.exists(model_cls), f\"Cannot remove model_cls that isn't registered: {model_cls}\"\n self._model_cls_list = [m for m in self._model_cls_list if m != model_cls]\n self._key_to_cls_map.pop(model_cls.ag_key)\n\n @property\n def model_cls_list(self) -> list[Type[AbstractModel]]:\n return self._model_cls_list\n\n @property\n def keys(self) -> list[str]:\n return [self.key(model_cls) for model_cls in self.model_cls_list]\n\n def key_to_cls_map(self) -> dict[str, Type[AbstractModel]]:\n return self._key_to_cls_map\n\n def key_to_cls(self, key: str) -> Type[AbstractModel]:\n if key not in self._key_to_cls_map:\n raise ValueError(\n f\"No registered model exists with provided key: {key}\"\n f\"\\n\\tValid keys: {list(self.key_to_cls_map().keys())}\"\n )\n return self.key_to_cls_map()[key]\n\n def priority_map(self, problem_type: str | None = None) -> dict[Type[AbstractModel], int]:\n return {model_cls: self.priority(model_cls, problem_type=problem_type) for model_cls in self._model_cls_list}\n\n def key(self, model_cls: Type[AbstractModel]) -> str:\n assert self.exists(model_cls), f\"Model class must be registered: {model_cls}\"\n return model_cls.ag_key\n\n def name_map(self) -> dict[Type[AbstractModel], str]:\n return {model_cls: model_cls.ag_name for model_cls in self._model_cls_list}\n\n def name(self, model_cls: Type[AbstractModel]) -> str:\n assert self.exists(model_cls), f\"Model class must be registered: {model_cls}\"\n return model_cls.ag_name\n\n def priority(self, model_cls: Type[AbstractModel], problem_type: str | None = None) -> int:\n assert self.exists(model_cls), f\"Model class must be registered: {model_cls}\"\n return model_cls.get_ag_priority(problem_type=problem_type)\n\n def docstring(self, model_cls: Type[AbstractModel]) -> str:\n assert self.exists(model_cls), f\"Model class must be registered: {model_cls}\"\n return model_cls.__doc__\n\n # TODO: Could add a lot of information here to track which features are supported for each model:\n # ag.early_stop support\n # refit_full support\n # GPU support\n # etc.\n def to_frame(self) -> pd.DataFrame:\n model_classes = self.model_cls_list\n cls_dict = {}\n for model_cls in model_classes:\n cls_dict[self.key(model_cls)] = {\n \"model_cls\": model_cls.__name__,\n \"ag_name\": self.name(model_cls),\n \"ag_priority\": self.priority(model_cls),\n }\n df = pd.DataFrame(cls_dict).T\n df.index.name = \"ag_key\"\n return df\n"
},
{
"path": "tabular/src/autogluon/tabular/testing/__init__.py",
"content": "from .fit_helper import FitHelper\nfrom .model_fit_helper import ModelFitHelper\n"
},
{
"path": "tabular/src/autogluon/tabular/testing/fit_helper.py",
"content": "from __future__ import annotations\n\nimport copy\nimport os\nimport shutil\nimport subprocess\nimport sys\nimport textwrap\nimport uuid\nfrom typing import Any, Type\n\nimport numpy as np\nimport pandas as pd\nimport pandas.testing as pdt\n\nfrom autogluon.common.utils.path_converter import PathConverter\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.metrics import METRICS\nfrom autogluon.core.models import AbstractModel, BaggedEnsembleModel\nfrom autogluon.core.stacked_overfitting.utils import check_stacked_overfitting_from_leaderboard\nfrom autogluon.core.testing.global_context_snapshot import GlobalContextSnapshot\nfrom autogluon.core.utils import download, generate_train_test_split_combined, infer_problem_type, unzip\nfrom autogluon.tabular import TabularDataset, TabularPredictor\nfrom autogluon.tabular.testing.generate_datasets import (\n generate_toy_binary_10_dataset,\n generate_toy_binary_dataset,\n generate_toy_multiclass_10_dataset,\n generate_toy_multiclass_30_dataset,\n generate_toy_multiclass_dataset,\n generate_toy_quantile_10_dataset,\n generate_toy_quantile_dataset,\n generate_toy_quantile_single_level_dataset,\n generate_toy_regression_10_dataset,\n generate_toy_regression_dataset,\n)\n\n\nclass DatasetLoaderHelper:\n dataset_info_dict = dict(\n # Binary dataset\n adult={\n \"url\": \"https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip\",\n \"name\": \"AdultIncomeBinaryClassification\",\n \"problem_type\": BINARY,\n \"label\": \"class\",\n },\n # Multiclass big dataset with 7 classes, all features are numeric. Runs SLOW.\n covertype={\n \"url\": \"https://autogluon.s3.amazonaws.com/datasets/CoverTypeMulticlassClassification.zip\",\n \"name\": \"CoverTypeMulticlassClassification\",\n \"problem_type\": MULTICLASS,\n \"label\": \"Cover_Type\",\n },\n # Subset of covertype dataset with 3k train/test rows. Ratio of labels is preserved.\n covertype_small={\n \"url\": \"https://autogluon.s3.amazonaws.com/datasets/CoverTypeMulticlassClassificationSmall.zip\",\n \"name\": \"CoverTypeMulticlassClassificationSmall\",\n \"problem_type\": MULTICLASS,\n \"label\": \"Cover_Type\",\n },\n # Regression with mixed feature-types, skewed Y-values.\n ames={\n \"url\": \"https://autogluon.s3.amazonaws.com/datasets/AmesHousingPriceRegression.zip\",\n \"name\": \"AmesHousingPriceRegression\",\n \"problem_type\": REGRESSION,\n \"label\": \"SalePrice\",\n },\n # Regression with multiple text field and categorical\n sts={\n \"url\": \"https://autogluon-text.s3.amazonaws.com/glue_sts.zip\",\n \"name\": \"glue_sts\",\n \"problem_type\": REGRESSION,\n \"label\": \"score\",\n },\n )\n\n toy_map = dict(\n toy_binary=generate_toy_binary_dataset,\n toy_multiclass=generate_toy_multiclass_dataset,\n toy_regression=generate_toy_regression_dataset,\n toy_quantile=generate_toy_quantile_dataset,\n toy_quantile_single_level=generate_toy_quantile_single_level_dataset,\n toy_binary_10=generate_toy_binary_10_dataset,\n toy_multiclass_10=generate_toy_multiclass_10_dataset,\n toy_regression_10=generate_toy_regression_10_dataset,\n toy_quantile_10=generate_toy_quantile_10_dataset,\n toy_multiclass_30=generate_toy_multiclass_30_dataset,\n )\n\n @staticmethod\n def load_dataset(name: str, directory_prefix: str = \"./datasets/\") -> tuple[pd.DataFrame, pd.DataFrame, dict]:\n if name in DatasetLoaderHelper.toy_map:\n return DatasetLoaderHelper.toy_map[name]()\n dataset_info = copy.deepcopy(DatasetLoaderHelper.dataset_info_dict[name])\n train_file = dataset_info.pop(\"train_file\", \"train_data.csv\")\n test_file = dataset_info.pop(\"test_file\", \"test_data.csv\")\n name_inner = dataset_info.pop(\"name\")\n url = dataset_info.pop(\"url\", None)\n train_data, test_data = DatasetLoaderHelper.load_data(\n directory_prefix=directory_prefix,\n train_file=train_file,\n test_file=test_file,\n name=name_inner,\n url=url,\n )\n\n return train_data, test_data, dataset_info\n\n # TODO: Refactor this eventually, this is old code from 2019 that can be improved (use consistent local path for datasets, don't assume zip files, etc.)\n @staticmethod\n def load_data(\n directory_prefix: str,\n train_file: str,\n test_file: str,\n name: str,\n url: str | None = None,\n ) -> tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"\n Will check if files exist locally for:\n directory_prefix/name/train_file\n directory_prefix/name/test_file\n If either don't exist, then download the files from the `url` location.\n Then, load both train and test data and return them.\n\n Parameters\n ----------\n directory_prefix\n train_file\n test_file\n name\n url\n\n Returns\n -------\n train_data: pd.DataFrame\n test_data: pd.DataFrame\n \"\"\"\n if not os.path.exists(directory_prefix):\n os.mkdir(directory_prefix)\n directory = directory_prefix + name + \"/\"\n train_file_path = directory + train_file\n test_file_path = directory + test_file\n if (not os.path.exists(train_file_path)) or (not os.path.exists(test_file_path)):\n # fetch files from s3:\n print(\"%s data not found locally, so fetching from %s\" % (name, url))\n zip_name = download(url, directory_prefix)\n unzip(zip_name, directory_prefix)\n os.remove(zip_name)\n\n train_data = TabularDataset(train_file_path)\n test_data = TabularDataset(test_file_path)\n return train_data, test_data\n\n\nclass FitHelper:\n \"\"\"\n Helper functions to test and verify predictors and models when fit through TabularPredictor's API.\n \"\"\"\n\n @staticmethod\n def fit_and_validate_dataset(\n dataset_name: str,\n fit_args: dict[str, Any],\n init_args: dict[str, Any] | None = None,\n sample_size: int | None = 1000, # FIXME: default to None\n refit_full: bool = True,\n delete_directory: bool = True,\n extra_metrics: list[str] | None = None,\n extra_info: bool = False,\n predictor_info: bool = False,\n expected_model_count: int | None = 2,\n fit_weighted_ensemble: bool = True,\n min_cls_count_train: int = 1,\n path_as_absolute: bool = False,\n compile: bool = False,\n compiler_configs: dict | None = None,\n allowed_dataset_features: list[str] | None = None,\n expected_stacked_overfitting_at_test: bool | None = None,\n expected_stacked_overfitting_at_val: bool | None = None,\n scikit_api: bool = False,\n use_test_data: bool = False,\n use_test_for_val: bool = False,\n raise_on_model_failure: bool | None = None,\n deepcopy_fit_args: bool = True,\n verify_model_seed: bool = False,\n verify_load_wo_cuda: bool = False,\n verify_single_prediction_equivalent_to_multi: bool = True,\n ) -> TabularPredictor:\n if compiler_configs is None:\n compiler_configs = {}\n directory_prefix = \"./datasets/\"\n train_data, test_data, dataset_info = DatasetLoaderHelper.load_dataset(\n name=dataset_name, directory_prefix=directory_prefix\n )\n if sample_size is not None and sample_size < len(test_data):\n test_data = test_data.sample(n=sample_size, random_state=0)\n label = dataset_info[\"label\"]\n problem_type = dataset_info[\"problem_type\"]\n _init_args = dict(\n label=label,\n problem_type=problem_type,\n )\n if \"init_kwargs\" in dataset_info:\n _init_args.update(dataset_info[\"init_kwargs\"])\n if allowed_dataset_features is not None:\n train_data = train_data[allowed_dataset_features + [label]]\n test_data = test_data[allowed_dataset_features + [label]]\n\n if init_args is None:\n init_args = _init_args\n else:\n init_args = copy.deepcopy(init_args)\n _init_args.update(init_args)\n init_args = _init_args\n if \"path\" not in init_args:\n init_args[\"path\"] = os.path.join(directory_prefix, dataset_name, f\"AutogluonOutput_{uuid.uuid4()}\")\n if path_as_absolute:\n init_args[\"path\"] = PathConverter.to_absolute(path=init_args[\"path\"])\n assert PathConverter._is_absolute(path=init_args[\"path\"])\n save_path = init_args[\"path\"]\n\n if deepcopy_fit_args:\n fit_args = copy.deepcopy(fit_args)\n if use_test_data:\n fit_args[\"test_data\"] = test_data\n if use_test_for_val:\n fit_args[\"tuning_data\"] = test_data\n if raise_on_model_failure is not None and \"raise_on_model_failure\" not in fit_args:\n fit_args[\"raise_on_model_failure\"] = raise_on_model_failure\n if \"fit_weighted_ensemble\" not in fit_args:\n if not fit_weighted_ensemble and expected_model_count is not None:\n expected_model_count -= 1\n fit_args[\"fit_weighted_ensemble\"] = fit_weighted_ensemble\n\n ctx_before = GlobalContextSnapshot.capture()\n\n predictor: TabularPredictor = FitHelper.fit_dataset(\n train_data=train_data,\n init_args=init_args,\n fit_args=fit_args,\n sample_size=sample_size,\n scikit_api=scikit_api,\n min_cls_count_train=min_cls_count_train,\n )\n\n ctx_after_fit = GlobalContextSnapshot.capture()\n ctx_before.assert_unchanged(ctx_after_fit)\n\n if compile:\n predictor.compile(models=\"all\", compiler_configs=compiler_configs)\n predictor.persist(models=\"all\")\n\n model_names = predictor.model_names()\n model_name = model_names[0]\n if expected_model_count is not None:\n assert len(model_names) == expected_model_count\n\n predictor.predict(test_data)\n\n ctx_after_predict = GlobalContextSnapshot.capture()\n ctx_after_fit.assert_unchanged(ctx_after_predict)\n\n predictor.evaluate(test_data)\n\n test_data_transform = predictor.transform_features(data=test_data, model=model_name)\n test_data_transform_before = test_data_transform.copy(deep=True)\n model = predictor._trainer.load_model(model_name=model_name)\n model.predict(X=test_data_transform)\n pdt.assert_frame_equal(test_data_transform, test_data_transform_before, check_dtype=True)\n\n if predictor.can_predict_proba:\n pred_proba = predictor.predict_proba(test_data)\n predictor.evaluate_predictions(y_true=test_data[label], y_pred=pred_proba)\n\n pred_proba_repeat = predictor.predict_proba(test_data)\n are_close = np.allclose(pred_proba, pred_proba_repeat, atol=1e-5)\n if not are_close:\n raise AssertionError(\n \"Predictions differ when predicting on the same data multiple times\\n\"\n f\"First Predict:\\n{pred_proba}\\n\"\n f\"Second Predict:\\n{pred_proba_repeat}\\n\"\n )\n\n pred_proba_1 = predictor.predict_proba(test_data.head(1)) # Verify model can predict on a single sample\n if verify_single_prediction_equivalent_to_multi:\n pred_proba_1_from_multi = pred_proba.head(1)\n are_close = np.allclose(pred_proba_1, pred_proba_1_from_multi, atol=1e-5)\n if not are_close:\n raise AssertionError(\n \"Predictions differ when predicting a single sample vs predicting multiple samples\\n\"\n f\"Single Sample:\\n{pred_proba_1}\\n\"\n f\"Multi Sample:\\n{pred_proba_1_from_multi}\\n\"\n )\n else:\n try:\n predictor.predict_proba(test_data)\n except AssertionError:\n pass # expected\n else:\n raise AssertionError(\"Expected `predict_proba` to raise AssertionError, but it didn't!\")\n\n if refit_full:\n refit_model_names = predictor.refit_full()\n if expected_model_count is not None:\n assert len(refit_model_names) == expected_model_count\n refit_model_name = refit_model_names[model_name]\n assert \"_FULL\" in refit_model_name\n predictor.predict(test_data, model=refit_model_name)\n if predictor.can_predict_proba:\n predictor.predict_proba(test_data, model=refit_model_name)\n\n # verify that val_in_fit is False if the model supports refit_full\n model = predictor._trainer.load_model(refit_model_name)\n if isinstance(model, BaggedEnsembleModel):\n model = model.load_child(model.models[0])\n model_info = model.get_info()\n can_refit_full = model._get_tags()[\"can_refit_full\"]\n if can_refit_full:\n assert not model_info[\"val_in_fit\"], (\n f\"val data must not be present in refit model if `can_refit_full=True`. Maybe an exception occurred?\"\n )\n else:\n assert model_info[\"val_in_fit\"], f\"val data must be present in refit model if `can_refit_full=False`\"\n if verify_model_seed:\n names = predictor.model_names()\n for name in names:\n model = predictor._trainer.load_model(name)\n _verify_model_seed(model=model)\n\n if predictor_info:\n predictor.info()\n lb_kwargs = {}\n if extra_info:\n lb_kwargs[\"extra_info\"] = True\n lb = predictor.leaderboard(test_data, extra_metrics=extra_metrics, **lb_kwargs)\n stacked_overfitting_assert(\n lb, predictor, expected_stacked_overfitting_at_val, expected_stacked_overfitting_at_test\n )\n\n predictor_load = predictor.load(path=predictor.path)\n predictor_load.predict(test_data)\n\n # TODO: This is expensive, only do this sparingly.\n if verify_load_wo_cuda:\n import torch\n\n if torch.cuda.is_available():\n # Checks if the model is able to predict w/o CUDA.\n # This verifies that a model artifact works on a CPU machine.\n predictor_path = predictor.path\n\n code = textwrap.dedent(f\"\"\"\n import os\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n from autogluon.tabular import TabularPredictor\n\n import torch\n assert torch.cuda.is_available() is False\n predictor = TabularPredictor.load(r\"{predictor_path}\")\n X, y = predictor.load_data_internal()\n predictor.persist(\"all\")\n predictor.predict_multi(X, transform_features=False)\n \"\"\")\n subprocess.run([sys.executable, \"-c\", code], check=True)\n\n assert os.path.realpath(save_path) == os.path.realpath(predictor.path)\n if delete_directory:\n shutil.rmtree(\n save_path, ignore_errors=True\n ) # Delete AutoGluon output directory to ensure runs' information has been removed.\n return predictor\n\n @staticmethod\n def load_dataset(name: str, directory_prefix: str = \"./datasets/\") -> tuple[pd.DataFrame, pd.DataFrame, dict]:\n return DatasetLoaderHelper.load_dataset(name=name, directory_prefix=directory_prefix)\n\n @staticmethod\n def fit_dataset(\n train_data: pd.DataFrame,\n init_args: dict[str, Any],\n fit_args: dict[str, Any],\n sample_size: int | None = None,\n min_cls_count_train: int = 1,\n scikit_api: bool = False,\n ) -> TabularPredictor:\n if \"problem_type\" in init_args:\n problem_type = init_args[\"problem_type\"]\n else:\n problem_type = infer_problem_type(train_data[init_args[\"label\"]])\n\n if sample_size is not None and sample_size < len(train_data):\n train_data, _ = generate_train_test_split_combined(\n data=train_data,\n label=init_args[\"label\"],\n problem_type=problem_type,\n test_size=len(train_data) - sample_size,\n min_cls_count_train=min_cls_count_train,\n )\n\n if scikit_api:\n from autogluon.tabular.experimental import TabularClassifier, TabularRegressor\n\n X = train_data.drop(columns=[init_args[\"label\"]])\n y = train_data[init_args[\"label\"]]\n if problem_type in [REGRESSION]:\n regressor = TabularRegressor(init_args=init_args, fit_args=fit_args)\n regressor.fit(X, y)\n return regressor.predictor_\n else:\n classifier = TabularClassifier(init_args=init_args, fit_args=fit_args)\n classifier.fit(X, y)\n return classifier.predictor_\n else:\n return TabularPredictor(**init_args).fit(train_data, **fit_args)\n\n @staticmethod\n def verify_model(\n model_cls: Type[AbstractModel],\n model_hyperparameters: dict[str, Any],\n bag: bool | str = \"first\",\n refit_full: bool | str = \"first\",\n extra_metrics: bool = False,\n require_known_problem_types: bool = True,\n raise_on_model_failure: bool = True,\n problem_types: list[str] | None = None,\n verify_model_seed: bool = True,\n verify_single_prediction_equivalent_to_multi: bool = True,\n use_larger_toy_datasets: bool = False,\n **kwargs,\n ):\n \"\"\"\n\n Parameters\n ----------\n model_cls\n model_hyperparameters\n bag\n refit_full\n extra_metrics\n require_known_problem_types\n raise_on_model_failure\n problem_types: list[str], optional\n If specified, checks the given problem_types.\n If None, checks `model_cls.supported_problem_types()`\n verify_model_seed: bool = True\n verify_single_prediction_equivalent_to_multi: bool = True\n **kwargs\n\n Returns\n -------\n\n \"\"\"\n if verify_model_seed and model_cls.seed_name is not None:\n # verify that the seed logic works\n model_hyperparameters = model_hyperparameters.copy()\n model_hyperparameters[model_cls.seed_name] = 42\n\n fit_args = dict(\n hyperparameters={model_cls: model_hyperparameters},\n )\n supported_problem_types = model_cls.supported_problem_types()\n if supported_problem_types is None:\n raise AssertionError(\n f\"Model must specify `cls.supported_problem_types`\"\n f\"\"\"\\nExample code:\n @classmethod\n def supported_problem_types(cls) -> list[str] | None:\n return [\"binary\", \"multiclass\", \"regression\", \"quantile\"]\n \"\"\"\n )\n assert isinstance(supported_problem_types, list)\n assert len(supported_problem_types) > 0\n\n known_problem_types = [\n \"binary\",\n \"multiclass\",\n \"regression\",\n \"quantile\",\n \"softclass\",\n ]\n\n if require_known_problem_types:\n for problem_type in supported_problem_types:\n if problem_type not in known_problem_types:\n raise AssertionError(\n f\"Model {model_cls.__name__} supports an unknown problem_type: {problem_type}\"\n f\"\\nKnown problem types: {known_problem_types}\"\n f\"\\nEither remove the unknown problem_type from `model_cls.supported_problem_types` or set `require_known_problem_types=False`\"\n )\n\n if use_larger_toy_datasets:\n problem_type_dataset_map = {\n \"binary\": [\"toy_binary_10\"],\n \"multiclass\": [\"toy_multiclass_10\"],\n \"regression\": [\"toy_regression_10\"],\n \"quantile\": [\"toy_quantile_10\"],\n }\n else:\n problem_type_dataset_map = {\n \"binary\": [\"toy_binary\"],\n \"multiclass\": [\"toy_multiclass\"],\n \"regression\": [\"toy_regression\"],\n \"quantile\": [\"toy_quantile\", \"toy_quantile_single_level\"],\n }\n\n problem_types_refit_full = []\n if refit_full:\n if isinstance(refit_full, bool):\n problem_types_refit_full = supported_problem_types\n elif refit_full == \"first\":\n problem_types_refit_full = supported_problem_types[:1]\n\n if problem_types is None:\n problem_types_to_check = supported_problem_types\n else:\n problem_types_to_check = problem_types\n\n for problem_type in problem_types_to_check:\n if problem_type not in problem_type_dataset_map:\n print(f\"WARNING: Skipping check on problem_type='{problem_type}': No dataset available\")\n continue\n _extra_metrics = None\n if extra_metrics:\n _extra_metrics = METRICS.get(problem_type, None)\n refit_full = problem_type in problem_types_refit_full\n for dataset_name in problem_type_dataset_map[problem_type]:\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n fit_weighted_ensemble=False,\n refit_full=refit_full,\n extra_metrics=_extra_metrics,\n raise_on_model_failure=raise_on_model_failure,\n verify_model_seed=verify_model_seed,\n verify_single_prediction_equivalent_to_multi=verify_single_prediction_equivalent_to_multi,\n **kwargs,\n )\n\n if bag:\n model_params_bag = copy.deepcopy(model_hyperparameters)\n model_params_bag[\"ag.ens.fold_fitting_strategy\"] = \"sequential_local\"\n fit_args_bag = dict(\n hyperparameters={model_cls: model_params_bag},\n num_bag_folds=2,\n num_bag_sets=1,\n )\n if isinstance(bag, bool):\n problem_types_bag = problem_types_to_check\n elif bag == \"first\":\n problem_types_bag = problem_types_to_check[:1]\n else:\n raise ValueError(f\"Unknown 'bag' value: {bag}\")\n\n for problem_type in problem_types_bag:\n _extra_metrics = None\n if extra_metrics:\n _extra_metrics = METRICS.get(problem_type, None)\n refit_full = problem_type in problem_types_refit_full\n for dataset_name in problem_type_dataset_map[problem_type]:\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args_bag,\n fit_weighted_ensemble=False,\n refit_full=refit_full,\n extra_metrics=_extra_metrics,\n raise_on_model_failure=raise_on_model_failure,\n verify_model_seed=verify_model_seed,\n verify_single_prediction_equivalent_to_multi=verify_single_prediction_equivalent_to_multi,\n **kwargs,\n )\n\n\ndef stacked_overfitting_assert(\n lb: pd.DataFrame,\n predictor: TabularPredictor,\n expected_stacked_overfitting_at_val: bool | None,\n expected_stacked_overfitting_at_test: bool | None,\n):\n if expected_stacked_overfitting_at_val is not None:\n assert predictor._stacked_overfitting_occurred == expected_stacked_overfitting_at_val, (\n \"Expected stacked overfitting at val mismatch!\"\n )\n\n if expected_stacked_overfitting_at_test is not None:\n stacked_overfitting = check_stacked_overfitting_from_leaderboard(lb)\n assert stacked_overfitting == expected_stacked_overfitting_at_test, (\n \"Expected stacked overfitting at test mismatch!\"\n )\n\n\ndef _verify_model_seed(model: AbstractModel):\n assert model.random_seed is None or isinstance(model.random_seed, int)\n if model.seed_name is not None:\n if model.seed_name in model._user_params:\n assert model.random_seed == model._user_params[model.seed_name]\n assert model.seed_name in model.params\n assert model.random_seed == model.params[model.seed_name]\n if isinstance(model, BaggedEnsembleModel):\n for child in model.models:\n child = model.load_child(child)\n _verify_model_seed(child)\n"
},
{
"path": "tabular/src/autogluon/tabular/testing/generate_datasets.py",
"content": "from __future__ import annotations\n\nimport pandas as pd\nfrom sklearn.datasets import make_blobs\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REGRESSION\n\n\ndef generate_toy_binary_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3],\n label: [0, 0, 1, 1],\n }\n\n dataset_info = {\n \"problem_type\": BINARY,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_multiclass_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3, 4, 5],\n label: [0, 0, 1, 1, 2, 2],\n }\n\n dataset_info = {\n \"problem_type\": MULTICLASS,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_regression_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3],\n label: [0.1, 0.9, 1.1, 1.9],\n }\n\n dataset_info = {\n \"problem_type\": REGRESSION,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_quantile_dataset():\n train_data, test_data, dataset_info = generate_toy_regression_dataset()\n dataset_info[\"problem_type\"] = QUANTILE\n dataset_info[\"init_kwargs\"] = {\"quantile_levels\": [0.25, 0.5, 0.75]}\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_quantile_single_level_dataset():\n train_data, test_data, dataset_info = generate_toy_regression_dataset()\n dataset_info[\"problem_type\"] = QUANTILE\n dataset_info[\"init_kwargs\"] = {\"quantile_levels\": [0.71]}\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_binary_10_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n label: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0],\n }\n\n dataset_info = {\n \"problem_type\": BINARY,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_multiclass_10_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n label: [0, 0, 1, 1, 2, 2, 0, 0, 1, 1],\n }\n\n dataset_info = {\n \"problem_type\": MULTICLASS,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_regression_10_dataset():\n label = \"label\"\n dummy_dataset = {\n \"int\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n label: [0.1, 0.9, 1.1, 1.9, 0.2, 0.8, 1.2, 1.8, -0.1, 0.7],\n }\n\n dataset_info = {\n \"problem_type\": REGRESSION,\n \"label\": label,\n }\n\n train_data = pd.DataFrame(dummy_dataset)\n test_data = train_data\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_quantile_10_dataset():\n train_data, test_data, dataset_info = generate_toy_regression_10_dataset()\n dataset_info[\"problem_type\"] = QUANTILE\n dataset_info[\"init_kwargs\"] = {\"quantile_levels\": [0.25, 0.5, 0.75]}\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_multiclass_30_dataset():\n label = \"label\"\n train_data = generate_toy_multiclass_n_dataset(n_samples=30, n_features=2, n_classes=3)\n test_data = train_data\n\n dataset_info = {\n \"problem_type\": MULTICLASS,\n \"label\": label,\n }\n return train_data, test_data, dataset_info\n\n\ndef generate_toy_multiclass_n_dataset(n_samples, n_features, n_classes) -> pd.DataFrame:\n X, y = make_blobs(centers=n_classes, n_samples=n_samples, n_features=n_features, cluster_std=0.5, random_state=0)\n data = pd.DataFrame(X)\n data[\"label\"] = y\n return data\n"
},
{
"path": "tabular/src/autogluon/tabular/testing/model_fit_helper.py",
"content": "from __future__ import annotations\n\nfrom typing import Tuple\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.core.data.label_cleaner import LabelCleaner\nfrom autogluon.core.models import AbstractModel, BaggedEnsembleModel\nfrom autogluon.core.utils import generate_train_test_split, infer_problem_type\nfrom autogluon.features.generators import AbstractFeatureGenerator, AutoMLPipelineFeatureGenerator\nfrom autogluon.tabular.testing.fit_helper import FitHelper\n\n\n# Helper functions for training models outside of predictors\nclass ModelFitHelper:\n \"\"\"\n Helper functions to test and verify models when fit outside TabularPredictor's API (aka as stand-alone models)\n \"\"\"\n\n @staticmethod\n def fit_and_validate_dataset(\n dataset_name: str,\n model: AbstractModel,\n fit_args: dict,\n sample_size: int = 1000,\n check_predict_children: bool = False,\n ) -> AbstractModel:\n directory_prefix = \"./datasets/\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(\n name=dataset_name, directory_prefix=directory_prefix\n )\n label = dataset_info[\"label\"]\n model, label_cleaner, feature_generator = ModelFitHelper.fit_dataset(\n train_data=train_data, model=model, label=label, fit_args=fit_args, sample_size=sample_size\n )\n if sample_size is not None and sample_size < len(test_data):\n test_data = test_data.sample(n=sample_size, random_state=0)\n\n X_test = test_data.drop(columns=[label])\n X_test = feature_generator.transform(X_test)\n\n y_pred = model.predict(X_test)\n assert isinstance(y_pred, np.ndarray), (\n f\"Expected np.ndarray as model.predict(X_test) output. Got: {y_pred.__class__}\"\n )\n\n y_pred_proba = model.predict_proba(X_test)\n assert isinstance(y_pred_proba, np.ndarray), (\n f\"Expected np.ndarray as model.predict_proba(X_test) output. Got: {y_pred.__class__}\"\n )\n model.get_info()\n\n if check_predict_children:\n assert isinstance(model, BaggedEnsembleModel)\n y_pred_children = model.predict_children(X_test)\n assert len(y_pred_children) == model.n_children\n if model.can_predict_proba:\n y_pred_proba_children = model.predict_proba_children(X_test)\n assert len(y_pred_proba_children) == model.n_children\n y_pred_proba_from_children = np.mean(y_pred_proba_children, axis=0)\n assert np.isclose(y_pred_proba_from_children, y_pred_proba).all()\n\n for y_pred_proba_child, y_pred_child in zip(y_pred_proba_children, y_pred_children):\n y_pred_child_from_proba = model.predict_from_proba(y_pred_proba=y_pred_proba_child)\n assert np.isclose(y_pred_child_from_proba, y_pred_child).all()\n\n return model\n\n @staticmethod\n def fit_dataset(\n train_data: pd.DataFrame,\n model: AbstractModel,\n label: str,\n fit_args: dict,\n sample_size: int = None,\n ) -> Tuple[AbstractModel, LabelCleaner, AbstractFeatureGenerator]:\n if sample_size is not None and sample_size < len(train_data):\n train_data = train_data.sample(n=sample_size, random_state=0)\n X = train_data.drop(columns=[label])\n y = train_data[label]\n\n problem_type = infer_problem_type(y)\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=y)\n y = label_cleaner.transform(y)\n feature_generator = AutoMLPipelineFeatureGenerator()\n X = feature_generator.fit_transform(X, y)\n\n X, X_val, y, y_val = generate_train_test_split(X, y, problem_type=problem_type, test_size=0.2, random_state=0)\n\n model.fit(X=X, y=y, X_val=X_val, y_val=y_val, **fit_args)\n return model, label_cleaner, feature_generator\n"
},
{
"path": "tabular/src/autogluon/tabular/trainer/__init__.py",
"content": "from .auto_trainer import AutoTrainer\n"
},
{
"path": "tabular/src/autogluon/tabular/trainer/abstract_trainer.py",
"content": "from __future__ import annotations\n\nimport copy\nimport logging\nimport os\nimport shutil\nimport time\nimport traceback\nfrom collections import defaultdict\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport networkx as nx\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.types import R_FLOAT, S_STACK\nfrom autogluon.common.utils.distribute_utils import DistributedContext\nfrom autogluon.common.utils.lite import disable_if_lite_mode\nfrom autogluon.common.utils.log_utils import convert_time_in_s_to_log_friendly, reset_logger_for_remote_call\nfrom autogluon.common.utils.resource_utils import ResourceManager, get_resource_manager\nfrom autogluon.common.utils.try_import import try_import_ray, try_import_torch\nfrom autogluon.core.augmentation.distill_utils import augment_data, format_distillation_labels\nfrom autogluon.core.calibrate import calibrate_decision_threshold\nfrom autogluon.core.calibrate.conformity_score import compute_conformity_score\nfrom autogluon.core.calibrate.temperature_scaling import apply_temperature_scaling, tune_temperature_scaling\nfrom autogluon.core.callbacks import AbstractCallback\nfrom autogluon.core.constants import BINARY, MULTICLASS, QUANTILE, REFIT_FULL_NAME, REGRESSION, SOFTCLASS\nfrom autogluon.core.data.label_cleaner import LabelCleaner, LabelCleanerMulticlassToBinary\nfrom autogluon.core.metrics import Scorer, compute_metric, get_metric\nfrom autogluon.core.models import (\n AbstractModel,\n BaggedEnsembleModel,\n GreedyWeightedEnsembleModel,\n SimpleWeightedEnsembleModel,\n StackerEnsembleModel,\n WeightedEnsembleModel,\n)\nfrom autogluon.core.pseudolabeling.pseudolabeling import assert_pseudo_column_match\nfrom autogluon.core.ray.distributed_jobs_managers import ParallelFitManager\nfrom autogluon.core.trainer import AbstractTrainer\nfrom autogluon.core.trainer.utils import process_hyperparameters\nfrom autogluon.core.utils import (\n compute_permutation_feature_importance,\n convert_pred_probas_to_df,\n default_holdout_frac,\n extract_column,\n generate_train_test_split,\n get_pred_from_proba,\n infer_eval_metric,\n)\nfrom autogluon.core.utils.exceptions import (\n InsufficientTime,\n NoGPUError,\n NoStackFeatures,\n NotEnoughCudaMemoryError,\n NotEnoughMemoryError,\n NotValidStacker,\n NoValidFeatures,\n TimeLimitExceeded,\n)\nfrom autogluon.core.utils.feature_selection import FeatureSelector\nfrom autogluon.core.utils.loaders import load_pkl\nfrom autogluon.core.utils.savers import save_pkl\n\nlogger = logging.getLogger(__name__)\n\n\nclass AbstractTabularTrainer(AbstractTrainer[AbstractModel]):\n \"\"\"\n AbstractTabularTrainer contains logic to train a variety of models under a variety of constraints and automatically generate a multi-layer stack ensemble.\n Beyond the basic functionality, it also has support for model refitting, distillation, pseudo-labelling, unlabeled data, and much more.\n\n It is not recommended to directly use Trainer. Instead, use Predictor or Learner which internally uses Trainer.\n This documentation is for developers. Users should avoid this class.\n\n Due to the complexity of the logic within this class, a text description will not give the full picture.\n It is recommended to carefully read the code and use a debugger to understand how it works.\n\n AbstractTabularTrainer makes much fewer assumptions about the problem than Learner and Predictor.\n It expects these ambiguities to have already been resolved upstream. For example, problem_type, feature_metadata, num_classes, etc.\n\n Parameters\n ----------\n path : str\n Path to save and load trainer artifacts to disk.\n Path should end in `/` or `os.path.sep()`.\n problem_type : str\n One of ['binary', 'multiclass', 'regression', 'quantile', 'softclass']\n num_classes : int\n The number of classes in the problem.\n If problem_type is in ['regression', 'quantile'], this must be None.\n If problem_type is 'binary', this must be 2.\n If problem_type is in ['multiclass', 'softclass'], this must be >= 2.\n feature_metadata : FeatureMetadata\n FeatureMetadata for X. Sent to each model during fit.\n eval_metric : Scorer, default = None\n Metric to optimize. If None, a default metric is used depending on the problem_type.\n quantile_levels : list[float] | np.ndarray, default = None\n # TODO: Add documentation, not documented in Predictor.\n Only used when problem_type=quantile\n low_memory : bool, default = True\n Deprecated parameter, likely to be removed in future versions.\n If True, caches models to disk separately instead of containing all models within memory.\n If False, may cause a variety of bugs.\n k_fold : int, default = 0\n If <2, then non-bagged mode is used.\n If >= 2, then bagged mode is used with num_bag_folds == k_fold for each model.\n Bagged mode changes the way models are trained and ensembled.\n Bagged mode enables multi-layer stacking and repeated bagging.\n n_repeats : int, default = 1\n The maximum repeats of bagging to do when in bagged mode.\n Larger values take linearly longer to train and infer, but improves quality slightly.\n sample_weight : str, default = None\n Column name of the sample weight in X\n weight_evaluation : bool, default = False\n If True, the eval_metric is calculated with sample_weight incorporated into the score.\n save_data : bool, default = True\n Whether to cache the data (X, y, X_val, y_val) to disk.\n Required for a variety of advanced post-fit functionality.\n It is recommended to keep as True.\n random_state : int, default = 0\n Random state for data splitting in bagged mode.\n verbosity : int, default = 2\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`,\n where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels).\n raise_on_model_failure : bool, default = False\n If True, Trainer will raise on any exception during model training.\n This is ideal when using a debugger during development.\n If False, Trainer will try to skip to the next model if an exception occurred during model training.\n\n .. versionadded:: 1.3.0\n \"\"\"\n\n distill_stackname = \"distill\" # name of stack-level for distilled student models\n\n def __init__(\n self,\n path: str,\n *,\n problem_type: str,\n num_classes: int | None = None,\n feature_metadata: FeatureMetadata | None = None,\n eval_metric: Scorer | None = None,\n quantile_levels: list[float] | np.ndarray | None = None,\n low_memory: bool = True,\n k_fold: int = 0,\n n_repeats: int = 1,\n sample_weight: str | None = None,\n weight_evaluation: bool = False,\n save_data: bool = False,\n random_state: int = 0,\n verbosity: int = 2,\n raise_on_model_failure: bool = False,\n ):\n super().__init__(\n path=path,\n low_memory=low_memory,\n save_data=save_data,\n )\n self._validate_num_classes(num_classes=num_classes, problem_type=problem_type)\n self._validate_quantile_levels(quantile_levels=quantile_levels, problem_type=problem_type)\n self.problem_type = problem_type\n self.feature_metadata = feature_metadata\n\n #: Integer value added to the stack level to get the random_state for kfold splits or the train/val split if bagging is disabled\n self.random_state = random_state\n self.verbosity = verbosity\n self.raise_on_model_failure = raise_on_model_failure\n\n # TODO: consider redesign where Trainer doesn't need sample_weight column name and weights are separate from X\n self.sample_weight = sample_weight\n self.weight_evaluation = weight_evaluation\n if eval_metric is not None:\n self.eval_metric = eval_metric\n else:\n self.eval_metric = infer_eval_metric(problem_type=self.problem_type)\n\n logger.log(\n 20, f\"AutoGluon will gauge predictive performance using evaluation metric: '{self.eval_metric.name}'\"\n )\n if not self.eval_metric.greater_is_better_internal:\n logger.log(\n 20,\n \"\\tThis metric's sign has been flipped to adhere to being higher_is_better. \"\n \"The metric score can be multiplied by -1 to get the metric value.\",\n )\n if not (self.eval_metric.needs_pred or self.eval_metric.needs_quantile):\n logger.log(\n 20,\n \"\\tThis metric expects predicted probabilities rather than predicted class labels, \"\n \"so you'll need to use predict_proba() instead of predict()\",\n )\n\n logger.log(20, \"\\tTo change this, specify the eval_metric parameter of Predictor()\")\n self.num_classes = num_classes\n self.quantile_levels = quantile_levels\n\n #: will be set to True if feature-pruning is turned on.\n self.feature_prune = False\n\n self.bagged_mode = True if k_fold >= 2 else False\n if self.bagged_mode:\n #: int number of folds to do model bagging, < 2 means disabled\n self.k_fold = k_fold\n self.n_repeats = n_repeats\n else:\n self.k_fold = 0\n self.n_repeats = 1\n\n #: Internal float of the total time limit allowed for a given fit call. Used in logging statements.\n self._time_limit = None\n #: Internal timestamp of the time training started for a given fit call. Used in logging statements.\n self._time_train_start = None\n #: Same as `self._time_train_start` except it is not reset to None after the fit call completes.\n self._time_train_start_last = None\n\n self._num_rows_train = None\n self._num_cols_train = None\n self._num_rows_val = None\n self._num_rows_test = None\n\n self.is_data_saved = False\n self._X_saved = False\n self._y_saved = False\n self._X_val_saved = False\n self._y_val_saved = False\n\n #: custom split indices\n self._groups = None\n\n #: whether to treat regression predictions as class-probabilities (during distillation)\n self._regress_preds_asprobas = False\n\n #: dict of model name -> model failure metadata\n self._models_failed_to_train_errors = dict()\n\n # self._exceptions_list = [] # TODO: Keep exceptions list for debugging during benchmarking.\n\n self.callbacks: list[AbstractCallback] = []\n self._callback_early_stop = False\n\n @property\n def _path_attr(self) -> str:\n \"\"\"Path to cached model graph attributes\"\"\"\n return os.path.join(self.path_utils, \"attr\")\n\n @property\n def has_val(self) -> bool:\n \"\"\"Whether the trainer uses validation data\"\"\"\n return self._num_rows_val is not None\n\n @property\n def num_rows_val_for_calibration(self) -> int:\n \"\"\"The number of rows available to optimize model calibration\"\"\"\n if self._num_rows_val is not None:\n return self._num_rows_val\n elif self.bagged_mode:\n assert self._num_rows_train is not None\n return self._num_rows_train\n else:\n return 0\n\n @property\n def time_left(self) -> float | None:\n \"\"\"\n Remaining time left in the fit call.\n None if time_limit was unspecified.\n \"\"\"\n if self._time_train_start is None:\n return None\n elif self._time_limit is None:\n return None\n time_elapsed = time.time() - self._time_train_start\n time_left = self._time_limit - time_elapsed\n return time_left\n\n @property\n def logger(self) -> logging.Logger:\n return logger\n\n def log(self, level: int, msg, *args, **kwargs):\n self.logger.log(level, msg, *args, **kwargs)\n\n def load_X(self):\n if self._X_saved:\n path = os.path.join(self.path_data, \"X.pkl\")\n return load_pkl.load(path=path)\n return None\n\n def load_X_val(self):\n if self._X_val_saved:\n path = os.path.join(self.path_data, \"X_val.pkl\")\n return load_pkl.load(path=path)\n return None\n\n def load_y(self):\n if self._y_saved:\n path = os.path.join(self.path_data, \"y.pkl\")\n return load_pkl.load(path=path)\n return None\n\n def load_y_val(self):\n if self._y_val_saved:\n path = os.path.join(self.path_data, \"y_val.pkl\")\n return load_pkl.load(path=path)\n return None\n\n def load_data(self):\n X = self.load_X()\n y = self.load_y()\n X_val = self.load_X_val()\n y_val = self.load_y_val()\n\n return X, y, X_val, y_val\n\n def save_X(self, X, verbose=True):\n path = os.path.join(self.path_data, \"X.pkl\")\n save_pkl.save(path=path, object=X, verbose=verbose)\n self._X_saved = True\n\n def save_X_val(self, X, verbose=True):\n path = os.path.join(self.path_data, \"X_val.pkl\")\n save_pkl.save(path=path, object=X, verbose=verbose)\n self._X_val_saved = True\n\n def save_X_test(self, X, verbose=True):\n path = os.path.join(self.path_data, \"X_test.pkl\")\n save_pkl.save(path=path, object=X, verbose=verbose)\n self._X_test_saved = True\n\n def save_y(self, y, verbose=True):\n path = os.path.join(self.path_data, \"y.pkl\")\n save_pkl.save(path=path, object=y, verbose=verbose)\n self._y_saved = True\n\n def save_y_val(self, y, verbose=True):\n path = os.path.join(self.path_data, \"y_val.pkl\")\n save_pkl.save(path=path, object=y, verbose=verbose)\n self._y_val_saved = True\n\n def save_y_test(self, y, verbose=True):\n path = os.path.join(self.path_data, \"y_test.pkl\")\n save_pkl.save(path=path, object=y, verbose=verbose)\n self._y_test_saved = True\n\n def get_model_names(\n self,\n stack_name: list[str] | str | None = None,\n level: list[int] | int | None = None,\n can_infer: bool | None = None,\n models: list[str] | None = None,\n ) -> list[str]:\n if models is None:\n models = list(self.model_graph.nodes)\n if stack_name is not None:\n if not isinstance(stack_name, list):\n stack_name = [stack_name]\n node_attributes: dict = self.get_models_attribute_dict(attribute=\"stack_name\", models=models)\n models = [model_name for model_name in models if node_attributes[model_name] in stack_name]\n if level is not None:\n if not isinstance(level, list):\n level = [level]\n node_attributes: dict = self.get_models_attribute_dict(attribute=\"level\", models=models)\n models = [model_name for model_name in models if node_attributes[model_name] in level]\n # TODO: can_infer is technically more complicated, if an ancestor can't infer then the model can't infer.\n if can_infer is not None:\n node_attributes = self.get_models_attribute_full(attribute=\"can_infer\", models=models, func=min)\n models = [model for model in models if node_attributes[model] == can_infer]\n return models\n\n def get_max_level(self, stack_name: str | None = None, models: list[str] | None = None) -> int:\n models = self.get_model_names(stack_name=stack_name, models=models)\n models_attribute_dict = self.get_models_attribute_dict(attribute=\"level\", models=models)\n if models_attribute_dict:\n return max(list(models_attribute_dict.values()))\n else:\n return -1\n\n def construct_model_templates(self, hyperparameters: dict[str, Any]) -> tuple[list[AbstractModel], dict]:\n \"\"\"Constructs a list of unfit models based on the hyperparameters dict.\"\"\"\n raise NotImplementedError\n\n def construct_model_templates_distillation(\n self, hyperparameters: dict, **kwargs\n ) -> tuple[list[AbstractModel], dict]:\n \"\"\"Constructs a list of unfit models based on the hyperparameters dict for softclass distillation.\"\"\"\n raise NotImplementedError\n\n def get_model_level(self, model_name: str) -> int:\n return self.get_model_attribute(model=model_name, attribute=\"level\")\n\n def fit(self, X, y, hyperparameters: dict, X_val=None, y_val=None, **kwargs):\n raise NotImplementedError\n\n # TODO: Enable easier re-mapping of trained models -> hyperparameters input (They don't share a key since name can change)\n def train_multi_levels(\n self,\n X,\n y,\n hyperparameters: dict,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n X_unlabeled=None,\n base_model_names: list[str] | None = None,\n core_kwargs: dict | None = None,\n aux_kwargs: dict | None = None,\n level_start=1,\n level_end=1,\n time_limit=None,\n name_suffix: str | None = None,\n relative_stack=True,\n level_time_modifier=0.333,\n infer_limit=None,\n infer_limit_batch_size=None,\n callbacks: list[AbstractCallback] | None = None,\n ) -> list[str]:\n \"\"\"\n Trains a multi-layer stack ensemble using the input data on the hyperparameters dict input.\n hyperparameters is used to determine the models used in each stack layer.\n If continuing a stack ensemble with level_start>1, ensure that base_model_names is set to the appropriate base models that will be used by the level_start level models.\n Trains both core and aux models.\n core models are standard models which are fit on the data features. Core models will also use model predictions if base_model_names was specified or if level != 1.\n aux models are ensemble models which only use the predictions of core models as features. These models never use the original features.\n\n level_time_modifier : float, default 0.333\n The amount of extra time given relatively to early stack levels compared to later stack levels.\n If 0, then all stack levels are given 100%/L of the time, where L is the number of stack levels.\n If 1, then all stack levels are given 100% of the time, meaning if the first level uses all of the time given to it, the other levels won't train.\n Time given to a level = remaining_time / remaining_levels * (1 + level_time_modifier), capped by total remaining time.\n\n Returns a list of the model names that were trained from this method call, in order of fit.\n \"\"\"\n self._fit_setup(time_limit=time_limit, callbacks=callbacks)\n time_train_start = self._time_train_start\n assert time_train_start is not None\n\n if self.callbacks:\n callback_classes = [c.__class__.__name__ for c in self.callbacks]\n logger.log(20, f\"User-specified callbacks ({len(self.callbacks)}): {callback_classes}\")\n\n hyperparameters = self._process_hyperparameters(hyperparameters=hyperparameters)\n\n if relative_stack:\n if level_start != 1:\n raise AssertionError(\n f\"level_start must be 1 when `relative_stack=True`. (level_start = {level_start})\"\n )\n level_add = 0\n if base_model_names:\n max_base_model_level = self.get_max_level(models=base_model_names)\n level_start = max_base_model_level + 1\n level_add = level_start - 1\n level_end += level_add\n if level_start != 1:\n hyperparameters_relative = {}\n for key in hyperparameters:\n if isinstance(key, int):\n hyperparameters_relative[key + level_add] = hyperparameters[key]\n else:\n hyperparameters_relative[key] = hyperparameters[key]\n hyperparameters = hyperparameters_relative\n\n core_kwargs = {} if core_kwargs is None else core_kwargs.copy()\n aux_kwargs = {} if aux_kwargs is None else aux_kwargs.copy()\n\n self._callbacks_setup(\n X=X,\n y=y,\n hyperparameters=hyperparameters,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n level_start=level_start,\n level_end=level_end,\n time_limit=time_limit,\n base_model_names=base_model_names,\n core_kwargs=core_kwargs,\n aux_kwargs=aux_kwargs,\n name_suffix=name_suffix,\n level_time_modifier=level_time_modifier,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n )\n # TODO: Add logic for callbacks to specify that the rest of the trainer logic should be skipped in the case where they are overriding the trainer logic.\n\n model_names_fit = []\n if level_start != level_end:\n logger.log(\n 20,\n f\"AutoGluon will fit {level_end - level_start + 1} stack levels (L{level_start} to L{level_end}) ...\",\n )\n for level in range(level_start, level_end + 1):\n core_kwargs_level = core_kwargs.copy()\n aux_kwargs_level = aux_kwargs.copy()\n full_weighted_ensemble = (\n aux_kwargs_level.pop(\"fit_full_last_level_weighted_ensemble\", True)\n and (level == level_end)\n and (level > 1)\n )\n additional_full_weighted_ensemble = (\n aux_kwargs_level.pop(\"full_weighted_ensemble_additionally\", False) and full_weighted_ensemble\n )\n if time_limit is not None:\n time_train_level_start = time.time()\n levels_left = level_end - level + 1\n time_left = time_limit - (time_train_level_start - time_train_start)\n time_limit_for_level = min(time_left / levels_left * (1 + level_time_modifier), time_left)\n time_limit_core = time_limit_for_level\n time_limit_aux = max(\n time_limit_for_level * 0.1, min(time_limit, 360)\n ) # Allows aux to go over time_limit, but only by a small amount\n core_kwargs_level[\"time_limit\"] = core_kwargs_level.get(\"time_limit\", time_limit_core)\n aux_kwargs_level[\"time_limit\"] = aux_kwargs_level.get(\"time_limit\", time_limit_aux)\n base_model_names, aux_models = self.stack_new_level(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n models=hyperparameters,\n level=level,\n base_model_names=base_model_names,\n core_kwargs=core_kwargs_level,\n aux_kwargs=aux_kwargs_level,\n name_suffix=name_suffix,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n full_weighted_ensemble=full_weighted_ensemble,\n additional_full_weighted_ensemble=additional_full_weighted_ensemble,\n )\n model_names_fit += base_model_names + aux_models\n if (self.model_best is None or infer_limit is not None) and len(model_names_fit) != 0:\n self.model_best = self.get_model_best(infer_limit=infer_limit, infer_limit_as_child=True)\n self._callbacks_conclude()\n self._fit_cleanup()\n self.save()\n return model_names_fit\n\n def _fit_setup(\n self, time_limit: float | None = None, callbacks: list[AbstractCallback | list | tuple] | None = None\n ):\n \"\"\"\n Prepare the trainer state at the start of / prior to a fit call.\n Should be paired with a `self._fit_cleanup()` at the conclusion of the fit call.\n \"\"\"\n self._time_train_start = time.time()\n self._time_train_start_last = self._time_train_start\n self._time_limit = time_limit\n self.reset_callbacks()\n callbacks_new = []\n if callbacks is not None:\n assert isinstance(callbacks, list), f\"`callbacks` must be a list. Found invalid type: `{type(callbacks)}`.\"\n for callback in callbacks:\n if isinstance(callback, (list, tuple)):\n assert len(callback) == 2, (\n f\"Callback must either be an initialized object or a tuple/list of length 2, found: {callback}\"\n )\n callback_cls = callback[0]\n if isinstance(callback_cls, str):\n from autogluon.core.callbacks._early_stopping_callback import EarlyStoppingCallback\n from autogluon.core.callbacks._early_stopping_count_callback import EarlyStoppingCountCallback\n from autogluon.core.callbacks._early_stopping_ensemble_callback import (\n EarlyStoppingEnsembleCallback,\n )\n\n _callback_cls_lst = [\n EarlyStoppingCallback,\n EarlyStoppingCountCallback,\n EarlyStoppingEnsembleCallback,\n ]\n\n _callback_cls_name_map = {c.__name__: c for c in _callback_cls_lst}\n\n assert callback_cls in _callback_cls_name_map.keys(), (\n f\"Unknown callback class: {callback_cls}. \"\n f\"Valid classes: {list(_callback_cls_name_map.keys())}\"\n )\n callback_cls = _callback_cls_name_map[callback_cls]\n\n callback_kwargs = callback[1]\n assert isinstance(callback_kwargs, dict), (\n f\"Callback kwargs must be a dictionary, found: {callback_kwargs}\"\n )\n callback = callback_cls(**callback_kwargs)\n else:\n assert isinstance(callback, AbstractCallback), (\n f\"Elements in `callbacks` must be of type AbstractCallback. Found invalid type: `{type(callback)}`.\"\n )\n callbacks_new.append(callback)\n else:\n callbacks_new = []\n self.callbacks = callbacks_new\n\n def _fit_cleanup(self):\n \"\"\"\n Cleanup the trainer state after fit call completes.\n This ensures that future fit calls are not corrupted by prior fit calls.\n Should be paired with an earlier `self._fit_setup()` call.\n \"\"\"\n self._time_limit = None\n self._time_train_start = None\n self.reset_callbacks()\n\n def _callbacks_setup(self, **kwargs):\n for callback in self.callbacks:\n callback.before_trainer_fit(trainer=self, **kwargs)\n\n def _callbacks_conclude(self):\n for callback in self.callbacks:\n callback.after_trainer_fit(trainer=self)\n\n def reset_callbacks(self):\n \"\"\"Deletes callback objects and resets `self._callback_early_stop` to False.\"\"\"\n self.callbacks = []\n self._callback_early_stop = False\n\n # TODO: Consider better greedy approximation method such as via fitting a weighted ensemble to evaluate the value of a subset.\n def _filter_base_models_via_infer_limit(\n self,\n base_model_names: list[str],\n infer_limit: float | None,\n infer_limit_modifier: float = 1.0,\n as_child: bool = True,\n verbose: bool = True,\n ) -> list[str]:\n \"\"\"\n Returns a subset of base_model_names whose combined prediction time for 1 row of data does not exceed infer_limit seconds.\n With the goal of selecting the best valid subset that is most valuable to stack ensembles who use them as base models,\n this is a variant of the constrained knapsack problem and is NP-Hard and infeasible to exactly solve even with fewer than 10 models.\n For practical purposes, this method applies a greedy approximation approach to selecting the subset\n by simply removing models in reverse order of validation score until the remaining subset is valid.\n\n Parameters\n ----------\n base_model_names: list[str]\n list of model names. These models must already be added to the trainer.\n infer_limit: float, optional\n Inference limit in seconds for 1 row of data. This is compared against values pre-computed during fit for the models.\n infer_limit_modifier: float, default = 1.0\n Modifier to multiply infer_limit by.\n Set to <1.0 to provide headroom for stack models who take the returned subset as base models\n so that the stack models are less likely to exceed infer_limit.\n as_child: bool, default = True\n If True, use the inference time of only 1 child model for bags instead of the overall inference time of the bag.\n This is useful if the intent is to refit the models, as this will best estimate the inference time of the refit model.\n verbose: bool, default = True\n Whether to log the models that are removed.\n\n Returns\n -------\n Returns valid subset of models that satisfy constraints.\n \"\"\"\n if infer_limit is None or not base_model_names:\n return base_model_names\n\n base_model_names = base_model_names.copy()\n num_models_og = len(base_model_names)\n infer_limit_threshold = infer_limit * infer_limit_modifier # Add headroom\n\n if as_child:\n attribute = \"predict_1_child_time\"\n else:\n attribute = \"predict_1_time\"\n\n predict_1_time_full_set = self.get_model_attribute_full(model=base_model_names, attribute=attribute)\n\n messages_to_log = []\n\n base_model_names_copy = base_model_names.copy()\n # Prune models that by themselves have larger inference latency than the infer_limit, as they can never be valid\n for base_model_name in base_model_names_copy:\n predict_1_time_full = self.get_model_attribute_full(model=base_model_name, attribute=attribute)\n if predict_1_time_full >= infer_limit_threshold:\n predict_1_time_full_set_old = predict_1_time_full_set\n base_model_names.remove(base_model_name)\n predict_1_time_full_set = self.get_model_attribute_full(model=base_model_names, attribute=attribute)\n if verbose:\n predict_1_time_full_set_log, time_unit = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_full_set\n )\n predict_1_time_full_set_old_log, time_unit_old = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_full_set_old\n )\n messages_to_log.append(\n f\"\\t{round(predict_1_time_full_set_old_log, 3)}{time_unit_old}\\t-> {round(predict_1_time_full_set_log, 3)}{time_unit}\\t({base_model_name})\"\n )\n\n score_val_dict = self.get_models_attribute_dict(attribute=\"val_score\", models=base_model_names)\n sorted_scores = sorted(score_val_dict.items(), key=lambda x: x[1])\n i = 0\n # Prune models by ascending validation score until the remaining subset's combined inference latency satisfies infer_limit\n while base_model_names and (predict_1_time_full_set >= infer_limit_threshold):\n # TODO: Incorporate score vs inference speed tradeoff in a smarter way\n base_model_to_remove = sorted_scores[i][0]\n predict_1_time_full_set_old = predict_1_time_full_set\n base_model_names.remove(base_model_to_remove)\n i += 1\n predict_1_time_full_set = self.get_model_attribute_full(model=base_model_names, attribute=attribute)\n if verbose:\n predict_1_time_full_set_log, time_unit = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_full_set\n )\n predict_1_time_full_set_old_log, time_unit_old = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_full_set_old\n )\n messages_to_log.append(\n f\"\\t{round(predict_1_time_full_set_old_log, 3)}{time_unit_old}\\t-> {round(predict_1_time_full_set_log, 3)}{time_unit}\\t({base_model_to_remove})\"\n )\n\n if messages_to_log:\n infer_limit_threshold_log, time_unit_threshold = convert_time_in_s_to_log_friendly(\n time_in_sec=infer_limit_threshold\n )\n logger.log(\n 20,\n f\"Removing {len(messages_to_log)}/{num_models_og} base models to satisfy inference constraint \"\n f\"(constraint={round(infer_limit_threshold_log, 3)}{time_unit_threshold}) ...\",\n )\n for msg in messages_to_log:\n logger.log(20, msg)\n\n return base_model_names\n\n def stack_new_level(\n self,\n X,\n y,\n models: list[AbstractModel] | dict,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n X_unlabeled=None,\n level=1,\n base_model_names: list[str] | None = None,\n core_kwargs: dict | None = None,\n aux_kwargs: dict | None = None,\n name_suffix: str | None = None,\n infer_limit=None,\n infer_limit_batch_size=None,\n full_weighted_ensemble: bool = False,\n additional_full_weighted_ensemble: bool = False,\n ) -> tuple[list[str], list[str]]:\n \"\"\"\n Similar to calling self.stack_new_level_core, except auxiliary models will also be trained via a call to self.stack_new_level_aux, with the models trained from self.stack_new_level_core used as base models.\n \"\"\"\n if base_model_names is None:\n base_model_names = []\n core_kwargs = {} if core_kwargs is None else core_kwargs.copy()\n aux_kwargs = {} if aux_kwargs is None else aux_kwargs.copy()\n if level < 1:\n raise AssertionError(f\"Stack level must be >= 1, but level={level}.\")\n if base_model_names and level == 1:\n raise AssertionError(\n f\"Stack level 1 models cannot have base models, but base_model_names={base_model_names}.\"\n )\n if name_suffix:\n core_kwargs[\"name_suffix\"] = core_kwargs.get(\"name_suffix\", \"\") + name_suffix\n aux_kwargs[\"name_suffix\"] = aux_kwargs.get(\"name_suffix\", \"\") + name_suffix\n core_models = self.stack_new_level_core(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n models=models,\n level=level,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n base_model_names=base_model_names,\n **core_kwargs,\n )\n\n aux_models = []\n if full_weighted_ensemble:\n full_aux_kwargs = aux_kwargs.copy()\n if additional_full_weighted_ensemble:\n full_aux_kwargs[\"name_extra\"] = \"_ALL\"\n all_base_model_names = self.get_model_names(\n stack_name=\"core\"\n ) # Fit weighted ensemble on all previously fitted core models\n aux_models += self._stack_new_level_aux(\n X_val, y_val, X, y, all_base_model_names, level, infer_limit, infer_limit_batch_size, **full_aux_kwargs\n )\n\n if (not full_weighted_ensemble) or additional_full_weighted_ensemble:\n aux_models += self._stack_new_level_aux(\n X_val, y_val, X, y, core_models, level, infer_limit, infer_limit_batch_size, **aux_kwargs\n )\n\n return core_models, aux_models\n\n def stack_new_level_core(\n self,\n X,\n y,\n models: list[AbstractModel] | dict,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n X_unlabeled=None,\n level=1,\n base_model_names: list[str] | None = None,\n fit_strategy: Literal[\"sequential\", \"parallel\"] = \"sequential\",\n stack_name=\"core\",\n ag_args=None,\n ag_args_fit=None,\n ag_args_ensemble=None,\n included_model_types=None,\n excluded_model_types=None,\n ensemble_type=StackerEnsembleModel,\n name_suffix: str | None = None,\n get_models_func=None,\n refit_full=False,\n infer_limit=None,\n infer_limit_batch_size=None,\n **kwargs,\n ) -> list[str]:\n \"\"\"\n Trains all models using the data provided.\n If level > 1, then the models will use base model predictions as additional features.\n The base models used can be specified via base_model_names.\n If self.bagged_mode, then models will be trained as StackerEnsembleModels.\n The data provided in this method should not contain stack features, as they will be automatically generated if necessary.\n \"\"\"\n if self._callback_early_stop:\n return []\n if get_models_func is None:\n get_models_func = self.construct_model_templates\n if base_model_names is None:\n base_model_names = []\n if not self.bagged_mode and level != 1:\n raise ValueError(\"Stack Ensembling is not valid for non-bagged mode.\")\n\n base_model_names = self._filter_base_models_via_infer_limit(\n base_model_names=base_model_names,\n infer_limit=infer_limit,\n infer_limit_modifier=0.8,\n )\n if ag_args_fit is None:\n ag_args_fit = {}\n ag_args_fit = ag_args_fit.copy()\n if infer_limit_batch_size is not None:\n ag_args_fit[\"predict_1_batch_size\"] = infer_limit_batch_size\n\n if isinstance(models, dict):\n get_models_kwargs = dict(\n level=level,\n name_suffix=name_suffix,\n ag_args=ag_args,\n ag_args_fit=ag_args_fit,\n included_model_types=included_model_types,\n excluded_model_types=excluded_model_types,\n )\n\n if self.bagged_mode:\n if level == 1:\n (base_model_names, base_model_paths, base_model_types) = (None, None, None)\n elif level > 1:\n base_model_names, base_model_paths, base_model_types = self._get_models_load_info(\n model_names=base_model_names\n )\n if len(base_model_names) == 0: # type: ignore\n logger.log(20, f\"No base models to train on, skipping stack level {level}...\")\n return []\n else:\n raise AssertionError(f\"Stack level cannot be less than 1! level = {level}\")\n\n ensemble_kwargs = {\n \"base_model_names\": base_model_names,\n \"base_model_paths_dict\": base_model_paths,\n \"base_model_types_dict\": base_model_types,\n \"base_model_types_inner_dict\": self.get_models_attribute_dict(\n attribute=\"type_inner\", models=base_model_names\n ),\n \"base_model_performances_dict\": self.get_models_attribute_dict(\n attribute=\"val_score\", models=base_model_names\n ),\n \"random_state\": level + self.random_state,\n }\n get_models_kwargs.update(\n dict(\n ag_args_ensemble=ag_args_ensemble,\n ensemble_type=ensemble_type,\n ensemble_kwargs=ensemble_kwargs,\n )\n )\n models, model_args_fit = get_models_func(hyperparameters=models, **get_models_kwargs)\n if model_args_fit:\n hyperparameter_tune_kwargs = {\n model_name: model_args_fit[model_name][\"hyperparameter_tune_kwargs\"]\n for model_name in model_args_fit\n if \"hyperparameter_tune_kwargs\" in model_args_fit[model_name]\n }\n kwargs[\"hyperparameter_tune_kwargs\"] = hyperparameter_tune_kwargs\n\n logger.log(\n 10 if ((not refit_full) and DistributedContext.is_distributed_mode()) else 20,\n f'Fitting {len(models)} L{level} models, fit_strategy=\"{fit_strategy}\" ...',\n )\n\n X_init = self.get_inputs_to_stacker(X, base_models=base_model_names, fit=True)\n feature_metadata = self.get_feature_metadata(use_orig_features=True, base_models=base_model_names)\n if X_val is not None:\n X_val = self.get_inputs_to_stacker(X_val, base_models=base_model_names, fit=False, use_val_cache=True)\n if X_test is not None:\n X_test = self.get_inputs_to_stacker(X_test, base_models=base_model_names, fit=False, use_val_cache=False)\n compute_score = not refit_full\n if refit_full and X_val is not None:\n X_init = pd.concat([X_init, X_val])\n y = pd.concat([y, y_val])\n X_val = None\n y_val = None\n if X_unlabeled is not None:\n X_unlabeled = self.get_inputs_to_stacker(X_unlabeled, base_models=base_model_names, fit=False)\n\n fit_kwargs = dict(\n num_classes=self.num_classes,\n feature_metadata=feature_metadata,\n )\n\n # FIXME: TODO: v0.1 X_unlabeled isn't cached so it won't be available during refit_full or fit_extra.\n return self._train_multi(\n X=X_init,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n models=models,\n level=level,\n stack_name=stack_name,\n compute_score=compute_score,\n fit_kwargs=fit_kwargs,\n fit_strategy=fit_strategy,\n **kwargs,\n )\n\n def _stack_new_level_aux(\n self, X_val, y_val, X, y, core_models, level, infer_limit, infer_limit_batch_size, **kwargs\n ):\n if X_val is None:\n aux_models = self.stack_new_level_aux(\n X=X,\n y=y,\n base_model_names=core_models,\n level=level + 1,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n **kwargs,\n )\n else:\n aux_models = self.stack_new_level_aux(\n X=X_val,\n y=y_val,\n fit=False,\n base_model_names=core_models,\n level=level + 1,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n **kwargs,\n )\n return aux_models\n\n # TODO: Consider making level be auto-determined based off of max(base_model_levels)+1\n # TODO: Remove name_suffix, hacked in\n # TODO: X can be optional because it isn't needed if fit=True\n def stack_new_level_aux(\n self,\n X,\n y,\n base_model_names: list[str],\n level: int | str = \"auto\",\n fit=True,\n stack_name=\"aux1\",\n time_limit=None,\n name_suffix: str | None = None,\n get_models_func=None,\n check_if_best=True,\n infer_limit=None,\n infer_limit_batch_size=None,\n use_val_cache=True,\n fit_weighted_ensemble: bool = True,\n name_extra: str | None = None,\n total_resources: dict | None = None,\n ) -> list[str]:\n \"\"\"\n Trains auxiliary models (currently a single weighted ensemble) using the provided base models.\n Level must be greater than the level of any of the base models.\n Auxiliary models never use the original features and only train with the predictions of other models as features.\n \"\"\"\n if self._callback_early_stop:\n return []\n if fit_weighted_ensemble is False:\n # Skip fitting of aux models\n return []\n\n base_model_names = self._filter_base_models_via_infer_limit(\n base_model_names=base_model_names, infer_limit=infer_limit, infer_limit_modifier=0.95\n )\n\n if len(base_model_names) == 0:\n logger.log(20, f\"No base models to train on, skipping auxiliary stack level {level}...\")\n return []\n\n if isinstance(level, str):\n assert level == \"auto\", f\"level must be 'auto' if str, found: {level}\"\n levels_dict = self.get_models_attribute_dict(attribute=\"level\", models=base_model_names)\n base_model_level_max = None\n for k, v in levels_dict.items():\n if base_model_level_max is None or v > base_model_level_max:\n base_model_level_max = v\n level = base_model_level_max + 1\n\n if infer_limit_batch_size is not None:\n ag_args_fit = dict()\n ag_args_fit[\"predict_1_batch_size\"] = infer_limit_batch_size\n else:\n ag_args_fit = None\n X_stack_preds = self.get_inputs_to_stacker(\n X, base_models=base_model_names, fit=fit, use_orig_features=False, use_val_cache=use_val_cache\n )\n if self.weight_evaluation:\n X, w = extract_column(\n X, self.sample_weight\n ) # TODO: consider redesign with w as separate arg instead of bundled inside X\n if w is not None:\n X_stack_preds[self.sample_weight] = w.values / w.mean()\n child_hyperparameters = None\n if name_extra is not None:\n child_hyperparameters = {\"ag_args\": {\"name_suffix\": name_extra}}\n return self.generate_weighted_ensemble(\n X=X_stack_preds,\n y=y,\n level=level,\n base_model_names=base_model_names,\n k_fold=1,\n n_repeats=1,\n ag_args_fit=ag_args_fit,\n stack_name=stack_name,\n time_limit=time_limit,\n name_suffix=name_suffix,\n get_models_func=get_models_func,\n check_if_best=check_if_best,\n child_hyperparameters=child_hyperparameters,\n total_resources=total_resources,\n )\n\n def predict(self, X: pd.DataFrame, model: str | None = None) -> np.ndarray:\n if model is None:\n model = self._get_best()\n return self._predict_model(X=X, model=model)\n\n def predict_proba(self, X: pd.DataFrame, model: str | None = None) -> np.ndarray:\n if model is None:\n model = self._get_best()\n return self._predict_proba_model(X=X, model=model)\n\n def _get_best(self) -> str:\n if self.model_best is not None:\n return self.model_best\n else:\n return self.get_model_best()\n\n # Note: model_pred_proba_dict is mutated in this function to minimize memory usage\n def get_inputs_to_model(\n self,\n model: str | AbstractModel,\n X: pd.DataFrame,\n model_pred_proba_dict: dict[str, np.ndarray] | None = None,\n fit: bool = False,\n preprocess_nonadaptive: bool = False,\n ) -> pd.DataFrame:\n \"\"\"\n For output X:\n If preprocess_nonadaptive=False, call model.predict(X)\n If preprocess_nonadaptive=True, call model.predict(X, preprocess_nonadaptive=False)\n \"\"\"\n if isinstance(model, str):\n # TODO: Remove unnecessary load when no stacking\n model = self.load_model(model)\n model_level = self.get_model_level(model.name)\n if model_level > 1 and isinstance(model, StackerEnsembleModel):\n if fit:\n model_pred_proba_dict = None\n else:\n model_set = self.get_minimum_model_set(model)\n model_set = [\n m for m in model_set if m != model.name\n ] # TODO: Can probably be faster, get this result from graph\n model_pred_proba_dict = self.get_model_pred_proba_dict(\n X=X, models=model_set, model_pred_proba_dict=model_pred_proba_dict\n )\n X = model.preprocess(\n X=X,\n preprocess_nonadaptive=preprocess_nonadaptive,\n fit=fit,\n model_pred_proba_dict=model_pred_proba_dict,\n )\n elif preprocess_nonadaptive:\n X = model.preprocess(X=X, preprocess_stateful=False)\n return X\n\n def score(\n self,\n X: pd.DataFrame,\n y: np.ndarray,\n model: str | None = None,\n metric: Scorer | None = None,\n weights: np.ndarray | None = None,\n as_error: bool = False,\n ) -> float:\n if metric is None:\n metric = self.eval_metric\n if metric.needs_pred or metric.needs_quantile:\n y_pred = self.predict(X=X, model=model)\n y_pred_proba = None\n else:\n y_pred = None\n y_pred_proba = self.predict_proba(X=X, model=model)\n return compute_metric(\n y=y,\n y_pred=y_pred,\n y_pred_proba=y_pred_proba,\n metric=metric,\n weights=weights,\n weight_evaluation=self.weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n def score_with_y_pred_proba(\n self,\n y: np.ndarray,\n y_pred_proba: np.ndarray,\n metric: Scorer | None = None,\n weights: np.ndarray | None = None,\n as_error: bool = False,\n ) -> float:\n if metric is None:\n metric = self.eval_metric\n if metric.needs_pred or metric.needs_quantile:\n y_pred = get_pred_from_proba(y_pred_proba=y_pred_proba, problem_type=self.problem_type)\n y_pred_proba = None\n else:\n y_pred = None\n return compute_metric(\n y=y,\n y_pred=y_pred,\n y_pred_proba=y_pred_proba,\n metric=metric,\n weights=weights,\n weight_evaluation=self.weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n def score_with_y_pred(\n self,\n y: np.ndarray,\n y_pred: np.ndarray,\n weights: np.ndarray | None = None,\n metric: Scorer | None = None,\n as_error: bool = False,\n ) -> float:\n if metric is None:\n metric = self.eval_metric\n return compute_metric(\n y=y,\n y_pred=y_pred,\n y_pred_proba=None,\n metric=metric,\n weights=weights,\n weight_evaluation=self.weight_evaluation,\n as_error=as_error,\n quantile_levels=self.quantile_levels,\n )\n\n # TODO: Slow if large ensemble with many models, could cache output result to speed up during inference\n def _construct_model_pred_order(self, models: list[str]) -> list[str]:\n \"\"\"\n Constructs a list of model names in order of inference calls required to infer on all the models.\n\n Parameters\n ----------\n models : list[str]\n The list of models to construct the prediction order from.\n If a model has dependencies, the dependency models will be put earlier in the output list.\n Models explicitly mentioned in the `models` input will be placed as early as possible in the output list.\n Models earlier in `models` will attempt to be placed earlier in the output list than those later in `models`.\n It is recommended that earlier elements do not have dependency models that are listed later in `models`.\n\n Returns\n -------\n Returns list of models in inference call order, including dependency models of those specified in the input.\n \"\"\"\n model_set = set()\n model_order = []\n for model in models:\n if model in model_set:\n continue\n min_models_set = set(self.get_minimum_model_set(model))\n models_to_load = list(min_models_set.difference(model_set))\n subgraph = nx.subgraph(self.model_graph, models_to_load)\n model_pred_order = list(nx.lexicographical_topological_sort(subgraph))\n model_order += [m for m in model_pred_order if m not in model_set]\n model_set = set(model_order)\n return model_order\n\n def _construct_model_pred_order_with_pred_dict(\n self, models: list[str], models_to_ignore: list[str] | None = None\n ) -> list[str]:\n \"\"\"\n Constructs a list of model names in order of inference calls required to infer on all the models.\n Unlike `_construct_model_pred_order`, this method's output is in undefined order when multiple models are valid to infer at the same time.\n\n Parameters\n ----------\n models : list[str]\n The list of models to construct the prediction order from.\n If a model has dependencies, the dependency models will be put earlier in the output list.\n models_to_ignore : list[str], optional\n A list of models that have already been computed and can be ignored.\n Models in this list and their dependencies (if not depended on by other models in `models`) will be pruned from the final output.\n\n Returns\n -------\n Returns list of models in inference call order, including dependency models of those specified in the input.\n \"\"\"\n model_set = set()\n for model in models:\n if model in model_set:\n continue\n min_model_set = set(self.get_minimum_model_set(model))\n model_set = model_set.union(min_model_set)\n if models_to_ignore is not None:\n model_set = model_set.difference(set(models_to_ignore))\n models_to_load = list(model_set)\n subgraph = nx.DiGraph(nx.subgraph(self.model_graph, models_to_load)) # Wrap subgraph in DiGraph to unfreeze it\n # For model in models_to_ignore, remove model node from graph and all ancestors that have no remaining descendants and are not in `models`\n models_to_ignore = [\n model for model in models_to_load if (model not in models) and (not list(subgraph.successors(model)))\n ]\n while models_to_ignore:\n model = models_to_ignore[0]\n predecessors = list(subgraph.predecessors(model))\n subgraph.remove_node(model)\n models_to_ignore = models_to_ignore[1:]\n for predecessor in predecessors:\n if (\n (predecessor not in models)\n and (not list(subgraph.successors(predecessor)))\n and (predecessor not in models_to_ignore)\n ):\n models_to_ignore.append(predecessor)\n\n # Get model prediction order\n return list(nx.lexicographical_topological_sort(subgraph))\n\n def get_models_attribute_dict(self, attribute: str, models: list | None = None) -> dict[str, Any]:\n \"\"\"Returns dictionary of model name -> attribute value for the provided attribute.\"\"\"\n models_attribute_dict = nx.get_node_attributes(self.model_graph, attribute)\n if models is not None:\n model_names = []\n for model in models:\n if not isinstance(model, str):\n model = model.name\n model_names.append(model)\n if attribute == \"path\":\n models_attribute_dict = {\n key: os.path.join(*val) for key, val in models_attribute_dict.items() if key in model_names\n }\n else:\n models_attribute_dict = {key: val for key, val in models_attribute_dict.items() if key in model_names}\n return models_attribute_dict\n\n # TODO: Consider adding persist to disk functionality for pred_proba dictionary to lessen memory burden on large multiclass problems.\n # For datasets with 100+ classes, this function could potentially run the system OOM due to each pred_proba numpy array taking significant amounts of space.\n # This issue already existed in the previous level-based version but only had the minimum required predictions in memory at a time, whereas this has all model predictions in memory.\n # TODO: Add memory optimal topological ordering -> Minimize amount of pred_probas in memory at a time, delete pred probas that are no longer required\n def get_model_pred_proba_dict(\n self,\n X: pd.DataFrame,\n models: list[str],\n model_pred_proba_dict: dict | None = None,\n model_pred_time_dict: dict | None = None,\n record_pred_time: bool = False,\n use_val_cache: bool = False,\n ):\n \"\"\"\n Optimally computes pred_probas (or predictions if regression) for each model in `models`.\n Will compute each necessary model only once and store predictions in a `model_pred_proba_dict` dictionary.\n Note: Mutates model_pred_proba_dict and model_pred_time_dict input if present to minimize memory usage.\n\n Parameters\n ----------\n X : pd.DataFrame\n Input data to predict on.\n models : list[str]\n The list of models to predict with.\n Note that if models have dependency models, their dependencies will also be predicted with and included in the output.\n model_pred_proba_dict : dict, optional\n A dict of predict_probas that could have been computed by a prior call to `get_model_pred_proba_dict` to avoid redundant computations.\n Models already present in model_pred_proba_dict will not be predicted on.\n get_model_pred_proba_dict(X, models=['A', 'B', 'C']) is equivalent to\n get_model_pred_proba_dict(X, models=['C'], model_pred_proba_dict=get_model_pred_proba_dict(X, models=['A', 'B']))\n Note: Mutated in-place to minimize memory usage\n model_pred_time_dict : dict, optional\n If `record_pred_time==True`, this is a dict of model name to marginal time taken in seconds for the prediction of X.\n Must be specified alongside `model_pred_proba_dict` if `record_pred_time=True` and `model_pred_proba_dict != None`.\n Ignored if `record_pred_time=False`.\n Note: Mutated in-place to minimize memory usage\n record_pred_time : bool, default = False\n Whether to store marginal inference times of each model as an extra output `model_pred_time_dict`.\n use_val_cache : bool, default = False\n Whether to fetch cached val prediction probabilities for models instead of predicting on the data.\n Only set to True if X is equal to the validation data and you want to skip live predictions.\n\n Returns\n -------\n If `record_pred_time==True`, outputs tuple of dicts (model_pred_proba_dict, model_pred_time_dict), else output only model_pred_proba_dict\n \"\"\"\n if model_pred_proba_dict is None:\n model_pred_proba_dict = {}\n if model_pred_time_dict is None:\n model_pred_time_dict = {}\n\n if use_val_cache:\n _, model_pred_proba_dict = self._update_pred_proba_dict_with_val_cache(\n model_set=set(models), model_pred_proba_dict=model_pred_proba_dict\n )\n if not model_pred_proba_dict:\n model_pred_order = self._construct_model_pred_order(models)\n else:\n model_pred_order = self._construct_model_pred_order_with_pred_dict(\n models, models_to_ignore=list(model_pred_proba_dict.keys())\n )\n if use_val_cache:\n model_set, model_pred_proba_dict = self._update_pred_proba_dict_with_val_cache(\n model_set=set(model_pred_order), model_pred_proba_dict=model_pred_proba_dict\n )\n model_pred_order = [model for model in model_pred_order if model in model_set]\n\n # Compute model predictions in topological order\n for model_name in model_pred_order:\n if record_pred_time:\n time_start = time.time()\n\n model = self.load_model(model_name=model_name)\n if isinstance(model, StackerEnsembleModel):\n preprocess_kwargs = dict(infer=False, model_pred_proba_dict=model_pred_proba_dict)\n model_pred_proba_dict[model_name] = model.predict_proba(X, **preprocess_kwargs)\n else:\n model_pred_proba_dict[model_name] = model.predict_proba(X)\n\n if record_pred_time:\n time_end = time.time()\n model_pred_time_dict[model_name] = time_end - time_start\n\n if record_pred_time:\n return model_pred_proba_dict, model_pred_time_dict\n else:\n return model_pred_proba_dict\n\n def get_model_oof_dict(self, models: list[str]) -> dict:\n \"\"\"\n Returns a dictionary of out-of-fold prediction probabilities, keyed by model name\n \"\"\"\n return {model: self.get_model_oof(model) for model in models}\n\n def get_model_pred_dict(self, X: pd.DataFrame, models: list[str], record_pred_time: bool = False, **kwargs):\n \"\"\"\n Optimally computes predictions for each model in `models`.\n Will compute each necessary model only once and store predictions in a `model_pred_dict` dictionary.\n Note: Mutates model_pred_proba_dict and model_pred_time_dict input if present to minimize memory usage.\n\n Acts as a wrapper to `self.get_model_pred_proba_dict`, converting the output to predictions.\n\n Parameters\n ----------\n X : pd.DataFrame\n Input data to predict on.\n models : list[str]\n The list of models to predict with.\n Note that if models have dependency models, their dependencies will also be predicted with and included in the output.\n record_pred_time : bool, default = False\n Whether to store marginal inference times of each model as an extra output `model_pred_time_dict`.\n **kwargs : dict, optional\n Refer to `self.get_model_pred_proba_dict` for documentation of remaining arguments.\n This method shares identical arguments.\n\n Returns\n -------\n If `record_pred_time==True`, outputs tuple of dicts (model_pred_dict, model_pred_time_dict), else output only model_pred_dict\n \"\"\"\n model_pred_proba_dict = self.get_model_pred_proba_dict(\n X=X, models=models, record_pred_time=record_pred_time, **kwargs\n )\n if record_pred_time:\n model_pred_proba_dict, model_pred_time_dict = model_pred_proba_dict\n else:\n model_pred_time_dict = None\n\n model_pred_dict = {}\n for m in model_pred_proba_dict:\n # Convert pred_proba to pred\n model_pred_dict[m] = get_pred_from_proba(\n y_pred_proba=model_pred_proba_dict[m], problem_type=self.problem_type\n )\n\n if record_pred_time:\n return model_pred_dict, model_pred_time_dict\n else:\n return model_pred_dict\n\n def get_model_oof(self, model: str, use_refit_parent: bool = False) -> np.ndarray:\n \"\"\"\n Gets the out of fold prediction probabilities for a bagged ensemble model\n\n Parameters\n ----------\n model : str\n Name of the model to get OOF.\n use_refit_parent: bool = False\n If True and the model is a refit model, will instead return the parent model's OOF.\n If False and the model is a refit model, an exception will be raised.\n\n Returns\n -------\n np.ndarray\n model OOF prediction probabilities (if classification) or predictions (if regression)\n \"\"\"\n if use_refit_parent and self.get_model_attribute(model=model, attribute=\"refit_full\", default=False):\n model = self.get_model_attribute(model=model, attribute=\"refit_full_parent\")\n model_type = self.get_model_attribute(model=model, attribute=\"type\")\n if issubclass(model_type, BaggedEnsembleModel):\n model_path = self.get_model_attribute(model=model, attribute=\"path\")\n return model_type.load_oof(path=os.path.join(self.path, model_path))\n else:\n raise AssertionError(f\"Model {model} must be a BaggedEnsembleModel to return oof_pred_proba\")\n\n def get_model_learning_curves(self, model: str) -> dict:\n model_type = self.get_model_attribute(model=model, attribute=\"type\")\n model_path = self.get_model_attribute(model=model, attribute=\"path\")\n return model_type.load_learning_curves(path=os.path.join(self.path, model_path))\n\n def _update_pred_proba_dict_with_val_cache(self, model_set: set, model_pred_proba_dict):\n \"\"\"For each model in model_set, check if y_pred_proba_val is cached to disk. If so, load and add it to model_pred_proba_dict\"\"\"\n for model in model_set:\n y_pred_proba = self.get_model_attribute(model, attribute=\"cached_y_pred_proba_val\", default=None)\n if isinstance(y_pred_proba, bool):\n if y_pred_proba:\n try:\n y_pred_proba = self._load_model_y_pred_proba_val(model)\n except FileNotFoundError:\n y_pred_proba = None\n else:\n y_pred_proba = None\n if y_pred_proba is not None:\n model_pred_proba_dict[model] = y_pred_proba\n model_set = model_set.difference(set(model_pred_proba_dict.keys()))\n return model_set, model_pred_proba_dict\n\n def get_inputs_to_stacker(\n self,\n X: pd.DataFrame,\n *,\n model: str | None = None,\n base_models: list[str] | None = None,\n model_pred_proba_dict: dict | None = None,\n fit: bool = False,\n use_orig_features: bool = True,\n use_val_cache: bool = False,\n ) -> pd.DataFrame:\n \"\"\"\n Returns the valid X input for a stacker model with base models equal to `base_models`.\n Pairs with `feature_metadata = self.get_feature_metadata(...)`. The contents of the returned `X` should reflect `feature_metadata`.\n\n Parameters\n ----------\n X : pd.DataFrame\n Input data to augment.\n model : str, default = None\n The model to derive `base_models` from.\n Cannot be specified alongside `base_models`.\n base_models : list[str], default = None\n The list of base models to augment X with.\n Base models will add their prediction probabilities as extra features to X.\n Cannot be specified alongside `model`.\n model_pred_proba_dict : dict, optional\n A dict of predict_probas that could have been computed by a prior call to `get_model_pred_proba_dict` to avoid redundant computations.\n Models already present in model_pred_proba_dict will not be predicted on.\n Note: Mutated in-place to minimize memory usage\n fit : bool, default = False\n If True, X represents the training data and the models will return their out-of-fold prediction probabilities.\n If False, X represents validation or test data and the models will predict directly on X to generate their prediction probabilities.\n use_orig_features : bool, default = True\n If True, the output DataFrame will include X's original features in addition to the new stack features.\n If False, the output DataFrame will only contain the new stack features.\n use_val_cache : bool, default = False\n Whether to fetch cached val prediction probabilities for models instead of predicting on the data.\n Only set to True if X is equal to the validation data and you want to skip live predictions.\n\n Returns\n -------\n X : DataFrame, an updated DataFrame with the additional stack features from `base_models`.\n \"\"\"\n if model is not None and base_models is not None:\n raise AssertionError(\"Only one of `model`, `base_models` is allowed to be set.\")\n\n if model is not None and base_models is None:\n base_models = self.get_base_model_names(model)\n if not base_models:\n return X\n if fit:\n model_pred_proba_dict = self.get_model_oof_dict(models=base_models)\n else:\n model_pred_proba_dict = self.get_model_pred_proba_dict(\n X=X, models=base_models, model_pred_proba_dict=model_pred_proba_dict, use_val_cache=use_val_cache\n )\n pred_proba_list = [model_pred_proba_dict[model] for model in base_models]\n stack_column_names, _ = self._get_stack_column_names(models=base_models)\n X_stacker = convert_pred_probas_to_df(\n pred_proba_list=pred_proba_list, problem_type=self.problem_type, columns=stack_column_names, index=X.index\n )\n if use_orig_features:\n X = pd.concat([X_stacker, X], axis=1)\n else:\n X = X_stacker\n return X\n\n def get_feature_metadata(\n self, use_orig_features: bool = True, model: str | None = None, base_models: list[str] | None = None\n ) -> FeatureMetadata:\n \"\"\"\n Returns the FeatureMetadata input to a `model.fit` call.\n Pairs with `X = self.get_inputs_to_stacker(...)`. The returned FeatureMetadata should reflect the contents of `X`.\n\n Parameters\n ----------\n use_orig_features : bool, default = True\n If True, will include the original features in the FeatureMetadata.\n If False, will only include the stack features in the FeatureMetadata.\n model : str, default = None\n If specified, it must be an already existing model.\n `base_models` will be set to the base models of `model`.\n base_models : list[str], default = None\n If specified, will add the stack features of the `base_models` to FeatureMetadata.\n\n Returns\n -------\n FeatureMetadata\n The FeatureMetadata that should be passed into a `model.fit` call.\n \"\"\"\n if model is not None and base_models is not None:\n raise AssertionError(\"Only one of `model`, `base_models` is allowed to be set.\")\n if model is not None and base_models is None:\n base_models = self.get_base_model_names(model)\n\n feature_metadata = None\n if use_orig_features:\n feature_metadata = self.feature_metadata\n if base_models:\n stack_column_names, _ = self._get_stack_column_names(models=base_models)\n stacker_type_map_raw = {column: R_FLOAT for column in stack_column_names}\n stacker_type_group_map_special = {S_STACK: stack_column_names}\n stacker_feature_metadata = FeatureMetadata(\n type_map_raw=stacker_type_map_raw, type_group_map_special=stacker_type_group_map_special\n )\n if feature_metadata is not None:\n feature_metadata = feature_metadata.join_metadata(stacker_feature_metadata)\n else:\n feature_metadata = stacker_feature_metadata\n if feature_metadata is None:\n feature_metadata = FeatureMetadata(type_map_raw={})\n return feature_metadata\n\n def _get_stack_column_names(self, models: list[str]) -> tuple[list[str], int]:\n \"\"\"\n Get the stack column names generated when the provided models are used as base models in a stack ensemble.\n Additionally output the number of columns per model as an int.\n \"\"\"\n if self.problem_type in [MULTICLASS, SOFTCLASS]:\n stack_column_names = [\n stack_column_prefix + \"_\" + str(cls)\n for stack_column_prefix in models\n for cls in range(self.num_classes)\n ]\n num_columns_per_model = self.num_classes\n elif self.problem_type == QUANTILE:\n stack_column_names = [\n stack_column_prefix + \"_\" + str(q) for stack_column_prefix in models for q in self.quantile_levels\n ]\n num_columns_per_model = len(self.quantile_levels)\n else:\n stack_column_names = models\n num_columns_per_model = 1\n return stack_column_names, num_columns_per_model\n\n # You must have previously called fit() with cache_data=True\n # Fits _FULL versions of specified models, but does NOT link them (_FULL stackers will still use normal models as input)\n def refit_single_full(\n self,\n X=None,\n y=None,\n X_val=None,\n y_val=None,\n X_unlabeled=None,\n models=None,\n fit_strategy: Literal[\"sequential\", \"parallel\"] = \"sequential\",\n **kwargs,\n ) -> list[str]:\n if fit_strategy == \"parallel\":\n logger.log(\n 30, f\"Note: refit_full does not yet support fit_strategy='parallel', switching to 'sequential'...\"\n )\n fit_strategy = \"sequential\"\n if X is None:\n X = self.load_X()\n if X_val is None:\n X_val = self.load_X_val()\n if y is None:\n y = self.load_y()\n if y_val is None:\n y_val = self.load_y_val()\n\n if models is None:\n models = self.get_model_names()\n\n model_levels = dict()\n ignore_models = []\n ignore_stack_names = [REFIT_FULL_NAME]\n for stack_name in ignore_stack_names:\n ignore_models += self.get_model_names(\n stack_name=stack_name\n ) # get_model_names returns [] if stack_name does not exist\n models = [model for model in models if model not in ignore_models]\n for model in models:\n model_level = self.get_model_level(model)\n if model_level not in model_levels:\n model_levels[model_level] = []\n model_levels[model_level].append(model)\n\n levels = sorted(model_levels.keys())\n models_trained_full = []\n model_refit_map = {} # FIXME: is this even used, remove?\n\n if fit_strategy == \"sequential\":\n for level in levels:\n models_level = model_levels[level]\n for model in models_level:\n model_name, models_trained = _detached_refit_single_full(\n _self=self,\n model=model,\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n level=level,\n kwargs=kwargs,\n fit_strategy=fit_strategy,\n )\n if len(models_trained) == 1:\n model_refit_map[model_name] = models_trained[0]\n for model_trained in models_trained:\n self._update_model_attr(\n model_trained,\n refit_full=True,\n refit_full_parent=model_name,\n refit_full_parent_val_score=self.get_model_attribute(model_name, \"val_score\"),\n )\n models_trained_full += models_trained\n elif fit_strategy == \"parallel\":\n # -- Parallel refit\n ray = try_import_ray()\n\n # FIXME: Need a common utility class for initializing ray so we don't duplicate code\n if not ray.is_initialized():\n ray.init(log_to_driver=False, logging_level=logging.ERROR)\n\n distributed_manager = ParallelFitManager(\n mode=\"refit\",\n func=_remote_refit_single_full,\n func_kwargs=dict(fit_strategy=fit_strategy),\n func_put_kwargs=dict(\n _self=self,\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n kwargs=kwargs,\n ),\n # TODO: check if this is available in the kwargs\n num_cpus=kwargs.get(\"total_resources\", {}).get(\"num_cpus\", 1),\n num_gpus=kwargs.get(\"total_resources\", {}).get(\"num_gpus\", 0),\n get_model_attribute_func=self.get_model_attribute,\n X=X,\n y=y,\n )\n\n for level in levels:\n models_trained_full_level = []\n distributed_manager.job_kwargs[\"level\"] = level\n models_level = model_levels[level]\n\n logger.log(\n 20, f\"Scheduling distributed model-workers for refitting {len(models_level)} L{level} models...\"\n )\n unfinished_job_refs = distributed_manager.schedule_jobs(models_to_fit=models_level)\n\n while unfinished_job_refs:\n finished, unfinished_job_refs = ray.wait(unfinished_job_refs, num_returns=1)\n refit_full_parent, model_trained, model_path, model_type = ray.get(finished[0])\n\n self._add_model(\n model_type.load(path=os.path.join(self.path, model_path), reset_paths=self.reset_paths),\n stack_name=REFIT_FULL_NAME,\n level=level,\n _is_refit=True,\n )\n model_refit_map[refit_full_parent] = model_trained\n self._update_model_attr(\n model_trained,\n refit_full=True,\n refit_full_parent=refit_full_parent,\n refit_full_parent_val_score=self.get_model_attribute(refit_full_parent, \"val_score\"),\n )\n models_trained_full_level.append(model_trained)\n\n logger.log(20, f\"Finished refit model for {refit_full_parent}\")\n unfinished_job_refs += distributed_manager.schedule_jobs()\n\n logger.log(20, f\"Finished distributed refitting for {len(models_trained_full_level)} L{level} models.\")\n models_trained_full += models_trained_full_level\n distributed_manager.clean_job_state(unfinished_job_refs=unfinished_job_refs)\n\n distributed_manager.clean_up_ray()\n else:\n raise ValueError(f\"Invalid value for fit_strategy: '{fit_strategy}'\")\n\n keys_to_del = []\n for model in model_refit_map.keys():\n if model_refit_map[model] not in models_trained_full:\n keys_to_del.append(model)\n for key in keys_to_del:\n del model_refit_map[key]\n self.save() # TODO: This could be more efficient by passing in arg to not save if called by refit_ensemble_full since it saves anyways later.\n return models_trained_full\n\n # Fits _FULL models and links them in the stack so _FULL models only use other _FULL models as input during stacking\n # If model is specified, will fit all _FULL models that are ancestors of the provided model, automatically linking them.\n # If no model is specified, all models are refit and linked appropriately.\n def refit_ensemble_full(self, model: str | list[str] = \"all\", **kwargs) -> dict:\n if model == \"all\":\n ensemble_set = self.get_model_names()\n elif isinstance(model, list):\n ensemble_set = self.get_minimum_models_set(model)\n else:\n if model == \"best\":\n model = self.get_model_best()\n ensemble_set = self.get_minimum_model_set(model)\n existing_models = self.get_model_names()\n ensemble_set_valid = []\n model_refit_map = self.model_refit_map()\n for model in ensemble_set:\n if model in model_refit_map and model_refit_map[model] in existing_models:\n logger.log(\n 20,\n f\"Model '{model}' already has a refit _FULL model: '{model_refit_map[model]}', skipping refit...\",\n )\n else:\n ensemble_set_valid.append(model)\n if ensemble_set_valid:\n models_trained_full = self.refit_single_full(models=ensemble_set_valid, **kwargs)\n else:\n models_trained_full = []\n\n model_refit_map = self.model_refit_map()\n for model_full in models_trained_full:\n # TODO: Consider moving base model info to a separate pkl file so that it can be edited without having to load/save the model again\n # Downside: Slower inference speed when models are not persisted in memory prior.\n model_loaded = self.load_model(model_full)\n if isinstance(model_loaded, StackerEnsembleModel):\n for stack_column_prefix in model_loaded.stack_column_prefix_lst:\n base_model = model_loaded.stack_column_prefix_to_model_map[stack_column_prefix]\n new_base_model = model_refit_map[base_model]\n new_base_model_type = self.get_model_attribute(model=new_base_model, attribute=\"type\")\n new_base_model_path = self.get_model_attribute(model=new_base_model, attribute=\"path\")\n\n model_loaded.base_model_paths_dict[new_base_model] = new_base_model_path\n model_loaded.base_model_types_dict[new_base_model] = new_base_model_type\n model_loaded.base_model_names.append(new_base_model)\n model_loaded.stack_column_prefix_to_model_map[stack_column_prefix] = new_base_model\n\n model_loaded.save() # TODO: Avoid this!\n\n # Remove old edges and add new edges\n edges_to_remove = list(self.model_graph.in_edges(model_loaded.name))\n self.model_graph.remove_edges_from(edges_to_remove)\n if isinstance(model_loaded, StackerEnsembleModel):\n for stack_column_prefix in model_loaded.stack_column_prefix_lst:\n base_model_name = model_loaded.stack_column_prefix_to_model_map[stack_column_prefix]\n self.model_graph.add_edge(base_model_name, model_loaded.name)\n\n self.save()\n return self.model_refit_map()\n\n def get_refit_full_parent(self, model: str) -> str:\n \"\"\"Get refit full model's parent. If model does not have a parent, return `model`.\"\"\"\n return self.get_model_attribute(model=model, attribute=\"refit_full_parent\", default=model)\n\n def get_model_best(\n self,\n can_infer: bool | None = None,\n allow_full: bool = True,\n infer_limit: float | None = None,\n infer_limit_as_child: bool = False,\n ) -> str:\n \"\"\"\n Returns the name of the model with the best validation score that satisfies all specified constraints.\n If no model satisfies the constraints, an AssertionError will be raised.\n\n Parameters\n ----------\n can_infer: bool, default = None\n If True, only consider models that can infer.\n If False, only consider models that can't infer.\n If None, consider all models.\n allow_full: bool, default = True\n If True, consider all models.\n If False, disallow refit_full models.\n infer_limit: float, default = None\n The maximum time in seconds per sample that a model is allowed to take during inference.\n If None, consider all models.\n If specified, consider only models that have a lower predict time per sample than `infer_limit`.\n infer_limit_as_child: bool, default = False\n If True, use the predict time per sample of the (theoretical) refit version of the model.\n If the model is already refit, the predict time per sample is unchanged.\n If False, use the predict time per sample of the model.\n\n Returns\n -------\n model: str\n The string name of the model with the best metric score that satisfies all constraints.\n \"\"\"\n models = self.get_model_names(can_infer=can_infer)\n if not models:\n raise AssertionError(\"Trainer has no fit models that can infer.\")\n models_full = self.get_models_attribute_dict(models=models, attribute=\"refit_full_parent\")\n if not allow_full:\n models = [model for model in models if model not in models_full]\n\n predict_1_time_attribute = None\n if infer_limit is not None:\n if infer_limit_as_child:\n predict_1_time_attribute = \"predict_1_child_time\"\n else:\n predict_1_time_attribute = \"predict_1_time\"\n models_predict_1_time = self.get_models_attribute_full(models=models, attribute=predict_1_time_attribute)\n models_og = copy.deepcopy(models)\n for model_key in models_predict_1_time:\n if models_predict_1_time[model_key] is None or models_predict_1_time[model_key] > infer_limit:\n models.remove(model_key)\n if models_og and not models:\n # get the fastest model\n models_predict_time_list = [models_predict_1_time[m] for m in models_og]\n min_time = np.array(models_predict_time_list).min()\n infer_limit_new = min_time * 1.2 # Give 20% lee-way\n logger.log(\n 30,\n f\"WARNING: Impossible to satisfy infer_limit constraint. Relaxing constraint from {infer_limit} to {infer_limit_new} ...\",\n )\n models = models_og\n for model_key in models_predict_1_time:\n if models_predict_1_time[model_key] > infer_limit_new:\n models.remove(model_key)\n if not models:\n raise AssertionError(\n f\"Trainer has no fit models that can infer while satisfying the constraints: (infer_limit={infer_limit}, allow_full={allow_full}).\"\n )\n model_performances = self.get_models_attribute_dict(models=models, attribute=\"val_score\")\n\n predict_time_attr = predict_1_time_attribute if predict_1_time_attribute is not None else \"predict_time\"\n models_predict_time = self.get_models_attribute_full(models=models, attribute=predict_time_attr)\n\n perfs = [\n (m, model_performances[m], models_predict_time[m]) for m in models if model_performances[m] is not None\n ]\n if not perfs:\n models = [m for m in models if m in models_full]\n perfs = [\n (m, self.get_model_attribute(model=m, attribute=\"refit_full_parent_val_score\"), models_predict_time[m])\n for m in models\n ]\n if not perfs:\n raise AssertionError(\n \"No fit models that can infer exist with a validation score to choose the best model.\"\n )\n elif not allow_full:\n raise AssertionError(\n \"No fit models that can infer exist with a validation score to choose the best model, but refit_full models exist. Set `allow_full=True` to get the best refit_full model.\"\n )\n return max(perfs, key=lambda i: (i[1], -i[2]))[0]\n\n def save_model(self, model, reduce_memory=True):\n # TODO: In future perhaps give option for the reduce_memory_size arguments, perhaps trainer level variables specified by user?\n if reduce_memory:\n model.reduce_memory_size(remove_fit=True, remove_info=False, requires_save=True)\n if self.low_memory:\n model.save()\n else:\n self.models[model.name] = model\n\n def save(self) -> None:\n models = self.models\n if self.low_memory:\n self.models = {}\n save_pkl.save(path=os.path.join(self.path, self.trainer_file_name), object=self)\n if self.low_memory:\n self.models = models\n\n def compile(self, model_names=\"all\", with_ancestors=False, compiler_configs=None) -> list[str]:\n \"\"\"\n Compile a list of models for accelerated prediction.\n\n Parameters\n ----------\n model_names : str or list\n A list of model names for model compilation. Alternatively, this can be 'all' or 'best'.\n compiler_configs: dict, default=None\n Model specific compiler options.\n This can be useful to specify the compiler backend for a specific model,\n e.g. {\"RandomForest\": {\"compiler\": \"onnx\"}}\n \"\"\"\n if model_names == \"all\":\n model_names = self.get_model_names(can_infer=True)\n elif model_names == \"best\":\n if self.model_best is not None:\n model_names = [self.model_best]\n else:\n model_names = [self.get_model_best(can_infer=True)]\n if not isinstance(model_names, list):\n raise ValueError(f\"model_names must be a list of model names. Invalid value: {model_names}\")\n if with_ancestors:\n model_names = self.get_minimum_models_set(model_names)\n\n logger.log(20, f\"Compiling {len(model_names)} Models ...\")\n total_compile_time = 0\n\n model_names_to_compile = []\n model_names_to_configs_dict = dict()\n for model_name in model_names:\n model_type_inner = self.get_model_attribute(model_name, \"type_inner\")\n # Get model specific compiler options\n # Model type can be described with either model type, or model name as string\n if model_name in compiler_configs:\n config = compiler_configs[model_name]\n elif model_type_inner in compiler_configs:\n config = compiler_configs[model_type_inner]\n else:\n config = None\n if config is not None:\n model_names_to_compile.append(model_name)\n model_names_to_configs_dict[model_name] = config\n else:\n logger.log(20, f\"Skipping compilation for {model_name} ... (No config specified)\")\n for model_name in model_names_to_compile:\n model = self.load_model(model_name)\n config = model_names_to_configs_dict[model_name]\n\n # Check if already compiled, or if can't compile due to missing dependencies,\n # or if model hasn't implemented compiling.\n if \"compiler\" in config and model.get_compiler_name() == config[\"compiler\"]:\n logger.log(\n 20,\n f'Skipping compilation for {model_name} ... (Already compiled with \"{model.get_compiler_name()}\" backend)',\n )\n elif model.can_compile(compiler_configs=config):\n logger.log(20, f\"Compiling model: {model.name} ... Config = {config}\")\n compile_start_time = time.time()\n model.compile(compiler_configs=config)\n compile_end_time = time.time()\n model.compile_time = compile_end_time - compile_start_time\n compile_type = model.get_compiler_name()\n total_compile_time += model.compile_time\n\n # Update model_graph in order to put compile_time into leaderboard,\n # since models are saved right after training.\n self.model_graph.nodes[model.name][\"compile_time\"] = model.compile_time\n self.save_model(model, reduce_memory=False)\n logger.log(20, f'\\tCompiled model with \"{compile_type}\" backend ...')\n logger.log(20, f\"\\t{round(model.compile_time, 2)}s\\t = Compile runtime\")\n else:\n logger.log(\n 20,\n f\"Skipping compilation for {model.name} ... (Unable to compile with the provided config: {config})\",\n )\n logger.log(20, f\"Finished compiling models, total runtime = {round(total_compile_time, 2)}s.\")\n self.save()\n return model_names\n\n def persist(self, model_names=\"all\", with_ancestors=False, max_memory=None) -> list[str]:\n if model_names == \"all\":\n model_names = self.get_model_names()\n elif model_names == \"best\":\n if self.model_best is not None:\n model_names = [self.model_best]\n else:\n model_names = [self.get_model_best(can_infer=True)]\n if not isinstance(model_names, list):\n raise ValueError(f\"model_names must be a list of model names. Invalid value: {model_names}\")\n if with_ancestors:\n model_names = self.get_minimum_models_set(model_names)\n model_names_already_persisted = [model_name for model_name in model_names if model_name in self.models]\n if model_names_already_persisted:\n logger.log(\n 30,\n f\"The following {len(model_names_already_persisted)} models were already persisted and will be ignored in the model loading process: {model_names_already_persisted}\",\n )\n model_names = [model_name for model_name in model_names if model_name not in model_names_already_persisted]\n if not model_names:\n logger.log(\n 30,\n f\"No valid unpersisted models were specified to be persisted, so no change in model persistence was performed.\",\n )\n return []\n if max_memory is not None:\n\n @disable_if_lite_mode(ret=True)\n def _check_memory():\n info = self.get_models_info(model_names)\n model_mem_size_map = {model: info[model][\"memory_size\"] for model in model_names}\n for model in model_mem_size_map:\n if \"children_info\" in info[model]:\n for child in info[model][\"children_info\"].values():\n model_mem_size_map[model] += child[\"memory_size\"]\n total_mem_required = sum(model_mem_size_map.values())\n available_mem = ResourceManager.get_available_virtual_mem()\n memory_proportion = total_mem_required / available_mem\n if memory_proportion > max_memory:\n logger.log(\n 30,\n f\"Models will not be persisted in memory as they are expected to require {round(memory_proportion * 100, 2)}% of memory, which is greater than the specified max_memory limit of {round(max_memory * 100, 2)}%.\",\n )\n logger.log(\n 30,\n f\"\\tModels will be loaded on-demand from disk to maintain safe memory usage, increasing inference latency. If inference latency is a concern, try to use smaller models or increase the value of max_memory.\",\n )\n return False\n else:\n logger.log(\n 20,\n f\"Persisting {len(model_names)} models in memory. Models will require {round(memory_proportion * 100, 2)}% of memory.\",\n )\n return True\n\n if not _check_memory():\n return []\n\n models = []\n for model_name in model_names:\n model = self.load_model(model_name)\n self.models[model.name] = model\n models.append(model)\n\n for model in models:\n # TODO: Move this to model code\n if isinstance(model, BaggedEnsembleModel):\n for fold, fold_model in enumerate(model.models):\n if isinstance(fold_model, str):\n model.models[fold] = model.load_child(fold_model)\n return model_names\n\n def unpersist(self, model_names=\"all\") -> list:\n if model_names == \"all\":\n model_names = list(self.models.keys())\n if not isinstance(model_names, list):\n raise ValueError(f\"model_names must be a list of model names. Invalid value: {model_names}\")\n unpersisted_models = []\n for model in model_names:\n if model in self.models:\n self.models.pop(model)\n unpersisted_models.append(model)\n if unpersisted_models:\n logger.log(20, f\"Unpersisted {len(unpersisted_models)} models: {unpersisted_models}\")\n else:\n logger.log(\n 30,\n f\"No valid persisted models were specified to be unpersisted, so no change in model persistence was performed.\",\n )\n return unpersisted_models\n\n def generate_weighted_ensemble(\n self,\n X,\n y,\n level,\n base_model_names,\n k_fold=1,\n n_repeats=1,\n stack_name=None,\n hyperparameters=None,\n ag_args_fit=None,\n time_limit=None,\n name_suffix: str | None = None,\n save_bag_folds=None,\n check_if_best=True,\n child_hyperparameters=None,\n get_models_func=None,\n total_resources: dict | None = None,\n ) -> list[str]:\n if get_models_func is None:\n get_models_func = self.construct_model_templates\n if len(base_model_names) == 0:\n logger.log(20, \"No base models to train on, skipping weighted ensemble...\")\n return []\n\n if child_hyperparameters is None:\n child_hyperparameters = {}\n\n if save_bag_folds is None:\n can_infer_dict = self.get_models_attribute_dict(\"can_infer\", models=base_model_names)\n if False in can_infer_dict.values():\n save_bag_folds = False\n else:\n save_bag_folds = True\n\n feature_metadata = self.get_feature_metadata(use_orig_features=False, base_models=base_model_names)\n\n base_model_paths_dict = self.get_models_attribute_dict(attribute=\"path\", models=base_model_names)\n base_model_paths_dict = {key: os.path.join(self.path, val) for key, val in base_model_paths_dict.items()}\n weighted_ensemble_model, _ = get_models_func(\n hyperparameters={\n \"default\": {\n \"ENS_WEIGHTED\": [child_hyperparameters],\n }\n },\n ensemble_type=WeightedEnsembleModel,\n ensemble_kwargs=dict(\n base_model_names=base_model_names,\n base_model_paths_dict=base_model_paths_dict,\n base_model_types_dict=self.get_models_attribute_dict(attribute=\"type\", models=base_model_names),\n base_model_types_inner_dict=self.get_models_attribute_dict(\n attribute=\"type_inner\", models=base_model_names\n ),\n base_model_performances_dict=self.get_models_attribute_dict(\n attribute=\"val_score\", models=base_model_names\n ),\n hyperparameters=hyperparameters,\n random_state=level + self.random_state,\n ),\n ag_args={\"name_bag_suffix\": \"\"},\n ag_args_fit=ag_args_fit,\n ag_args_ensemble={\"save_bag_folds\": save_bag_folds},\n name_suffix=name_suffix,\n level=level,\n )\n weighted_ensemble_model = weighted_ensemble_model[0]\n w = None\n if self.weight_evaluation:\n X, w = extract_column(X, self.sample_weight)\n models = self._train_multi(\n X=X,\n y=y,\n X_val=None,\n y_val=None,\n models=[weighted_ensemble_model],\n k_fold=k_fold,\n n_repeats=n_repeats,\n hyperparameter_tune_kwargs=None,\n stack_name=stack_name,\n level=level,\n time_limit=time_limit,\n ens_sample_weight=w,\n fit_kwargs=dict(\n feature_metadata=feature_metadata, num_classes=self.num_classes, groups=None\n ), # FIXME: Is this the right way to do this?\n total_resources=total_resources,\n )\n for weighted_ensemble_model_name in models:\n if check_if_best and weighted_ensemble_model_name in self.get_model_names():\n if self.model_best is None:\n self.model_best = weighted_ensemble_model_name\n else:\n best_score = self.get_model_attribute(self.model_best, \"val_score\")\n cur_score = self.get_model_attribute(weighted_ensemble_model_name, \"val_score\")\n if best_score is not None and cur_score > best_score:\n # new best model\n self.model_best = weighted_ensemble_model_name\n return models\n\n def _train_single(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n model: AbstractModel,\n X_val: pd.DataFrame | None = None,\n y_val: pd.Series | None = None,\n X_test: pd.DataFrame | None = None,\n y_test: pd.Series | None = None,\n total_resources: dict = None,\n **model_fit_kwargs,\n ) -> AbstractModel:\n \"\"\"\n Trains model but does not add the trained model to this Trainer.\n Returns trained model object.\n \"\"\"\n model = model.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n total_resources=total_resources,\n **model_fit_kwargs,\n )\n return model\n\n def _train_and_save(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n model: AbstractModel,\n X_val: pd.DataFrame | None = None,\n y_val: pd.Series | None = None,\n X_test: pd.DataFrame | None = None,\n y_test: pd.Series | None = None,\n X_pseudo: pd.DataFrame | None = None,\n y_pseudo: pd.DataFrame | None = None,\n time_limit: float | None = None,\n stack_name: str = \"core\",\n level: int = 1,\n compute_score: bool = True,\n total_resources: dict | None = None,\n errors: Literal[\"ignore\", \"raise\"] = \"ignore\",\n errors_ignore: list | None = None,\n errors_raise: list | None = None,\n is_ray_worker: bool = False,\n **model_fit_kwargs,\n ) -> list[str]:\n \"\"\"\n Trains model and saves it to disk, returning a list with a single element: The name of the model, or no elements if training failed.\n If the model name is returned:\n The model can be accessed via self.load_model(model.name).\n The model will have metadata information stored in self.model_graph.\n The model's name will be appended to self.models_level[stack_name][level]\n The model will be accessible and usable through any Trainer function that takes as input 'model' or 'model_name'.\n Note: self._train_and_save should not be used outside of self._train_single_full\n\n Parameters\n ----------\n errors: Literal[\"ignore\", \"raise\"], default = \"ignore\"\n Determines how model fit exceptions are handled.\n If \"ignore\", will ignore all model exceptions during fit. If an exception occurs, an empty list is returned.\n If \"raise\", will raise the model exception if it occurs.\n Can be overwritten by `errors_ignore` and `errors_raise`.\n errors_ignore: list[str], optional\n The exception types specified in `errors_ignore` will be treated as if `errors=\"ignore\"`.\n errors_raise: list[str], optional\n The exception types specified in `errors_raise` will be treated as if `errors=\"raise\"`.\n\n \"\"\"\n fit_start_time = time.time()\n model_names_trained = []\n y_pred_proba_val = None\n\n is_distributed_mode = DistributedContext.is_distributed_mode() or is_ray_worker\n\n fit_log_message = f\"Fitting model: {model.name} ...\"\n if time_limit is not None:\n time_left_total = time_limit\n not_enough_time = False\n if time_limit <= 0:\n not_enough_time = True\n elif self._time_limit is not None and self._time_train_start is not None:\n time_left_total = self._time_limit - (fit_start_time - self._time_train_start)\n # If only a very small amount of time remains, skip training\n min_time_required = min(self._time_limit * 0.01, 10)\n if (time_left_total < min_time_required) and (time_limit < min_time_required):\n not_enough_time = True\n if not_enough_time:\n skip_msg = f\"Skipping {model.name} due to lack of time remaining.\"\n not_enough_time_exception = InsufficientTime(skip_msg)\n if self._check_raise_exception(\n exception=not_enough_time_exception,\n errors=errors,\n errors_ignore=errors_ignore,\n errors_raise=errors_raise,\n ):\n raise not_enough_time_exception\n else:\n logger.log(15, skip_msg)\n return []\n fit_log_message += (\n f\" Training model for up to {time_limit:.2f}s of the {time_left_total:.2f}s of remaining time.\"\n )\n logger.log(10 if is_distributed_mode else 20, fit_log_message)\n\n if isinstance(model, BaggedEnsembleModel) and not compute_score:\n # Do not perform OOF predictions when we don't compute a score.\n model_fit_kwargs[\"_skip_oof\"] = True\n if not isinstance(model, BaggedEnsembleModel):\n model_fit_kwargs.setdefault(\"log_resources\", True)\n\n model_fit_kwargs = dict(\n model=model,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n time_limit=time_limit,\n total_resources=total_resources,\n **model_fit_kwargs,\n )\n\n # If model is not bagged model and not stacked then pseudolabeled data needs to be incorporated at this level\n # Bagged model does validation on the fit level where as single models do it separately. Hence this if statement\n # is required\n if (\n not isinstance(model, BaggedEnsembleModel)\n and X_pseudo is not None\n and y_pseudo is not None\n and X_pseudo.columns.equals(X.columns)\n ):\n assert_pseudo_column_match(X=X, X_pseudo=X_pseudo)\n # Needs .astype(X.dtypes) because pd.concat will convert categorical features to int/float unexpectedly. Need to convert them back to original.\n X_w_pseudo = pd.concat([X, X_pseudo], ignore_index=True).astype(X.dtypes)\n y_w_pseudo = pd.concat([y, y_pseudo], ignore_index=True)\n logger.log(15, f\"{len(X_pseudo)} extra rows of pseudolabeled data added to training set for {model.name}\")\n model_fit_kwargs[\"X\"] = X_w_pseudo\n model_fit_kwargs[\"y\"] = y_w_pseudo\n else:\n model_fit_kwargs[\"X\"] = X\n model_fit_kwargs[\"y\"] = y\n if level > 1:\n if X_pseudo is not None and y_pseudo is not None:\n logger.log(15, f\"Dropping pseudo in stacking layer due to missing out-of-fold predictions\")\n else:\n model_fit_kwargs[\"X_pseudo\"] = X_pseudo\n model_fit_kwargs[\"y_pseudo\"] = y_pseudo\n\n exception = None\n try:\n model = self._train_single(**model_fit_kwargs)\n\n fit_end_time = time.time()\n if self.weight_evaluation:\n w = model_fit_kwargs.get(\"sample_weight\", None)\n w_val = model_fit_kwargs.get(\"sample_weight_val\", None)\n else:\n w = None\n w_val = None\n if not compute_score:\n score = None\n model.predict_time = None\n elif X_val is not None and y_val is not None:\n y_pred_proba_val = model.predict_proba(X_val, record_time=True)\n score = model.score_with_y_pred_proba(y=y_val, y_pred_proba=y_pred_proba_val, sample_weight=w_val)\n elif isinstance(model, BaggedEnsembleModel):\n if model.is_valid_oof() or isinstance(model, WeightedEnsembleModel):\n score = model.score_with_oof(y=y, sample_weight=w)\n else:\n score = None\n else:\n score = None\n pred_end_time = time.time()\n if model.fit_time is None:\n model.fit_time = fit_end_time - fit_start_time\n if model.predict_time is None and score is not None:\n model.predict_time = pred_end_time - fit_end_time\n model.val_score = score\n # TODO: Add recursive=True to avoid repeatedly loading models each time this is called for bagged ensembles (especially during repeated bagging)\n self.save_model(model=model)\n except Exception as exc:\n if self.raise_on_model_failure:\n # immediately raise instead of skipping to next model, useful for debugging during development\n logger.warning(\n \"Model failure occurred... Raising exception instead of continuing to next model. (raise_on_model_failure=True)\"\n )\n raise exc\n exception = exc # required to reference exc outside of `except` statement\n del_model = True\n if isinstance(exception, TimeLimitExceeded):\n logger.log(20, f\"\\tTime limit exceeded... Skipping {model.name}.\")\n elif isinstance(exception, NotEnoughMemoryError):\n logger.warning(f\"\\tNot enough memory to train {model.name}... Skipping this model.\")\n elif isinstance(exception, NoStackFeatures):\n logger.warning(f\"\\tNo stack features to train {model.name}... Skipping this model. {exception}\")\n elif isinstance(exception, NotValidStacker):\n logger.warning(f\"\\tStacking disabled for {model.name}... Skipping this model. {exception}\")\n elif isinstance(exception, NoValidFeatures):\n logger.warning(f\"\\tNo valid features to train {model.name}... Skipping this model.\")\n elif isinstance(exception, NoGPUError):\n logger.warning(f\"\\tNo GPUs available to train {model.name}... Skipping this model.\")\n elif isinstance(exception, NotEnoughCudaMemoryError):\n logger.warning(f\"\\tNot enough CUDA memory available to train {model.name}... Skipping this model.\")\n elif isinstance(exception, ImportError):\n logger.error(\n f\"\\tWarning: Exception caused {model.name} to fail during training (ImportError)... Skipping this model.\"\n )\n logger.error(f\"\\t\\t{exception}\")\n del_model = False\n if self.verbosity > 2:\n logger.exception(\"Detailed Traceback:\")\n else: # all other exceptions\n logger.error(\n f\"\\tWarning: Exception caused {model.name} to fail during training... Skipping this model.\"\n )\n logger.error(f\"\\t\\t{exception}\")\n if self.verbosity > 0:\n logger.exception(\"Detailed Traceback:\")\n crash_time = time.time()\n total_time = crash_time - fit_start_time\n tb = traceback.format_exc()\n model_info = self.get_model_info(model=model)\n self._models_failed_to_train_errors[model.name] = dict(\n exc_type=exception.__class__.__name__,\n exc_str=str(exception),\n exc_traceback=tb,\n model_info=model_info,\n total_time=total_time,\n )\n\n if del_model:\n del model\n else:\n self._add_model(\n model=model,\n stack_name=stack_name,\n level=level,\n y_pred_proba_val=y_pred_proba_val,\n is_ray_worker=is_ray_worker,\n )\n model_names_trained.append(model.name)\n if self.low_memory:\n del model\n if exception is not None:\n if self._check_raise_exception(\n exception=exception, errors=errors, errors_ignore=errors_ignore, errors_raise=errors_raise\n ):\n raise exception\n return model_names_trained\n\n # FIXME: v1.0 Move to AbstractModel for most fields\n def _get_model_metadata(self, model: AbstractModel, stack_name: str = \"core\", level: int = 1) -> dict[str, Any]:\n \"\"\"\n Returns the model metadata used to initialize a node in the DAG (self.model_graph).\n \"\"\"\n if isinstance(model, BaggedEnsembleModel):\n type_inner = model._child_type\n else:\n type_inner = type(model)\n num_children = len(model.models) if hasattr(model, \"models\") else 1\n predict_child_time = model.predict_time / num_children if model.predict_time is not None else None\n predict_1_child_time = model.predict_1_time / num_children if model.predict_1_time is not None else None\n fit_metadata = model.get_fit_metadata()\n\n model_param_aux = getattr(model, \"_params_aux_child\", model.params_aux)\n model_metadata = dict(\n fit_time=model.fit_time,\n compile_time=model.compile_time,\n predict_time=model.predict_time,\n predict_1_time=model.predict_1_time,\n predict_child_time=predict_child_time,\n predict_1_child_time=predict_1_child_time,\n predict_n_time_per_row=model.predict_n_time_per_row,\n predict_n_size=model.predict_n_size,\n val_score=model.val_score,\n eval_metric=model.eval_metric.name,\n stopping_metric=model.stopping_metric.name,\n path=os.path.relpath(model.path, self.path).split(os.sep), # model's relative path to trainer\n type=type(model), # Outer type, can be BaggedEnsemble, StackEnsemble (Type that is able to load the model)\n type_inner=type_inner, # Inner type, if Ensemble then it is the type of the inner model (May not be able to load with this type)\n can_infer=model.can_infer(),\n can_fit=model.can_fit(),\n is_valid=model.is_valid(),\n stack_name=stack_name,\n level=level,\n num_children=num_children,\n fit_num_cpus=model.fit_num_cpus,\n fit_num_gpus=model.fit_num_gpus,\n fit_num_cpus_child=model.fit_num_cpus_child,\n fit_num_gpus_child=model.fit_num_gpus_child,\n refit_full_requires_gpu=(model.fit_num_gpus_child is not None)\n and (model.fit_num_gpus_child >= 1)\n and model._user_params.get(\"refit_folds\", False),\n **fit_metadata,\n )\n return model_metadata\n\n def _add_model(\n self,\n model: AbstractModel,\n stack_name: str = \"core\",\n level: int = 1,\n y_pred_proba_val=None,\n _is_refit=False,\n is_distributed_main=False,\n is_ray_worker: bool = False,\n ) -> bool:\n \"\"\"\n Registers the fit model in the Trainer object. Stores information such as model performance, save path, model type, and more.\n To use a model in Trainer, self._add_model must be called.\n If self.low_memory, then the model object will be deleted after this call. Use Trainer directly to leverage the model further.\n\n Parameters\n ----------\n model : AbstractModel\n Model which has been fit. This model will be registered to the Trainer.\n stack_name : str, default 'core'\n Stack name to assign the model to. This is used for advanced functionality.\n level : int, default 1\n Stack level of the stack name to assign the model to. This is used for advanced functionality.\n The model's name is appended to self.models_level[stack_name][level]\n The model's base_models (if it has any) must all be a lower level than the model.\n is_distributed_main: bool, default = False\n If True, the main process in distributed training is calling this function.\n This is used to avoid redundant logging in distributed training.\n\n Returns\n -------\n boolean, True if model was registered, False if model was found to be invalid and not registered.\n \"\"\"\n if model.val_score is not None and np.isnan(model.val_score):\n msg = (\n f\"WARNING: {model.name} has a val_score of {model.val_score} (NaN)! \"\n f\"This should never happen. The model will not be saved to avoid instability.\"\n )\n if self.raise_on_model_failure:\n raise AssertionError(\n f\"{msg} Raising an exception because `raise_on_model_failure={self.raise_on_model_failure}`.\"\n )\n else:\n logger.warning(msg)\n return False\n # TODO: Add to HPO\n\n node_attributes = self._get_model_metadata(model=model, stack_name=stack_name, level=level)\n if y_pred_proba_val is not None:\n # Cache y_pred_proba_val for later reuse to avoid redundant predict calls\n self._save_model_y_pred_proba_val(model=model.name, y_pred_proba_val=y_pred_proba_val)\n node_attributes[\"cached_y_pred_proba_val\"] = True\n\n self.model_graph.add_node(\n model.name,\n **node_attributes,\n )\n if isinstance(model, StackerEnsembleModel):\n prior_models = self.get_model_names()\n # TODO: raise exception if no base models and level != 1?\n for stack_column_prefix in model.stack_column_prefix_lst:\n base_model_name = model.stack_column_prefix_to_model_map[stack_column_prefix]\n if base_model_name not in prior_models:\n raise AssertionError(\n f\"Model '{model.name}' depends on model '{base_model_name}', but '{base_model_name}' is not registered as a trained model! Valid models: {prior_models}\"\n )\n elif level <= self.model_graph.nodes[base_model_name][\"level\"]:\n raise AssertionError(\n f\"Model '{model.name}' depends on model '{base_model_name}', but '{base_model_name}' is not in a lower stack level. ('{model.name}' level: {level}, '{base_model_name}' level: {self.model_graph.nodes[base_model_name]['level']})\"\n )\n self.model_graph.add_edge(base_model_name, model.name)\n self._log_model_stats(\n model, _is_refit=_is_refit, is_distributed_main=is_distributed_main, is_ray_worker=is_ray_worker\n )\n if self.low_memory:\n del model\n return True\n\n def _path_attr_model(self, model: str):\n \"\"\"Returns directory where attributes are cached\"\"\"\n return os.path.join(self._path_attr, model)\n\n def _path_to_model_attr(self, model: str, attribute: str):\n \"\"\"Returns pkl file path for a cached model attribute\"\"\"\n return os.path.join(self._path_attr_model(model), f\"{attribute}.pkl\")\n\n def _save_model_y_pred_proba_val(self, model: str, y_pred_proba_val):\n \"\"\"Cache y_pred_proba_val for later reuse to avoid redundant predict calls\"\"\"\n save_pkl.save(\n path=self._path_to_model_attr(model=model, attribute=\"y_pred_proba_val\"), object=y_pred_proba_val\n )\n\n def _load_model_y_pred_proba_val(self, model: str):\n \"\"\"Load cached y_pred_proba_val for a given model\"\"\"\n return load_pkl.load(path=self._path_to_model_attr(model=model, attribute=\"y_pred_proba_val\"))\n\n # TODO: Once Python min-version is 3.8, can refactor to use positional-only argument for model\n # https://peps.python.org/pep-0570/#empowering-library-authors\n # Currently this method cannot accept the attribute key 'model' without making usage ugly.\n def _update_model_attr(self, model: str, **attributes):\n \"\"\"Updates model node in graph with the input attributes dictionary\"\"\"\n if model not in self.model_graph:\n raise AssertionError(f'\"{model}\" is not a key in self.model_graph, cannot add attributes: {attributes}')\n self.model_graph.nodes[model].update(attributes)\n\n def _log_model_stats(self, model, _is_refit=False, is_distributed_main=False, is_ray_worker: bool = False):\n \"\"\"Logs model fit time, val score, predict time, and predict_1_time\"\"\"\n model = self.load_model(model)\n print_weights = model._get_tags().get(\"print_weights\", False)\n\n is_log_during_distributed_fit = DistributedContext.is_distributed_mode() and (not is_distributed_main)\n if is_ray_worker:\n is_log_during_distributed_fit = True\n log_level = 10 if is_log_during_distributed_fit else 20\n\n if print_weights:\n model_weights = model._get_model_weights()\n model_weights = {k: round(v, 3) for k, v in model_weights.items()}\n msg_weights = \"\"\n is_first = True\n for key, value in sorted(model_weights.items(), key=lambda x: x[1], reverse=True):\n if not is_first:\n msg_weights += \", \"\n msg_weights += f\"'{key}': {value}\"\n is_first = False\n logger.log(log_level, f\"\\tEnsemble Weights: {{{msg_weights}}}\")\n if model.val_score is not None:\n if model.eval_metric.name != self.eval_metric.name:\n logger.log(log_level, f\"\\tNote: model has different eval_metric than default.\")\n if not model.eval_metric.greater_is_better_internal:\n sign_str = \"-\"\n else:\n sign_str = \"\"\n logger.log(\n log_level, f\"\\t{round(model.val_score, 4)}\\t = Validation score ({sign_str}{model.eval_metric.name})\"\n )\n if model.fit_time is not None:\n logger.log(log_level, f\"\\t{round(model.fit_time, 2)}s\\t = Training runtime\")\n if model.predict_time is not None:\n logger.log(log_level, f\"\\t{round(model.predict_time, 2)}s\\t = Validation runtime\")\n predict_n_time_per_row = self.get_model_attribute_full(model=model.name, attribute=\"predict_n_time_per_row\")\n predict_n_size = self.get_model_attribute_full(model=model.name, attribute=\"predict_n_size\", func=min)\n if predict_n_time_per_row is not None and predict_n_size is not None:\n logger.log(\n 15,\n f\"\\t{round(1 / (predict_n_time_per_row if predict_n_time_per_row else np.finfo(np.float16).eps), 1)}\"\n f\"\\t = Inference throughput (rows/s | {int(predict_n_size)} batch size)\",\n )\n if model.predict_1_time is not None:\n fit_metadata = model.get_fit_metadata()\n predict_1_batch_size = fit_metadata.get(\"predict_1_batch_size\", None)\n assert predict_1_batch_size is not None, (\n \"predict_1_batch_size cannot be None if predict_1_time is not None\"\n )\n\n if _is_refit:\n predict_1_time = self.get_model_attribute(model=model.name, attribute=\"predict_1_child_time\")\n predict_1_time_full = self.get_model_attribute_full(model=model.name, attribute=\"predict_1_child_time\")\n else:\n predict_1_time = model.predict_1_time\n predict_1_time_full = self.get_model_attribute_full(model=model.name, attribute=\"predict_1_time\")\n\n predict_1_time_log, time_unit = convert_time_in_s_to_log_friendly(time_in_sec=predict_1_time)\n logger.log(\n log_level,\n f\"\\t{round(predict_1_time_log, 3)}{time_unit}\\t = Validation runtime (1 row | {predict_1_batch_size} batch size | MARGINAL)\",\n )\n\n predict_1_time_full_log, time_unit = convert_time_in_s_to_log_friendly(time_in_sec=predict_1_time_full)\n logger.log(\n log_level,\n f\"\\t{round(predict_1_time_full_log, 3)}{time_unit}\\t = Validation runtime (1 row | {predict_1_batch_size} batch size)\",\n )\n\n if not _is_refit:\n predict_1_time_child = self.get_model_attribute(model=model.name, attribute=\"predict_1_child_time\")\n predict_1_time_child_log, time_unit = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_child\n )\n logger.log(\n log_level,\n f\"\\t{round(predict_1_time_child_log, 3)}{time_unit}\\t = Validation runtime (1 row | {predict_1_batch_size} batch size | REFIT | MARGINAL)\",\n )\n\n predict_1_time_full_child = self.get_model_attribute_full(\n model=model.name, attribute=\"predict_1_child_time\"\n )\n predict_1_time_full_child_log, time_unit = convert_time_in_s_to_log_friendly(\n time_in_sec=predict_1_time_full_child\n )\n logger.log(\n log_level,\n f\"\\t{round(predict_1_time_full_child_log, 3)}{time_unit}\\t = Validation runtime (1 row | {predict_1_batch_size} batch size | REFIT)\",\n )\n\n # TODO: Split this to avoid confusion, HPO should go elsewhere?\n def _train_single_full(\n self,\n X,\n y,\n model: AbstractModel,\n X_unlabeled=None,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n X_pseudo=None,\n y_pseudo=None,\n hyperparameter_tune_kwargs=None,\n stack_name=\"core\",\n k_fold=None,\n k_fold_start=0,\n k_fold_end=None,\n n_repeats=None,\n n_repeat_start=0,\n level=1,\n time_limit=None,\n fit_kwargs=None,\n compute_score=True,\n total_resources: dict | None = None,\n errors: Literal[\"ignore\", \"raise\"] = \"ignore\",\n errors_ignore: list | None = None,\n errors_raise: list | None = None,\n is_ray_worker: bool = False,\n label_cleaner: None | LabelCleaner = None,\n **kwargs,\n ) -> list[str]:\n \"\"\"\n Trains a model, with the potential to train multiple versions of this model with hyperparameter tuning and feature pruning.\n Returns a list of successfully trained and saved model names.\n Models trained from this method will be accessible in this Trainer.\n\n Parameters\n ----------\n errors: Literal[\"ignore\", \"raise\"], default = \"ignore\"\n Determines how model fit exceptions are handled.\n If \"ignore\", will ignore all model exceptions during fit. If an exception occurs, an empty list is returned.\n If \"raise\", will raise the model exception if it occurs.\n Can be overwritten by `errors_ignore` and `errors_raise`.\n errors_ignore: list[str], optional\n The exception types specified in `errors_ignore` will be treated as if `errors=\"ignore\"`.\n errors_raise: list[str], optional\n The exception types specified in `errors_raise` will be treated as if `errors=\"raise\"`.\n \"\"\"\n if self._callback_early_stop:\n return []\n check_callbacks = k_fold_start == 0 and n_repeat_start == 0 and not is_ray_worker\n skip_model = False\n if self.callbacks and check_callbacks:\n skip_model, time_limit = self._callbacks_before_fit(\n model=model,\n time_limit=time_limit,\n stack_name=stack_name,\n level=level,\n )\n if self._callback_early_stop or skip_model:\n return []\n\n model_fit_kwargs = self._get_model_fit_kwargs(\n X=X,\n X_val=X_val,\n time_limit=time_limit,\n k_fold=k_fold,\n fit_kwargs=fit_kwargs,\n ens_sample_weight=kwargs.get(\"ens_sample_weight\", None),\n label_cleaner=label_cleaner,\n )\n exception = None\n if hyperparameter_tune_kwargs:\n if n_repeat_start != 0:\n raise ValueError(f\"n_repeat_start must be 0 to hyperparameter_tune, value = {n_repeat_start}\")\n elif k_fold_start != 0:\n raise ValueError(f\"k_fold_start must be 0 to hyperparameter_tune, value = {k_fold_start}\")\n # hpo_models (dict): keys = model_names, values = model_paths\n fit_log_message = f\"Hyperparameter tuning model: {model.name} ...\"\n if time_limit is not None:\n if time_limit <= 0:\n logger.log(15, f\"Skipping {model.name} due to lack of time remaining.\")\n return []\n fit_start_time = time.time()\n if self._time_limit is not None and self._time_train_start is not None:\n time_left_total = self._time_limit - (fit_start_time - self._time_train_start)\n else:\n time_left_total = time_limit\n fit_log_message += f\" Tuning model for up to {round(time_limit, 2)}s of the {round(time_left_total, 2)}s of remaining time.\"\n logger.log(20, fit_log_message)\n try:\n if isinstance(model, BaggedEnsembleModel):\n bagged_model_fit_kwargs = self._get_bagged_model_fit_kwargs(\n k_fold=k_fold,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeats=n_repeats,\n n_repeat_start=n_repeat_start,\n )\n model_fit_kwargs.update(bagged_model_fit_kwargs)\n hpo_models, hpo_results = model.hyperparameter_tune(\n X=X,\n y=y,\n model=model,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n stack_name=stack_name,\n level=level,\n compute_score=compute_score,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n total_resources=total_resources,\n **model_fit_kwargs,\n )\n else:\n hpo_models, hpo_results = model.hyperparameter_tune(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n total_resources=total_resources,\n **model_fit_kwargs,\n )\n if len(hpo_models) == 0:\n logger.warning(\n f\"No model was trained during hyperparameter tuning {model.name}... Skipping this model.\"\n )\n except Exception as exc:\n exception = exc # required to provide exc outside of `except` statement\n if isinstance(exception, NoStackFeatures):\n logger.warning(f\"\\tNo stack features to train {model.name}... Skipping this model. {exception}\")\n elif isinstance(exception, NotValidStacker):\n logger.warning(f\"\\tStacking disabled for {model.name}... Skipping this model. {exception}\")\n elif isinstance(exception, NoValidFeatures):\n logger.warning(f\"\\tNo valid features to train {model.name}... Skipping this model.\")\n else:\n logger.exception(\n f\"Warning: Exception caused {model.name} to fail during hyperparameter tuning... Skipping this model.\"\n )\n logger.warning(exception)\n del model\n model_names_trained = []\n else:\n # Commented out because it takes too much space (>>5 GB if run for an hour on a small-medium sized dataset)\n # self.hpo_results[model.name] = hpo_results\n model_names_trained = []\n self._extra_banned_names.add(model.name)\n for model_hpo_name, model_info in hpo_models.items():\n model_hpo = self.load_model(\n model_hpo_name, path=os.path.relpath(model_info[\"path\"], self.path), model_type=type(model)\n )\n logger.log(20, f\"Fitted model: {model_hpo.name} ...\")\n if self._add_model(model=model_hpo, stack_name=stack_name, level=level):\n model_names_trained.append(model_hpo.name)\n else:\n model_fit_kwargs.update(dict(X_pseudo=X_pseudo, y_pseudo=y_pseudo))\n if isinstance(model, BaggedEnsembleModel):\n bagged_model_fit_kwargs = self._get_bagged_model_fit_kwargs(\n k_fold=k_fold,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeats=n_repeats,\n n_repeat_start=n_repeat_start,\n )\n model_fit_kwargs.update(bagged_model_fit_kwargs)\n model_names_trained = self._train_and_save(\n X=X,\n y=y,\n model=model,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n stack_name=stack_name,\n level=level,\n compute_score=compute_score,\n total_resources=total_resources,\n errors=errors,\n errors_ignore=errors_ignore,\n errors_raise=errors_raise,\n is_ray_worker=is_ray_worker,\n **model_fit_kwargs,\n )\n if self.callbacks and check_callbacks:\n self._callbacks_after_fit(model_names=model_names_trained, stack_name=stack_name, level=level)\n self.save()\n if exception is not None:\n if self._check_raise_exception(\n exception=exception, errors=errors, errors_ignore=errors_ignore, errors_raise=errors_raise\n ):\n raise exception\n return model_names_trained\n\n # TODO: Move to a utility function outside of AbstractTabularTrainer\n @staticmethod\n def _check_raise_exception(\n exception: Exception,\n errors: Literal[\"ignore\", \"raise\"] = \"ignore\",\n errors_ignore: list | None = None,\n errors_raise: list | None = None,\n ) -> bool:\n \"\"\"\n Check if an exception should be raised based on the provided error handling logic.\n\n Parameters\n ----------\n exception: Exception\n The exception to check\n errors: Literal[\"ignore\", \"raise\"], default = \"ignore\"\n Determines how exceptions are handled.\n If \"ignore\", will return False.\n If \"raise\", will return True.\n Can be overwritten by `errors_ignore` and `errors_raise`.\n errors_ignore: list[str], optional\n The exception types specified in `errors_ignore` will be treated as if `errors=\"ignore\"`.\n errors_raise: list[str], optional\n The exception types specified in `errors_raise` will be treated as if `errors=\"raise\"`.\n\n Returns\n -------\n raise_exception: bool\n If True, indicates that the exception should be raised based on the provided error handling rules.\n \"\"\"\n raise_exception = None\n if errors_raise is not None:\n for err_type in errors_raise:\n if isinstance(exception, err_type):\n raise_exception = True\n break\n if errors_ignore is not None and raise_exception is None:\n for err_type in errors_ignore:\n if isinstance(exception, err_type):\n raise_exception = False\n break\n if raise_exception is None:\n if errors == \"ignore\":\n raise_exception = False\n elif errors == \"raise\":\n raise_exception = True\n else:\n raise ValueError(f\"Invalid `errors` value: {errors} (valid values: ['ignore', 'raise']\")\n return raise_exception\n\n def _callbacks_before_fit(\n self,\n *,\n model: AbstractModel,\n time_limit: float | None,\n stack_name: str,\n level: int,\n ):\n skip_model = False\n ts = time.time()\n for callback in self.callbacks:\n callback_early_stop, callback_skip_model = callback.before_model_fit(\n trainer=self,\n model=model,\n time_limit=time_limit,\n stack_name=stack_name,\n level=level,\n )\n if callback_early_stop:\n self._callback_early_stop = True\n if callback_skip_model:\n skip_model = True\n if time_limit is not None:\n te = time.time()\n time_limit -= te - ts\n ts = te\n return skip_model, time_limit\n\n def _callbacks_after_fit(\n self,\n *,\n model_names: list[str],\n stack_name: str,\n level: int,\n ):\n for callback in self.callbacks:\n callback_early_stop = callback.after_model_fit(\n self,\n model_names=model_names,\n stack_name=stack_name,\n level=level,\n )\n if callback_early_stop:\n self._callback_early_stop = True\n\n # TODO: How to deal with models that fail during this? They have trained valid models before, but should we still use those models or remove the entire model? Currently we still use models.\n # TODO: Time allowance can be made better by only using time taken during final model training and not during HPO and feature pruning.\n # TODO: Time allowance not accurate if running from fit_continue\n # TODO: Remove level and stack_name arguments, can get them automatically\n # TODO: Make sure that pretraining on X_unlabeled only happens 1 time rather than every fold of bagging. (Do during pretrain API work?)\n def _train_multi_repeats(\n self, X, y, models: list, n_repeats, n_repeat_start=1, time_limit=None, time_limit_total_level=None, **kwargs\n ) -> list[str]:\n \"\"\"\n Fits bagged ensemble models with additional folds and/or bagged repeats.\n Models must have already been fit prior to entering this method.\n This method should only be called in self._train_multi\n Returns a list of successfully trained and saved model names.\n \"\"\"\n if time_limit_total_level is None:\n time_limit_total_level = time_limit\n models_valid = models\n models_valid_next = []\n repeats_completed = 0\n time_start = time.time()\n for n in range(n_repeat_start, n_repeats):\n if not models_valid:\n break # No models to repeat\n if time_limit is not None:\n time_start_repeat = time.time()\n time_left = time_limit - (time_start_repeat - time_start)\n if n == n_repeat_start:\n time_required = time_limit_total_level * 0.575 # Require slightly over 50% to be safe\n else:\n time_required = (time_start_repeat - time_start) / repeats_completed * (0.575 / 0.425)\n if time_left < time_required:\n logger.log(15, \"Not enough time left to finish repeated k-fold bagging, stopping early ...\")\n break\n logger.log(20, f\"Repeating k-fold bagging: {n + 1}/{n_repeats}\")\n for i, model in enumerate(models_valid):\n if self._callback_early_stop:\n break\n if not self.get_model_attribute(model=model, attribute=\"can_fit\"):\n if isinstance(model, str):\n models_valid_next.append(model)\n else:\n models_valid_next.append(model.name)\n continue\n\n if isinstance(model, str):\n model = self.load_model(model)\n if not isinstance(model, BaggedEnsembleModel):\n raise AssertionError(\n f\"{model.name} must inherit from BaggedEnsembleModel to perform repeated k-fold bagging. Model type: {type(model).__name__}\"\n )\n if time_limit is None:\n time_left = None\n else:\n time_start_model = time.time()\n time_left = time_limit - (time_start_model - time_start)\n\n models_valid_next += self._train_single_full(\n X=X,\n y=y,\n model=model,\n k_fold_start=0,\n k_fold_end=None,\n n_repeats=n + 1,\n n_repeat_start=n,\n time_limit=time_left,\n **kwargs,\n )\n models_valid = copy.deepcopy(models_valid_next)\n models_valid_next = []\n repeats_completed += 1\n logger.log(20, f\"Completed {n_repeat_start + repeats_completed}/{n_repeats} k-fold bagging repeats ...\")\n return models_valid\n\n def _train_multi_initial(\n self,\n X,\n y,\n models: list[AbstractModel],\n k_fold,\n n_repeats,\n hyperparameter_tune_kwargs=None,\n time_limit=None,\n feature_prune_kwargs=None,\n **kwargs,\n ):\n \"\"\"\n Fits models that have not previously been fit.\n This method should only be called in self._train_multi\n Returns a list of successfully trained and saved model names.\n \"\"\"\n multi_fold_time_start = time.time()\n fit_args = dict(\n X=X,\n y=y,\n k_fold=k_fold,\n )\n fit_args.update(kwargs)\n\n hpo_enabled = False\n if hyperparameter_tune_kwargs:\n for key in hyperparameter_tune_kwargs:\n if hyperparameter_tune_kwargs[key] is not None:\n hpo_enabled = True\n break\n\n hpo_time_ratio = 0.9\n if hpo_enabled:\n time_split = True\n else:\n time_split = False\n k_fold_start = 0\n bagged = k_fold > 0\n if not bagged:\n time_ratio = hpo_time_ratio if hpo_enabled else 1\n models = self._train_multi_fold(\n models=models,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n time_limit=time_limit,\n time_split=time_split,\n time_ratio=time_ratio,\n **fit_args,\n )\n else:\n time_ratio = hpo_time_ratio if hpo_enabled else 1\n models = self._train_multi_fold(\n models=models,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n k_fold_start=0,\n k_fold_end=k_fold,\n n_repeats=n_repeats,\n n_repeat_start=0,\n time_limit=time_limit,\n time_split=time_split,\n time_ratio=time_ratio,\n **fit_args,\n )\n\n multi_fold_time_elapsed = time.time() - multi_fold_time_start\n if time_limit is not None:\n time_limit = time_limit - multi_fold_time_elapsed\n\n if feature_prune_kwargs is not None and len(models) > 0:\n feature_prune_time_start = time.time()\n model_fit_kwargs = self._get_model_fit_kwargs(\n X=X,\n X_val=kwargs.get(\"X_val\", None),\n time_limit=None,\n k_fold=k_fold,\n fit_kwargs=kwargs.get(\"fit_kwargs\", {}),\n ens_sample_weight=kwargs.get(\"ens_sample_weight\"),\n )\n model_fit_kwargs.update(dict(X=X, y=y, X_val=kwargs.get(\"X_val\", None), y_val=kwargs.get(\"y_val\", None)))\n if bagged:\n bagged_model_fit_kwargs = self._get_bagged_model_fit_kwargs(\n k_fold=k_fold, k_fold_start=k_fold_start, k_fold_end=k_fold, n_repeats=n_repeats, n_repeat_start=0\n )\n model_fit_kwargs.update(bagged_model_fit_kwargs)\n\n # FIXME: v1.3: X.columns incorrectly includes sample_weight column\n # FIXME: v1.3: Move sample_weight logic into fit_stack_core level methods, currently we are editing X too many times in self._get_model_fit_kwargs\n candidate_features = self._proxy_model_feature_prune(\n time_limit=time_limit,\n layer_fit_time=multi_fold_time_elapsed,\n level=kwargs[\"level\"],\n features=X.columns.tolist(),\n model_fit_kwargs=model_fit_kwargs,\n **feature_prune_kwargs,\n )\n if time_limit is not None:\n time_limit = time_limit - (time.time() - feature_prune_time_start)\n\n fit_args[\"X\"] = X[candidate_features]\n fit_args[\"X_val\"] = (\n kwargs[\"X_val\"][candidate_features]\n if isinstance(kwargs.get(\"X_val\", None), pd.DataFrame)\n else kwargs.get(\"X_val\", None)\n )\n\n if len(candidate_features) < len(X.columns):\n unfit_models = []\n original_prune_map = {}\n for model in models:\n unfit_model = self.load_model(model).convert_to_template()\n unfit_model.rename(f\"{unfit_model.name}_Prune\")\n unfit_models.append(unfit_model)\n original_prune_map[unfit_model.name] = model\n pruned_models = self._train_multi_fold(\n models=unfit_models,\n hyperparameter_tune_kwargs=None,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold,\n n_repeats=n_repeats,\n n_repeat_start=0,\n time_limit=time_limit,\n **fit_args,\n )\n force_prune = feature_prune_kwargs.get(\"force_prune\", False)\n models = self._retain_better_pruned_models(\n pruned_models=pruned_models, original_prune_map=original_prune_map, force_prune=force_prune\n )\n return models\n\n # TODO: Ban KNN from being a Stacker model outside of aux. Will need to ensemble select on all stack layers ensemble selector to make it work\n # TODO: Robert dataset, LightGBM is super good but RF and KNN take all the time away from it on 1h despite being much worse\n # TODO: Add time_limit_per_model\n # TODO: Rename for v0.1\n def _train_multi_fold(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n models: list[AbstractModel],\n time_limit: float | None = None,\n time_split: bool = False,\n time_ratio: float = 1,\n hyperparameter_tune_kwargs: dict | None = None,\n fit_strategy: Literal[\"sequential\", \"parallel\"] = \"sequential\",\n **kwargs,\n ) -> list[str]:\n \"\"\"\n Trains and saves a list of models sequentially.\n This method should only be called in self._train_multi_initial\n Returns a list of trained model names.\n \"\"\"\n time_start = time.time()\n if time_limit is not None:\n time_limit = time_limit * time_ratio\n if time_limit is not None and len(models) > 0:\n time_limit_model_split = time_limit / len(models)\n else:\n time_limit_model_split = time_limit\n\n if fit_strategy == \"parallel\" and hyperparameter_tune_kwargs is not None and hyperparameter_tune_kwargs:\n for k, v in hyperparameter_tune_kwargs.items():\n if v is not None and (not isinstance(v, dict) or len(v) != 0):\n logger.log(\n 30,\n f\"WARNING: fit_strategy='parallel', but `hyperparameter_tune_kwargs` is specified for model '{k}' with value {v}. \"\n f\"Hyperparameter tuning does not yet support `parallel` fit_strategy. \"\n f\"Falling back to fit_strategy='sequential' ... \",\n )\n fit_strategy = \"sequential\"\n break\n if fit_strategy == \"parallel\":\n num_cpus = kwargs.get(\"total_resources\", {}).get(\"num_cpus\", \"auto\")\n if isinstance(num_cpus, str) and num_cpus == \"auto\":\n num_cpus = get_resource_manager().get_cpu_count()\n if num_cpus < 12:\n force_parallel = os.environ.get(\"AG_FORCE_PARALLEL\", \"False\") == \"True\"\n if not force_parallel:\n logger.log(\n 30,\n f\"Note: fit_strategy='parallel', but `num_cpus={num_cpus}`. \"\n f\"Running parallel mode with fewer than 12 CPUs is not recommended and has been disabled. \"\n f'You can override this by specifying `os.environ[\"AG_FORCE_PARALLEL\"] = \"True\"`. '\n f\"Falling back to fit_strategy='sequential' ...\",\n )\n fit_strategy = \"sequential\"\n if fit_strategy == \"parallel\":\n num_gpus = kwargs.get(\"total_resources\", {}).get(\"num_gpus\", 0)\n if isinstance(num_gpus, str) and num_gpus == \"auto\":\n num_gpus = get_resource_manager().get_gpu_count()\n if isinstance(num_gpus, (float, int)) and num_gpus > 0:\n logger.log(\n 30,\n f\"WARNING: fit_strategy='parallel', but `num_gpus={num_gpus}` is specified. \"\n f\"GPU is not yet supported for `parallel` fit_strategy. To enable parallel, ensure you specify `num_gpus=0` in the fit call. \"\n f\"Falling back to fit_strategy='sequential' ... \",\n )\n fit_strategy = \"sequential\"\n if fit_strategy == \"parallel\":\n try:\n try_import_ray()\n except Exception as e:\n logger.log(\n 30,\n f\"WARNING: Exception encountered when trying to import ray (fit_strategy='parallel'). \"\n f\"ray is required for 'parallel' fit_strategy. Falling back to fit_strategy='sequential' ... \"\n f\"\\n\\tException details: {e.__class__.__name__}: {e}\",\n )\n fit_strategy = \"sequential\"\n\n if fit_strategy == \"sequential\":\n models_valid = []\n for model in models:\n if self._callback_early_stop:\n return models_valid\n\n models_valid += _detached_train_multi_fold(\n _self=self,\n model=model,\n X=X,\n y=y,\n time_start=time_start,\n time_split=time_split,\n time_limit=time_limit,\n time_limit_model_split=time_limit_model_split,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n is_ray_worker=False,\n kwargs=kwargs,\n )\n elif fit_strategy == \"parallel\":\n models_valid = self._train_multi_fold_parallel(\n X=X,\n y=y,\n models=models,\n time_start=time_start,\n time_limit_model_split=time_limit_model_split,\n time_limit=time_limit,\n time_split=time_split,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n **kwargs,\n )\n else:\n raise ValueError(f\"Invalid value for fit_strategy: '{fit_strategy}'\")\n return models_valid\n\n def _train_multi_fold_parallel(\n self,\n X: pd.DataFrame,\n y: pd.Series,\n models: list[AbstractModel],\n time_start: float,\n time_limit_model_split: float | None,\n time_limit: float | None = None,\n time_split: bool = False,\n hyperparameter_tune_kwargs: dict | None = None,\n **kwargs,\n ) -> list[str]:\n # -- Parallel or Distributed training\n ray = try_import_ray()\n\n # FIXME: Need a common utility class for initializing ray so we don't duplicate code\n if not ray.is_initialized():\n ray.init(log_to_driver=False, logging_level=logging.ERROR)\n\n models_valid = []\n\n if time_limit is not None:\n # Give models less than the full time limit to account for overheads (predict, cache, ray, etc.)\n time_limit_models = time_limit * 0.9\n else:\n time_limit_models = None\n\n logger.log(20, \"Scheduling parallel model-workers for training...\")\n distributed_manager = ParallelFitManager(\n mode=\"fit\",\n X=X, # FIXME: REMOVE\n y=y, # FIXME: REMOVE\n func=_remote_train_multi_fold,\n func_kwargs=dict(\n time_split=time_split,\n time_limit_model_split=time_limit_model_split,\n time_limit=time_limit_models,\n time_start=time_start,\n errors=\"raise\",\n ),\n func_put_kwargs=dict(\n _self=self,\n X=X,\n y=y,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n kwargs=kwargs,\n ),\n num_cpus=kwargs.get(\"total_resources\", {}).get(\"num_cpus\", 1),\n num_gpus=kwargs.get(\"total_resources\", {}).get(\"num_gpus\", 0),\n num_splits=kwargs.get(\"k_fold\", 1) * kwargs.get(\"n_repeats\", 1),\n problem_type=self.problem_type, # FIXME: Should this be passed here?\n num_classes=self.num_classes, # FIXME: Should this be passed here?\n )\n jobs_finished = 0\n jobs_total = len(models)\n\n ordered_model_names = [m.name for m in models] # Use to ensure same model order is returned\n expected_model_names = set(ordered_model_names)\n unfinished_job_refs = distributed_manager.schedule_jobs(models_to_fit=models)\n\n timeout = None\n\n if time_limit is not None:\n # allow between 5 and 60 seconds overhead before force killing jobs to give some leniency to jobs with overhead.\n time_overhead = min(max(time_limit * 0.01, 5), 60)\n min_time_required_base = min(\n self._time_limit * 0.01, 10\n ) # This is checked in the worker thread, will skip if not satisfied\n # If time remaining is less than min_time_required, avoid scheduling new jobs and only wait for existing ones to finish.\n min_time_required = (\n min_time_required_base * 1.5 + 1\n ) # Add 50% buffer and 1 second to account for ray overhead\n else:\n time_overhead = None\n min_time_required = None\n\n can_schedule_jobs = True\n while unfinished_job_refs:\n if time_limit is not None:\n time_left = time_limit - (time.time() - time_start)\n timeout = int(time_left + time_overhead) # include overhead.\n if timeout <= 0:\n logger.log(20, \"Ran into timeout while waiting for model training to finish. Stopping now.\")\n break\n finished, unfinished_job_refs = ray.wait(unfinished_job_refs, num_returns=1, timeout=timeout)\n\n if not finished:\n logger.log(20, \"Ran into timeout while waiting for model training to finish. Stopping now.\")\n break\n\n distributed_manager.deallocate_resources(job_ref=finished[0])\n model_name, model_path, model_type, exc, model_failure_info = ray.get(finished[0])\n assert model_name in expected_model_names, (\n f\"Unexpected model name outputted during parallel fit: {model_name}\\n\"\n f\"Valid Names: {expected_model_names}\\n\"\n f\"This should never happen. Please create a GitHub Issue.\"\n )\n jobs_finished += 1\n\n if exc is not None or model_path is None:\n if exc is None:\n if model_failure_info is not None:\n exc_type = model_failure_info[\"exc_type\"]\n exc_str = model_failure_info[\"exc_str\"]\n else:\n exc_type = None\n exc_str = None\n else:\n exc_type = exc.__class__\n exc_str = str(exc)\n if exc_type is not None:\n extra_log = f\": {exc_type.__name__}: {exc_str}\"\n else:\n extra_log = \"\"\n if exc_type is not None and issubclass(exc_type, InsufficientTime):\n logger.log(20, exc_str)\n else:\n logger.log(\n 20,\n f\"Skipping {model_name if isinstance(model_name, str) else model_name.name} due to exception{extra_log}\",\n )\n if model_failure_info is not None:\n self._models_failed_to_train_errors[model_name] = model_failure_info\n else:\n logger.log(20, f\"Fitted {model_name}:\")\n\n # TODO: figure out a way to avoid calling _add_model in the worker-process to save overhead time.\n # - Right now, we need to call it within _add_model to be able to pass the model path to the main process without changing\n # the return signature of _train_single_full. This can be a lot of work to change.\n # TODO: determine if y_pred_proba_val was cached in the worker-process. Right now, we re-do predictions for holdout data.\n # Self object is not permanently mutated during worker execution, so we need to add model to the \"main\" self (again).\n # This is the synchronization point between the distributed and main processes.\n if self._add_model(\n model_type.load(path=os.path.join(self.path, model_path), reset_paths=self.reset_paths),\n stack_name=kwargs[\"stack_name\"],\n level=kwargs[\"level\"],\n ):\n jobs_running = len(unfinished_job_refs)\n if can_schedule_jobs:\n remaining_task_word = \"pending\"\n else:\n remaining_task_word = \"skipped\"\n parallel_status_log = (\n f\"\\tJobs: {jobs_running} running, \"\n f\"{jobs_total - (jobs_finished + jobs_running)} {remaining_task_word}, \"\n f\"{jobs_finished}/{jobs_total} finished\"\n )\n if time_limit is not None:\n time_left = time_limit - (time.time() - time_start)\n parallel_status_log += f\" | {time_left:.0f}s remaining\"\n logger.log(20, parallel_status_log)\n models_valid.append(model_name)\n else:\n logger.log(\n 40,\n f\"Failed to add {model_name} to model graph. This should never happen. Please create a GitHub issue.\",\n )\n\n if not unfinished_job_refs and not distributed_manager.models_to_schedule:\n # Completed all jobs\n break\n\n # TODO: look into what this does / how this works for distributed training\n if self._callback_early_stop:\n logger.log(\n 20,\n \"Callback triggered in parallel setting. Stopping model training and cancelling remaining jobs.\",\n )\n break\n\n # Stop due to time limit after adding model\n if time_limit is not None:\n time_elapsed = time.time() - time_start\n time_left = time_limit - time_elapsed\n time_left_models = time_limit_models - time_elapsed\n if (time_left + time_overhead) <= 0:\n logger.log(\n 20,\n \"Time limit reached for this stacking layer. Stopping model training and cancelling remaining jobs.\",\n )\n break\n elif time_left_models < min_time_required:\n if can_schedule_jobs:\n if len(distributed_manager.models_to_schedule) > 0:\n logger.log(\n 20,\n f\"Low on time, skipping {len(distributed_manager.models_to_schedule)} \"\n f\"pending jobs and waiting for running jobs to finish... ({time_left:.0f}s remaining time)\",\n )\n can_schedule_jobs = False\n\n if can_schedule_jobs:\n # Re-schedule jobs\n unfinished_job_refs += distributed_manager.schedule_jobs()\n\n distributed_manager.clean_up_ray(unfinished_job_refs=unfinished_job_refs)\n logger.log(20, \"Finished all parallel work for this stacking layer.\")\n\n models_valid = set(models_valid)\n models_valid = [m for m in ordered_model_names if m in models_valid] # maintain original order\n\n return models_valid\n\n def _train_multi(\n self,\n X,\n y,\n models: list[AbstractModel],\n hyperparameter_tune_kwargs=None,\n feature_prune_kwargs=None,\n k_fold=None,\n n_repeats=None,\n n_repeat_start=0,\n time_limit=None,\n delay_bag_sets: bool = False,\n **kwargs,\n ) -> list[str]:\n \"\"\"\n Train a list of models using the same data.\n Assumes that input data has already been processed in the form the models will receive as input (including stack feature generation).\n Trained models are available in the trainer object.\n Note: Consider using public APIs instead of this.\n Returns a list of trained model names.\n \"\"\"\n time_limit_total_level = time_limit\n if k_fold is None:\n k_fold = self.k_fold\n if n_repeats is None:\n n_repeats = self.n_repeats\n if (k_fold == 0) and (n_repeats != 1):\n raise ValueError(f\"n_repeats must be 1 when k_fold is 0, values: ({n_repeats}, {k_fold})\")\n if (time_limit is None and feature_prune_kwargs is None) or (not delay_bag_sets):\n n_repeats_initial = n_repeats\n else:\n n_repeats_initial = 1\n if n_repeat_start == 0:\n time_start = time.time()\n model_names_trained = self._train_multi_initial(\n X=X,\n y=y,\n models=models,\n k_fold=k_fold,\n n_repeats=n_repeats_initial,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n feature_prune_kwargs=feature_prune_kwargs,\n time_limit=time_limit,\n **kwargs,\n )\n n_repeat_start = n_repeats_initial\n if time_limit is not None:\n time_limit = time_limit - (time.time() - time_start)\n else:\n model_names_trained = models\n if (n_repeats > 1) and (n_repeat_start < n_repeats):\n model_names_trained = self._train_multi_repeats(\n X=X,\n y=y,\n models=model_names_trained,\n k_fold=k_fold,\n n_repeats=n_repeats,\n n_repeat_start=n_repeat_start,\n time_limit=time_limit,\n time_limit_total_level=time_limit_total_level,\n **kwargs,\n )\n return model_names_trained\n\n def _train_multi_and_ensemble(\n self,\n X,\n y,\n X_val,\n y_val,\n X_test=None,\n y_test=None,\n hyperparameters: dict | None = None,\n X_unlabeled=None,\n num_stack_levels=0,\n time_limit=None,\n groups=None,\n **kwargs,\n ) -> list[str]:\n \"\"\"Identical to self.train_multi_levels, but also saves the data to disk. This should only ever be called once.\"\"\"\n if time_limit is not None and time_limit <= 0:\n raise AssertionError(\n f\"Not enough time left to train models. Consider specifying a larger time_limit. Time remaining: {round(time_limit, 2)}s\"\n )\n if self.save_data and not self.is_data_saved:\n self.save_X(X)\n self.save_y(y)\n if X_val is not None:\n self.save_X_val(X_val)\n if y_val is not None:\n self.save_y_val(y_val)\n if X_test is not None:\n self.save_X_test(X_test)\n if y_test is not None:\n self.save_y_test(y_test)\n self.is_data_saved = True\n if self._groups is None:\n self._groups = groups\n self._num_rows_train = len(X)\n if X_val is not None:\n self._num_rows_val = len(X_val)\n if X_test is not None:\n self._num_rows_test = len(X_test)\n self._num_cols_train = len(list(X.columns))\n model_names_fit = self.train_multi_levels(\n X,\n y,\n hyperparameters=hyperparameters,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n level_start=1,\n level_end=num_stack_levels + 1,\n time_limit=time_limit,\n **kwargs,\n )\n if len(self.get_model_names()) == 0:\n # TODO v1.0: Add toggle to raise exception if no models trained\n logger.log(30, \"Warning: AutoGluon did not successfully train any models\")\n return model_names_fit\n\n def _predict_model(self, X: pd.DataFrame, model: str, model_pred_proba_dict: dict | None = None) -> np.ndarray:\n y_pred_proba = self._predict_proba_model(X=X, model=model, model_pred_proba_dict=model_pred_proba_dict)\n return get_pred_from_proba(y_pred_proba=y_pred_proba, problem_type=self.problem_type)\n\n def _predict_proba_model(\n self, X: pd.DataFrame, model: str, model_pred_proba_dict: dict | None = None\n ) -> np.ndarray:\n model_pred_proba_dict = self.get_model_pred_proba_dict(\n X=X, models=[model], model_pred_proba_dict=model_pred_proba_dict\n )\n if not isinstance(model, str):\n model = model.name\n return model_pred_proba_dict[model]\n\n def _proxy_model_feature_prune(\n self,\n model_fit_kwargs: dict,\n time_limit: float,\n layer_fit_time: float,\n level: int,\n features: list[str],\n **feature_prune_kwargs: dict,\n ) -> list[str]:\n \"\"\"\n Uses the best LightGBM-based base learner of this layer to perform time-aware permutation feature importance based feature pruning.\n If all LightGBM models fail, use the model that achieved the highest validation accuracy. Feature pruning gets the smaller of the\n remaining layer time limit and k times (default=2) it took to fit the base learners of this layer as its resource. Note that feature pruning can\n exit earlier based on arguments in feature_prune_kwargs. The method returns the list of feature names that survived the pruning procedure.\n\n Parameters\n ----------\n feature_prune_kwargs : dict\n Feature pruning kwarg arguments. Should contain arguments passed to FeatureSelector.select_features. One can optionally attach the following\n additional kwargs that are consumed at this level: 'proxy_model_class' to use a model of particular type with the highest validation score as the\n proxy model, 'feature_prune_time_limit' to manually specify how long we should perform the feature pruning procedure for, 'k' to specify how long\n we should perform feature pruning for if 'feature_prune_time_limit' has not been set (feature selection time budget is set to k * layer_fit_time),\n and 'raise_exception' to signify that AutoGluon should throw an exception if feature pruning errors out.\n time_limit : float\n Time limit left within the current stack layer in seconds. Feature pruning should never take more than this time.\n layer_fit_time : float\n How long it took to fit all the models in this layer once. Used to calculate how long to feature prune for.\n level : int\n Level of this stack layer.\n features: list[str]\n The list of feature names in the inputted dataset.\n\n Returns\n -------\n candidate_features : list[str]\n Feature names that survived the pruning procedure.\n \"\"\"\n k = feature_prune_kwargs.pop(\"k\", 2)\n proxy_model_class = feature_prune_kwargs.pop(\"proxy_model_class\", self._get_default_proxy_model_class())\n feature_prune_time_limit = feature_prune_kwargs.pop(\"feature_prune_time_limit\", None)\n raise_exception_on_fail = feature_prune_kwargs.pop(\"raise_exception\", False)\n\n proxy_model = self._get_feature_prune_proxy_model(proxy_model_class=proxy_model_class, level=level)\n if proxy_model is None:\n return features\n\n if feature_prune_time_limit is not None:\n feature_prune_time_limit = min(max(time_limit - layer_fit_time, 0), feature_prune_time_limit)\n elif time_limit is not None:\n feature_prune_time_limit = min(\n max(time_limit - layer_fit_time, 0), max(k * layer_fit_time, 0.05 * time_limit)\n )\n else:\n feature_prune_time_limit = max(k * layer_fit_time, 300)\n\n if feature_prune_time_limit < 2 * proxy_model.fit_time:\n logger.warning(\n f\"Insufficient time to train even a single feature pruning model (remaining: {feature_prune_time_limit}, \"\n f\"needed: {proxy_model.fit_time}). Skipping feature pruning.\"\n )\n return features\n selector = FeatureSelector(\n model=proxy_model,\n time_limit=feature_prune_time_limit,\n raise_exception=raise_exception_on_fail,\n problem_type=self.problem_type,\n )\n candidate_features = selector.select_features(**feature_prune_kwargs, **model_fit_kwargs)\n return candidate_features\n\n def _get_default_proxy_model_class(self):\n return None\n\n def _retain_better_pruned_models(\n self, pruned_models: list[str], original_prune_map: dict, force_prune: bool = False\n ) -> list[str]:\n \"\"\"\n Compares models fit on the pruned set of features with their counterpart, models fit on full set of features.\n Take the model that achieved a higher validation set score and delete the other from self.model_graph.\n\n Parameters\n ----------\n pruned_models : list[str]\n A list of pruned model names.\n original_prune_map : dict\n A dictionary mapping the names of models fitted on pruned features to the names of models fitted on original features.\n force_prune : bool, default = False\n If set to true, force all base learners to work with the pruned set of features.\n\n Returns\n ----------\n models : list[str]\n A list of model names.\n \"\"\"\n models = []\n for pruned_model in pruned_models:\n original_model = original_prune_map[pruned_model]\n leaderboard = self.leaderboard()\n original_score = leaderboard[leaderboard[\"model\"] == original_model][\"score_val\"].item()\n pruned_score = leaderboard[leaderboard[\"model\"] == pruned_model][\"score_val\"].item()\n score_str = f\"({round(pruned_score, 4)} vs {round(original_score, 4)})\"\n if force_prune:\n logger.log(\n 30,\n f\"Pruned score vs original score is {score_str}. Replacing original model since force_prune=True...\",\n )\n self.delete_models(models_to_delete=original_model, dry_run=False)\n models.append(pruned_model)\n elif pruned_score > original_score:\n logger.log(\n 30,\n f\"Model trained with feature pruning score is better than original model's score {score_str}. Replacing original model...\",\n )\n self.delete_models(models_to_delete=original_model, dry_run=False)\n models.append(pruned_model)\n else:\n logger.log(\n 30,\n f\"Model trained with feature pruning score is not better than original model's score {score_str}. Keeping original model...\",\n )\n self.delete_models(models_to_delete=pruned_model, dry_run=False)\n models.append(original_model)\n return models\n\n # TODO: Enable raw=True for bagged models when X=None\n # This is non-trivial to implement for multi-layer stacking ensembles on the OOF data.\n # TODO: Consider limiting X to 10k rows here instead of inside the model call\n def get_feature_importance(self, model=None, X=None, y=None, raw=True, **kwargs) -> pd.DataFrame:\n if model is None:\n model = self.model_best\n model: AbstractModel = self.load_model(model)\n if X is None and model.val_score is None:\n raise AssertionError(\n f\"Model {model.name} is not valid for generating feature importances on original training data because no validation data was used during training, please specify new test data to compute feature importances.\"\n )\n\n if X is None:\n if isinstance(model, WeightedEnsembleModel):\n if self.bagged_mode:\n if raw:\n raise AssertionError(\n \"`feature_stage='transformed'` feature importance on the original training data is not yet supported when bagging is enabled, please specify new test data to compute feature importances.\"\n )\n X = None\n is_oof = True\n else:\n if raw:\n X = self.load_X_val()\n else:\n X = None\n is_oof = False\n elif isinstance(model, BaggedEnsembleModel):\n if raw:\n raise AssertionError(\n \"`feature_stage='transformed'` feature importance on the original training data is not yet supported when bagging is enabled, please specify new test data to compute feature importances.\"\n )\n X = self.load_X()\n X = self.get_inputs_to_model(model=model, X=X, fit=True)\n is_oof = True\n else:\n X = self.load_X_val()\n if not raw:\n X = self.get_inputs_to_model(model=model, X=X, fit=False)\n is_oof = False\n else:\n is_oof = False\n if not raw:\n X = self.get_inputs_to_model(model=model, X=X, fit=False)\n\n if y is None and X is not None:\n if is_oof:\n y = self.load_y()\n else:\n y = self.load_y_val()\n\n if raw:\n return self._get_feature_importance_raw(X=X, y=y, model=model, **kwargs)\n else:\n if is_oof:\n kwargs[\"is_oof\"] = is_oof\n return model.compute_feature_importance(X=X, y=y, **kwargs)\n\n # TODO: Can get feature importances of all children of model at no extra cost, requires scoring the values after predict_proba on each model\n # Could solve by adding a self.score_all() function which takes model as input and also returns scores of all children models.\n # This would be best solved after adding graph representation, it lives most naturally in AbstractModel\n # TODO: Can skip features which were pruned on all models that model depends on (Complex to implement, requires graph representation)\n # TODO: Note that raw importance will not equal non-raw importance for bagged models, even if raw features are identical to the model features.\n # This is because for non-raw, we do an optimization where each fold model calls .compute_feature_importance(), and then the feature importances are averaged across the folds.\n # This is different from raw, where the predictions of the folds are averaged and then feature importance is computed.\n # Consider aligning these methods so they produce the same result.\n # The output of this function is identical to non-raw when model is level 1 and non-bagged\n def _get_feature_importance_raw(self, X, y, model, eval_metric=None, **kwargs) -> pd.DataFrame:\n if eval_metric is None:\n eval_metric = self.eval_metric\n if model is None:\n model = self._get_best()\n if eval_metric.needs_pred:\n predict_func = self.predict\n else:\n predict_func = self.predict_proba\n model: AbstractModel = self.load_model(model)\n predict_func_kwargs = dict(model=model)\n return compute_permutation_feature_importance(\n X=X,\n y=y,\n predict_func=predict_func,\n predict_func_kwargs=predict_func_kwargs,\n eval_metric=eval_metric,\n quantile_levels=self.quantile_levels,\n **kwargs,\n )\n\n def _get_models_load_info(self, model_names):\n model_names = copy.deepcopy(model_names)\n model_paths = self.get_models_attribute_dict(attribute=\"path\", models=model_names)\n model_types = self.get_models_attribute_dict(attribute=\"type\", models=model_names)\n return model_names, model_paths, model_types\n\n def get_model_attribute_full(self, model: str | list[str], attribute: str, func=sum) -> float | int:\n \"\"\"\n Sums the attribute value across all models that the provided model depends on, including itself.\n For instance, this function can return the expected total predict_time of a model.\n attribute is the name of the desired attribute to be summed,\n or a dictionary of model name -> attribute value if the attribute is not present in the graph.\n \"\"\"\n if isinstance(model, list):\n base_model_set = self.get_minimum_models_set(model)\n else:\n base_model_set = self.get_minimum_model_set(model)\n if isinstance(attribute, dict):\n is_dict = True\n else:\n is_dict = False\n if len(base_model_set) == 1:\n if is_dict:\n return attribute[model]\n else:\n return self.model_graph.nodes[base_model_set[0]][attribute]\n # attribute_full = 0\n attribute_lst = []\n for base_model in base_model_set:\n if is_dict:\n attribute_base_model = attribute[base_model]\n else:\n attribute_base_model = self.model_graph.nodes[base_model][attribute]\n if attribute_base_model is None:\n return None\n attribute_lst.append(attribute_base_model)\n # attribute_full += attribute_base_model\n if attribute_lst:\n attribute_full = func(attribute_lst)\n else:\n attribute_full = 0\n return attribute_full\n\n def get_models_attribute_full(self, models: list[str], attribute: str, func=sum):\n \"\"\"\n For each model in models, returns the output of self.get_model_attribute_full mapped to a dict.\n \"\"\"\n d = dict()\n for model in models:\n d[model] = self.get_model_attribute_full(model=model, attribute=attribute, func=func)\n return d\n\n # Gets the minimum set of models that the provided models depend on, including themselves\n # Returns a list of model names\n def get_minimum_models_set(self, models: list) -> list:\n models_set = set()\n for model in models:\n models_set = models_set.union(self.get_minimum_model_set(model))\n return list(models_set)\n\n # Gets the set of base models used directly by the provided model\n # Returns a list of model names\n def get_base_model_names(self, model) -> list:\n if not isinstance(model, str):\n model = model.name\n base_model_set = list(self.model_graph.predecessors(model))\n return base_model_set\n\n def model_refit_map(self, inverse=False) -> dict[str, str]:\n \"\"\"\n Returns dict of parent model -> refit model\n\n If inverse=True, return dict of refit model -> parent model\n \"\"\"\n model_refit_map = self.get_models_attribute_dict(attribute=\"refit_full_parent\")\n if not inverse:\n model_refit_map = {parent: refit for refit, parent in model_refit_map.items()}\n return model_refit_map\n\n def model_exists(self, model: str) -> bool:\n return model in self.get_model_names()\n\n def _flatten_model_info(self, model_info: dict) -> dict:\n \"\"\"\n Flattens the model_info nested dictionary into a shallow dictionary to convert to a pandas DataFrame row.\n\n Parameters\n ----------\n model_info: dict\n A nested dictionary of model metadata information\n\n Returns\n -------\n A flattened dictionary of model info.\n \"\"\"\n model_info_keys = [\n \"num_features\",\n \"model_type\",\n \"hyperparameters\",\n \"hyperparameters_fit\",\n \"ag_args_fit\",\n \"features\",\n \"is_initialized\",\n \"is_fit\",\n \"is_valid\",\n \"can_infer\",\n ]\n model_info_flat = {k: v for k, v in model_info.items() if k in model_info_keys}\n\n custom_info = {}\n bagged_info = model_info.get(\"bagged_info\", {})\n custom_info[\"num_models\"] = bagged_info.get(\"num_child_models\", 1)\n custom_info[\"memory_size\"] = bagged_info.get(\"max_memory_size\", model_info[\"memory_size\"])\n custom_info[\"memory_size_min\"] = bagged_info.get(\"min_memory_size\", model_info[\"memory_size\"])\n custom_info[\"compile_time\"] = bagged_info.get(\"compile_time\", model_info[\"compile_time\"])\n custom_info[\"child_model_type\"] = bagged_info.get(\"child_model_type\", None)\n custom_info[\"child_hyperparameters\"] = bagged_info.get(\"child_hyperparameters\", None)\n custom_info[\"child_hyperparameters_fit\"] = bagged_info.get(\"child_hyperparameters_fit\", None)\n custom_info[\"child_ag_args_fit\"] = bagged_info.get(\"child_ag_args_fit\", None)\n\n model_info_keys = [\n \"num_models\",\n \"memory_size\",\n \"memory_size_min\",\n \"compile_time\",\n \"child_model_type\",\n \"child_hyperparameters\",\n \"child_hyperparameters_fit\",\n \"child_ag_args_fit\",\n ]\n for key in model_info_keys:\n model_info_flat[key] = custom_info[key]\n return model_info_flat\n\n def leaderboard(self, extra_info=False, refit_full: bool | None = None, set_refit_score_to_parent: bool = False):\n model_names = self.get_model_names()\n models_full_dict = self.get_models_attribute_dict(models=model_names, attribute=\"refit_full_parent\")\n if refit_full is not None:\n if refit_full:\n model_names = [model for model in model_names if model in models_full_dict]\n else:\n model_names = [model for model in model_names if model not in models_full_dict]\n score_val = []\n eval_metric = []\n stopping_metric = []\n fit_time_marginal = []\n pred_time_val_marginal = []\n stack_level = []\n fit_time = []\n pred_time_val = []\n can_infer = []\n fit_order = list(range(1, len(model_names) + 1))\n score_val_dict = self.get_models_attribute_dict(\"val_score\")\n eval_metric_dict = self.get_models_attribute_dict(\"eval_metric\")\n stopping_metric_dict = self.get_models_attribute_dict(\"stopping_metric\")\n fit_time_marginal_dict = self.get_models_attribute_dict(\"fit_time\")\n predict_time_marginal_dict = self.get_models_attribute_dict(\"predict_time\")\n fit_time_dict = self.get_models_attribute_full(attribute=\"fit_time\", models=model_names, func=sum)\n pred_time_val_dict = self.get_models_attribute_full(attribute=\"predict_time\", models=model_names, func=sum)\n can_infer_dict = self.get_models_attribute_full(attribute=\"can_infer\", models=model_names, func=min)\n for model_name in model_names:\n if set_refit_score_to_parent and (model_name in models_full_dict):\n if models_full_dict[model_name] not in score_val_dict:\n raise AssertionError(\n f\"Model parent is missing from leaderboard when `set_refit_score_to_parent=True`, \"\n f\"this is invalid. The parent model may have been deleted. \"\n f\"(model='{model_name}', parent='{models_full_dict[model_name]}')\"\n )\n score_val.append(score_val_dict[models_full_dict[model_name]])\n else:\n score_val.append(score_val_dict[model_name])\n eval_metric.append(eval_metric_dict[model_name])\n stopping_metric.append(stopping_metric_dict[model_name])\n fit_time_marginal.append(fit_time_marginal_dict[model_name])\n fit_time.append(fit_time_dict[model_name])\n pred_time_val_marginal.append(predict_time_marginal_dict[model_name])\n pred_time_val.append(pred_time_val_dict[model_name])\n stack_level.append(self.get_model_level(model_name))\n can_infer.append(can_infer_dict[model_name])\n\n model_info_dict = defaultdict(list)\n extra_info_dict = dict()\n if extra_info:\n # TODO: feature_metadata\n # TODO: disk size\n # TODO: load time\n # TODO: Add persist_if_mem_safe() function to persist in memory all models if reasonable memory size (or a specific model+ancestors)\n # TODO: Add is_persisted() function to check which models are persisted in memory\n # TODO: package_dependencies, package_dependencies_full\n\n info = self.get_info(include_model_info=True)\n model_info = info[\"model_info\"]\n custom_model_info = {}\n for model_name in model_info:\n custom_info = {}\n bagged_info = model_info[model_name].get(\"bagged_info\", {})\n custom_info[\"num_models\"] = bagged_info.get(\"num_child_models\", 1)\n custom_info[\"memory_size\"] = bagged_info.get(\"max_memory_size\", model_info[model_name][\"memory_size\"])\n custom_info[\"memory_size_min\"] = bagged_info.get(\n \"min_memory_size\", model_info[model_name][\"memory_size\"]\n )\n custom_info[\"compile_time\"] = bagged_info.get(\"compile_time\", model_info[model_name][\"compile_time\"])\n custom_info[\"child_model_type\"] = bagged_info.get(\"child_model_type\", None)\n custom_info[\"child_hyperparameters\"] = bagged_info.get(\"child_hyperparameters\", None)\n custom_info[\"child_hyperparameters_fit\"] = bagged_info.get(\"child_hyperparameters_fit\", None)\n custom_info[\"child_ag_args_fit\"] = bagged_info.get(\"child_ag_args_fit\", None)\n custom_model_info[model_name] = custom_info\n\n model_info_keys = [\n \"num_features\",\n \"model_type\",\n \"hyperparameters\",\n \"hyperparameters_fit\",\n \"ag_args_fit\",\n \"features\",\n ]\n model_info_sum_keys = []\n for key in model_info_keys:\n model_info_dict[key] = [model_info[model_name][key] for model_name in model_names]\n if key in model_info_sum_keys:\n key_dict = {model_name: model_info[model_name][key] for model_name in model_names}\n model_info_dict[key + \"_full\"] = [\n self.get_model_attribute_full(model=model_name, attribute=key_dict)\n for model_name in model_names\n ]\n\n model_info_keys = [\n \"num_models\",\n \"memory_size\",\n \"memory_size_min\",\n \"compile_time\",\n \"child_model_type\",\n \"child_hyperparameters\",\n \"child_hyperparameters_fit\",\n \"child_ag_args_fit\",\n ]\n model_info_full_keys = {\n \"memory_size\": [(\"memory_size_w_ancestors\", sum)],\n \"memory_size_min\": [(\"memory_size_min_w_ancestors\", max)],\n \"num_models\": [(\"num_models_w_ancestors\", sum)],\n }\n for key in model_info_keys:\n model_info_dict[key] = [custom_model_info[model_name][key] for model_name in model_names]\n if key in model_info_full_keys:\n key_dict = {model_name: custom_model_info[model_name][key] for model_name in model_names}\n for column_name, func in model_info_full_keys[key]:\n model_info_dict[column_name] = [\n self.get_model_attribute_full(model=model_name, attribute=key_dict, func=func)\n for model_name in model_names\n ]\n\n ancestors = [list(nx.dag.ancestors(self.model_graph, model_name)) for model_name in model_names]\n descendants = [list(nx.dag.descendants(self.model_graph, model_name)) for model_name in model_names]\n\n model_info_dict[\"num_ancestors\"] = [len(ancestor_lst) for ancestor_lst in ancestors]\n model_info_dict[\"num_descendants\"] = [len(descendant_lst) for descendant_lst in descendants]\n model_info_dict[\"ancestors\"] = ancestors\n model_info_dict[\"descendants\"] = descendants\n\n extra_info_dict = {\n \"stopping_metric\": stopping_metric,\n }\n\n df = pd.DataFrame(\n data={\n \"model\": model_names,\n \"score_val\": score_val,\n \"eval_metric\": eval_metric,\n \"pred_time_val\": pred_time_val,\n \"fit_time\": fit_time,\n \"pred_time_val_marginal\": pred_time_val_marginal,\n \"fit_time_marginal\": fit_time_marginal,\n \"stack_level\": stack_level,\n \"can_infer\": can_infer,\n \"fit_order\": fit_order,\n **extra_info_dict,\n **model_info_dict,\n }\n )\n df_sorted = df.sort_values(\n by=[\"score_val\", \"pred_time_val\", \"model\"], ascending=[False, True, False]\n ).reset_index(drop=True)\n\n df_columns_lst = df_sorted.columns.tolist()\n explicit_order = [\n \"model\",\n \"score_val\",\n \"eval_metric\",\n \"pred_time_val\",\n \"fit_time\",\n \"pred_time_val_marginal\",\n \"fit_time_marginal\",\n \"stack_level\",\n \"can_infer\",\n \"fit_order\",\n \"num_features\",\n \"num_models\",\n \"num_models_w_ancestors\",\n \"memory_size\",\n \"memory_size_w_ancestors\",\n \"memory_size_min\",\n \"memory_size_min_w_ancestors\",\n \"num_ancestors\",\n \"num_descendants\",\n \"model_type\",\n \"child_model_type\",\n ]\n explicit_order = [column for column in explicit_order if column in df_columns_lst]\n df_columns_other = [column for column in df_columns_lst if column not in explicit_order]\n df_columns_new = explicit_order + df_columns_other\n df_sorted = df_sorted[df_columns_new]\n\n return df_sorted\n\n def model_failures(self) -> pd.DataFrame:\n \"\"\"\n [Advanced] Get the model failures that occurred during the fitting of this predictor, in the form of a pandas DataFrame.\n\n This is useful for in-depth debugging of model failures and identifying bugs.\n\n Returns\n -------\n model_failures_df: pd.DataFrame\n A DataFrame of model failures. Each row corresponds to a model failure, and columns correspond to meta information about that model.\n \"\"\"\n model_infos = dict()\n for i, (model_name, model_info) in enumerate(self._models_failed_to_train_errors.items()):\n model_info = copy.deepcopy(model_info)\n model_info_inner = model_info[\"model_info\"]\n\n model_info_inner = self._flatten_model_info(model_info_inner)\n\n valid_keys = [\n \"exc_type\",\n \"exc_str\",\n \"exc_traceback\",\n \"total_time\",\n ]\n valid_keys_inner = [\n \"model_type\",\n \"hyperparameters\",\n \"hyperparameters_fit\",\n \"is_initialized\",\n \"is_fit\",\n \"is_valid\",\n \"can_infer\",\n \"num_features\",\n \"memory_size\",\n \"num_models\",\n \"child_model_type\",\n \"child_hyperparameters\",\n \"child_hyperparameters_fit\",\n ]\n model_info_out = {k: v for k, v in model_info.items() if k in valid_keys}\n model_info_inner_out = {k: v for k, v in model_info_inner.items() if k in valid_keys_inner}\n\n model_info_out.update(model_info_inner_out)\n model_info_out[\"model\"] = model_name\n model_info_out[\"exc_order\"] = i + 1\n\n model_infos[model_name] = model_info_out\n\n df = pd.DataFrame(\n data=model_infos,\n ).T\n\n explicit_order = [\n \"model\",\n \"exc_type\",\n \"total_time\",\n \"model_type\",\n \"child_model_type\",\n \"is_initialized\",\n \"is_fit\",\n \"is_valid\",\n \"can_infer\",\n \"num_features\",\n \"num_models\",\n \"memory_size\",\n \"hyperparameters\",\n \"hyperparameters_fit\",\n \"child_hyperparameters\",\n \"child_hyperparameters_fit\",\n \"exc_str\",\n \"exc_traceback\",\n \"exc_order\",\n ]\n\n df_columns_lst = list(df.columns)\n explicit_order = [column for column in explicit_order if column in df_columns_lst]\n df_columns_other = [column for column in df_columns_lst if column not in explicit_order]\n df_columns_new = explicit_order + df_columns_other\n df_sorted = df[df_columns_new]\n df_sorted = df_sorted.reset_index(drop=True)\n\n return df_sorted\n\n def get_info(self, include_model_info=False, include_model_failures=True) -> dict:\n num_models_trained = len(self.get_model_names())\n if self.model_best is not None:\n best_model = self.model_best\n else:\n try:\n best_model = self.get_model_best()\n except AssertionError:\n best_model = None\n if best_model is not None:\n best_model_score_val = self.get_model_attribute(model=best_model, attribute=\"val_score\")\n best_model_stack_level = self.get_model_level(best_model)\n else:\n best_model_score_val = None\n best_model_stack_level = None\n # fit_time = None\n num_bag_folds = self.k_fold\n max_core_stack_level = self.get_max_level(\"core\")\n max_stack_level = self.get_max_level()\n\n problem_type = self.problem_type\n eval_metric = self.eval_metric.name\n time_train_start = self._time_train_start_last\n num_rows_train = self._num_rows_train\n num_cols_train = self._num_cols_train\n num_rows_val = self._num_rows_val\n num_rows_test = self._num_rows_test\n num_classes = self.num_classes\n # TODO:\n # Disk size of models\n # Raw feature count\n # HPO time\n # Bag time\n # Feature prune time\n # Exception count / models failed count\n # True model count (models * kfold)\n # AutoGluon version fit on\n # Max memory usage\n # CPU count used / GPU count used\n\n info = {\n \"time_train_start\": time_train_start,\n \"num_rows_train\": num_rows_train,\n \"num_cols_train\": num_cols_train,\n \"num_rows_val\": num_rows_val,\n \"num_rows_test\": num_rows_test,\n \"num_classes\": num_classes,\n \"problem_type\": problem_type,\n \"eval_metric\": eval_metric,\n \"best_model\": best_model,\n \"best_model_score_val\": best_model_score_val,\n \"best_model_stack_level\": best_model_stack_level,\n \"num_models_trained\": num_models_trained,\n \"num_bag_folds\": num_bag_folds,\n \"max_stack_level\": max_stack_level,\n \"max_core_stack_level\": max_core_stack_level,\n }\n\n if include_model_info:\n info[\"model_info\"] = self.get_models_info()\n if include_model_failures:\n info[\"model_info_failures\"] = copy.deepcopy(self._models_failed_to_train_errors)\n\n return info\n\n def reduce_memory_size(\n self,\n remove_data=True,\n remove_fit_stack=False,\n remove_fit=True,\n remove_info=False,\n requires_save=True,\n reduce_children=False,\n **kwargs,\n ):\n if remove_data and self.is_data_saved:\n data_files = [\n os.path.join(self.path_data, \"X.pkl\"),\n os.path.join(self.path_data, \"X_val.pkl\"),\n os.path.join(self.path_data, \"y.pkl\"),\n os.path.join(self.path_data, \"y_val.pkl\"),\n ]\n for data_file in data_files:\n try:\n os.remove(data_file)\n except FileNotFoundError:\n pass\n if requires_save:\n self.is_data_saved = False\n try:\n os.rmdir(self.path_data)\n except OSError:\n pass\n shutil.rmtree(path=Path(self._path_attr), ignore_errors=True)\n try:\n os.rmdir(self.path_utils)\n except OSError:\n pass\n if remove_info and requires_save:\n # Remove model failure info artifacts\n self._models_failed_to_train_errors = dict()\n models = self.get_model_names()\n for model in models:\n model = self.load_model(model)\n model.reduce_memory_size(\n remove_fit_stack=remove_fit_stack,\n remove_fit=remove_fit,\n remove_info=remove_info,\n requires_save=requires_save,\n reduce_children=reduce_children,\n **kwargs,\n )\n if requires_save:\n self.save_model(model, reduce_memory=False)\n if requires_save:\n self.save()\n\n # TODO: Also enable deletion of models which didn't succeed in training (files may still be persisted)\n # This includes the original HPO fold for stacking\n # Deletes specified models from trainer and from disk (if delete_from_disk=True).\n def delete_models(\n self,\n models_to_keep=None,\n models_to_delete=None,\n allow_delete_cascade=False,\n delete_from_disk=True,\n dry_run=True,\n ):\n if models_to_keep is not None and models_to_delete is not None:\n raise ValueError(\"Exactly one of [models_to_keep, models_to_delete] must be set.\")\n if models_to_keep is not None:\n if not isinstance(models_to_keep, list):\n models_to_keep = [models_to_keep]\n minimum_model_set = set()\n for model in models_to_keep:\n minimum_model_set.update(self.get_minimum_model_set(model))\n minimum_model_set = list(minimum_model_set)\n models_to_remove = [model for model in self.get_model_names() if model not in minimum_model_set]\n elif models_to_delete is not None:\n if not isinstance(models_to_delete, list):\n models_to_delete = [models_to_delete]\n minimum_model_set = set(models_to_delete)\n minimum_model_set_orig = copy.deepcopy(minimum_model_set)\n for model in models_to_delete:\n minimum_model_set.update(nx.algorithms.dag.descendants(self.model_graph, model))\n if not allow_delete_cascade:\n if minimum_model_set != minimum_model_set_orig:\n raise AssertionError(\n \"models_to_delete contains models which cause a delete cascade due to other models being dependent on them. Set allow_delete_cascade=True to enable the deletion.\"\n )\n minimum_model_set = list(minimum_model_set)\n models_to_remove = [model for model in self.get_model_names() if model in minimum_model_set]\n else:\n raise ValueError(\"Exactly one of [models_to_keep, models_to_delete] must be set.\")\n\n if dry_run:\n logger.log(30, f\"Dry run enabled, AutoGluon would have deleted the following models: {models_to_remove}\")\n if delete_from_disk:\n for model in models_to_remove:\n model = self.load_model(model)\n logger.log(30, f\"\\tDirectory {model.path} would have been deleted.\")\n logger.log(30, \"To perform the deletion, set dry_run=False\")\n return\n\n if delete_from_disk:\n for model in models_to_remove:\n model = self.load_model(model)\n model.delete_from_disk()\n\n for model in models_to_remove:\n self._delete_model_from_graph(model=model)\n\n models_kept = self.get_model_names()\n\n if self.model_best is not None and self.model_best not in models_kept:\n try:\n self.model_best = self.get_model_best()\n except AssertionError:\n self.model_best = None\n\n # TODO: Delete from all the other model dicts\n self.save()\n\n def _delete_model_from_graph(self, model: str):\n self.model_graph.remove_node(model)\n if model in self.models:\n self.models.pop(model)\n path_attr_model = Path(self._path_attr_model(model))\n shutil.rmtree(path=path_attr_model, ignore_errors=True)\n\n @staticmethod\n def _process_hyperparameters(hyperparameters: dict) -> dict:\n return process_hyperparameters(hyperparameters=hyperparameters)\n\n def distill(\n self,\n X=None,\n y=None,\n X_val=None,\n y_val=None,\n X_unlabeled=None,\n time_limit=None,\n hyperparameters=None,\n holdout_frac=None,\n verbosity=None,\n models_name_suffix=None,\n teacher=None,\n teacher_preds=\"soft\",\n augmentation_data=None,\n augment_method=\"spunge\",\n augment_args={\"size_factor\": 5, \"max_size\": int(1e5)},\n augmented_sample_weight=1.0,\n ):\n \"\"\"Various distillation algorithms.\n Args:\n X, y: pd.DataFrame and pd.Series of training data.\n If None, original training data used during predictor.fit() will be loaded.\n This data is split into train/validation if X_val, y_val are None.\n X_val, y_val: pd.DataFrame and pd.Series of validation data.\n time_limit, hyperparameters, holdout_frac: defined as in predictor.fit()\n teacher (None or str):\n If None, uses the model with the highest validation score as the teacher model, otherwise use the specified model name as the teacher.\n teacher_preds (None or str): If None, we only train with original labels (no data augmentation, overrides augment_method)\n If 'hard', labels are hard teacher predictions given by: teacher.predict()\n If 'soft', labels are soft teacher predictions given by: teacher.predict_proba()\n Note: 'hard' and 'soft' are equivalent for regression problems.\n If augment_method specified, teacher predictions are only used to label augmented data (training data keeps original labels).\n To apply label-smoothing: teacher_preds='onehot' will use original training data labels converted to one-hots for multiclass (no data augmentation). # TODO: expose smoothing-hyperparameter.\n models_name_suffix (str): Suffix to append to each student model's name, new names will look like: 'MODELNAME_dstl_SUFFIX'\n augmentation_data: pd.DataFrame of additional data to use as \"augmented data\" (does not contain labels).\n When specified, augment_method, augment_args are ignored, and this is the only augmented data that is used (teacher_preds cannot be None).\n augment_method (None or str): specifies which augmentation strategy to utilize. Options: [None, 'spunge','munge']\n If None, no augmentation gets applied.\n }\n augment_args (dict): args passed into the augmentation function corresponding to augment_method.\n augmented_sample_weight (float): Nonnegative value indicating how much to weight augmented samples. This is only considered if sample_weight was initially specified in Predictor.\n \"\"\"\n if verbosity is None:\n verbosity = self.verbosity\n\n if teacher is None:\n teacher = self._get_best()\n\n hyperparameter_tune = False # TODO: add as argument with scheduler options.\n if augmentation_data is not None and teacher_preds is None:\n raise ValueError(\"augmentation_data must be None if teacher_preds is None\")\n\n logger.log(\n 20,\n f\"Distilling with teacher='{teacher}', teacher_preds={str(teacher_preds)}, augment_method={str(augment_method)} ...\",\n )\n if teacher not in self.get_model_names(can_infer=True):\n raise AssertionError(\n f\"Teacher model '{teacher}' is not a valid teacher model! Either it does not exist or it cannot infer on new data.\\n\"\n f\"Valid teacher models: {self.get_model_names(can_infer=True)}\"\n )\n if X is None:\n if y is not None:\n raise ValueError(\"X cannot be None when y specified.\")\n X = self.load_X()\n X_val = self.load_X_val()\n\n if y is None:\n y = self.load_y()\n y_val = self.load_y_val()\n\n if X_val is None:\n if y_val is not None:\n raise ValueError(\"X_val cannot be None when y_val specified.\")\n if holdout_frac is None:\n holdout_frac = default_holdout_frac(len(X), hyperparameter_tune)\n X, X_val, y, y_val = generate_train_test_split(\n X, y, problem_type=self.problem_type, test_size=holdout_frac\n )\n\n y_val_og = y_val.copy()\n og_bagged_mode = self.bagged_mode\n og_verbosity = self.verbosity\n self.bagged_mode = False # turn off bagging\n self.verbosity = verbosity # change verbosity for distillation\n\n if self.sample_weight is not None:\n X, w = extract_column(X, self.sample_weight)\n\n if teacher_preds is None or teacher_preds == \"onehot\":\n augment_method = None\n logger.log(\n 20,\n \"Training students without a teacher model. Set teacher_preds = 'soft' or 'hard' to distill using the best AutoGluon predictor as teacher.\",\n )\n\n if teacher_preds in [\"onehot\", \"soft\"]:\n y = format_distillation_labels(y, self.problem_type, self.num_classes)\n y_val = format_distillation_labels(y_val, self.problem_type, self.num_classes)\n\n if augment_method is None and augmentation_data is None:\n if teacher_preds == \"hard\":\n y_pred = pd.Series(self.predict(X, model=teacher))\n if (self.problem_type != REGRESSION) and (\n len(y_pred.unique()) < len(y.unique())\n ): # add missing labels\n logger.log(\n 15, \"Adding missing labels to distillation dataset by including some real training examples\"\n )\n indices_to_add = []\n for clss in y.unique():\n if clss not in y_pred.unique():\n logger.log(15, f\"Fetching a row with label={clss} from training data\")\n clss_index = y[y == clss].index[0]\n indices_to_add.append(clss_index)\n X_extra = X.loc[indices_to_add].copy()\n y_extra = y.loc[indices_to_add].copy() # these are actually real training examples\n X = pd.concat([X, X_extra])\n y_pred = pd.concat([y_pred, y_extra])\n if self.sample_weight is not None:\n w = pd.concat([w, w[indices_to_add]])\n y = y_pred\n elif teacher_preds == \"soft\":\n y = self.predict_proba(X, model=teacher)\n if self.problem_type == MULTICLASS:\n y = pd.DataFrame(y)\n else:\n y = pd.Series(y)\n else:\n X_aug = augment_data(\n X=X,\n feature_metadata=self.feature_metadata,\n augmentation_data=augmentation_data,\n augment_method=augment_method,\n augment_args=augment_args,\n )\n if len(X_aug) > 0:\n if teacher_preds == \"hard\":\n y_aug = pd.Series(self.predict(X_aug, model=teacher))\n elif teacher_preds == \"soft\":\n y_aug = self.predict_proba(X_aug, model=teacher)\n if self.problem_type == MULTICLASS:\n y_aug = pd.DataFrame(y_aug)\n else:\n y_aug = pd.Series(y_aug)\n else:\n raise ValueError(f\"Unknown teacher_preds specified: {teacher_preds}\")\n\n X = pd.concat([X, X_aug])\n y = pd.concat([y, y_aug])\n if self.sample_weight is not None:\n w = pd.concat([w, pd.Series([augmented_sample_weight] * len(X_aug))])\n\n X.reset_index(drop=True, inplace=True)\n y.reset_index(drop=True, inplace=True)\n if self.sample_weight is not None:\n w.reset_index(drop=True, inplace=True)\n X[self.sample_weight] = w\n\n name_suffix = \"_DSTL\" # all student model names contain this substring\n if models_name_suffix is not None:\n name_suffix = name_suffix + \"_\" + models_name_suffix\n\n if hyperparameters is None:\n hyperparameters = {\"GBM\": {}, \"CAT\": {}, \"NN_TORCH\": {}, \"RF\": {}}\n hyperparameters = self._process_hyperparameters(\n hyperparameters=hyperparameters\n ) # TODO: consider exposing ag_args_fit, excluded_model_types as distill() arguments.\n if teacher_preds is not None and teacher_preds != \"hard\" and self.problem_type != REGRESSION:\n self._regress_preds_asprobas = True\n\n core_kwargs = {\n \"stack_name\": self.distill_stackname,\n \"get_models_func\": self.construct_model_templates_distillation,\n }\n aux_kwargs = {\n \"get_models_func\": self.construct_model_templates_distillation,\n \"check_if_best\": False,\n }\n\n # self.bagged_mode = True # TODO: Add options for bagging\n models = self.train_multi_levels(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n hyperparameters=hyperparameters,\n time_limit=time_limit, # FIXME: Also limit augmentation time\n name_suffix=name_suffix,\n core_kwargs=core_kwargs,\n aux_kwargs=aux_kwargs,\n )\n\n distilled_model_names = []\n w_val = None\n if self.weight_evaluation:\n X_val, w_val = extract_column(X_val, self.sample_weight)\n for model_name in models: # finally measure original metric on validation data and overwrite stored val_scores\n model_score = self.score(X_val, y_val_og, model=model_name, weights=w_val)\n model_obj = self.load_model(model_name)\n model_obj.val_score = model_score\n model_obj.save() # TODO: consider omitting for sake of efficiency\n self.model_graph.nodes[model_name][\"val_score\"] = model_score\n distilled_model_names.append(model_name)\n leaderboard = self.leaderboard()\n logger.log(20, \"Distilled model leaderboard:\")\n leaderboard_distilled = leaderboard[leaderboard[\"model\"].isin(models)].reset_index(drop=True)\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n logger.log(20, leaderboard_distilled)\n\n # reset trainer to old state before distill() was called:\n self.bagged_mode = og_bagged_mode # TODO: Confirm if safe to train future models after training models in both bagged and non-bagged modes\n self.verbosity = og_verbosity\n return distilled_model_names\n\n def _get_model_fit_kwargs(\n self,\n X: pd.DataFrame,\n X_val: pd.DataFrame,\n time_limit: float,\n k_fold: int,\n fit_kwargs: dict,\n ens_sample_weight: list | None = None,\n label_cleaner: None | LabelCleaner = None,\n ) -> dict:\n # Returns kwargs to be passed to AbstractModel's fit function\n if fit_kwargs is None:\n fit_kwargs = dict()\n\n model_fit_kwargs = dict(time_limit=time_limit, verbosity=self.verbosity, **fit_kwargs)\n if self.sample_weight is not None:\n X, w_train = extract_column(X, self.sample_weight)\n if w_train is not None: # may be None for ensemble\n # TODO: consider moving weight normalization into AbstractModel.fit()\n model_fit_kwargs[\"sample_weight\"] = (\n w_train.values / w_train.mean()\n ) # normalization can affect gradient algorithms like boosting\n if X_val is not None:\n X_val, w_val = extract_column(X_val, self.sample_weight)\n if (\n self.weight_evaluation and w_val is not None\n ): # ignore validation sample weights unless weight_evaluation specified\n model_fit_kwargs[\"sample_weight_val\"] = w_val.values / w_val.mean()\n if ens_sample_weight is not None:\n model_fit_kwargs[\"sample_weight\"] = (\n ens_sample_weight # sample weights to use for weighted ensemble only\n )\n if self._groups is not None and \"groups\" not in model_fit_kwargs:\n if k_fold == self.k_fold: # don't do this on refit full\n model_fit_kwargs[\"groups\"] = self._groups\n\n if label_cleaner is not None:\n model_fit_kwargs[\"label_cleaner\"] = label_cleaner\n\n # FIXME: Sample weight `extract_column` is a hack, have to compute feature_metadata here because sample weight column could be in X upstream, extract sample weight column upstream instead.\n if \"feature_metadata\" not in model_fit_kwargs:\n raise AssertionError(f\"Missing expected parameter 'feature_metadata'.\")\n return model_fit_kwargs\n\n def _get_bagged_model_fit_kwargs(\n self, k_fold: int, k_fold_start: int, k_fold_end: int, n_repeats: int, n_repeat_start: int\n ) -> dict:\n # Returns additional kwargs (aside from _get_model_fit_kwargs) to be passed to BaggedEnsembleModel's fit function\n if k_fold is None:\n k_fold = self.k_fold\n if n_repeats is None:\n n_repeats = self.n_repeats\n return dict(\n k_fold=k_fold,\n k_fold_start=k_fold_start,\n k_fold_end=k_fold_end,\n n_repeats=n_repeats,\n n_repeat_start=n_repeat_start,\n compute_base_preds=False,\n )\n\n def _get_feature_prune_proxy_model(self, proxy_model_class: AbstractModel | None, level: int) -> AbstractModel:\n \"\"\"\n Returns proxy model to be used for feature pruning - the base learner that has the highest validation score in a particular stack layer.\n Ties are broken by inference speed. If proxy_model_class is not None, take the best base learner belonging to proxy_model_class.\n proxy_model_class is an AbstractModel class (ex. LGBModel).\n \"\"\"\n proxy_model = None\n if isinstance(proxy_model_class, str):\n raise AssertionError(\n f\"proxy_model_class must be a subclass of AbstractModel. Was instead a string: {proxy_model_class}\"\n )\n banned_models = [GreedyWeightedEnsembleModel, SimpleWeightedEnsembleModel]\n assert proxy_model_class not in banned_models, (\n \"WeightedEnsemble models cannot be feature pruning proxy models.\"\n )\n\n leaderboard = self.leaderboard()\n banned_names = []\n candidate_model_rows = leaderboard[(~leaderboard[\"score_val\"].isna()) & (leaderboard[\"stack_level\"] == level)]\n candidate_models_type_inner = self.get_models_attribute_dict(\n attribute=\"type_inner\", models=candidate_model_rows[\"model\"]\n )\n for model_name, type_inner in candidate_models_type_inner.copy().items():\n if type_inner in banned_models:\n banned_names.append(model_name)\n candidate_models_type_inner.pop(model_name, None)\n banned_names = set(banned_names)\n candidate_model_rows = candidate_model_rows[~candidate_model_rows[\"model\"].isin(banned_names)]\n if proxy_model_class is not None:\n candidate_model_names = [\n model_name\n for model_name, model_class in candidate_models_type_inner.items()\n if model_class == proxy_model_class\n ]\n candidate_model_rows = candidate_model_rows[candidate_model_rows[\"model\"].isin(candidate_model_names)]\n if len(candidate_model_rows) == 0:\n if proxy_model_class is None:\n logger.warning(f\"No models from level {level} have been successfully fit. Skipping feature pruning.\")\n else:\n logger.warning(\n f\"No models of type {proxy_model_class} have finished training in level {level}. Skipping feature pruning.\"\n )\n return proxy_model\n best_candidate_model_rows = candidate_model_rows.loc[\n candidate_model_rows[\"score_val\"] == candidate_model_rows[\"score_val\"].max()\n ]\n return self.load_model(best_candidate_model_rows.loc[best_candidate_model_rows[\"fit_time\"].idxmin()][\"model\"])\n\n def calibrate_model(\n self, model_name: str | None = None, lr: float = 0.1, max_iter: int = 200, init_val: float = 1.0\n ):\n \"\"\"\n Applies temperature scaling to a model.\n Applies inverse softmax to predicted probs then trains temperature scalar\n on validation data to maximize negative log likelihood.\n Inversed softmaxes are divided by temperature scalar\n then softmaxed to return predicted probs.\n\n Parameters:\n -----------\n model_name: str: default = None\n model name to tune temperature scaling on.\n If None, will tune best model only. Best model chosen by validation score\n lr: float: default = 0.1\n The learning rate for temperature scaling algorithm\n max_iter: int: default = 200\n Number of iterations optimizer should take for\n tuning temperature scaler\n init_val: float: default = 1.0\n The initial value for temperature scalar term\n \"\"\"\n # TODO: Note that temperature scaling is known to worsen calibration in the face of shifted test data.\n try:\n # FIXME: Avoid depending on torch for temp scaling\n try_import_torch()\n except ImportError:\n logger.log(30, \"Warning: Torch is not installed, skipping calibration step...\")\n return\n\n if model_name is None:\n if self.has_val:\n can_infer = True\n else:\n can_infer = None\n if self.model_best is not None:\n models = self.get_model_names(can_infer=can_infer)\n if self.model_best in models:\n model_name = self.model_best\n if model_name is None:\n model_name = self.get_model_best(can_infer=can_infer)\n\n model_refit_map = self.model_refit_map()\n model_name_og = model_name\n for m, m_full in model_refit_map.items():\n if m_full == model_name:\n model_name_og = m\n break\n if self.has_val:\n X_val = self.load_X_val()\n y_val_probs = self.predict_proba(X_val, model_name_og)\n y_val = self.load_y_val().to_numpy()\n else: # bagged mode\n y_val_probs = self.get_model_oof(model_name_og)\n y_val = self.load_y().to_numpy()\n\n y_val_probs_og = y_val_probs\n if self.problem_type == BINARY:\n # Convert one-dimensional array to be in the form of a 2-class multiclass predict_proba output\n y_val_probs = LabelCleanerMulticlassToBinary.convert_binary_proba_to_multiclass_proba(y_val_probs)\n\n model = self.load_model(model_name=model_name)\n if self.problem_type == QUANTILE:\n logger.log(15, f\"Conformity scores being computed to calibrate model: {model_name}\")\n conformalize = compute_conformity_score(\n y_val_pred=y_val_probs, y_val=y_val, quantile_levels=self.quantile_levels\n )\n model.conformalize = conformalize\n model.save()\n else:\n logger.log(15, f\"Temperature scaling term being tuned for model: {model_name}\")\n temp_scalar = tune_temperature_scaling(\n y_val_probs=y_val_probs, y_val=y_val, init_val=init_val, max_iter=max_iter, lr=lr\n )\n if temp_scalar is None:\n logger.log(\n 15,\n f\"Warning: Infinity found during calibration, skipping calibration on {model.name}! \"\n f\"This can occur when the model is absolutely certain of a validation prediction (1.0 pred_proba).\",\n )\n elif temp_scalar <= 0:\n logger.log(\n 30,\n f\"Warning: Temperature scaling found optimal at a negative value ({temp_scalar}). Disabling temperature scaling to avoid overfitting.\",\n )\n else:\n # Check that scaling improves performance for the target metric\n score_without_temp = self.score_with_y_pred_proba(y=y_val, y_pred_proba=y_val_probs_og, weights=None)\n scaled_y_val_probs = apply_temperature_scaling(\n y_val_probs, temp_scalar, problem_type=self.problem_type, transform_binary_proba=False\n )\n score_with_temp = self.score_with_y_pred_proba(y=y_val, y_pred_proba=scaled_y_val_probs, weights=None)\n\n if score_with_temp > score_without_temp:\n logger.log(15, f\"Temperature term found is: {temp_scalar}\")\n model.params_aux[\"temperature_scalar\"] = temp_scalar\n model.save()\n else:\n logger.log(15, \"Temperature did not improve performance, skipping calibration.\")\n\n def calibrate_decision_threshold(\n self,\n X: pd.DataFrame | None = None,\n y: np.ndarray | None = None,\n metric: str | Scorer | None = None,\n model: str = \"best\",\n weights=None,\n decision_thresholds: int | list[float] = 25,\n secondary_decision_thresholds: int | None = 19,\n verbose: bool = True,\n **kwargs,\n ) -> float:\n # TODO: Docstring\n assert self.problem_type == BINARY, (\n f'calibrate_decision_threshold is only available for `problem_type=\"{BINARY}\"`'\n )\n\n if metric is None:\n metric = self.eval_metric\n elif isinstance(metric, str):\n metric = get_metric(metric, self.problem_type, \"eval_metric\")\n\n if model == \"best\":\n model = self.get_model_best()\n\n if y is None:\n # If model is refit_full, use its parent to avoid over-fitting\n model_parent = self.get_refit_full_parent(model=model)\n if not self.model_exists(model_parent):\n raise AssertionError(\n f\"Unable to calibrate the decision threshold on the internal data because the \"\n f'model \"{model}\" is a refit_full model trained on all of the internal data, '\n f'whose parent model \"{model_parent}\" does not exist or was deleted.\\n'\n f\"It may have been deleted due to `predictor.fit(..., keep_only_best=True)`. \"\n f\"Ensure `keep_only_best=False` to be able to calibrate refit_full models.\"\n )\n model = model_parent\n\n # TODO: Add helpful logging when data is not available, for example post optimize for deployment\n if self.has_val:\n # Use validation data\n X = self.load_X_val()\n if self.weight_evaluation:\n X, weights = extract_column(X=X, col_name=self.sample_weight)\n y: np.array = self.load_y_val()\n y_pred_proba = self.predict_proba(X=X, model=model)\n else:\n # Use out-of-fold data\n if self.weight_evaluation:\n X = self.load_X()\n X, weights = extract_column(X=X, col_name=self.sample_weight)\n y: np.array = self.load_y()\n y_pred_proba = self.get_model_oof(model=model)\n else:\n y_pred_proba = self.predict_proba(X=X, model=model)\n\n if not metric.needs_pred:\n logger.warning(\n f'WARNING: The provided metric \"{metric.name}\" does not use class predictions for scoring, '\n f\"and thus is invalid for decision threshold calibration. \"\n f\"Falling back to `decision_threshold=0.5`.\"\n )\n return 0.5\n\n return calibrate_decision_threshold(\n y=y,\n y_pred_proba=y_pred_proba,\n metric=lambda y, y_pred: self.score_with_y_pred(y=y, y_pred=y_pred, weights=weights, metric=metric),\n decision_thresholds=decision_thresholds,\n secondary_decision_thresholds=secondary_decision_thresholds,\n metric_name=metric.name,\n verbose=verbose,\n **kwargs,\n )\n\n @staticmethod\n def _validate_num_classes(num_classes: int | None, problem_type: str):\n if problem_type == BINARY:\n assert num_classes is not None and num_classes == 2, (\n f\"num_classes must be 2 when problem_type='{problem_type}' (num_classes={num_classes})\"\n )\n elif problem_type in [MULTICLASS, SOFTCLASS]:\n assert num_classes is not None and num_classes >= 2, (\n f\"num_classes must be >=2 when problem_type='{problem_type}' (num_classes={num_classes})\"\n )\n elif problem_type in [REGRESSION, QUANTILE]:\n assert num_classes is None, (\n f\"num_classes must be None when problem_type='{problem_type}' (num_classes={num_classes})\"\n )\n else:\n raise AssertionError(\n f\"Unknown problem_type: '{problem_type}'. Valid problem types: {[BINARY, MULTICLASS, REGRESSION, SOFTCLASS, QUANTILE]}\"\n )\n\n @staticmethod\n def _validate_quantile_levels(quantile_levels: list[float] | np.ndarray | None, problem_type: str):\n if problem_type == QUANTILE:\n assert quantile_levels is not None, (\n f\"quantile_levels must not be None when problem_type='{problem_type}' (quantile_levels={quantile_levels})\"\n )\n assert isinstance(quantile_levels, (list, np.ndarray)), (\n f\"quantile_levels must be a list or np.ndarray (quantile_levels={quantile_levels})\"\n )\n assert len(quantile_levels) > 0, (\n f\"quantile_levels must not be an empty list (quantile_levels={quantile_levels})\"\n )\n else:\n assert quantile_levels is None, (\n f\"quantile_levels must be None when problem_type='{problem_type}' (quantile_levels={quantile_levels})\"\n )\n\n\ndef _detached_train_multi_fold(\n *,\n _self: AbstractTabularTrainer,\n model: str | AbstractModel,\n X: pd.DataFrame,\n y: pd.Series,\n time_split: bool,\n time_start: float,\n time_limit: float | None,\n time_limit_model_split: float | None,\n hyperparameter_tune_kwargs: dict,\n is_ray_worker: bool = False,\n kwargs: dict,\n) -> list[str]:\n \"\"\"Dedicated class-detached function to train a single model on multiple folds.\"\"\"\n if isinstance(model, str):\n model = _self.load_model(model)\n elif _self.low_memory:\n model = copy.deepcopy(model)\n if hyperparameter_tune_kwargs is not None and isinstance(hyperparameter_tune_kwargs, dict):\n hyperparameter_tune_kwargs_model = hyperparameter_tune_kwargs.get(model.name, None)\n else:\n hyperparameter_tune_kwargs_model = None\n # TODO: Only update scores when finished, only update model as part of final models if finished!\n if time_split:\n time_left = time_limit_model_split\n else:\n if time_limit is None:\n time_left = None\n else:\n time_start_model = time.time()\n time_left = time_limit - (time_start_model - time_start)\n\n model_name_trained_lst = _self._train_single_full(\n X,\n y,\n model,\n time_limit=time_left,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs_model,\n is_ray_worker=is_ray_worker,\n **kwargs,\n )\n\n if _self.low_memory:\n del model\n\n return model_name_trained_lst\n\n\ndef _remote_train_multi_fold(\n *,\n _self: AbstractTabularTrainer,\n model: str | AbstractModel,\n X: pd.DataFrame,\n y: pd.Series,\n time_split: bool,\n time_start: float,\n time_limit: float | None,\n time_limit_model_split: float | None,\n hyperparameter_tune_kwargs: dict,\n kwargs: dict,\n errors: Literal[\"ignore\", \"raise\"] | None = None,\n) -> tuple[str, str | None, str | None, Exception | None, dict | None]:\n reset_logger_for_remote_call(verbosity=_self.verbosity)\n\n if errors is not None:\n kwargs[\"errors\"] = errors\n\n exception = None\n try:\n model_name_list = _detached_train_multi_fold(\n _self=_self,\n model=model,\n X=X,\n y=y,\n time_start=time_start,\n time_split=time_split,\n time_limit=time_limit,\n time_limit_model_split=time_limit_model_split,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n is_ray_worker=True,\n kwargs=kwargs,\n )\n except Exception as exc:\n model_name_list = []\n if errors is not None and errors == \"raise\":\n # If training fails and exception is returned, collect the exception information and return\n exception = exc # required to use in outer scope\n else:\n raise exc\n\n if not model_name_list:\n model_name = model if isinstance(model, str) else model.name\n # Get model_failure metadata if it exists\n model_failure_info = None\n if model_name in _self._models_failed_to_train_errors:\n model_failure_info = _self._models_failed_to_train_errors[model_name]\n return model_name, None, None, exception, model_failure_info\n\n # Fallback, return original model name if training failed.\n if not model_name_list:\n model_name = model if isinstance(model, str) else model.name\n return model_name, None, None, None, None\n model_name = model_name_list[0]\n return (\n model_name,\n _self.get_model_attribute(model=model_name, attribute=\"path\"),\n _self.get_model_attribute(model=model_name, attribute=\"type\"),\n None,\n None,\n )\n\n\ndef _detached_refit_single_full(\n *,\n _self: AbstractTabularTrainer,\n model: str,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame,\n y_val: pd.Series,\n X_unlabeled: pd.DataFrame,\n level: int,\n kwargs: dict,\n fit_strategy: Literal[\"sequential\", \"parallel\"] = \"sequential\",\n) -> tuple[str, list[str]]:\n # TODO: loading the model is the reasons we must allocate GPU resources for this job in cases where models require GPU when loaded from disk\n model = _self.load_model(model)\n model_name = model.name\n reuse_first_fold = False\n\n if isinstance(model, BaggedEnsembleModel):\n # Reuse if model is already _FULL and no X_val\n if X_val is None:\n reuse_first_fold = not model._bagged_mode\n\n if not reuse_first_fold:\n if isinstance(model, BaggedEnsembleModel):\n can_refit_full = model._get_tags_child().get(\"can_refit_full\", False)\n else:\n can_refit_full = model._get_tags().get(\"can_refit_full\", False)\n reuse_first_fold = not can_refit_full\n\n if not reuse_first_fold:\n model_full = model.convert_to_refit_full_template()\n # Mitigates situation where bagged models barely had enough memory and refit requires more. Worst case results in OOM, but this lowers chance of failure.\n model_full._user_params_aux[\"max_memory_usage_ratio\"] = model.params_aux[\"max_memory_usage_ratio\"] * 1.15\n # Re-set user specified training resources.\n # FIXME: this is technically also a bug for non-distributed mode, but there it is good to use more/all resources per refit.\n # FIXME: Unsure if it is better to do model.fit_num_cpus or model.fit_num_cpus_child,\n # (Nick): I'm currently leaning towards model.fit_num_cpus, it is also less memory intensive\n # Better to not specify this for sequential fits, since we want the models to use the optimal amount of resources,\n # which could be less than the available resources (ex: LightGBM fits faster using 50% of the cores)\n if fit_strategy == \"parallel\":\n # FIXME: Why use `model.fit_num_cpus_child` when we can use the same values as was passed to `ray` for the process, just pass those values as kwargs. Eliminates chance of inconsistency.\n if model.fit_num_cpus_child is not None:\n model_full._user_params_aux[\"num_cpus\"] = model.fit_num_cpus_child\n if model.fit_num_gpus_child is not None:\n model_full._user_params_aux[\"num_gpus\"] = model.fit_num_gpus_child\n # TODO: Do it for all models in the level at once to avoid repeated processing of data?\n base_model_names = _self.get_base_model_names(model_name)\n # FIXME: Logs for inference speed (1 row) are incorrect because\n # parents are non-refit models in this sequence and later correct after logging.\n # Avoiding fix at present to minimize hacks in the code.\n # Return to this later when Trainer controls all stacking logic to map correct parent.\n models_trained = _self.stack_new_level_core(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n models=[model_full],\n base_model_names=base_model_names,\n level=level,\n stack_name=REFIT_FULL_NAME,\n hyperparameter_tune_kwargs=None,\n feature_prune=False,\n k_fold=1,\n n_repeats=1,\n ensemble_type=type(model),\n refit_full=True,\n **kwargs,\n )\n if len(models_trained) == 0:\n reuse_first_fold = True\n logger.log(\n 30,\n f\"WARNING: Refit training failure detected for '{model_name}'... \"\n f\"Falling back to using first fold to avoid downstream exception.\"\n f\"\\n\\tThis is likely due to an out-of-memory error or other memory related issue. \"\n f\"\\n\\tPlease create a GitHub issue if this was triggered from a non-memory related problem.\",\n )\n if not model.params.get(\"save_bag_folds\", True):\n raise AssertionError(\n f\"Cannot avoid training failure during refit for '{model_name}' by falling back to \"\n f\"copying the first fold because it does not exist! (save_bag_folds=False)\"\n f\"\\n\\tPlease specify `save_bag_folds=True` in the `.fit` call to avoid this exception.\"\n )\n\n if reuse_first_fold:\n # Perform fallback black-box refit logic that doesn't retrain.\n model_full = model.convert_to_refit_full_via_copy()\n # FIXME: validation time not correct for infer 1 batch time, needed to hack _is_refit=True to fix\n logger.log(20, f\"Fitting model: {model_full.name} | Skipping fit via cloning parent ...\")\n _self._add_model(model_full, stack_name=REFIT_FULL_NAME, level=level, _is_refit=True)\n _self.save_model(model_full)\n models_trained = [model_full.name]\n\n return model_name, models_trained\n\n\ndef _remote_refit_single_full(\n *,\n _self: AbstractTabularTrainer,\n model: str,\n X: pd.DataFrame,\n y: pd.Series,\n X_val: pd.DataFrame,\n y_val: pd.Series,\n X_unlabeled: pd.DataFrame,\n level: int,\n kwargs: dict,\n fit_strategy: Literal[\"sequential\", \"parallel\"],\n) -> tuple[str, str, list[str], str, str]:\n reset_logger_for_remote_call(verbosity=_self.verbosity)\n\n model_name, models_trained = _detached_refit_single_full(\n _self=_self,\n model=model,\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_unlabeled=X_unlabeled,\n level=level,\n kwargs=kwargs,\n fit_strategy=fit_strategy,\n )\n\n # We always just refit one model per call, so this must be the case.\n assert len(models_trained) == 1\n refitted_model_name = models_trained[0]\n return (\n model_name,\n refitted_model_name,\n _self.get_model_attribute(model=refitted_model_name, attribute=\"path\"),\n _self.get_model_attribute(model=refitted_model_name, attribute=\"type\"),\n )\n"
},
{
"path": "tabular/src/autogluon/tabular/trainer/auto_trainer.py",
"content": "import logging\n\nfrom autogluon.core.models import AbstractModel\nfrom autogluon.core.utils import generate_train_test_split\n\nfrom ..models.lgb.lgb_model import LGBModel\nfrom ..registry import ag_model_registry\nfrom .abstract_trainer import AbstractTabularTrainer\nfrom .model_presets.presets import get_preset_models\nfrom .model_presets.presets_distill import get_preset_models_distillation\n\nlogger = logging.getLogger(__name__)\n\n\n# This Trainer handles model training details\nclass AutoTrainer(AbstractTabularTrainer):\n def construct_model_templates(self, hyperparameters, **kwargs):\n path = kwargs.pop(\"path\", self.path)\n problem_type = kwargs.pop(\"problem_type\", self.problem_type)\n eval_metric = kwargs.pop(\"eval_metric\", self.eval_metric)\n quantile_levels = kwargs.pop(\"quantile_levels\", self.quantile_levels)\n invalid_model_names = kwargs.pop(\"invalid_model_names\", self._get_banned_model_names())\n silent = kwargs.pop(\"silent\", self.verbosity < 3)\n ag_args_fit = kwargs.pop(\"ag_args_fit\", None)\n if quantile_levels is not None:\n if ag_args_fit is None:\n ag_args_fit = dict()\n ag_args_fit = ag_args_fit.copy()\n ag_args_fit[\"quantile_levels\"] = quantile_levels\n\n return get_preset_models(\n path=path,\n problem_type=problem_type,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n ag_args_fit=ag_args_fit,\n invalid_model_names=invalid_model_names,\n silent=silent,\n **kwargs,\n )\n\n def fit(\n self,\n X,\n y,\n hyperparameters,\n X_val=None,\n y_val=None,\n X_test=None,\n y_test=None,\n X_unlabeled=None,\n holdout_frac=0.1,\n num_stack_levels=0,\n core_kwargs: dict = None,\n aux_kwargs: dict = None,\n time_limit=None,\n infer_limit=None,\n infer_limit_batch_size=None,\n use_bag_holdout=False,\n groups=None,\n callbacks: list[callable] = None,\n label_cleaner=None,\n **kwargs,\n ):\n for key in kwargs:\n logger.warning(f\"Warning: Unknown argument passed to `AutoTrainer.fit()`. Argument: {key}\")\n\n if use_bag_holdout:\n if self.bagged_mode:\n logger.log(\n 20,\n f\"use_bag_holdout={use_bag_holdout}, will use tuning_data as holdout (will not be used for early stopping).\",\n )\n else:\n logger.warning(\n f\"Warning: use_bag_holdout={use_bag_holdout}, but bagged mode is not enabled. use_bag_holdout will be ignored.\"\n )\n\n if (y_val is None) or (X_val is None):\n if not self.bagged_mode or use_bag_holdout:\n if groups is not None:\n raise AssertionError(\n f\"Validation data must be manually specified if use_bag_holdout and groups are both specified.\"\n )\n if self.bagged_mode:\n # Need at least 2 samples of each class in train data after split for downstream k-fold splits\n # to ensure each k-fold has at least 1 sample of each class in training data\n min_cls_count_train = 2\n else:\n min_cls_count_train = 1\n X, X_val, y, y_val = generate_train_test_split(\n X,\n y,\n problem_type=self.problem_type,\n test_size=holdout_frac,\n random_state=self.random_state,\n min_cls_count_train=min_cls_count_train,\n )\n logger.log(\n 20,\n f\"Automatically generating train/validation split with holdout_frac={holdout_frac}, Train Rows: {len(X)}, Val Rows: {len(X_val)}\",\n )\n elif self.bagged_mode:\n if not use_bag_holdout:\n # TODO: User could be intending to blend instead. Add support for blend stacking.\n # This error message is necessary because when calculating out-of-fold predictions for user, we want to return them in the form given in train_data,\n # but if we merge train and val here, it becomes very confusing from a users perspective, especially because we reset index, making it impossible to match\n # the original train_data to the out-of-fold predictions from `predictor.predict_proba_oof()`.\n raise AssertionError(\n \"X_val, y_val is not None, but bagged mode was specified. \"\n \"If calling from `TabularPredictor.fit()`, `tuning_data` should be None.\\n\"\n \"Default bagged mode does not use tuning data / validation data. \"\n \"Instead, all data (`train_data` and `tuning_data`) should be combined and specified as `train_data`.\\n\"\n \"To avoid this error and use `tuning_data` as holdout data in bagged mode, \"\n \"specify the following:\\n\"\n \"\\tpredictor.fit(..., tuning_data=tuning_data, use_bag_holdout=True)\"\n )\n\n # Log the hyperparameters dictionary so it easy to edit if the user wants.\n n_configs = sum([len(hyperparameters[k]) for k in hyperparameters.keys()])\n extra_log_str = \"\"\n display_all = (n_configs < 20) or (self.verbosity >= 3)\n if not display_all:\n # FIXME: This isn't correct\n extra_log_str = (\n f\"Large model count detected ({n_configs} configs) ... \"\n f\"Only displaying the first 3 models of each family. To see all, set `verbosity=3`.\\n\"\n )\n log_str = f\"{extra_log_str}User-specified model hyperparameters to be fit:\\n{{\\n\"\n if display_all:\n for k in hyperparameters.keys():\n # TODO: Make hyperparameters[k] be a list upstream to avoid needing these edge-cases\n if not isinstance(hyperparameters[k], list):\n log_str += f\"\\t'{k}': {[hyperparameters[k]]},\\n\"\n else:\n log_str += f\"\\t'{k}': {hyperparameters[k]},\\n\"\n else:\n for k in hyperparameters.keys():\n if not isinstance(hyperparameters[k], list):\n log_str += f\"\\t'{k}': {[hyperparameters[k]]},\\n\"\n else:\n log_str += f\"\\t'{k}': {hyperparameters[k][:3]},\\n\"\n log_str += \"}\"\n logger.log(20, log_str)\n\n if label_cleaner is not None:\n core_kwargs[\"label_cleaner\"] = label_cleaner\n\n self._train_multi_and_ensemble(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n X_test=X_test,\n y_test=y_test,\n X_unlabeled=X_unlabeled,\n hyperparameters=hyperparameters,\n num_stack_levels=num_stack_levels,\n time_limit=time_limit,\n core_kwargs=core_kwargs,\n aux_kwargs=aux_kwargs,\n infer_limit=infer_limit,\n infer_limit_batch_size=infer_limit_batch_size,\n groups=groups,\n callbacks=callbacks,\n )\n\n def construct_model_templates_distillation(self, hyperparameters, **kwargs):\n path = kwargs.pop(\"path\", self.path)\n problem_type = kwargs.pop(\"problem_type\", self.problem_type)\n eval_metric = kwargs.pop(\"eval_metric\", self.eval_metric)\n invalid_model_names = kwargs.pop(\"invalid_model_names\", self._get_banned_model_names())\n silent = kwargs.pop(\"silent\", self.verbosity < 3)\n\n # TODO: QUANTILE VERSION?\n\n return get_preset_models_distillation(\n path=path,\n problem_type=problem_type,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n invalid_model_names=invalid_model_names,\n silent=silent,\n **kwargs,\n )\n\n def _get_default_proxy_model_class(self):\n return LGBModel\n\n def compile(self, model_names=\"all\", with_ancestors=False, compiler_configs: dict = None) -> list[str]:\n \"\"\"Ensures that compiler_configs maps to the correct models if the user specified the same keys as in hyperparameters such as RT, XT, etc.\"\"\"\n if compiler_configs is not None:\n model_types_map = self._get_model_types_map()\n compiler_configs_new = dict()\n for k in compiler_configs:\n if k in model_types_map:\n compiler_configs_new[model_types_map[k]] = compiler_configs[k]\n else:\n compiler_configs_new[k] = compiler_configs[k]\n compiler_configs = compiler_configs_new\n return super().compile(\n model_names=model_names, with_ancestors=with_ancestors, compiler_configs=compiler_configs\n )\n\n def _get_model_types_map(self) -> dict[str, AbstractModel]:\n return ag_model_registry.key_to_cls_map()\n"
},
{
"path": "tabular/src/autogluon/tabular/trainer/model_presets/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/trainer/model_presets/presets.py",
"content": "from __future__ import annotations\n\nimport copy\nimport inspect\nimport logging\nfrom collections import defaultdict\n\nfrom packaging import version\n\nfrom autogluon.common.model_filter import ModelFilter\nfrom autogluon.common.utils.hyperparameter_utils import (\n get_deprecated_lightgbm_large_hyperparameters,\n get_hyperparameter_str_deprecation_msg,\n)\nfrom autogluon.core.constants import (\n AG_ARGS,\n AG_ARGS_ENSEMBLE,\n AG_ARGS_FIT,\n BINARY,\n MULTICLASS,\n QUANTILE,\n REGRESSION,\n SOFTCLASS,\n)\nfrom autogluon.core.models import (\n AbstractModel,\n StackerEnsembleModel,\n)\nfrom autogluon.core.trainer.utils import process_hyperparameters\n\nfrom ...registry import ag_model_registry\nfrom ...version import __version__\n\nlogger = logging.getLogger(__name__)\n\nDEFAULT_CUSTOM_MODEL_PRIORITY = 0\n\nVALID_AG_ARGS_KEYS = {\n \"name\",\n \"name_main\",\n \"name_prefix\",\n \"name_suffix\",\n \"name_bag_suffix\",\n \"model_type\",\n \"priority\",\n \"problem_types\",\n \"disable_in_hpo\",\n \"valid_stacker\",\n \"valid_base\",\n \"hyperparameter_tune_kwargs\",\n}\n\n\n# DONE: Add levels, including 'default'\n# DONE: Add lists\n# DONE: Add custom which can append to lists\n# DONE: Add special optional AG args for things like name prefix, name suffix, name, etc.\n# DONE: Move creation of stack ensemble internally into this function? Requires passing base models in as well.\n# DONE: Add special optional AG args for training order\n# DONE: Add special optional AG args for base models\n# TODO: Consider making hyperparameters arg in fit() accept lists, concatenate hyperparameter sets together.\n# TODO: Consider adding special optional AG args for #cores,#gpus,num_early_stopping_iterations,etc.\n# DONE: Consider adding special optional AG args for max train time, max memory size, etc.\n# TODO: Consider adding special optional AG args for use_original_features,features_to_use,etc.\n# TODO: Consider adding optional AG args to dynamically disable models such as valid_num_classes_range, valid_row_count_range, valid_feature_count_range, etc.\n# TODO: Args such as max_repeats, num_folds\n# DONE: Add banned_model_types arg\n# TODO: Add option to update hyperparameters with only added keys, so disabling CatBoost would just be {'CAT': []}, which keeps the other models as is.\n# TODO: special optional AG arg for only training model if eval_metric in list / not in list. Useful for F1 and 'is_unbalanced' arg in LGBM.\ndef get_preset_models(\n path,\n problem_type,\n eval_metric,\n hyperparameters,\n level: int = 1,\n ensemble_type=StackerEnsembleModel,\n ensemble_kwargs: dict = None,\n ag_args_fit=None,\n ag_args=None,\n ag_args_ensemble=None,\n name_suffix: str = None,\n default_priorities=None,\n invalid_model_names: list = None,\n included_model_types: list = None,\n excluded_model_types: list = None,\n hyperparameter_preprocess_func=None,\n hyperparameter_preprocess_kwargs=None,\n silent=True,\n):\n hyperparameters = process_hyperparameters(hyperparameters)\n if hyperparameter_preprocess_func is not None:\n if hyperparameter_preprocess_kwargs is None:\n hyperparameter_preprocess_kwargs = dict()\n hyperparameters = hyperparameter_preprocess_func(hyperparameters, **hyperparameter_preprocess_kwargs)\n if problem_type not in [BINARY, MULTICLASS, REGRESSION, SOFTCLASS, QUANTILE]:\n raise NotImplementedError\n invalid_name_set = set()\n if invalid_model_names is not None:\n invalid_name_set.update(invalid_model_names)\n\n if default_priorities is None:\n priority_cls_map = ag_model_registry.priority_map(problem_type=problem_type)\n default_priorities = {\n ag_model_registry.key(model_cls): priority for model_cls, priority in priority_cls_map.items()\n }\n\n level_key = level if level in hyperparameters.keys() else \"default\"\n if level_key not in hyperparameters.keys() and level_key == \"default\":\n hyperparameters = {\"default\": hyperparameters}\n hp_level = hyperparameters[level_key]\n hp_level = ModelFilter.filter_models(\n models=hp_level, included_model_types=included_model_types, excluded_model_types=excluded_model_types\n )\n model_cfg_priority_dict = defaultdict(list)\n model_type_list = list(hp_level.keys())\n for model_type in model_type_list:\n models_of_type = hp_level[model_type]\n if not isinstance(models_of_type, list):\n models_of_type = [models_of_type]\n model_cfgs_to_process = []\n for model_cfg in models_of_type:\n model_cfgs_to_process.append(model_cfg)\n for model_cfg in model_cfgs_to_process:\n model_cfg = clean_model_cfg(\n model_cfg=model_cfg,\n model_type=model_type,\n ag_args=ag_args,\n ag_args_ensemble=ag_args_ensemble,\n ag_args_fit=ag_args_fit,\n problem_type=problem_type,\n )\n model_cfg[AG_ARGS][\"priority\"] = model_cfg[AG_ARGS].get(\n \"priority\", default_priorities.get(model_type, DEFAULT_CUSTOM_MODEL_PRIORITY)\n )\n model_priority = model_cfg[AG_ARGS][\"priority\"]\n # Check if model_cfg is valid\n is_valid = is_model_cfg_valid(model_cfg, level=level, problem_type=problem_type)\n if AG_ARGS_FIT in model_cfg and not model_cfg[AG_ARGS_FIT]:\n model_cfg.pop(AG_ARGS_FIT)\n if is_valid:\n model_cfg_priority_dict[model_priority].append(model_cfg)\n\n model_cfg_priority_list = [\n model\n for priority in sorted(model_cfg_priority_dict.keys(), reverse=True)\n for model in model_cfg_priority_dict[priority]\n ]\n\n if not silent:\n logger.log(20, \"Model configs that will be trained (in order):\")\n models = []\n model_args_fit = {}\n for model_cfg in model_cfg_priority_list:\n model = model_factory(\n model_cfg,\n path=path,\n problem_type=problem_type,\n eval_metric=eval_metric,\n name_suffix=name_suffix,\n ensemble_type=ensemble_type,\n ensemble_kwargs=ensemble_kwargs,\n invalid_name_set=invalid_name_set,\n level=level,\n )\n invalid_name_set.add(model.name)\n if \"hyperparameter_tune_kwargs\" in model_cfg[AG_ARGS]:\n model_args_fit[model.name] = {\n \"hyperparameter_tune_kwargs\": model_cfg[AG_ARGS][\"hyperparameter_tune_kwargs\"]\n }\n if \"ag_args_ensemble\" in model_cfg and not model_cfg[\"ag_args_ensemble\"]:\n model_cfg.pop(\"ag_args_ensemble\")\n if not silent:\n logger.log(20, f\"\\t{model.name}: \\t{model_cfg}\")\n models.append(model)\n return models, model_args_fit\n\n\ndef clean_model_cfg(\n model_cfg: dict, model_type=None, ag_args=None, ag_args_ensemble=None, ag_args_fit=None, problem_type=None\n):\n model_cfg = _verify_model_cfg(model_cfg=model_cfg, model_type=model_type)\n model_cfg = copy.deepcopy(model_cfg)\n if AG_ARGS not in model_cfg:\n model_cfg[AG_ARGS] = dict()\n if \"model_type\" not in model_cfg[AG_ARGS]:\n model_cfg[AG_ARGS][\"model_type\"] = model_type\n if model_cfg[AG_ARGS][\"model_type\"] is None:\n raise AssertionError(f\"model_type was not specified for model! Model: {model_cfg}\")\n model_type = model_cfg[AG_ARGS][\"model_type\"]\n model_types = ag_model_registry.key_to_cls_map()\n if not inspect.isclass(model_type):\n if model_type not in model_types:\n raise AssertionError(\n f\"Unknown model type specified in hyperparameters: '{model_type}'. Valid model types: {list(model_types.keys())}\"\n )\n model_type = model_types[model_type]\n elif not issubclass(model_type, AbstractModel):\n logger.warning(\n f\"Warning: Custom model type {model_type} does not inherit from {AbstractModel}. This may lead to instability. Consider wrapping {model_type} with an implementation of {AbstractModel}!\"\n )\n else:\n if not ag_model_registry.exists(model_type):\n logger.log(20, f\"Custom Model Type Detected: {model_type}\")\n model_cfg[AG_ARGS][\"model_type\"] = model_type\n model_type_real = model_cfg[AG_ARGS][\"model_type\"]\n if not inspect.isclass(model_type_real):\n model_type_real = model_types[model_type_real]\n default_ag_args = model_type_real._get_default_ag_args()\n if ag_args is not None:\n model_extra_ag_args = ag_args.copy()\n model_extra_ag_args.update(model_cfg[AG_ARGS])\n model_cfg[AG_ARGS] = model_extra_ag_args\n default_ag_args_ensemble = model_type_real._get_default_ag_args_ensemble(problem_type=problem_type)\n if ag_args_ensemble is not None:\n model_extra_ag_args_ensemble = ag_args_ensemble.copy()\n model_extra_ag_args_ensemble.update(model_cfg.get(AG_ARGS_ENSEMBLE, dict()))\n model_cfg[AG_ARGS_ENSEMBLE] = model_extra_ag_args_ensemble\n if ag_args_fit is not None:\n if AG_ARGS_FIT not in model_cfg:\n model_cfg[AG_ARGS_FIT] = dict()\n model_extra_ag_args_fit = ag_args_fit.copy()\n model_extra_ag_args_fit.update(model_cfg[AG_ARGS_FIT])\n model_cfg[AG_ARGS_FIT] = model_extra_ag_args_fit\n if default_ag_args is not None:\n default_ag_args.update(model_cfg[AG_ARGS])\n model_cfg[AG_ARGS] = default_ag_args\n if default_ag_args_ensemble is not None:\n default_ag_args_ensemble.update(model_cfg.get(AG_ARGS_ENSEMBLE, dict()))\n model_cfg[AG_ARGS_ENSEMBLE] = default_ag_args_ensemble\n return model_cfg\n\n\ndef _verify_model_cfg(model_cfg, model_type) -> dict:\n \"\"\"\n Ensures that model_cfg is of the correct type, or else raises an exception.\n Returns model_cfg\n \"\"\"\n if not isinstance(model_cfg, dict):\n extra_msg = \"\"\n error = True\n if isinstance(model_cfg, str) and model_cfg == \"GBMLarge\":\n extra_msg = get_hyperparameter_str_deprecation_msg()\n if version.parse(__version__) >= version.parse(\"1.3.0\"):\n error = True\n extra_msg = \"\\n\" + extra_msg\n else:\n error = False\n model_cfg = get_deprecated_lightgbm_large_hyperparameters()\n logger.warning(\n f\"#######################################################\"\n f\"\\nWARNING: {extra_msg}\"\n f\"\\n#######################################################\"\n )\n if error:\n raise AssertionError(\n f\"Invalid model hyperparameters, expecting dict, but found {type(model_cfg)}! Model Type: {model_type} | Value: {model_cfg}{extra_msg}\"\n )\n return model_cfg\n\n\n# Check if model is valid\ndef is_model_cfg_valid(model_cfg, level=1, problem_type=None):\n is_valid = True\n for key in model_cfg.get(AG_ARGS, {}):\n if key not in VALID_AG_ARGS_KEYS:\n logger.warning(f\"WARNING: Unknown ag_args key: {key}\")\n if AG_ARGS not in model_cfg:\n is_valid = False # AG_ARGS is required\n elif model_cfg[AG_ARGS].get(\"model_type\", None) is None:\n is_valid = False # model_type is required\n elif model_cfg[AG_ARGS].get(\"hyperparameter_tune_kwargs\", None) and model_cfg[AG_ARGS].get(\n \"disable_in_hpo\", False\n ):\n is_valid = False\n elif not model_cfg[AG_ARGS].get(\"valid_stacker\", True) and level > 1:\n is_valid = False # Not valid as a stacker model\n elif not model_cfg[AG_ARGS].get(\"valid_base\", True) and level == 1:\n is_valid = False # Not valid as a base model\n elif problem_type is not None and problem_type not in model_cfg[AG_ARGS].get(\"problem_types\", [problem_type]):\n is_valid = False # Not valid for this problem_type\n return is_valid\n\n\ndef model_factory(\n model,\n path,\n problem_type,\n eval_metric,\n name_suffix=None,\n ensemble_type=StackerEnsembleModel,\n ensemble_kwargs=None,\n invalid_name_set=None,\n level=1,\n):\n if invalid_name_set is None:\n invalid_name_set = set()\n model_type = model[AG_ARGS][\"model_type\"]\n if not inspect.isclass(model_type):\n model_type = ag_model_registry.key_to_cls(model_type)\n name_orig = model[AG_ARGS].get(\"name\", None)\n if name_orig is None:\n ag_name = model_type.ag_name\n if ag_name is None:\n ag_name = model_type.__name__\n name_main = model[AG_ARGS].get(\"name_main\", ag_name)\n name_prefix = model[AG_ARGS].get(\"name_prefix\", \"\")\n name_suff = model[AG_ARGS].get(\"name_suffix\", \"\")\n name_orig = name_prefix + name_main + name_suff\n name_stacker = None\n num_increment = 2\n if name_suffix is None:\n name_suffix = \"\"\n if ensemble_kwargs is None:\n name = f\"{name_orig}{name_suffix}\"\n while name in invalid_name_set: # Ensure name is unique\n name = f\"{name_orig}_{num_increment}{name_suffix}\"\n num_increment += 1\n else:\n name = name_orig\n name_bag_suffix = model[AG_ARGS].get(\"name_bag_suffix\", \"_BAG\")\n name_stacker = f\"{name}{name_bag_suffix}_L{level}{name_suffix}\"\n while name_stacker in invalid_name_set: # Ensure name is unique\n name = f\"{name_orig}_{num_increment}\"\n name_stacker = f\"{name}{name_bag_suffix}_L{level}{name_suffix}\"\n num_increment += 1\n model_params = copy.deepcopy(model)\n model_params.pop(AG_ARGS, None)\n model_params.pop(AG_ARGS_ENSEMBLE, None)\n\n extra_ensemble_hyperparameters = copy.deepcopy(model.get(AG_ARGS_ENSEMBLE, dict()))\n\n # Enable user to pass ensemble hyperparameters via `\"ag.ens.fold_fitting_strategy\": \"sequential_local\"`\n ag_args_ensemble_prefix = \"ag.ens.\"\n model_param_keys = list(model_params.keys())\n for key in model_param_keys:\n if key.startswith(ag_args_ensemble_prefix):\n key_suffix = key.split(ag_args_ensemble_prefix, 1)[-1]\n extra_ensemble_hyperparameters[key_suffix] = model_params.pop(key)\n\n model_init_kwargs = dict(\n path=path,\n name=name,\n problem_type=problem_type,\n eval_metric=eval_metric,\n hyperparameters=model_params,\n )\n\n if ensemble_kwargs is not None:\n ensemble_kwargs_model = copy.deepcopy(ensemble_kwargs)\n ensemble_kwargs_model[\"hyperparameters\"] = ensemble_kwargs_model.get(\"hyperparameters\", {})\n if ensemble_kwargs_model[\"hyperparameters\"] is None:\n ensemble_kwargs_model[\"hyperparameters\"] = {}\n ensemble_kwargs_model[\"hyperparameters\"].update(extra_ensemble_hyperparameters)\n model_init = ensemble_type(\n path=path,\n name=name_stacker,\n model_base=model_type,\n model_base_kwargs=model_init_kwargs,\n **ensemble_kwargs_model,\n )\n else:\n model_init = model_type(**model_init_kwargs)\n\n return model_init\n\n\n# TODO: v0.1 cleanup and avoid hardcoded logic with model names\ndef get_preset_models_softclass(hyperparameters, invalid_model_names: list = None, **kwargs):\n from autogluon.core.metrics.softclass_metrics import soft_log_loss\n\n model_types_standard = [\"GBM\", \"NN_TORCH\", \"CAT\", \"ENS_WEIGHTED\"]\n hyperparameters = copy.deepcopy(hyperparameters)\n\n hyperparameters_standard = {key: hyperparameters[key] for key in hyperparameters if key in model_types_standard}\n hyperparameters_rf = {key: hyperparameters[key] for key in hyperparameters if key == \"RF\"}\n\n # Swap RF criterion for MSE:\n if \"RF\" in hyperparameters_rf:\n rf_params = hyperparameters_rf[\"RF\"]\n rf_newparams = {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\"}}\n for i in range(len(rf_params)):\n rf_params[i].update(rf_newparams)\n rf_params = [\n j for n, j in enumerate(rf_params) if j not in rf_params[(n + 1) :]\n ] # Remove duplicates which may arise after overwriting criterion\n hyperparameters_standard[\"RF\"] = rf_params\n models, model_args_fit = get_preset_models(\n problem_type=SOFTCLASS,\n eval_metric=soft_log_loss,\n hyperparameters=hyperparameters_standard,\n invalid_model_names=invalid_model_names,\n **kwargs,\n )\n if len(models) == 0:\n raise ValueError(\n \"At least one of the following model-types must be present in hyperparameters: ['GBM','CAT','RF'], \"\n \"These are the only supported models for softclass prediction problems. \"\n \"Softclass problems are also not yet supported for fit() with per-stack level hyperparameters.\"\n )\n for model in models:\n model.normalize_pred_probas = True # FIXME: Do we need to do this for child models too?\n\n return models, model_args_fit\n"
},
{
"path": "tabular/src/autogluon/tabular/trainer/model_presets/presets_distill.py",
"content": "import logging\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.metrics import mean_squared_error\nfrom autogluon.core.trainer.utils import process_hyperparameters\n\nfrom .presets import get_preset_models, get_preset_models_softclass\n\nlogger = logging.getLogger(__name__)\n\n# Higher values indicate higher priority, priority dictates the order models are trained (which matters if time runs out).\nDEFAULT_DISTILL_PRIORITY = dict(\n GBM=100,\n CAT=80,\n RF=70,\n custom=0,\n)\n\n\ndef get_preset_models_distillation(\n path,\n problem_type,\n eval_metric,\n hyperparameters,\n level=1,\n name_suffix=\"_DSTL\",\n invalid_model_names: list = None,\n **kwargs,\n):\n hyperparameters = process_hyperparameters(hyperparameters)\n level_key = level if level in hyperparameters.keys() else \"default\"\n if level_key not in hyperparameters.keys() and level_key == \"default\":\n hyperparameters = {\"default\": hyperparameters}\n hyperparameters = hyperparameters[level_key]\n if problem_type == BINARY: # convert to regression in distillation\n eval_metric = mean_squared_error\n # Constrain output-range of NN:\n nn_outputrange = {\"y_range\": (0.0, 1.0), \"y_range_extend\": 0.0}\n nn_hyperparameters = None\n if isinstance(nn_hyperparameters, list):\n for i in range(len(nn_hyperparameters)):\n nn_hyperparameters[i].update(nn_outputrange)\n elif nn_hyperparameters is not None:\n nn_hyperparameters.update(nn_outputrange)\n # Swap RF criterion for MSE:\n rf_newparams = {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\"}}\n if \"RF\" in hyperparameters:\n rf_hyperparameters = hyperparameters[\"RF\"]\n else:\n rf_hyperparameters = None\n if isinstance(rf_hyperparameters, list):\n for i in range(len(rf_hyperparameters)):\n rf_hyperparameters[i].update(rf_newparams)\n rf_hyperparameters = [\n j for n, j in enumerate(rf_hyperparameters) if j not in rf_hyperparameters[(n + 1) :]\n ] # Remove duplicates which may arise after overwriting criterion\n elif rf_hyperparameters is not None:\n rf_hyperparameters.update(rf_newparams)\n if \"RF\" in hyperparameters:\n hyperparameters[\"RF\"] = rf_hyperparameters\n\n if problem_type == MULTICLASS:\n models, model_args_fit = get_preset_models_softclass(\n path=path,\n hyperparameters=hyperparameters,\n level=level,\n name_suffix=name_suffix,\n invalid_model_names=invalid_model_names,\n **kwargs,\n )\n else: # BINARY or REGRESSION\n models, model_args_fit = get_preset_models(\n path=path,\n problem_type=REGRESSION,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n level=level,\n name_suffix=name_suffix,\n default_priorities=DEFAULT_DISTILL_PRIORITY,\n invalid_model_names=invalid_model_names,\n **kwargs,\n )\n\n if problem_type in [MULTICLASS, BINARY]:\n for model in models:\n model.normalize_pred_probas = True\n\n logger.log(20, f\"Distilling with each of these student models: {[model.name for model in models]}\")\n return models, model_args_fit\n"
},
{
"path": "tabular/src/autogluon/tabular/tuning/__init__.py",
"content": ""
},
{
"path": "tabular/src/autogluon/tabular/tuning/feature_pruner.py",
"content": "import copy\nimport logging\n\nimport pandas as pd\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: currently is buggy\nclass FeaturePruner:\n def __init__(self, model_base, threshold_baseline=0.004, is_fit=False):\n self.model_base = model_base\n self.threshold_baseline = threshold_baseline\n self.is_fit = is_fit\n\n self.best_score = 0\n self.best_iteration = 0\n self.features_in_iter = []\n self.score_in_iter = []\n self.valid_feature_counts = []\n self.gain_dfs = []\n self.banned_features_in_iter = []\n self.thresholds = []\n self.threshold = self.threshold_baseline\n self.cur_iteration = 0\n self.tuned = False\n self.early_stopping_rounds = 2\n\n def evaluate(self):\n untuned_score = self.score_in_iter[0]\n logger.debug(\"untuned_score: %s\" % untuned_score)\n logger.debug(\"best_score: %s\" % self.best_score)\n logger.debug(\"best_iteration: %s\" % self.best_iteration)\n\n data_dict = {\n \"score\": self.score_in_iter,\n \"feature_count\": self.valid_feature_counts,\n \"threshold\": self.thresholds,\n }\n logger.debug(self.gain_dfs[self.best_iteration])\n z = pd.DataFrame(data=data_dict)\n # logger.debug(z)\n z_sorted = z.sort_values(by=[\"score\"], ascending=False)\n # logger.debug(z_sorted)\n\n # TODO: CV5 instead of holdout? Should be better\n # TODO: Add holdout here, it is overfitting with Logistic Regression\n def tune(self, X, y, X_val, y_val, X_holdout, y_holdout, total_runs=999):\n objective_goal_is_negative = False # Fixed to false if using sklearn scorers # self.model_base.problem_type == REGRESSION # TODO: if objective function goal = lower (logloss, MAE, etc.)\n logger.log(15, \"Feature-pruning \" + str(self.model_base.name) + \" for \" + str(total_runs) + \" runs...\")\n\n if len(self.features_in_iter) == 0:\n valid_features = X.columns.values\n else:\n valid_features = self.features_in_iter[-1]\n\n iter_since_best = 0\n iter_start = self.cur_iteration\n for iteration in range(self.cur_iteration, total_runs):\n self.cur_iteration = iteration\n logger.debug(\"iteration: %s \" % iteration)\n X_subset = X[valid_features].copy()\n X_val_subset = X_val[valid_features].copy()\n self.thresholds.append(self.threshold)\n self.valid_feature_counts.append(len(valid_features))\n self.features_in_iter.append(valid_features)\n\n if self.is_fit and (iteration == iter_start):\n model_iter = self.model_base\n else:\n model_iter = copy.deepcopy(self.model_base)\n model_iter.fit(X=X_subset, y=y, X_val=X_val_subset, y_val=y_val)\n\n banned_features = []\n\n feature_importance = None\n if hasattr(model_iter.model, \"feature_importances_\"):\n feature_importance = model_iter.model.feature_importances_\n elif hasattr(model_iter.model, \"feature_importance\"):\n feature_importance = model_iter.model.feature_importance()\n\n if feature_importance is not None:\n a = pd.Series(data=feature_importance, index=X_subset.columns)\n unused_features = a[a == 0]\n\n logger.log(15, \"Unused features after pruning: \" + str(list(unused_features.index)))\n features_to_use = [feature for feature in valid_features if feature not in unused_features.index]\n banned_features += list(unused_features.index)\n else:\n features_to_use = list(valid_features)\n\n cur_score_val = model_iter.score(X=X_val_subset, y=y_val)\n cur_score = model_iter.score(X=X_holdout[valid_features], y=y_holdout)\n\n logger.log(15, \"Iter \" + str(iteration) + \" Score: \" + str(cur_score))\n logger.log(15, \"Iter \" + str(iteration) + \" Score Val: \" + str(cur_score_val))\n if objective_goal_is_negative:\n cur_score = -cur_score\n\n if self.best_score == 0 or cur_score > self.best_score:\n logger.log(15, \"New best score found!\")\n logger.log(15, str(cur_score) + \" > \" + str(self.best_score))\n self.best_score = cur_score\n self.best_iteration = iteration\n iter_since_best = 0\n else:\n iter_since_best += 1\n\n self.score_in_iter.append(cur_score)\n\n if iter_since_best >= self.early_stopping_rounds:\n logger.log(15, \"Early stopping on iter %s\" % iteration)\n logger.log(15, \"Best Iteration: %s\" % self.best_iteration)\n self.tuned = True\n break\n\n gain_df = model_iter.compute_feature_importance(X=X_val_subset, y=y_val, features=features_to_use)\n if not objective_goal_is_negative:\n gain_df = -gain_df\n\n self.gain_dfs.append(gain_df)\n\n # TODO: Save gain_df, banned_features\n\n self.threshold = self.adjust_threshold(gain_df, self.threshold)\n\n banned_df = gain_df.loc[gain_df >= self.threshold].index\n banned_features += list(banned_df)\n\n logger.log(15, \"Banned features: \" + str(banned_features))\n self.banned_features_in_iter.append(banned_features)\n valid_features = [feature for feature in valid_features if feature not in banned_features]\n\n if len(valid_features) == 0:\n logger.log(15, \"No more features to remove, Feature pruning complete\")\n logger.log(15, \"score_in_iter: \" + str(self.score_in_iter))\n self.tuned = True\n break\n self.tuned = True\n\n @staticmethod\n def adjust_threshold(gain_df, threshold):\n if gain_df.shape[0] == 0:\n raise BaseException(\"ran out of features to prune!\")\n banned_df = gain_df.loc[gain_df >= threshold].index\n banned_features = list(banned_df)\n\n if len(banned_features) == 0:\n if threshold < -100000000: # FIXME: Hacked for regression\n raise BaseException(\"threshold already max!\")\n elif threshold > 0.000001:\n threshold_new = threshold / 2\n elif threshold > 0:\n threshold_new = 0\n elif threshold == 0:\n threshold_new = -0.00000001\n elif (threshold < 0) and (threshold > -0.0001):\n threshold_new = threshold * 2\n else:\n threshold_new = gain_df.max()\n\n logger.log(10, \"Adjusting threshold to %s\" % threshold_new)\n return FeaturePruner.adjust_threshold(gain_df=gain_df, threshold=threshold_new)\n else:\n return threshold\n"
},
{
"path": "tabular/tests/__init__.py",
"content": ""
},
{
"path": "tabular/tests/conftest.py",
"content": "from __future__ import annotations\n\nimport os\nfrom contextlib import contextmanager\n\nimport pytest\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\n\ndef pytest_addoption(parser):\n parser.addoption(\"--runslow\", action=\"store_true\", default=False, help=\"run slow tests\")\n parser.addoption(\"--runregression\", action=\"store_true\", default=False, help=\"run regression tests\")\n parser.addoption(\"--runpyodide\", action=\"store_true\", default=False, help=\"run Pyodide tests\")\n parser.addoption(\"--run-multi-gpu\", action=\"store_true\", default=False, help=\"run multi-GPU tests\")\n\n\ndef pytest_configure(config):\n config.addinivalue_line(\"markers\", \"slow: mark test as slow to run\")\n config.addinivalue_line(\"markers\", \"regression: mark test as regression test\")\n config.addinivalue_line(\"markers\", \"pyodide: mark test as pyodide test\")\n config.addinivalue_line(\"markers\", \"multi_gpu: mark test to run on multi-GPU CI only\")\n\n\ndef pytest_collection_modifyitems(config, items):\n skip_slow = pytest.mark.skip(reason=\"need --runslow option to run\")\n skip_regression = pytest.mark.skip(reason=\"need --runregression option to run\")\n skip_pyodide = pytest.mark.skip(reason=\"need --runpyodide option to run\")\n skip_multi_gpu = pytest.mark.skip(reason=\"need --run-multi-gpu option to run\")\n custom_markers = dict(\n slow=skip_slow,\n regression=skip_regression,\n pyodide=skip_pyodide,\n multi_gpu=skip_multi_gpu,\n )\n if config.getoption(\"--runslow\"):\n # --runslow given in cli: do not skip slow tests\n custom_markers.pop(\"slow\", None)\n if config.getoption(\"--runregression\"):\n # --runregression given in cli: do not skip slow tests\n custom_markers.pop(\"regression\", None)\n if config.getoption(\"--runpyodide\"):\n # --runpyodide given in cli: do not skip pyodide tests\n custom_markers.pop(\"pyodide\", None)\n if config.getoption(\"--run-multi-gpu\"):\n # --run-multi-gpu given in cli: do not skip multi-GPU tests\n custom_markers.pop(\"multi_gpu\", None)\n\n for item in items:\n for marker in custom_markers:\n if marker in item.keywords:\n item.add_marker(custom_markers[marker])\n\n # Normalize the file paths and use a consistent comparison method\n normalized_path = lambda p: os.path.normpath(str(p))\n resource_allocation_path = normalized_path(\"tests/unittests/resource_allocation\")\n\n # Reordering logic to ensure tests under ./unittests/resource_allocation run last\n # TODO: Fix this once resource_allocation tests are robost enough to run with other tests without ordering issues\n resource_allocation_tests = [item for item in items if resource_allocation_path in normalized_path(item.fspath)]\n other_tests = [item for item in items if resource_allocation_path not in normalized_path(item.fspath)]\n\n items.clear()\n items.extend(other_tests)\n items.extend(resource_allocation_tests)\n\n\n@contextmanager\ndef mock_system_resourcses(num_cpus=None, num_gpus=None):\n original_get_cpu_count = ResourceManager.get_cpu_count\n original_get_gpu_count = ResourceManager.get_gpu_count\n if num_cpus is not None:\n ResourceManager.get_cpu_count = lambda: num_cpus\n if num_gpus is not None:\n ResourceManager.get_gpu_count = lambda: num_gpus\n yield\n ResourceManager.get_cpu_count = original_get_cpu_count\n ResourceManager.get_gpu_count = original_get_gpu_count\n\n\n@pytest.fixture\ndef mock_system_resources_ctx_mgr():\n return mock_system_resourcses\n\n\n@pytest.fixture\ndef mock_num_cpus():\n return 16\n\n\n@pytest.fixture\ndef mock_num_gpus():\n return 2\n\n\n@pytest.fixture\ndef k_fold():\n return 2\n"
},
{
"path": "tabular/tests/regressiontests/__init__.py",
"content": ""
},
{
"path": "tabular/tests/regressiontests/test_tabular_lite.py",
"content": "\"\"\"Test autogluon.tabular on pyodide.\nThe test needs to be run via pyodide/tools/pytest_wrapper.py,\nand using conftest.py from pyodide root directory.\n\"\"\"\n\nfrom pathlib import Path\n\nimport pytest\n\nROOT_PATH = Path(__file__).parent.parent.parent.parent\nWHL_PREFIX = \"autogluon_lite\"\n\n\n@pytest.fixture\ndef selenium_standalone_micropip(selenium_standalone):\n wheel_paths = []\n for regex_path_str in [\n (\"common\", f\"{WHL_PREFIX}.common-*.whl\"),\n (\"core\", f\"{WHL_PREFIX}.core-*.whl\"),\n (\"features\", f\"{WHL_PREFIX}.features-*.whl\"),\n (\"tabular\", f\"{WHL_PREFIX}.tabular-*.whl\"),\n (\"autogluon\", f\"{WHL_PREFIX}-*.whl\"),\n ]:\n wheel_path = [\n f\"{regex_path_str[0]}/dist/{w.name}\"\n for w in (ROOT_PATH / regex_path_str[0] / \"dist\").glob(regex_path_str[1])\n ]\n assert len(wheel_path) == 1\n wheel_path = wheel_path[0]\n wheel_paths.append(wheel_path)\n\n from pytest_pyodide import spawn_web_server\n\n with spawn_web_server(ROOT_PATH) as server:\n server_hostname, server_port, _ = server\n base_url = f\"http://{server_hostname}:{server_port}/\"\n selenium_standalone.run_js(\n f\"\"\"\n await pyodide.loadPackage(\"micropip\");\n pyodide.runPython(\"import micropip\");\n await pyodide.runPythonAsync(`\n import micropip\n await micropip.install('{base_url + wheel_paths[0]}')\n await micropip.install('{base_url + wheel_paths[1]}')\n await micropip.install('{base_url + wheel_paths[2]}')\n await micropip.install('{base_url + wheel_paths[3]}')\n await micropip.install('{base_url + wheel_paths[4]}')\n `);\n \"\"\"\n )\n yield selenium_standalone\n\n\n@pytest.mark.skip_refcount_check\n@pytest.mark.driver_timeout(60)\n@pytest.mark.pyodide\ndef test_train_classifier(selenium_standalone_micropip):\n from pytest_pyodide import run_in_pyodide\n\n @run_in_pyodide(packages=[\"xgboost\", \"pandas\", \"numpy\", \"scipy\", \"scikit-learn\", \"lightgbm\"])\n async def run(selenium, test, train_data, test_data):\n expected_score_range = test[\"expected_score_range\"]\n expected_score_range = {\n k: (v[0], 0.02)\n for k, v in expected_score_range.items()\n if k\n not in [\n \"CatBoost\",\n \"NeuralNetFastAI\",\n \"NeuralNetTorch\",\n ]\n }\n\n from autogluon.tabular import TabularPredictor\n\n print(train_data.head())\n label = \"label\"\n print(\"Summary of class variable: \\n\", train_data[label].describe())\n save_path = \"agModels-predictClass\" # specifies folder to store trained models\n predictor = TabularPredictor(label=label, path=save_path).fit(train_data)\n\n y_test = test_data[label] # values to predict\n test_data_nolab = test_data.drop(columns=[label]) # delete label column to prove we're not cheating\n test_data_nolab.head()\n predictor = TabularPredictor.load(save_path)\n\n y_pred = predictor.predict(test_data_nolab)\n print(\"Predictions: \\n\", y_pred)\n perf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)\n leaderboard = predictor.leaderboard(test_data)\n print(leaderboard[[\"model\", \"score_test\", \"score_val\"]])\n trained_models = leaderboard[leaderboard.columns[0]]\n print(trained_models)\n assert set(trained_models) == set(expected_score_range.keys()), \"Not all models were trained\"\n for model in leaderboard[\"model\"]:\n score = leaderboard[leaderboard[\"model\"] == model][\"score_test\"].values[0]\n score_range = expected_score_range[model]\n assert score_range[0] - score_range[1] <= score <= score_range[0] + score_range[1], (\n f\"Score for {model} ({score}) is out of expected range {score_range}\"\n )\n\n from .utils import make_dataset, tests\n\n test_case = tests[-1]\n data_train, data_test = make_dataset(request=test_case, seed=0)\n run(selenium_standalone_micropip, test_case, data_train, data_test)\n"
},
{
"path": "tabular/tests/regressiontests/test_tabular_regression.py",
"content": "\"\"\"Runs autogluon.tabular on synthetic classification and regression datasets.\nWe test various parameters to TabularPredictor() and fit()\nWe then check the leaderboard:\n Did we run the expected list of models\n Did each model have the expected score (within given range)\n Did the ensembling produce the expected score (within given range)\nThis helps us spot any change in model performance.\nIf any changes are spotted, the script does its best to dump out new proposed score ranges\nthat you can cut and paste into the tests. Only do this once you've identified the cause!\n\nPotential naming confusion:\n - this is a *regression* test, to make sure no functionality has accidentally got worse (testing terminology)\n - it runs two types of TabularPredictor tests : *regression* and classification (ML terminology)\n\nThese tests are designed to run fast, to permit them to be run on a github hook.\nCurrently the 11 tests, calling TabularPredictor.fit() 20 times, run in ~8 minutes on an 8 vcore machine with no GPU.\n\nTesting by @willsmithorg on master AG as of 2022-02-22 - 2022-02-23:\nTested on AWS Linux instance m5.2xlarge, amzn2-ami-kernel-5.10-hvm-2.0.20211223.0-x86_64-gp2 with\n (8 vcore, no GPU, Python==3.7.10, scikit-learn==1.0.2, torch==1.10.2),\nTested on Github jenkins Linux:\n (? vcore, 0 GPU, Python==3.9.10, scikit-learn==1.0.2, torch==1.10.2),\nTested on AWS Windows instance t3.xlarge,\n (4 vcore, 0 GPU, Python==3.9.7 , scikit-learn==1.0.2, torch==1.10.2),\n - Pytorch scores are slightly different, all else same.\n\n\"\"\"\n\nimport hashlib\nimport math\nimport sys\n\nimport numpy as np\nimport pytest\n\nfrom autogluon.tabular import TabularDataset, TabularPredictor\n\nfrom .utils import make_dataset, tests\n\n\n# Lots of test data since inference is fast and we want a score that's very reflective of model quality,\n# despite very fast training times.\n# Round to given accuracy. The 8 is to remove floating point rounding errors.\ndef myfloor(x, base=0.01):\n return round(base * math.floor(float(x) / base), 8)\n\n\n@pytest.mark.regression\ndef inner_test_tabular(testname):\n # Find the named test\n test = None\n for t in tests:\n if t[\"name\"] == testname:\n test = t\n assert test is not None, f\"Could not find test {testname}\"\n\n # Build the dataset\n (dftrain, dftest) = make_dataset(request=test, seed=0)\n\n # Check the synthetic dataset itself hasn't changed. We round it to 3dp otherwise tiny floating point differences\n # between platforms can give a different hash that still yields same prediction scores.\n # Ultimately it doesn't matter how we do this as long as the same dataset gives the same hash function on\n # different python versions and architectures.\n current_hash = hashlib.sha256(dftrain.round(decimals=3).values.tobytes()).hexdigest()[0:10]\n proposedconfig = \"Proposed new config:\\n\"\n proposedconfig += f\"'dataset_hash' : '{current_hash}',\"\n assert current_hash == test[\"dataset_hash\"], (\n f\"Test '{testname}' input dataset has changed. All scores will change.\\n\" + proposedconfig\n )\n\n # Now run the Predictor 1 or more times with various parameters, and make sure we get\n # back the expected results.\n\n # Params can either omitted, or a single run, or a list of runs.\n if \"params\" not in test:\n test[\"params\"] = {\"predict\": {}, \"fit\": {}}\n if not isinstance(test[\"params\"], list):\n test[\"params\"] = [test[\"params\"]]\n for params in test[\"params\"]:\n # Run this model and set of params\n predictor = TabularPredictor(label=\"label\", **params[\"predict\"])\n predictor.fit(dftrain, **params[\"fit\"])\n leaderboard = predictor.leaderboard(dftest)\n leaderboard = leaderboard.sort_values(by=\"model\") # So we can pre-generate sample config in alphabetical order\n\n # Store proposed new config based on the current run, in case the developer wants to keep thee results (just cut and paste).\n proposedconfig = \"Proposed new config:\\n\"\n proposedconfig += \"'expected_score_range' : {\\n\"\n for model in leaderboard[\"model\"]:\n midx_in_leaderboard = leaderboard.index.values[leaderboard[\"model\"] == model][0]\n if np.isnan(leaderboard[\"score_test\"][midx_in_leaderboard]):\n values = \"np.nan, np.nan\"\n else:\n if model in test[\"expected_score_range\"] and not np.isnan(test[\"expected_score_range\"][model][1]):\n currentprecision = test[\"expected_score_range\"][model][1]\n else:\n currentprecision = 0.01\n values = \"{}, {}\".format(\n myfloor(leaderboard[\"score_test\"][midx_in_leaderboard], currentprecision), currentprecision\n )\n proposedconfig += f\" '{model}': ({values}),\\n\"\n proposedconfig += \"},\\n\"\n\n # First validate the model list was as expected.\n assert set(leaderboard[\"model\"]) == set(test[\"expected_score_range\"].keys()), (\n f\"Test '{testname}' params {params} got unexpected model list.\\n\" + proposedconfig\n )\n\n # Now validate the scores for each model were as expected.\n all_assertions_met = True\n currentconfig = \"Existing config:\\n\"\n currentconfig += \"'expected_score_range' : {\\n\"\n for model in sorted(test[\"expected_score_range\"]):\n midx_in_leaderboard = leaderboard.index.values[leaderboard[\"model\"] == model][0]\n assert leaderboard[\"model\"][midx_in_leaderboard] == model\n expectedrange = test[\"expected_score_range\"][model][1]\n expectedmin = test[\"expected_score_range\"][model][0]\n expectedmax = expectedmin + expectedrange\n\n if np.isnan(expectedmin):\n values = \"np.nan, np.nan\"\n else:\n values = \"{}, {}\".format(expectedmin, expectedrange)\n\n if (\n (leaderboard[\"score_test\"][midx_in_leaderboard] >= expectedmin)\n and (leaderboard[\"score_test\"][midx_in_leaderboard] <= expectedmax)\n ) or (np.isnan(leaderboard[\"score_test\"][midx_in_leaderboard]) and np.isnan(expectedmin)):\n currentconfig += f\" '{model}': ({values}),\\n\"\n else:\n currentconfig += f\" '{model}': ({values}), # <--- not met, got {leaderboard['score_test'][midx_in_leaderboard]} \\n\"\n all_assertions_met = False\n currentconfig += \"},\\n\"\n\n assert all_assertions_met, (\n f\"Test '{testname}', params {params} had unexpected scores:\\n\" + currentconfig + proposedconfig\n )\n\n # Clean up this model created with specific params.\n predictor.delete_models(models_to_keep=[], dry_run=False)\n\n\n# The tests are all run individually rather than in 1 big loop that simply goes through the tests dictionary.\n# This is so we easily remove some tests if necessary.\n@pytest.mark.parametrize(\n \"testname\",\n [\n \"small regression\",\n \"small regression excluded models\",\n \"small regression light hyperparameters\",\n \"small regression very light hyperparameters\",\n \"small regression toy hyperparameters\",\n \"small regression high quality\",\n \"small regression best quality\",\n \"small regression with categorical\",\n \"small regression metric mae\",\n \"small classification\",\n \"small classification boolean\",\n ],\n)\n# These results have only been confirmed for Linux. Windows is known to give different results for Pytorch.\n@pytest.mark.skipif(sys.platform != \"linux\", reason=\"Scores only confirmed on Linux\")\n@pytest.mark.regression\ndef test_tabular_score(testname):\n inner_test_tabular(testname)\n"
},
{
"path": "tabular/tests/regressiontests/utils.py",
"content": "import random\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.datasets import make_classification, make_regression\nfrom sklearn.model_selection import train_test_split\n\n\"\"\"Format of a test:\n { # Default regression model on a small dataset\n 'name': # some unique name\n 'type': # either 'regression' or 'classification'. We make a dataset using\n # scikit-learn.make_{regression,classification}\n 'n_samples': # number of rows in the training dataset. With TEST_SIZE default of 0.5,\n # we make an additional n_samples for testing.\n 'n_features': 2, # number of columns.\n 'n_categorical': 0, # number of categorical (discrete not continuous) columns.\n ''dataset_hash' : # Hash of synthetic dataset to ensure the dataset itself didn't change.\n 'params' : [ { 'predict' : {}, 'fit' : {} }, # If an array, we call TabularPredictor multiple times with\n # different parameters.\n { 'predict' : {}, 'fit' : {} }, # Pass the additional parameters to predict(), fit() or both\n # in the dicts.\n # If a scalar, we only call TabularPredictor once.\n ],\n 'expected_score_range' : { # A list of models we expect to run, and a valid score range we\n # expect from each model.\n 'CatBoost': (-7.86, 0.01), # The first value is the lower bound, the 2nd value is a delta\n 'ExtraTreesMSE': (-7.88, 0.01), # to compute the upper bound, e.g. ( -8.12, 0.01 ) means we\n # expect the score to be from -8.12 to -8.11 inclusive.\n 'CatBoost_BAG_L1': (np.nan, np.nan), # If np.nan, we expect this model to return np.nan as the score.\n },\n },\n\"\"\"\ntests = [\n #\n # Regressions\n #\n { # Default regression model on a small dataset\n \"name\": \"small regression\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": [\n {\n \"predict\": {},\n \"fit\": {},\n }, # All of the following params should return same results because they're defaults\n {\"predict\": {}, \"fit\": {\"presets\": \"medium_quality_faster_train\"}},\n {\"predict\": {}, \"fit\": {\"presets\": \"ignore_text\"}},\n {\"predict\": {}, \"fit\": {\"hyperparameters\": \"default\"}},\n {\"predict\": {\"eval_metric\": \"root_mean_squared_error\"}, \"fit\": {}},\n ],\n \"expected_score_range\": {\n \"CatBoost\": (-7.86, 0.01),\n \"ExtraTreesMSE\": (-7.88, 0.01),\n \"KNeighborsDist\": (-8.69, 0.01),\n \"KNeighborsUnif\": (-9.06, 0.01),\n \"LightGBM\": (-15.55, 0.01),\n \"LightGBMLarge\": (-10.43, 0.01),\n \"LightGBMXT\": (-16.32, 0.01),\n \"NeuralNetFastAI\": (-6.12, 0.01),\n \"NeuralNetTorch\": (-4.96, 0.01),\n \"RandomForestMSE\": (-9.63, 0.01),\n \"WeightedEnsemble_L2\": (-5.66, 0.01),\n \"XGBoost\": (-10.8, 0.01),\n },\n },\n { # If we explicitly exclude some models the others should return unchanged and the ensemble result will be changed.\n \"name\": \"small regression excluded models\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"excluded_model_types\": [\"KNN\", \"RF\", \"XT\", \"GBM\", \"CAT\", \"XGB\"]}},\n \"expected_score_range\": {\n \"NeuralNetFastAI\": (-6.12, 0.01),\n \"NeuralNetTorch\": (-4.96, 0.01),\n \"WeightedEnsemble_L2\": (-5.09, 0.01),\n },\n },\n { # Small regression, hyperparameters = light removes some models\n \"name\": \"small regression light hyperparameters\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"hyperparameters\": \"light\"}},\n \"expected_score_range\": {\n \"CatBoost\": (-7.86, 0.01),\n \"ExtraTreesMSE\": (-7.87, 0.01),\n \"LightGBM\": (-15.55, 0.01),\n \"LightGBMLarge\": (-10.43, 0.01),\n \"LightGBMXT\": (-16.32, 0.01),\n \"NeuralNetFastAI\": (-6.12, 0.01),\n \"NeuralNetTorch\": (-4.96, 0.01),\n \"RandomForestMSE\": (-9.63, 0.01),\n \"WeightedEnsemble_L2\": (-5.66, 0.01),\n \"XGBoost\": (-10.8, 0.01),\n },\n },\n { # Small regression, hyperparameters = very_light removes some models\n \"name\": \"small regression very light hyperparameters\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"hyperparameters\": \"very_light\"}},\n \"expected_score_range\": {\n \"CatBoost\": (-7.86, 0.01),\n \"LightGBM\": (-15.55, 0.01),\n \"LightGBMLarge\": (-10.43, 0.01),\n \"LightGBMXT\": (-16.32, 0.01),\n \"NeuralNetFastAI\": (-6.12, 0.01),\n \"NeuralNetTorch\": (-4.96, 0.01),\n \"WeightedEnsemble_L2\": (-5.58, 0.01),\n \"XGBoost\": (-10.8, 0.01),\n },\n },\n { # Small regression, hyperparameters = toy removes almost all models and runs very fast\n \"name\": \"small regression toy hyperparameters\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"hyperparameters\": \"toy\"}},\n \"expected_score_range\": {\n \"CatBoost\": (-28.39, 0.01),\n \"LightGBM\": (-27.81, 0.01),\n \"NeuralNetTorch\": (-27.11, 0.01),\n \"WeightedEnsemble_L2\": (-19.12, 0.01),\n \"XGBoost\": (-19.12, 0.01),\n },\n },\n { # High quality preset on small dataset.\n \"name\": \"small regression high quality\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"presets\": \"high_quality_fast_inference_only_refit\"}},\n \"expected_score_range\": {\n \"CatBoost_BAG_L1\": (np.nan, np.nan),\n \"CatBoost_BAG_L1_FULL\": (-7.75, 0.01),\n \"ExtraTreesMSE_BAG_L1\": (-7.52, 0.01),\n \"ExtraTreesMSE_BAG_L1_FULL\": (-7.52, 0.01),\n \"KNeighborsDist_BAG_L1\": (-8.21, 0.01),\n \"KNeighborsDist_BAG_L1_FULL\": (-8.21, 0.01),\n \"KNeighborsUnif_BAG_L1\": (-8.7, 0.01),\n \"KNeighborsUnif_BAG_L1_FULL\": (-8.7, 0.01),\n \"LightGBMLarge_BAG_L1\": (np.nan, np.nan),\n \"LightGBMLarge_BAG_L1_FULL\": (-9.94, 0.01),\n \"LightGBMXT_BAG_L1\": (np.nan, np.nan),\n \"LightGBMXT_BAG_L1_FULL\": (-13.03, 0.01),\n \"LightGBM_BAG_L1\": (np.nan, np.nan),\n \"LightGBM_BAG_L1_FULL\": (-14.17, 0.01),\n \"NeuralNetFastAI_BAG_L1\": (np.nan, np.nan),\n \"NeuralNetFastAI_BAG_L1_FULL\": (-5.48, 0.01),\n \"NeuralNetTorch_BAG_L1\": (np.nan, np.nan),\n \"NeuralNetTorch_BAG_L1_FULL\": (-5.29, 0.01),\n \"RandomForestMSE_BAG_L1\": (-9.5, 0.01),\n \"RandomForestMSE_BAG_L1_FULL\": (-9.5, 0.01),\n \"WeightedEnsemble_L2\": (np.nan, np.nan),\n \"WeightedEnsemble_L2_FULL\": (-5.29, 0.01),\n \"XGBoost_BAG_L1\": (np.nan, np.nan),\n \"XGBoost_BAG_L1_FULL\": (-9.76, 0.01),\n },\n },\n { # Best quality preset on small dataset.\n \"name\": \"small regression best quality\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {}, \"fit\": {\"presets\": \"best_quality\"}},\n \"expected_score_range\": {\n \"CatBoost_BAG_L1\": (-7.85, 0.01),\n \"ExtraTreesMSE_BAG_L1\": (-7.52, 0.01),\n \"KNeighborsDist_BAG_L1\": (-8.21, 0.01),\n \"KNeighborsUnif_BAG_L1\": (-8.70, 0.01),\n \"LightGBMLarge_BAG_L1\": (-9.44, 0.01),\n \"LightGBMXT_BAG_L1\": (-14.78, 0.01),\n \"LightGBM_BAG_L1\": (-14.92, 0.01),\n \"NeuralNetFastAI_BAG_L1\": (-5.55, 0.01),\n \"NeuralNetTorch_BAG_L1\": (-5.07, 0.01),\n \"RandomForestMSE_BAG_L1\": (-9.5, 0.01),\n \"WeightedEnsemble_L2\": (-5.05, 0.01), # beats default, as expected\n \"XGBoost_BAG_L1\": (-9.74, 0.01),\n },\n },\n { # Default regression model, add some categorical features.\n \"name\": \"small regression with categorical\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 1,\n \"dataset_hash\": \"3e26d128e0\",\n \"params\": {\"predict\": {}, \"fit\": {}}, # Default params\n \"expected_score_range\": {\n \"CatBoost\": (-22.58, 0.01),\n \"ExtraTreesMSE\": (-25.09, 0.01),\n \"KNeighborsDist\": (-39.45, 0.01),\n \"KNeighborsUnif\": (-35.64, 0.01),\n \"LightGBM\": (-32.96, 0.01),\n \"LightGBMLarge\": (-34.86, 0.01),\n \"LightGBMXT\": (-32.69, 0.01),\n \"NeuralNetFastAI\": (-22.11, 0.01),\n \"NeuralNetTorch\": (-19.76, 0.01),\n \"RandomForestMSE\": (-27.49, 0.01),\n \"WeightedEnsemble_L2\": (-19.76, 0.01),\n \"XGBoost\": (-24.93, 0.01),\n },\n },\n { # Default regression model different metric\n \"name\": \"small regression metric mae\",\n \"type\": \"regression\",\n \"n_samples\": 100,\n \"n_features\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"5850a1c21a\",\n \"params\": {\"predict\": {\"eval_metric\": \"mean_absolute_error\"}, \"fit\": {}},\n \"expected_score_range\": {\n \"CatBoost\": (-5.23, 0.01),\n \"ExtraTreesMSE\": (-5.48, 0.01),\n \"KNeighborsDist\": (-6.16, 0.01),\n \"KNeighborsUnif\": (-6.61, 0.01),\n \"LightGBM\": (-11.97, 0.01),\n \"LightGBMLarge\": (-7.69, 0.01),\n \"LightGBMXT\": (-12.37, 0.01),\n \"NeuralNetFastAI\": (-4.74, 0.01),\n \"NeuralNetTorch\": (-3.77, 0.01),\n \"RandomForestMSE\": (-6.96, 0.01),\n \"WeightedEnsemble_L2\": (-4.03, 0.01),\n \"XGBoost\": (-8.32, 0.01),\n },\n },\n #\n # Classifications\n #\n { # Default classification model on a small dataset\n \"name\": \"small classification\",\n \"type\": \"classification\",\n \"n_samples\": 400, # With only 8 classes it's hard to compare model quality unless we have a biggest test set (and therefore train set).\n \"n_features\": 10,\n \"n_informative\": 5,\n \"n_classes\": 8,\n \"n_categorical\": 0,\n \"dataset_hash\": \"be1f16df80\",\n \"params\": [\n {\"predict\": {}, \"fit\": {}}, # All of the following params should return same results\n {\"predict\": {}, \"fit\": {\"presets\": \"medium_quality_faster_train\"}},\n {\"predict\": {}, \"fit\": {\"presets\": \"ignore_text\"}},\n {\"predict\": {}, \"fit\": {\"hyperparameters\": \"default\"}},\n {\"predict\": {\"eval_metric\": \"accuracy\"}, \"fit\": {}},\n ],\n \"expected_score_range\": {\n \"CatBoost\": (0.245, 0.001), # Classification scores are low numbers so we decrease the\n \"ExtraTreesEntr\": (0.327, 0.001), # tolerance to 0.001 to make sure we pick up changes.\n \"ExtraTreesGini\": (0.32, 0.001),\n \"KNeighborsDist\": (0.337, 0.001),\n \"KNeighborsUnif\": (0.322, 0.001),\n \"LightGBM\": (0.197, 0.001),\n \"LightGBMLarge\": (0.265, 0.001),\n \"LightGBMXT\": (0.23, 0.001),\n \"NeuralNetFastAI\": (0.34, 0.001),\n \"NeuralNetTorch\": (0.232, 0.001),\n \"RandomForestEntr\": (0.305, 0.001),\n \"RandomForestGini\": (0.295, 0.001),\n \"WeightedEnsemble_L2\": (0.34, 0.001),\n \"XGBoost\": (0.227, 0.001),\n },\n },\n { # There's different logic for boolean classification so let's test that with n_classes = 2.\n \"name\": \"small classification boolean\",\n \"type\": \"classification\",\n \"n_samples\": 400,\n \"n_features\": 10,\n \"n_informative\": 5,\n \"n_classes\": 2,\n \"n_categorical\": 0,\n \"dataset_hash\": \"79e634aac3\",\n \"params\": [\n {\"predict\": {}, \"fit\": {}}, # All of the following params should return same results\n {\"predict\": {\"eval_metric\": \"accuracy\"}, \"fit\": {}},\n ],\n \"expected_score_range\": {\n \"CatBoost\": (0.61, 0.001),\n \"ExtraTreesEntr\": (0.607, 0.001),\n \"ExtraTreesGini\": (0.6, 0.001),\n \"KNeighborsDist\": (0.61, 0.001),\n \"KNeighborsUnif\": (0.61, 0.001),\n \"LightGBM\": (0.632, 0.001),\n \"LightGBMLarge\": (0.552, 0.001),\n \"LightGBMXT\": (0.612, 0.001),\n \"NeuralNetFastAI\": (0.62, 0.001),\n \"NeuralNetTorch\": (0.597, 0.001),\n \"RandomForestEntr\": (0.607, 0.001),\n \"RandomForestGini\": (0.582, 0.001),\n \"WeightedEnsemble_L2\": (0.61, 0.001),\n \"XGBoost\": (0.58, 0.001),\n },\n },\n]\n\n\ndef make_dataset(request, seed):\n TEST_SIZE = 0.5\n # Ensure our datasets and model calls remain deterministic.\n random.seed(seed)\n np.random.seed(seed)\n if request[\"type\"] == \"regression\":\n x, y = make_regression(\n n_samples=int(request[\"n_samples\"] * (1 / (1 - TEST_SIZE))), n_features=request[\"n_features\"], noise=4\n ) # To make it hard enough that we get better performance on slower models\n elif request[\"type\"] == \"classification\":\n x, y = make_classification(\n n_samples=int(request[\"n_samples\"] * (1 / (1 - TEST_SIZE))),\n n_features=request[\"n_features\"],\n n_informative=request[\"n_informative\"],\n n_redundant=request[\"n_classes\"] - request[\"n_informative\"],\n n_classes=request[\"n_classes\"],\n class_sep=0.4,\n ) # To make it hard enough that we get better performance on slower models\n else:\n assert False, \"Unrecognised request type '{request['type'}'\"\n\n dfx = pd.DataFrame(x)\n dfy = pd.DataFrame(y, columns=[\"label\"])\n\n # Make some columns categorical if required.\n if request[\"n_categorical\"] > 0:\n cols_to_convert = random.sample(set(dfx.columns.values), k=request[\"n_categorical\"])\n for col in cols_to_convert:\n dfx[col] = dfx[col].astype(int)\n vals = np.unique(dfx[col])\n # Shuffle the categoricals so there's no pattern in their ordering.\n vals2 = vals.copy() - min(vals)\n np.random.shuffle(vals2)\n mapper = dict(zip(vals, vals2))\n dfx[col] = dfx[col].map(mapper)\n dfx[col] = dfx[col].astype(\"category\")\n\n x_train, x_test, y_train, y_test = train_test_split(dfx, dfy, test_size=TEST_SIZE)\n dftrain = pd.concat([x_train, y_train], axis=1)\n dftest = pd.concat([x_test, y_test], axis=1)\n\n return (dftrain, dftest)\n"
},
{
"path": "tabular/tests/test_check_style.py",
"content": "import logging\nimport warnings\nfrom subprocess import PIPE, Popen\n\n\ndef test_check_style():\n logging.getLogger().setLevel(logging.INFO)\n logging.info(\"PEP8 Style check\")\n flake8_proc = Popen([\"flake8\", \"--count\", \"--max-line-length\", \"300\"], stdout=PIPE)\n flake8_out = flake8_proc.communicate()[0]\n lines = flake8_out.splitlines()\n count = int(lines[-1].decode())\n if count > 0:\n warnings.warn(f\"{count} PEP8 warnings remaining\")\n assert count < 1000, \"Too many PEP8 warnings found, improve code quality to pass test.\"\n"
},
{
"path": "tabular/tests/unittests/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/calibrate/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/calibrate/test_calibrate.py",
"content": "from autogluon.tabular.testing import FitHelper\n\n\ndef test_calibrate_binary():\n \"\"\"Tests that calibrate=True doesn't crash in binary\"\"\"\n fit_args = dict(\n hyperparameters={\"GBM\": {}},\n calibrate=True,\n )\n dataset_name = \"toy_binary\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_calibrate_binary_bag():\n \"\"\"Tests that calibrate=True doesn't crash in binary w/ bagging\"\"\"\n fit_args = dict(\n hyperparameters={\"GBM\": {\"ag_args_ensemble\": {\"fold_fitting_strategy\": \"sequential_local\"}}},\n calibrate=True,\n num_bag_folds=3,\n )\n dataset_name = \"toy_binary\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_calibrate_multiclass():\n \"\"\"Tests that calibrate=True doesn't crash in multiclass\"\"\"\n fit_args = dict(\n hyperparameters={\"GBM\": {}},\n calibrate=True,\n )\n dataset_name = \"toy_multiclass\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_calibrate_multiclass_bag():\n \"\"\"Tests that calibrate=True doesn't crash in multiclass w/ bagging\"\"\"\n fit_args = dict(\n hyperparameters={\"GBM\": {\"ag_args_ensemble\": {\"fold_fitting_strategy\": \"sequential_local\"}}},\n calibrate=True,\n num_bag_folds=3,\n )\n dataset_name = \"toy_multiclass\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_calibrate_quantile():\n \"\"\"Tests that calibrate=True doesn't crash in quantile\"\"\"\n fit_args = dict(\n hyperparameters={\"RF\": {}},\n calibrate=True,\n )\n dataset_name = \"toy_quantile\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_calibrate_quantile_bag():\n \"\"\"Tests that calibrate=True doesn't crash in quantile w/ bagging\"\"\"\n fit_args = dict(\n hyperparameters={\"RF\": {\"ag_args_ensemble\": {\"fold_fitting_strategy\": \"sequential_local\"}}},\n calibrate=True,\n num_bag_folds=3,\n )\n dataset_name = \"toy_quantile\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n"
},
{
"path": "tabular/tests/unittests/callbacks/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/callbacks/test_callbacks.py",
"content": "from autogluon.core.callbacks import (\n EarlyStoppingCallback,\n EarlyStoppingCountCallback,\n EarlyStoppingEnsembleCallback,\n ExampleCallback,\n)\nfrom autogluon.core.models import DummyModel\nfrom autogluon.tabular.models.lgb.lgb_model import LGBModel\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_early_stopping_count_callback():\n callback = EarlyStoppingCountCallback(patience=1)\n\n fit_args = dict(\n hyperparameters={\n DummyModel: {\"ag_args\": {\"priority\": 100}},\n LGBModel: {\"ag_args\": {\"priority\": 90}},\n },\n callbacks=[callback],\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=2, refit_full=False, deepcopy_fit_args=False\n )\n\n\ndef test_early_stopping_count_callback_as_list():\n callback = [\"EarlyStoppingCountCallback\", {\"patience\": 1}]\n\n fit_args = dict(\n hyperparameters={\n DummyModel: {\"ag_args\": {\"priority\": 100}},\n LGBModel: {\"ag_args\": {\"priority\": 90}},\n },\n callbacks=[callback],\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=2, refit_full=False, deepcopy_fit_args=False\n )\n\n\ndef test_early_stopping_callback():\n callback = EarlyStoppingCallback()\n\n fit_args = dict(\n hyperparameters={\n DummyModel: {},\n LGBModel: {},\n },\n infer_limit=100,\n infer_limit_batch_size=1000,\n callbacks=[callback],\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=3, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"LightGBM\"\n assert callback.infer_limit is not None\n\n\ndef test_early_stopping_callback_v2():\n \"\"\"\n Tests EarlyStoppingCallback early stops prior to fitting LightGBM.\n Tests `patience_per_level=True`\n \"\"\"\n callback = EarlyStoppingCallback(patience=2, patience_per_level=True)\n\n fit_args = dict(\n hyperparameters={\n DummyModel: [{}, {}, {}, {}],\n LGBModel: {},\n },\n callbacks=[callback],\n num_bag_folds=2,\n num_stack_levels=1,\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=6, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"Dummy_BAG_L1\"\n assert callback.score_best == 0.76\n assert callback.infer_limit is None\n\n\ndef test_early_stopping_callback_v3():\n \"\"\"\n Test EarlyStoppingCallback early stops prior to fitting LightGBM.\n Tests `patience_per_level=False`\n Tests passing multiple callbacks.\n \"\"\"\n callback = EarlyStoppingCallback(patience=2, patience_per_level=False)\n\n fit_args = dict(\n hyperparameters={\n DummyModel: [{}, {}, {}, {}],\n LGBModel: {},\n },\n callbacks=[callback, ExampleCallback()],\n num_bag_folds=2,\n num_stack_levels=1,\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=3, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"Dummy_BAG_L1\"\n assert callback.score_best == 0.76\n assert callback.infer_limit is None\n\n\ndef test_early_stopping_ensemble_callback():\n callback = EarlyStoppingEnsembleCallback()\n\n fit_args = dict(\n hyperparameters={\n DummyModel: {},\n LGBModel: {},\n },\n infer_limit=100,\n infer_limit_batch_size=1000,\n callbacks=[callback],\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=4, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"LightGBM\"\n assert callback.infer_limit is not None\n assert callback.infer_limit_batch_size == 1000\n\n\ndef test_early_stopping_ensemble_callback_v2():\n \"\"\"\n Tests EarlyStoppingEnsembleCallback early stops prior to fitting LightGBM.\n Tests `patience_per_level=True`\n \"\"\"\n callback = EarlyStoppingEnsembleCallback(patience=2, patience_per_level=True)\n\n fit_args = dict(\n hyperparameters={\n DummyModel: [{}, {}, {}, {}],\n LGBModel: {},\n },\n callbacks=[callback],\n num_bag_folds=2,\n num_stack_levels=1,\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=9, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"Dummy_BAG_L1\"\n assert callback.score_best == 0.76\n assert callback.infer_limit is None\n assert callback.infer_limit_batch_size is None\n\n\ndef test_early_stopping_ensemble_callback_v3():\n \"\"\"\n Test EarlyStoppingEnsembleCallback early stops prior to fitting LightGBM.\n Tests `patience_per_level=False`\n Tests passing multiple callbacks.\n \"\"\"\n callback = EarlyStoppingEnsembleCallback(patience=2, patience_per_level=False)\n\n fit_args = dict(\n hyperparameters={\n DummyModel: [{}, {}, {}, {}],\n LGBModel: {},\n },\n callbacks=[callback, ExampleCallback()],\n num_bag_folds=2,\n num_stack_levels=1,\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=5, refit_full=False, deepcopy_fit_args=False\n )\n\n assert callback.model_best == \"Dummy_BAG_L1\"\n assert callback.score_best == 0.76\n assert callback.infer_limit is None\n assert callback.infer_limit_batch_size is None\n"
},
{
"path": "tabular/tests/unittests/configs/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/configs/test_config_helper.py",
"content": "import numpy as np\nimport pytest\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.features import TextNgramFeatureGenerator\nfrom autogluon.tabular.configs.config_helper import ConfigBuilder, FeatureGeneratorBuilder\nfrom autogluon.tabular.models import KNNModel\nfrom autogluon.tabular.registry import ag_model_registry\n\n\ndef test_presets():\n expected_config = dict(presets=[\"best_quality\"])\n actual_config = ConfigBuilder().presets(\"best_quality\").build()\n assert actual_config == expected_config\n\n actual_config = ConfigBuilder().presets([\"best_quality\"]).build()\n assert actual_config == expected_config\n\n expected_config = dict(presets=[\"best_quality\", \"optimize_for_deployment\"])\n actual_config = ConfigBuilder().presets([\"best_quality\", \"optimize_for_deployment\"]).build()\n assert actual_config == expected_config\n\n expected_config = dict(presets={\"a\": 42})\n actual_config = ConfigBuilder().presets({\"a\": 42}).build()\n assert actual_config == expected_config\n\n\ndef test_presets_invalid_option():\n with pytest.raises(\n AssertionError,\n match=r\"The following presets are not recognized: .'unknown1'. - use one of the valid presets: .*\",\n ):\n ConfigBuilder().presets(\"unknown1\").build()\n\n with pytest.raises(\n AssertionError,\n match=r\"The following presets are not recognized: .'unknown2', 'unknown3'. - use one of the valid presets: .*\",\n ):\n ConfigBuilder().presets([\"best_quality\", \"unknown2\", \"unknown3\"]).build()\n\n\ndef test_excluded_model_types():\n expected_config = dict(excluded_model_types=[\"RF\"])\n actual_config = ConfigBuilder().excluded_model_types(\"RF\").build()\n assert actual_config == expected_config\n\n actual_config = ConfigBuilder().excluded_model_types([\"RF\"]).build()\n assert actual_config == expected_config\n\n expected_config = dict(excluded_model_types=[\"LR\", \"RF\"])\n actual_config = ConfigBuilder().excluded_model_types([\"RF\", \"LR\"]).build()\n assert actual_config == expected_config\n\n expected_config = dict(excluded_model_types=[\"RF\"])\n actual_config = ConfigBuilder().excluded_model_types([\"RF\", \"RF\"]).build()\n assert actual_config == expected_config\n\n\ndef test_excluded_model_types_invalid_option():\n with pytest.raises(AssertionError, match=r\"unknown1 is not one of the valid models .*\"):\n ConfigBuilder().excluded_model_types(\"unknown1\").build()\n\n with pytest.raises(AssertionError, match=r\"unknown2 is not one of the valid models .*\"):\n ConfigBuilder().excluded_model_types([\"unknown2\"]).build()\n\n\ndef test_included_model_types():\n model_keys = ag_model_registry.keys\n model_keys_no_rf = [k for k in model_keys if k not in [\"RF\", \"ENS_WEIGHTED\", \"SIMPLE_ENS_WEIGHTED\"]]\n\n expected_config = dict(excluded_model_types=model_keys_no_rf)\n actual_config = ConfigBuilder().included_model_types(\"RF\").build()\n assert actual_config == expected_config\n\n actual_config = ConfigBuilder().included_model_types([\"RF\"]).build()\n assert actual_config == expected_config\n\n actual_config = ConfigBuilder().included_model_types([\"RF\", \"RF\"]).build()\n assert actual_config == expected_config\n\n model_keys_no_lr = [k for k in model_keys_no_rf if k != \"LR\"]\n expected_config = dict(excluded_model_types=model_keys_no_lr)\n actual_config = ConfigBuilder().included_model_types([\"RF\", \"LR\"]).build()\n assert actual_config == expected_config\n\n class CustomKNN(KNNModel):\n pass\n\n expected_config = dict(excluded_model_types=model_keys_no_rf)\n actual_config = ConfigBuilder().included_model_types([CustomKNN, \"RF\"]).build()\n assert actual_config == expected_config\n\n\ndef test_included_model_types_invalid_option():\n with pytest.raises(\n AssertionError,\n match=r\"The following model types are not recognized: .'unknown1'. - use one of the valid models: .*\",\n ):\n ConfigBuilder().included_model_types(\"unknown1\").build()\n\n with pytest.raises(\n AssertionError,\n match=r\"The following model types are not recognized: .'unknown2', 'unknown3'. - use one of the valid models: .*\",\n ):\n ConfigBuilder().included_model_types([\"RF\", \"unknown2\", \"unknown3\"]).build()\n\n\ndef test_time_limit():\n expected_config = dict(time_limit=10)\n actual_config = ConfigBuilder().time_limit(10).build()\n assert actual_config == expected_config\n\n expected_config = dict(time_limit=None)\n actual_config = ConfigBuilder().time_limit(None).build()\n assert actual_config == expected_config\n\n\ndef test_time_limit_invalid_option():\n with pytest.raises(AssertionError, match=r\"time_limit must be greater than zero\"):\n ConfigBuilder().time_limit(-1).build()\n\n\ndef test_hyperparameters_str():\n expected_config = dict(hyperparameters=\"very_light\")\n actual_config = ConfigBuilder().hyperparameters(\"very_light\").build()\n assert actual_config == expected_config\n\n\ndef test_hyperparameters_dict():\n expected_config = dict(hyperparameters={\"RF\": {}})\n actual_config = ConfigBuilder().hyperparameters({\"RF\": {}}).build()\n assert actual_config == expected_config\n\n class CustomKNN(KNNModel):\n pass\n\n expected_config = dict(hyperparameters={CustomKNN: [{}, {\"prop\": 42}]})\n actual_config = ConfigBuilder().hyperparameters({CustomKNN: [{}, {\"prop\": 42}]}).build()\n assert actual_config == expected_config\n\n\ndef test_hyperparameters__invalid_option():\n with pytest.raises(ValueError, match=r\"hyperparameters must be either str: .* or dict with keys of .*\"):\n ConfigBuilder().hyperparameters(42).build()\n\n with pytest.raises(AssertionError, match=r\"unknown is not one of the valid presets .*\"):\n ConfigBuilder().hyperparameters(\"unknown\").build()\n\n with pytest.raises(\n AssertionError,\n match=r\"The following model types are not recognized: .'unknown'. - use one of the valid models: .*\",\n ):\n ConfigBuilder().hyperparameters({\"unknown\": []}).build()\n\n\ndef test_auto_stack():\n assert ConfigBuilder().auto_stack().build() == dict(auto_stack=True)\n assert ConfigBuilder().auto_stack(False).build() == dict(auto_stack=False)\n\n\ndef test_use_bag_holdout():\n assert ConfigBuilder().use_bag_holdout().build() == dict(use_bag_holdout=True)\n assert ConfigBuilder().use_bag_holdout(False).build() == dict(use_bag_holdout=False)\n\n\ndef test_num_bag_folds():\n assert ConfigBuilder().num_bag_folds(0).build() == dict(num_bag_folds=0)\n with pytest.raises(AssertionError, match=r\"num_bag_folds must be greater or equal than zero\"):\n ConfigBuilder().num_bag_folds(-1).build()\n\n\ndef test_num_bag_sets():\n assert ConfigBuilder().num_bag_sets(1).build() == dict(num_bag_sets=1)\n with pytest.raises(AssertionError, match=r\"num_bag_sets must be greater than zero\"):\n ConfigBuilder().num_bag_sets(0).build()\n\n\ndef test_num_stack_levels():\n assert ConfigBuilder().num_stack_levels(0).build() == dict(num_stack_levels=0)\n with pytest.raises(AssertionError, match=r\"num_stack_levels must be greater or equal than zero\"):\n ConfigBuilder().num_stack_levels(-1).build()\n\n\ndef test_holdout_frac():\n assert ConfigBuilder().holdout_frac(0).build() == dict(holdout_frac=0)\n assert ConfigBuilder().holdout_frac(1).build() == dict(holdout_frac=1)\n with pytest.raises(AssertionError, match=r\"holdout_frac must be between 0 and 1\"):\n ConfigBuilder().holdout_frac(-0.1).build()\n with pytest.raises(AssertionError, match=r\"holdout_frac must be between 0 and 1\"):\n ConfigBuilder().holdout_frac(1.1).build()\n\n\ndef test_hyperparameter_tune_kwargs():\n assert ConfigBuilder().hyperparameter_tune_kwargs(\"auto\").build() == dict(hyperparameter_tune_kwargs=\"auto\")\n assert ConfigBuilder().hyperparameter_tune_kwargs(\"random\").build() == dict(hyperparameter_tune_kwargs=\"random\")\n assert ConfigBuilder().hyperparameter_tune_kwargs({\"props\": 42}).build() == dict(\n hyperparameter_tune_kwargs={\"props\": 42}\n )\n with pytest.raises(AssertionError, match=r\"unknown string must be one of .*\"):\n ConfigBuilder().hyperparameter_tune_kwargs(\"unknown\").build()\n with pytest.raises(ValueError, match=r\"hyperparameter_tune_kwargs must be either str: .* or dict\"):\n ConfigBuilder().hyperparameter_tune_kwargs(42).build()\n\n\ndef test_ag_args():\n assert ConfigBuilder().ag_args({\"param\": 42}).build() == dict(ag_args={\"param\": 42})\n\n\ndef test_ag_args_fit():\n assert ConfigBuilder().ag_args_fit({\"param\": 42}).build() == dict(ag_args_fit={\"param\": 42})\n\n\ndef test_ag_args_ensemble():\n assert ConfigBuilder().ag_args_ensemble({\"param\": 42}).build() == dict(ag_args_ensemble={\"param\": 42})\n\n\ndef test_set_best_to_refit_full():\n assert ConfigBuilder().set_best_to_refit_full().build() == dict(set_best_to_refit_full=True)\n assert ConfigBuilder().set_best_to_refit_full(False).build() == dict(set_best_to_refit_full=False)\n\n\ndef test_keep_only_best():\n assert ConfigBuilder().keep_only_best().build() == dict(keep_only_best=True)\n assert ConfigBuilder().keep_only_best(False).build() == dict(keep_only_best=False)\n\n\ndef test_save_space():\n assert ConfigBuilder().save_space().build() == dict(save_space=True)\n assert ConfigBuilder().save_space(False).build() == dict(save_space=False)\n\n\ndef test_calibrate():\n assert ConfigBuilder().calibrate().build() == dict(calibrate=True)\n assert ConfigBuilder().calibrate(False).build() == dict(calibrate=False)\n\n\ndef test_use_bag_holdout():\n assert ConfigBuilder().use_bag_holdout().build() == dict(use_bag_holdout=True)\n assert ConfigBuilder().use_bag_holdout(False).build() == dict(use_bag_holdout=False)\n\n\ndef test_refit_full():\n assert ConfigBuilder().refit_full().build() == dict(refit_full=True)\n assert ConfigBuilder().refit_full(False).build() == dict(refit_full=False)\n assert ConfigBuilder().refit_full(\"best\").build() == dict(refit_full=\"best\")\n\n\ndef test_feature_generator():\n vectorizer = CountVectorizer(min_df=7, ngram_range=(2, 3), max_features=11, dtype=np.uint8)\n\n config = (\n ConfigBuilder()\n .feature_generator()\n .enable_numeric_features()\n .enable_categorical_features()\n .enable_datetime_features()\n .enable_text_special_features()\n .enable_text_ngram_features()\n .enable_raw_text_features()\n .enable_vision_features()\n .vectorizer(vectorizer)\n .text_ngram_params({\"vectorizer_strategy\": \"both\"})\n .build()\n .build()\n )\n\n assert config[\"feature_generator\"].enable_numeric_features is True\n assert config[\"feature_generator\"].enable_categorical_features is True\n assert config[\"feature_generator\"].enable_datetime_features is True\n assert config[\"feature_generator\"].enable_text_special_features is True\n assert config[\"feature_generator\"].enable_text_ngram_features is True\n assert config[\"feature_generator\"].enable_raw_text_features is True\n assert config[\"feature_generator\"].enable_vision_features is True\n\n text_gen = None\n generators_classes = []\n for gl in config[\"feature_generator\"].generators:\n for g in gl:\n if isinstance(g, TextNgramFeatureGenerator):\n text_gen = g\n generators_classes.append(g.__class__.__name__)\n print(generators_classes)\n assert str(text_gen.vectorizer_default_raw) == str(vectorizer)\n assert text_gen.vectorizer_strategy == \"both\"\n assert sorted(list(set(generators_classes))) == [\n \"AsTypeFeatureGenerator\",\n \"CategoryFeatureGenerator\",\n \"DatetimeFeatureGenerator\",\n \"DropDuplicatesFeatureGenerator\",\n \"DropUniqueFeatureGenerator\",\n \"FillNaFeatureGenerator\",\n \"IdentityFeatureGenerator\",\n \"IsNanFeatureGenerator\",\n \"TextNgramFeatureGenerator\",\n \"TextSpecialFeatureGenerator\",\n ]\n\n\ndef test_feature_generator_2():\n config = (\n ConfigBuilder()\n .feature_generator()\n .enable_numeric_features(False)\n .enable_categorical_features(False)\n .enable_datetime_features(False)\n .enable_text_special_features(False)\n .enable_text_ngram_features(False)\n .enable_raw_text_features(False)\n .enable_vision_features(False)\n .build()\n .build()\n )\n\n assert config[\"feature_generator\"].enable_numeric_features is False\n assert config[\"feature_generator\"].enable_categorical_features is False\n assert config[\"feature_generator\"].enable_datetime_features is False\n assert config[\"feature_generator\"].enable_text_special_features is False\n assert config[\"feature_generator\"].enable_text_ngram_features is False\n assert config[\"feature_generator\"].enable_raw_text_features is False\n assert config[\"feature_generator\"].enable_vision_features is False\n\n text_gen = None\n generators_classes = []\n for gl in config[\"feature_generator\"].generators:\n for g in gl:\n if isinstance(g, TextNgramFeatureGenerator):\n text_gen = g\n generators_classes.append(g.__class__.__name__)\n assert text_gen is None\n assert sorted(list(set(generators_classes))) == [\n \"AsTypeFeatureGenerator\",\n \"DropDuplicatesFeatureGenerator\",\n \"DropUniqueFeatureGenerator\",\n \"FillNaFeatureGenerator\",\n ]\n\n\ndef test_feature_generator_builder_standalone():\n vectorizer = CountVectorizer(min_df=7, ngram_range=(2, 3), max_features=11, dtype=np.uint8)\n\n generator = (\n FeatureGeneratorBuilder()\n .enable_numeric_features()\n .enable_categorical_features()\n .enable_datetime_features()\n .enable_text_special_features()\n .enable_text_ngram_features()\n .enable_raw_text_features()\n .enable_vision_features()\n .vectorizer(vectorizer)\n .text_ngram_params({\"vectorizer_strategy\": \"both\"})\n .build()\n )\n assert generator.enable_numeric_features is True\n assert generator.enable_categorical_features is True\n assert generator.enable_datetime_features is True\n assert generator.enable_text_special_features is True\n assert generator.enable_text_ngram_features is True\n assert generator.enable_raw_text_features is True\n assert generator.enable_vision_features is True\n"
},
{
"path": "tabular/tests/unittests/configs/test_pipline_presets.py",
"content": "import pytest\n\nfrom autogluon.core.constants import BINARY\nfrom autogluon.tabular.configs.pipeline_presets import get_validation_and_stacking_method\n\n\ndef test_default_get_validation_and_stacking_method():\n (\n num_bag_folds,\n num_bag_sets,\n num_stack_levels,\n dynamic_stacking,\n use_bag_holdout,\n holdout_frac,\n refit_full,\n ) = get_validation_and_stacking_method(\n auto_stack=False,\n # Default\n num_bag_folds=None,\n num_bag_sets=None,\n use_bag_holdout=None,\n holdout_frac=None,\n num_stack_levels=None,\n dynamic_stacking=None,\n refit_full=None,\n # Can change the default behavior\n num_train_rows=1000,\n hpo_enabled=False,\n # Is ignored due to `auto_stack=False`\n problem_type=\"N/A\",\n )\n\n assert num_bag_folds == 0\n assert num_bag_sets == 1\n assert use_bag_holdout is False\n assert holdout_frac == 0.2\n assert refit_full is False\n assert dynamic_stacking is True\n\n\n@pytest.mark.parametrize(\n \"metadata_and_expected_result\",\n [\n # Reaction to dataset size checks\n (dict(num_train_rows=40), dict(num_bag_folds=5)),\n (dict(num_train_rows=70), dict(num_bag_folds=7)),\n (dict(num_train_rows=10_000), dict(holdout_frac=0.1)),\n # (also checks that dynamic stacking is disabled for holdout validation)\n (\n dict(num_train_rows=1_000_000),\n dict(holdout_frac=0.01, dynamic_stacking=False, use_bag_holdout=True),\n ),\n # HPO On check\n (\n dict(num_train_rows=1_000_000, hpo_enabled=True),\n dict(holdout_frac=0.02, dynamic_stacking=False, use_bag_holdout=True),\n ),\n # No dynamic stacking, auto_stack check\n (\n dict(num_train_rows=749, dynamic_stacking=False),\n dict(dynamic_stacking=False, num_stack_levels=0),\n ),\n (\n dict(num_train_rows=750, dynamic_stacking=False),\n dict(dynamic_stacking=False, num_stack_levels=1),\n ),\n (\n dict(num_train_rows=750, dynamic_stacking=False, problem_type=BINARY),\n dict(dynamic_stacking=False, num_stack_levels=0),\n ),\n (\n dict(num_train_rows=750, dynamic_stacking=False, problem_type=BINARY, use_bag_holdout=True),\n dict(dynamic_stacking=False, num_stack_levels=1, use_bag_holdout=True),\n ),\n ],\n)\ndef test_auto_stack_get_validation_and_stacking_method(metadata_and_expected_result):\n metadata, expected_result = metadata_and_expected_result\n\n metadata[\"hpo_enabled\"] = metadata.get(\"hpo_enabled\", False)\n metadata[\"problem_type\"] = metadata.get(\"problem_type\", \"N/A\")\n metadata[\"dynamic_stacking\"] = metadata.get(\"dynamic_stacking\", None)\n metadata[\"use_bag_holdout\"] = metadata.get(\"use_bag_holdout\", None)\n\n num_bag_folds = expected_result.get(\"num_bag_folds\", 8)\n num_bag_sets = expected_result.get(\"num_bag_sets\", 1)\n num_stack_levels = expected_result.get(\"num_stack_levels\", 1)\n dynamic_stacking = expected_result.get(\"dynamic_stacking\", True)\n use_bag_holdout = expected_result.get(\"use_bag_holdout\", False)\n holdout_frac = expected_result.get(\"holdout_frac\", 0.2)\n\n assert get_validation_and_stacking_method(\n auto_stack=True,\n # Default\n num_bag_folds=None,\n num_bag_sets=None,\n holdout_frac=None,\n num_stack_levels=None,\n refit_full=None,\n **metadata,\n ) == (\n num_bag_folds,\n num_bag_sets,\n num_stack_levels,\n dynamic_stacking,\n use_bag_holdout,\n holdout_frac,\n False,\n )\n"
},
{
"path": "tabular/tests/unittests/configs/test_presets.py",
"content": "import unittest\n\nfrom autogluon.common.utils.decorators import apply_presets\nfrom autogluon.tabular.configs.presets_configs import tabular_presets_alias, tabular_presets_dict\n\n\nclass TestPresets(unittest.TestCase):\n def test_presets(self):\n @apply_presets(tabular_presets_dict, tabular_presets_alias)\n def get_presets(presets=None, **kwargs):\n return kwargs\n\n # assert presets are applying their values correctly\n for preset_real in tabular_presets_dict:\n assert get_presets(presets=preset_real) == tabular_presets_dict[preset_real]\n\n # assert preset aliases are applying their values correctly\n for preset_alias, preset_real in tabular_presets_alias.items():\n assert get_presets(presets=preset_alias) == tabular_presets_dict[preset_real]\n\n # assert the quality presets exist\n for preset in [\"extreme_quality\", \"best_quality\", \"high_quality\", \"good_quality\", \"medium_quality\"]:\n assert preset in tabular_presets_dict\n"
},
{
"path": "tabular/tests/unittests/data/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/data/test_label_cleaner.py",
"content": "import numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS\nfrom autogluon.core.data.label_cleaner import (\n LabelCleaner,\n LabelCleanerBinary,\n LabelCleanerDummy,\n LabelCleanerMulticlass,\n LabelCleanerMulticlassToBinary,\n)\n\n\ndef test_label_cleaner_binary():\n # Given\n problem_type = BINARY\n input_labels_numpy = np.array([\"l1\", \"l2\", \"l2\", \"l1\", \"l1\", \"l2\"])\n input_labels = pd.Series(input_labels_numpy)\n input_labels_category = input_labels.astype(\"category\")\n input_labels_with_shifted_index = input_labels.copy()\n input_labels_with_shifted_index.index += 5\n input_labels_new = np.array([\"new\", \"l1\", \"l2\"])\n expected_output_labels = pd.Series([0, 1, 1, 0, 0, 1], dtype=\"uint8\")\n expected_output_labels_pos_class_l1 = pd.Series([1, 0, 0, 1, 1, 0], dtype=\"uint8\")\n expected_output_labels_new = pd.Series([np.nan, 0, 1])\n expected_output_labels_new_pos_class_l1 = pd.Series([np.nan, 1, 0])\n expected_output_labels_new_inverse = pd.Series([np.nan, \"l1\", \"l2\"])\n\n # When\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=input_labels) # positive_class='l2'\n label_cleaner_pos_class_l1 = LabelCleaner.construct(problem_type=problem_type, y=input_labels, positive_class=\"l1\")\n\n # Raise exception\n with pytest.raises(ValueError):\n LabelCleaner.construct(problem_type=problem_type, y=input_labels, positive_class=\"unknown_class\")\n\n # Raise exception\n with pytest.raises(AssertionError):\n LabelCleaner.construct(problem_type=problem_type, y=input_labels_new)\n\n # Then\n assert isinstance(label_cleaner, LabelCleanerBinary)\n assert label_cleaner.problem_type_transform == BINARY\n assert label_cleaner.cat_mappings_dependent_var == {0: \"l1\", 1: \"l2\"}\n assert label_cleaner_pos_class_l1.cat_mappings_dependent_var == {0: \"l2\", 1: \"l1\"}\n\n output_labels = label_cleaner.transform(input_labels)\n output_labels_pos_class_l1 = label_cleaner_pos_class_l1.transform(input_labels)\n output_labels_with_numpy = label_cleaner.transform(input_labels_numpy)\n output_labels_category = label_cleaner.transform(input_labels_category)\n output_labels_with_shifted_index = label_cleaner.transform(input_labels_with_shifted_index)\n output_labels_new = label_cleaner.transform(input_labels_new)\n output_labels_new_pos_class_l1 = label_cleaner_pos_class_l1.transform(input_labels_new)\n\n output_labels_inverse = label_cleaner.inverse_transform(output_labels)\n output_labels_inverse_pos_class_l1 = label_cleaner_pos_class_l1.inverse_transform(output_labels_pos_class_l1)\n output_labels_with_shifted_index_inverse = label_cleaner.inverse_transform(output_labels_with_shifted_index)\n output_labels_new_inverse = label_cleaner.inverse_transform(output_labels_new)\n output_labels_new_inverse_pos_class_l1 = label_cleaner_pos_class_l1.inverse_transform(\n output_labels_new_pos_class_l1\n )\n\n assert expected_output_labels.equals(output_labels)\n assert expected_output_labels_pos_class_l1.equals(output_labels_pos_class_l1)\n assert expected_output_labels.equals(output_labels_with_numpy)\n assert expected_output_labels.equals(output_labels_category)\n assert not expected_output_labels.equals(output_labels_with_shifted_index)\n output_labels_with_shifted_index.index -= 5\n assert expected_output_labels.equals(output_labels_with_shifted_index)\n assert expected_output_labels_new.equals(output_labels_new)\n assert expected_output_labels_new_pos_class_l1.equals(output_labels_new_pos_class_l1)\n\n assert input_labels.equals(output_labels_inverse)\n assert input_labels.equals(output_labels_inverse_pos_class_l1)\n assert input_labels_with_shifted_index.equals(output_labels_with_shifted_index_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_new_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_new_inverse_pos_class_l1)\n\n\ndef test_label_cleaner_multiclass():\n # Given\n problem_type = MULTICLASS\n input_labels_numpy = np.array([2, 4, 2, 2, 4, 1], dtype=\"int32\")\n input_labels = pd.Series(input_labels_numpy, dtype=\"int32\")\n input_labels_category = input_labels.astype(\"category\")\n input_labels_with_shifted_index = input_labels.copy()\n input_labels_with_shifted_index.index += 5\n input_labels_new = np.array([3, 5, 2], dtype=\"int32\")\n expected_output_labels = pd.Series([1, 2, 1, 1, 2, 0], dtype=\"uint8\")\n expected_output_labels_new = pd.Series([np.nan, np.nan, 1])\n expected_output_labels_new_inverse = pd.Series([np.nan, np.nan, 2])\n\n # When\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=input_labels, y_uncleaned=input_labels)\n\n # Then\n assert isinstance(label_cleaner, LabelCleanerMulticlass)\n assert label_cleaner.problem_type_transform == MULTICLASS\n assert label_cleaner.cat_mappings_dependent_var == {0: 1, 1: 2, 2: 4}\n\n output_labels = label_cleaner.transform(input_labels)\n output_labels_with_numpy = label_cleaner.transform(input_labels_numpy)\n output_labels_category = label_cleaner.transform(input_labels_category)\n output_labels_with_shifted_index = label_cleaner.transform(input_labels_with_shifted_index)\n output_labels_new = label_cleaner.transform(input_labels_new)\n\n output_labels_inverse = label_cleaner.inverse_transform(output_labels)\n output_labels_with_shifted_index_inverse = label_cleaner.inverse_transform(output_labels_with_shifted_index)\n output_labels_new_inverse = label_cleaner.inverse_transform(output_labels_new)\n\n output_labels_uncleaned = label_cleaner.transform_pred_uncleaned(y=input_labels)\n output_labels_uncleaned_inverse = label_cleaner.inverse_transform_pred_uncleaned(y=output_labels_uncleaned)\n\n output_labels_uncleaned_new = label_cleaner.transform_pred_uncleaned(y=input_labels_new)\n output_labels_uncleaned_new_inverse = label_cleaner.inverse_transform_pred_uncleaned(y=output_labels_uncleaned_new)\n\n input_labels = input_labels.astype(\"int32\")\n output_labels_uncleaned_inverse = output_labels_uncleaned_inverse.astype(\"int32\")\n\n assert expected_output_labels.equals(output_labels)\n assert expected_output_labels.equals(output_labels_with_numpy)\n assert expected_output_labels.equals(output_labels_category)\n assert not expected_output_labels.equals(output_labels_with_shifted_index)\n output_labels_with_shifted_index.index -= 5\n assert expected_output_labels.equals(output_labels_with_shifted_index)\n assert expected_output_labels_new.equals(output_labels_new)\n\n assert input_labels.equals(output_labels_inverse)\n assert input_labels_with_shifted_index.equals(output_labels_with_shifted_index_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_new_inverse)\n assert input_labels.equals(output_labels_uncleaned_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_uncleaned_new_inverse)\n\n\ndef test_label_cleaner_multiclass_to_binary():\n # Given\n problem_type = MULTICLASS\n input_labels_numpy = np.array([\"l1\", \"l2\", \"l2\", \"l1\", \"l1\", \"l2\"])\n input_labels = pd.Series(input_labels_numpy)\n input_labels_uncleaned = pd.Series([\"l0\", \"l1\", \"l2\", \"l2\", \"l1\", \"l1\", \"l2\", \"l3\", \"l4\"])\n input_labels_category = input_labels.astype(\"category\")\n input_labels_with_shifted_index = input_labels.copy()\n input_labels_with_shifted_index.index += 5\n input_labels_new = pd.Series([\"l0\", \"l1\", \"l2\"])\n input_labels_new_with_unknown = pd.Series([\"l0\", \"l1\", \"l2\", \"UNKNOWN_1\", \"l4\", \"UNKNOWN_2\"])\n input_labels_proba_transformed = pd.Series([0.7, 0.2, 0.5], index=[5, 2, 8])\n expected_output_labels = pd.Series([0, 1, 1, 0, 0, 1], dtype=\"uint8\")\n expected_output_labels_new = pd.Series([np.nan, 0, 1])\n expected_output_labels_new_inverse = pd.Series([np.nan, \"l1\", \"l2\"])\n expected_output_labels_proba_transformed_inverse = pd.DataFrame(\n data=[[0, 0.3, 0.7, 0, 0], [0, 0.8, 0.2, 0, 0], [0, 0.5, 0.5, 0, 0]],\n index=[5, 2, 8],\n columns=[\"l0\", \"l1\", \"l2\", \"l3\", \"l4\"],\n dtype=np.float32,\n )\n expected_output_labels_new_with_unknown = pd.Series([0, 1, 2, np.nan, 4, np.nan])\n expected_output_labels_new_with_unknown_inverse = pd.Series([\"l0\", \"l1\", \"l2\", np.nan, \"l4\", np.nan])\n\n # When\n label_cleaner = LabelCleaner.construct(\n problem_type=problem_type, y=input_labels, y_uncleaned=input_labels_uncleaned\n )\n\n # Then\n assert isinstance(label_cleaner, LabelCleanerMulticlassToBinary)\n assert label_cleaner.problem_type_transform == BINARY\n assert label_cleaner.cat_mappings_dependent_var == {0: \"l1\", 1: \"l2\"}\n\n output_labels = label_cleaner.transform(input_labels)\n output_labels_with_numpy = label_cleaner.transform(input_labels_numpy)\n output_labels_category = label_cleaner.transform(input_labels_category)\n output_labels_with_shifted_index = label_cleaner.transform(input_labels_with_shifted_index)\n output_labels_new = label_cleaner.transform(input_labels_new)\n\n output_labels_inverse = label_cleaner.inverse_transform(output_labels)\n output_labels_with_shifted_index_inverse = label_cleaner.inverse_transform(output_labels_with_shifted_index)\n output_labels_new_inverse = label_cleaner.inverse_transform(output_labels_new)\n\n output_labels_uncleaned = label_cleaner.transform_pred_uncleaned(y=input_labels)\n output_labels_uncleaned_inverse = label_cleaner.inverse_transform_pred_uncleaned(y=output_labels_uncleaned)\n\n output_labels_uncleaned_new = label_cleaner.transform_pred_uncleaned(y=input_labels_new)\n output_labels_uncleaned_new_inverse = label_cleaner.inverse_transform_pred_uncleaned(y=output_labels_uncleaned_new)\n\n output_labels_uncleaned_new_with_unknown = label_cleaner.transform_pred_uncleaned(y=input_labels_new_with_unknown)\n output_labels_uncleaned_new_with_unknown_inverse = label_cleaner.inverse_transform_pred_uncleaned(\n y=output_labels_uncleaned_new_with_unknown\n )\n\n assert expected_output_labels.equals(output_labels)\n assert expected_output_labels.equals(output_labels_with_numpy)\n assert expected_output_labels.equals(output_labels_category)\n assert not expected_output_labels.equals(output_labels_with_shifted_index)\n output_labels_with_shifted_index.index -= 5\n assert expected_output_labels.equals(output_labels_with_shifted_index)\n assert expected_output_labels_new.equals(output_labels_new)\n\n assert input_labels.equals(output_labels_inverse)\n assert input_labels_with_shifted_index.equals(output_labels_with_shifted_index_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_new_inverse)\n\n assert input_labels.equals(output_labels_uncleaned_inverse)\n assert input_labels_new.equals(output_labels_uncleaned_new_inverse)\n\n assert expected_output_labels_new_with_unknown.equals(output_labels_uncleaned_new_with_unknown)\n assert expected_output_labels_new_with_unknown_inverse.equals(output_labels_uncleaned_new_with_unknown_inverse)\n\n output_labels_proba_transformed_inverse = label_cleaner.inverse_transform_proba(\n input_labels_proba_transformed, as_pandas=True\n )\n\n pd.testing.assert_frame_equal(\n expected_output_labels_proba_transformed_inverse, output_labels_proba_transformed_inverse\n )\n\n\ndef test_label_cleaner_regression():\n # Given\n problem_type = REGRESSION\n input_labels_numpy = np.array([2, 4, 2, 2, 4, 1])\n input_labels = pd.Series(input_labels_numpy)\n input_labels_new = pd.Series([3, 5, 2])\n expected_output_labels = input_labels.copy()\n expected_output_labels_new = input_labels_new.copy()\n expected_output_labels_new_inverse = input_labels_new.copy()\n\n # When\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=input_labels, y_uncleaned=None)\n\n # Then\n assert isinstance(label_cleaner, LabelCleanerDummy)\n assert label_cleaner.problem_type_transform == REGRESSION\n\n output_labels = label_cleaner.transform(input_labels)\n output_labels_with_numpy = label_cleaner.transform(input_labels_numpy)\n output_labels_new = label_cleaner.transform(input_labels_new)\n\n output_labels_inverse = label_cleaner.inverse_transform(output_labels)\n output_labels_new_inverse = label_cleaner.inverse_transform(output_labels_new)\n\n assert expected_output_labels.equals(output_labels)\n assert expected_output_labels.equals(output_labels_with_numpy)\n assert expected_output_labels_new.equals(output_labels_new)\n\n assert input_labels.equals(output_labels_inverse)\n assert expected_output_labels_new_inverse.equals(output_labels_new_inverse)\n\n\ndef test_label_softclass():\n # Given\n problem_type = SOFTCLASS\n input_labels = pd.DataFrame(\n [\n [0, 1, 0, 0, 0, 0],\n [1, 0, 0, 0, 0, 0],\n [0, 0, 0.3, 0.6, 0.1, 0],\n ]\n )\n\n # When\n label_cleaner = LabelCleaner.construct(problem_type=problem_type, y=input_labels, y_uncleaned=None)\n\n # Then\n assert input_labels.equals(label_cleaner.transform(input_labels))\n assert input_labels.equals(label_cleaner.inverse_transform(input_labels))\n assert label_cleaner.num_classes == 6\n"
},
{
"path": "tabular/tests/unittests/data/test_learning_curves.py",
"content": "import math\n\nimport numpy as np\nimport pytest\nfrom sklearn.metrics import accuracy_score\n\nfrom autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.metrics import METRICS, get_metric, make_scorer\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.models import LGBModel, TabularNeuralNetTorchModel, XGBoostModel\nfrom autogluon.tabular.models.tabprep.prep_mixin import ModelAgnosticPrepMixin\nfrom autogluon.tabular.registry import ag_model_registry\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef get_default_model_name(model: str) -> str:\n return ag_model_registry.key_to_cls(model).ag_name\n\n\nmodel_key_to_cls_map = ag_model_registry.key_to_cls_map()\nMODELS = [\n name\n for name, model in model_key_to_cls_map.items()\n if model._get_class_tags().get(\"supports_learning_curves\", False) and not issubclass(model, ModelAgnosticPrepMixin)\n]\nPROBLEM_TYPES = [BINARY, MULTICLASS, REGRESSION]\n\ncommon_args = {\"sample_size\": 50, \"delete_directory\": False, \"refit_full\": False, \"raise_on_model_failure\": True}\n\nearly_stop = 999999\nlong_run = 5\nshort_run = 3\n\nmodel_iterations = {\n \"LightGBM\": long_run,\n \"XGBoost\": long_run,\n \"NeuralNetTorch\": short_run,\n}\n\nextended_run_hyperparams = {\n \"GBM\": {\n \"ag.early_stop\": early_stop,\n \"num_boost_round\": long_run,\n },\n \"XGB\": {\n \"ag.early_stop\": early_stop,\n \"n_estimators\": long_run,\n },\n \"NN_TORCH\": {\n \"epochs_wo_improve\": early_stop,\n \"num_epochs\": short_run,\n },\n}\n\nmetrics_to_test = {\n BINARY: [\"roc_auc\"],\n MULTICLASS: [\"log_loss\"],\n REGRESSION: [\"root_mean_squared_error\"],\n}\n\nfor model in MODELS:\n if model not in extended_run_hyperparams:\n extended_run_hyperparams[model] = {}\n\n # TODO: Not sure this will be correct for additional models\n if get_default_model_name(model) not in extended_run_hyperparams:\n extended_run_hyperparams[get_default_model_name(model)] = long_run\n\n\ndef get_one_model_problem():\n problem_type = PROBLEM_TYPES[0]\n model = MODELS[0]\n return [(problem_type, model)]\n\n\ndef get_all_models():\n n = len(PROBLEM_TYPES)\n return [(PROBLEM_TYPES[i % n], model) for i, model in enumerate(MODELS)]\n\n\ndef get_all_problems():\n n = len(MODELS)\n return [(problem_type, MODELS[i % n]) for i, problem_type in enumerate(PROBLEM_TYPES)]\n\n\ndef get_all_model_problems():\n return [(problem, model) for problem in PROBLEM_TYPES for model in MODELS]\n\n\ndef get_all_model_problem_metrics():\n return [(problem, model, metric) for model in MODELS for problem in PROBLEM_TYPES for metric in METRICS[problem]]\n\n\n# This is much faster to run than `get_all_model_problem_metrics`, but isn't fully comprehensive\n# This makes tests run in 79s , vs 1200s with `get_all_model_problem_metrics`.\ndef get_subset_model_problem_metrics():\n output = [\n (problem, model, metric)\n for model in MODELS\n for problem in PROBLEM_TYPES\n for metric in metrics_to_test[problem]\n ]\n return output\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_one_model_problem())\ndef test_off(problem_type, model, get_dataset_map):\n fit_args = dict(\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, **common_args)\n\n _, data = predictor.learning_curves()\n assert data == {}\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_one_model_problem())\ndef test_flag_false(problem_type, model, get_dataset_map):\n fit_args = dict(\n learning_curves=False,\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, **common_args)\n\n _, data = predictor.learning_curves()\n assert data == {}\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_all_model_problems())\ndef test_flag_true(problem_type, model, get_dataset_map):\n fit_args = dict(\n learning_curves=True,\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, **common_args)\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n _, model_metrics, data = model_data[model]\n metric_count, eval_set_count, _ = np.array(data).shape\n\n assert metric_count == 1\n assert eval_set_count == 2\n assert model_metrics[0] == predictor.eval_metric.name\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_all_problems())\ndef test_metrics(problem_type, model, get_dataset_map):\n # get all unique metrics\n metrics = list(set([metric.name for metric in METRICS[problem_type].values()]))\n metrics = [get_metric(name, problem_type) for name in metrics]\n\n fit_args = dict(\n learning_curves={\n \"metrics\": metrics,\n },\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n # FIXME: This is needed due to a bug: https://github.com/autogluon/autogluon/issues/4423\n min_cls_count_train = 10\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, min_cls_count_train=min_cls_count_train, **common_args\n )\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n _, model_metrics, data = model_data[model]\n\n metrics = list(set([get_metric(metric, problem_type, \"eval_metric\").name for metric in metrics]))\n assert sorted(model_metrics) == sorted(metrics)\n\n\ndef custom_metric(y_true, y_pred):\n return accuracy_score(y_true, y_pred) * 100\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_one_model_problem())\ndef test_custom_metrics(problem_type, model, get_dataset_map):\n myaccuracy = make_scorer(\"myaccuracy\", custom_metric, needs_class=True)\n\n fit_args = dict(\n learning_curves={\"metrics\": [myaccuracy, \"accuracy\"]},\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n args = common_args.copy()\n del args[\"sample_size\"]\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, **args)\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n _, model_metrics, data = model_data[model]\n\n idx, myidx = model_metrics.index(\"accuracy\"), model_metrics.index(\"myaccuracy\")\n myaccuracy_scores = data[myidx]\n accuracy_scores = [[metric * 100 for metric in eval_set] for eval_set in data[idx]]\n assert myaccuracy_scores == accuracy_scores\n\n\n# TODO: can't the error = True tests these be checked at the same time as the\n# correctness tests?\n# @pytest.mark.parametrize(\"problem_type, model\", get_all_problems())\n@pytest.mark.parametrize(\"problem_type, model, metric\", get_subset_model_problem_metrics())\n@pytest.mark.parametrize(\"use_error\", [True, False])\ndef test_metric_format(problem_type, model, metric, use_error, get_dataset_map):\n metric = get_metric(metric, problem_type, \"eval_metric\")\n\n init_args = {\n \"eval_metric\": metric,\n # \"verbosity\": 4,\n }\n\n fit_args = dict(\n learning_curves={\n \"metrics\": metric,\n \"use_error\": use_error,\n },\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n args = common_args.copy()\n args[\"sample_size\"] = 500\n\n # FIXME: Avoid needing this, using this to avoid bug: https://github.com/autogluon/autogluon/issues/4423\n args[\"min_cls_count_train\"] = 10\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, init_args=init_args, fit_args=fit_args, **args\n )\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n _, _, data = model_data[model]\n\n curve = np.array(data[0][1])\n\n if not use_error:\n curve = np.array([metric.convert_score_to_error(val) for val in curve])\n\n assert np.all(curve >= 0)\n\n\n@pytest.mark.parametrize(\"problem_type, model\", get_all_models())\ndef test_with_test_data(problem_type, model, get_dataset_map, get_default_metrics):\n init_args = {\n \"eval_metric\": get_default_metrics[problem_type],\n \"verbosity\": 4,\n }\n\n fit_args = dict(\n learning_curves=True,\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n common_args[\"sample_size\"] = 1000\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n init_args=init_args,\n fit_args=fit_args,\n use_test_data=True,\n use_test_for_val=True,\n **common_args,\n )\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n eval_sets, _, data = model_data[model]\n metric_count, eval_set_count, _ = np.array(data).shape\n\n assert eval_set_count == 3\n\n # ensure test_data preprocessing is same as val_data preprocessing\n eval_sets = [[], [], []]\n\n for m in range(metric_count):\n for e in range(eval_set_count):\n eval_sets[e].append(data[m][e])\n\n _, val, test = eval_sets\n assert val == test\n\n\n# TODO: how should we limit test parameters here? 22.5 min is much too long, but curve correctness\n# is a crucial aspect of learning curve generation that should be tested well\n# takes 8.7 minutes for full test run (only correctness tests)\n@pytest.mark.parametrize(\"problem_type, model, metric\", get_subset_model_problem_metrics())\ndef test_correctness(problem_type, model, metric, get_dataset_map):\n metric = get_metric(metric, problem_type, \"eval_metric\")\n\n init_args = {\n \"eval_metric\": metric,\n \"verbosity\": 4,\n }\n\n fit_args = dict(\n learning_curves={\n \"metrics\": metric,\n \"use_error\": True,\n },\n hyperparameters={\n model: extended_run_hyperparams[model],\n },\n )\n\n args = common_args.copy()\n args[\"sample_size\"] = 1000\n if metric.name == \"roc_auc\":\n args[\"sample_size\"] = 10000\n\n dataset_name = get_dataset_map[problem_type]\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, init_args=init_args, fit_args=fit_args, **args\n )\n\n model = get_default_model_name(model)\n _, model_data = predictor.learning_curves()\n eval_sets, _, data = model_data[model]\n\n def error(tabular_predictor: TabularPredictor):\n df = tabular_predictor.leaderboard(score_format=\"error\")\n return list(df[df[\"model\"] == model][\"metric_error_val\"])[0]\n\n def equal(a, b):\n if metric.needs_proba or metric.needs_threshold or problem_type == \"regression\":\n tol = 1e-06 if metric.name != \"roc_auc\" else 1e-02\n return math.isclose(a, b, rel_tol=tol)\n return a == b\n\n val_index = eval_sets.index(\"val\")\n curve = data[0][val_index] # get default eval_metric curve on validation dataset\n best = min(curve)\n\n assert len(curve) == model_iterations[model]\n assert equal(best, error(predictor))\n\n fit_args[\"learning_curves\"] = False\n clean_predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, init_args=init_args, fit_args=fit_args, **args\n )\n\n assert equal(error(predictor), error(clean_predictor))\n\n\n@pytest.mark.parametrize(\"learning_curve_supported_class\", [LGBModel, XGBoostModel, TabularNeuralNetTorchModel])\ndef test_supported_class_tags(learning_curve_supported_class):\n assert learning_curve_supported_class._get_class_tags().get(\"supports_learning_curves\", False)\n\n\n@pytest.fixture()\ndef get_dataset_map():\n return {\n BINARY: \"toy_binary_10\",\n MULTICLASS: \"toy_multiclass_30\",\n REGRESSION: \"toy_regression_10\",\n }\n\n\n@pytest.fixture()\ndef get_default_metrics():\n return {\n BINARY: \"accuracy\",\n MULTICLASS: \"log_loss\",\n REGRESSION: \"rmse\",\n }\n"
},
{
"path": "tabular/tests/unittests/dynamic_stacking/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/dynamic_stacking/test_dynamic_stacking.py",
"content": "import shutil\n\nimport pytest\n\nfrom autogluon.core.constants import BINARY\nfrom autogluon.core.metrics import METRICS\nfrom autogluon.tabular.testing import FitHelper\nfrom autogluon.tabular.testing.fit_helper import stacked_overfitting_assert\n\nDS_ARGS_TEST_DEFAULTS = dict(\n validation_procedure=\"holdout\",\n detection_time_frac=1 / 4,\n holdout_frac=1 / 9,\n n_folds=2,\n n_repeats=1,\n memory_safe_fits=True,\n clean_up_fits=True,\n holdout_data=None,\n)\n\n\ndef test_spot_and_avoid_stacked_overfitting():\n \"\"\"Tests that dynamic stacking works.\"\"\"\n fit_args = dict(\n hyperparameters={\"RF\": {}, \"GBM\": {}},\n fit_weighted_ensemble=False,\n dynamic_stacking=True,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n time_limit=None,\n ds_args=DS_ARGS_TEST_DEFAULTS,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"adult\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n extra_metrics=extra_metrics,\n expected_model_count=2,\n refit_full=False,\n allowed_dataset_features=[\"age\"],\n expected_stacked_overfitting_at_test=False,\n expected_stacked_overfitting_at_val=True,\n )\n\n\ndef test_dynamic_stacking_hps():\n \"\"\"Tests dynamic stacking arguments.\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=False,\n dynamic_stacking=True,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n time_limit=None,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n\n # Get custom val data (the test data)\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=\"adult\", directory_prefix=\"./datasets/\")\n label = dataset_info[\"label\"]\n allowed_cols = [\"age\", label]\n train_data = train_data[allowed_cols]\n test_data = test_data[allowed_cols]\n n_test_data = len(test_data)\n\n for ds_args_update in [\n dict(validation_procedure=\"holdout\", holdout_frac=1 / 5), # holdout\n dict(validation_procedure=\"cv\"), # 2-fold CV\n dict(validation_procedure=\"cv\", n_repeats=2), # 2-repeated 2-fold CV\n dict(memory_safe_fits=False, clean_up_fits=False), # fit options False\n dict(holdout_data=test_data),\n dict(holdout_data=test_data, validation_procedure=\"cv\", expect_raise=ValueError),\n ]:\n expect_raise = ds_args_update.pop(\"expect_raise\", None)\n tmp_ds_args = DS_ARGS_TEST_DEFAULTS.copy()\n if ds_args_update is not None:\n tmp_ds_args.update(ds_args_update)\n tmp_fit_args = fit_args.copy()\n tmp_fit_args[\"ds_args\"] = tmp_ds_args\n if expect_raise is None:\n predictor = FitHelper.fit_dataset(\n train_data=train_data, init_args=dict(label=label), fit_args=tmp_fit_args, sample_size=1000\n )\n if (\"holdout_data\" in ds_args_update) and (ds_args_update[\"holdout_data\"] is not None):\n n_expected = 1000 + n_test_data\n assert len(predictor.predict_oof()) == n_expected, \"Verify that holdout data was used for training\"\n lb = predictor.leaderboard(test_data, extra_info=True)\n stacked_overfitting_assert(lb, predictor, False, False)\n shutil.rmtree(predictor.path)\n else:\n with pytest.raises(expect_raise):\n FitHelper.fit_dataset(\n train_data=train_data, init_args=dict(label=label), fit_args=tmp_fit_args, sample_size=1000\n )\n\n\ndef test_no_dynamic_stacking():\n \"\"\"Tests that dynamic stacking does not run if stacking is disabled.\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n dynamic_stacking=True,\n fit_weighted_ensemble=False,\n num_stack_levels=0,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"adult\"\n extra_metrics = list(METRICS[BINARY])\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n extra_metrics=extra_metrics,\n expected_model_count=1,\n refit_full=False,\n )\n assert predictor._stacked_overfitting_occurred is None\n\n\ndef test_dynamic_stacking_fit_extra():\n \"\"\"Tests that fit_extra works after dynamic stacking.\"\"\"\n fit_args = dict(\n hyperparameters={\"RF\": {}},\n dynamic_stacking=True,\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=1,\n num_stack_levels=1,\n ds_args=DS_ARGS_TEST_DEFAULTS,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"adult\"\n extra_metrics = list(METRICS[BINARY])\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n extra_metrics=extra_metrics,\n expected_model_count=1,\n refit_full=False,\n delete_directory=False,\n allowed_dataset_features=[\"age\"],\n expected_stacked_overfitting_at_test=False,\n # This also check that we only consider something to be stacked overfitting if the dynamic stacking holdout score gets worse.\n expected_stacked_overfitting_at_val=True,\n )\n\n fit_extra_args = dict(\n hyperparameters={\"GBM\": {}},\n fit_weighted_ensemble=False,\n )\n\n predictor.fit_extra(**fit_extra_args)\n\n assert len(predictor.model_names()) == 2\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_dynamic_stacking_with_time_limit():\n \"\"\"Tests that dynamic stacking does not run if stacking is disabled.\"\"\"\n ds_args = DS_ARGS_TEST_DEFAULTS.copy()\n ds_args[\"holdout_frac\"] = 0.5\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n dynamic_stacking=True,\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=1,\n num_stack_levels=1,\n time_limit=60, # won't take 60s, but we need a number here instead of None.\n ds_args=ds_args,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"adult\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n extra_metrics=extra_metrics,\n expected_model_count=2,\n refit_full=False,\n delete_directory=False,\n allowed_dataset_features=[\"age\"],\n expected_stacked_overfitting_at_test=False,\n expected_stacked_overfitting_at_val=False,\n )\n\n\n@pytest.mark.timeout(\n 120\n) # if running AutoGluon twice fails due to a multiprocessing bug, we want to hang up and crash.\ndef test_dynamic_stacking_run_twice_parallel_fold_fitting_strategy():\n \"\"\"Tests that dynamic stacking memory save fit works.\"\"\"\n ds_args = DS_ARGS_TEST_DEFAULTS.copy()\n ds_args[\"memory_safe_fits\"] = True # guarantee for sanity\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=False,\n dynamic_stacking=True,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n time_limit=None,\n ds_args=ds_args,\n )\n\n # Get custom val data (the test data)\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=\"adult\", directory_prefix=\"./datasets/\")\n label = dataset_info[\"label\"]\n allowed_cols = [\"age\", label]\n train_data = train_data[allowed_cols]\n test_data = test_data[allowed_cols]\n\n for _ in range(2):\n predictor = FitHelper.fit_dataset(\n train_data=train_data, init_args=dict(label=label), fit_args=fit_args, sample_size=1000\n )\n lb = predictor.leaderboard(test_data, extra_info=True)\n stacked_overfitting_assert(lb, predictor, False, False)\n shutil.rmtree(predictor.path)\n"
},
{
"path": "tabular/tests/unittests/edgecases/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/edgecases/test_edgecases.py",
"content": "import shutil\nfrom pathlib import Path\n\nimport pytest\n\nfrom autogluon.core.constants import BINARY\nfrom autogluon.core.metrics import METRICS\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_no_weighted_ensemble():\n \"\"\"Tests that fit_weighted_ensemble=False works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=False,\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=1\n )\n\n\ndef test_no_full_last_level_weighted_ensemble():\n \"\"\"Tests that fit_weighted_ensemble=False works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=True,\n fit_full_last_level_weighted_ensemble=False,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=4\n )\n\n\ndef test_no_full_last_level_weighted_ensemble_additionally():\n \"\"\"Tests that fit_weighted_ensemble=False works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=True,\n fit_full_last_level_weighted_ensemble=False,\n full_weighted_ensemble_additionally=False,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=4\n )\n\n\ndef test_full_last_level_weighted_ensemble_additionally():\n \"\"\"Tests that fit_weighted_ensemble=False works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=True,\n fit_full_last_level_weighted_ensemble=True,\n full_weighted_ensemble_additionally=True,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=5\n )\n\n fit_args[\"num_stack_levels\"] = 0\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=2\n )\n\n\ndef test_full_last_level_weighted_ensemble():\n \"\"\"Tests that fit_weighted_ensemble=False works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=True,\n fit_full_last_level_weighted_ensemble=True,\n num_stack_levels=1,\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, expected_model_count=4\n )\n\n\ndef test_max_sets():\n \"\"\"Tests that max_sets works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {\"ag_args_ensemble\": {\"max_sets\": 3}}},\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=5,\n )\n dataset_name = \"toy_binary\"\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=1,\n refit_full=False,\n delete_directory=False,\n )\n leaderboard = predictor.leaderboard(extra_info=True)\n # 2 folds * 3 sets = 6\n assert leaderboard.iloc[0][\"num_models\"] == 6\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_num_folds():\n \"\"\"Tests that num_folds works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {\"ag_args_ensemble\": {\"num_folds\": 3}}},\n fit_weighted_ensemble=False,\n num_bag_folds=7,\n num_bag_sets=2,\n )\n dataset_name = \"toy_binary\"\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=1,\n refit_full=False,\n delete_directory=False,\n )\n leaderboard = predictor.leaderboard(extra_info=True)\n # 3 folds * 2 sets = 6\n assert leaderboard.iloc[0][\"num_models\"] == 6\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_num_folds_hpo():\n \"\"\"Tests that num_folds works\"\"\"\n fit_args = dict(\n hyperparameters={\"GBM\": {\"ag_args_ensemble\": {\"num_folds\": 2}}},\n fit_weighted_ensemble=False,\n num_bag_folds=5,\n num_bag_sets=2,\n hyperparameter_tune_kwargs={\n \"searcher\": \"random\",\n \"scheduler\": \"local\",\n \"num_trials\": 2,\n },\n )\n dataset_name = \"toy_binary\"\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=2,\n refit_full=False,\n delete_directory=False,\n )\n leaderboard = predictor.leaderboard(extra_info=True)\n # 2 folds * 2 sets = 4\n assert leaderboard.iloc[0][\"num_models\"] == 4\n assert leaderboard.iloc[1][\"num_models\"] == 4\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_use_bag_holdout_calibrate():\n \"\"\"\n Test that use_bag_holdout=True works for calibration\n Ensures the bug is fixed in https://github.com/autogluon/autogluon/issues/2674\n \"\"\"\n init_args = dict(eval_metric=\"log_loss\")\n\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n num_bag_folds=2,\n use_bag_holdout=True,\n calibrate=True,\n )\n\n dataset_name = \"adult\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n init_args=init_args,\n fit_args=fit_args,\n expected_model_count=2,\n refit_full=False,\n )\n\n\ndef test_num_folds_parallel(capsys):\n \"\"\"Tests that num_folds_parallel equal to 1 works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble=dict(num_folds_parallel=1),\n )\n dataset_name = \"toy_binary\"\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=1,\n refit_full=False,\n delete_directory=False,\n )\n leaderboard = predictor.leaderboard(extra_info=True)\n assert leaderboard.iloc[0][\"num_models\"] == 2\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_raises_num_cpus_float():\n \"\"\"Tests that num_cpus specified as a float raises a TypeError\"\"\"\n fit_args = dict(num_cpus=1.0)\n dataset_name = \"toy_binary\"\n with pytest.raises(TypeError, match=r\"`num_cpus` must be an int or 'auto'. Found: .*\"):\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=None,\n delete_directory=True,\n )\n\n\ndef test_raises_num_cpus_zero():\n \"\"\"Tests that num_cpus=0 raises a ValueError\"\"\"\n fit_args = dict(num_cpus=0)\n dataset_name = \"toy_binary\"\n with pytest.raises(ValueError, match=r\"`num_cpus` must be greater than or equal to 1. .*\"):\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=None,\n delete_directory=True,\n )\n\n\ndef test_raises_num_gpus_neg():\n \"\"\"Tests that num_gpus<0 raises a ValueError\"\"\"\n fit_args = dict(num_gpus=-1)\n dataset_name = \"toy_binary\"\n with pytest.raises(ValueError, match=r\"`num_gpus` must be greater than or equal to 0. .*\"):\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=None,\n delete_directory=True,\n )\n\n\n@pytest.mark.parametrize(\"delay_bag_sets\", [True, False])\ndef test_delay_bag_sets(delay_bag_sets):\n \"\"\"Tests that max_sets works\"\"\"\n fit_args = dict(\n hyperparameters={\"DUMMY\": [{}, {}]},\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=2,\n time_limit=30, # has no impact, but otherwise `delay_bag_sets` is ignored.\n delay_bag_sets=delay_bag_sets,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"toy_binary\"\n\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=2,\n refit_full=False,\n delete_directory=False,\n )\n\n # Verify fit order is correct.\n model_1 = predictor._trainer.load_model(\"Dummy_BAG_L1\")\n max_model_times_1 = max([(Path(model_1.path) / bm).stat().st_mtime_ns for bm in model_1.models])\n\n model_2 = predictor._trainer.load_model(\"Dummy_2_BAG_L1\")\n min_model_times_2 = min([(Path(model_2.path) / bm).stat().st_mtime_ns for bm in model_2.models])\n\n if delay_bag_sets:\n # Last model of model 1 should be trained after fist model of model 2\n assert max_model_times_1 > min_model_times_2\n else:\n # Last model of model 1 should be trained before fist model of model 2\n assert max_model_times_1 <= min_model_times_2\n\n shutil.rmtree(predictor.path, ignore_errors=True)\n"
},
{
"path": "tabular/tests/unittests/experimental/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/experimental/test_scikit_api.py",
"content": "from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION\nfrom autogluon.core.metrics import METRICS\nfrom autogluon.tabular.models.lgb.lgb_model import LGBModel\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_scikit_api_binary():\n \"\"\"Additionally tests that all binary metrics work\"\"\"\n fit_args = dict(\n hyperparameters={LGBModel: {}},\n )\n dataset_name = \"toy_binary\"\n extra_metrics = list(METRICS[BINARY])\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, scikit_api=True\n )\n\n\ndef test_scikit_api_multiclass():\n \"\"\"Additionally tests that all multiclass metrics work\"\"\"\n fit_args = dict(\n hyperparameters={LGBModel: {}},\n )\n extra_metrics = list(METRICS[MULTICLASS])\n\n dataset_name = \"toy_multiclass\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, scikit_api=True\n )\n\n\ndef test_scikit_api_regression():\n \"\"\"Additionally tests that all regression metrics work\"\"\"\n fit_args = dict(\n hyperparameters={LGBModel: {}},\n )\n extra_metrics = list(METRICS[REGRESSION])\n\n dataset_name = \"toy_regression\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, extra_metrics=extra_metrics, scikit_api=True\n )\n"
},
{
"path": "tabular/tests/unittests/models/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/models/advanced/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/models/advanced/test_bagged_deterministic.py",
"content": "\"\"\"\nUnit tests to ensure correctness of internal stacking logic.\n\"\"\"\n\nimport os\nimport shutil\nimport uuid\n\nfrom pandas.testing import assert_frame_equal, assert_series_equal\n\nfrom autogluon.core.utils import generate_train_test_split_combined\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper\n\n\n# TODO: Note that parallel fit can change predictor.model_names order depending on which model finishes fitting first, rather than by priority order.\n# We may want to find a way to make it be the same `predictor.model_names` order based on priority, or at least make input to stacker models be the same\n# This issue can theoretically lead to non-deterministic results in rare cases / edge cases\n# Might be able to fix by not using `list(model_graph.nodes)` to produce model names, but instead keeping track of ordered model names as a variable in trainer\n# Note: FASTAI produces a different result when fit with parallel mode when the cpus per fold/model differ.\n# The difference is usually extremely small (seems to be numerical precision), unless it impacts the early stopping iteration.\ndef test_bagged_deterministic():\n \"\"\"\n Tests that bagged models get a deterministic result, regardless of how they are trained\n\n Tests 4 configs:\n sequential fit + sequential bag\n sequential fit + parallel bag\n parallel fit + parallel bag\n parallel fit + sequential bag\n \"\"\"\n sample_size = 100\n dataset_name = \"adult\"\n directory_prefix = \"./datasets/\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name, directory_prefix=directory_prefix)\n\n label = dataset_info[\"label\"]\n\n init_args = dict(label=label, eval_metric=\"log_loss\", problem_type=dataset_info[\"problem_type\"])\n\n save_path = os.path.join(directory_prefix, dataset_name, f\"AutogluonOutput_{uuid.uuid4()}\")\n\n train_data, _ = generate_train_test_split_combined(\n data=train_data,\n label=init_args[\"label\"],\n problem_type=init_args[\"problem_type\"],\n train_size=sample_size,\n )\n test_data, _ = generate_train_test_split_combined(\n data=test_data,\n label=init_args[\"label\"],\n problem_type=init_args[\"problem_type\"],\n train_size=sample_size,\n )\n\n fit_args = dict(\n train_data=train_data,\n hyperparameters={\n \"GBM\": {},\n \"DUMMY\": {},\n # \"FASTAI\": {}, # Note: FASTAI is not identical between sequential and parallel fit if CPU counts differ during model fit\n },\n num_bag_folds=4,\n num_stack_levels=1,\n calibrate=True, # ensure calibration is also deterministic\n )\n\n # sequential fit with sequential bag\n predictor_seq_1 = TabularPredictor(path=save_path + \"seq_1\", **init_args).fit(\n fit_strategy=\"sequential\", ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"}, **fit_args\n )\n\n # sequential fit with parallel bag\n predictor_seq_2 = TabularPredictor(path=save_path + \"seq_2\", **init_args).fit(\n fit_strategy=\"sequential\", **fit_args\n )\n\n # parallel fit with sequential bag\n predictor_par_1 = TabularPredictor(path=save_path + \"par_1\", **init_args).fit(\n fit_strategy=\"parallel\", ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"}, **fit_args\n )\n\n # parallel fit with parallel bag\n predictor_par_2 = TabularPredictor(path=save_path + \"par_2\", **init_args).fit(fit_strategy=\"parallel\", **fit_args)\n\n p = predictor_seq_1\n\n y_pred = p.predict(test_data)\n y_pred_proba = p.predict_proba(test_data)\n scores = p.evaluate(test_data)\n model_names = p.model_names()\n\n features_per_model = {}\n child_order_per_model = {}\n for model_name in model_names:\n m = p._trainer.load_model(model_name)\n features_per_model[model_name] = m.features\n child_order_per_model[model_name] = p._trainer.load_model(model_name).models\n\n for p2 in [predictor_seq_2, predictor_par_1, predictor_par_2]:\n y_pred_2 = p2.predict(test_data)\n y_pred_proba_2 = p2.predict_proba(test_data)\n scores_2 = p2.evaluate(test_data)\n model_names_2 = p2.model_names()\n\n # TODO: To truly ensure equivalence, these should be the same order.\n # Currently this is not guaranteed because the order is based on the model fit order which is non-deterministic for parallel fit.\n # assert model_names == model_names_2\n assert sorted(model_names) == sorted(model_names_2)\n\n for model_name in model_names:\n m = p2._trainer.load_model(model_name)\n # To ensure equivalence, these should be the same order.\n # Regardless of which order models finish training in parallel,\n # they must be specified in the same order for the out-of-fold predictions to stacker models\n # This matters for edge-cases where a model would use the column order to determine what to do.\n # For example, during tie-breaks in the weighted ensemble.\n assert features_per_model[model_name] == m.features\n assert sorted(features_per_model[model_name]) == sorted(m.features)\n\n child_models = child_order_per_model[model_name]\n child_models_2 = m.models\n\n if not isinstance(child_models[0], str):\n child_models = [c.name for c in child_models]\n child_models_2 = [c.name for c in child_models_2]\n\n # This order must be equivalent, otherwise when averaging the predictions of the fold models\n # numerical imprecision causes slight differences in results\n assert child_models == child_models_2\n\n assert_series_equal(y_pred, y_pred_2, check_exact=True)\n assert_frame_equal(y_pred_proba, y_pred_proba_2, check_exact=True)\n assert scores == scores_2\n\n for predictor in [predictor_seq_1, predictor_seq_2, predictor_par_1, predictor_par_2]:\n shutil.rmtree(predictor.path, ignore_errors=True)\n"
},
{
"path": "tabular/tests/unittests/models/advanced/test_model_random_seed.py",
"content": "\"\"\"\nUnit tests to ensure correctness random seed logic\n\"\"\"\n\nimport os\nimport shutil\nimport uuid\nfrom copy import deepcopy\n\nimport pytest\n\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper\n\nTEST_CASES = []\n\nfor model_key, model_hps in [\n (\"GBM\", {\"num_boost_round\": 10}),\n (\"KNN\", {\"n_neighbors\": 2, \"ag_args_ensemble\": {\"use_child_oof\": True}}),\n (\"GBM\", {\"num_boost_round\": 10, \"ag_args_ensemble\": {\"refit_folds\": True}}),\n # # The below is only for full extended tests only for sanity check, not the CI\n # (\"CAT\", {\"iterations\": 10}),\n # (\"FASTAI\", {\"epochs\": 10}),\n # (\"LR\", {\"max_iter\": 10}),\n # (\"REALMLP\", {\"n_epochs\": 5}),\n # (\"TABM\", {\"n_epochs\": 10}),\n # (\"TABPFNMIX\", {}),\n # (\"TABPFNV2\", {}),\n # (\"NN_TORCH\", {\"num_epochs\": 10}),\n # (\"XGB\", {\"n_estimators\": 10}),\n # # Refit\n # (\"TABICL\", {\"ag_args_ensemble\": {\"refit_folds\": False}}),\n # (\"TABPFNV2\", {\"ag_args_ensemble\": {\"refit_folds\": False}}),\n # (\"MITRA\", {\"ag_args_ensemble\": {\"refit_folds\": False}}),\n # # OOF fit\n # (\"KNN\", {\"n_neighbors\": 2, \"ag_args_ensemble\": {\"use_child_oof\": False}}), # not enough samples?\n # (\"RF\", {\"n_estimators\": 10, \"ag_args_ensemble\": {\"use_child_oof\": False}}), # counts for RF and XT\n]:\n # Different fixed seed\n TEST_CASES.append(({model_key: model_hps}, {\"vary_seed_across_folds\": False, \"model_random_seed\": 42}, [42] * 3))\n # Vary default\n TEST_CASES.append(({model_key: model_hps}, {\"vary_seed_across_folds\": True}, list(range(3))))\n # No vary default\n TEST_CASES.append(\n ({model_key: model_hps}, {\"vary_seed_across_folds\": False}, [0] * 3),\n )\n # Two models, two different sets of seeds\n model_hps_2nd_model = deepcopy(model_hps)\n if \"ag_args_ensemble\" not in model_hps_2nd_model:\n model_hps_2nd_model[\"ag_args_ensemble\"] = {}\n model_hps_2nd_model[\"ag_args_ensemble\"][\"model_random_seed\"] = 42\n TEST_CASES.append(\n ({model_key: [model_hps, model_hps_2nd_model]}, {\"vary_seed_across_folds\": True}, ([0, 1, 2], [42, 43, 44]))\n )\n # Vary via model HPs instead of fit args\n model_hps = deepcopy(model_hps)\n if \"ag_args_ensemble\" not in model_hps:\n model_hps[\"ag_args_ensemble\"] = {}\n model_hps[\"ag_args_ensemble\"][\"vary_seed_across_folds\"] = False\n model_hps[\"ag_args_ensemble\"][\"model_random_seed\"] = 42\n TEST_CASES.append(({model_key: model_hps}, {}, [42] * 3))\n\n\n@pytest.mark.parametrize(\n \"hyperparameters, ag_args_ensemble, expected_random_seeds\",\n TEST_CASES,\n)\ndef test_bagged_random_seed(hyperparameters, ag_args_ensemble, expected_random_seeds):\n \"\"\"\n Tests that the random seeds for bagged models are correct.\n\n Tests 4 fit types:\n sequential fit + sequential bag\n sequential fit + parallel bag\n parallel fit + parallel bag\n parallel fit + sequential bag\n \"\"\"\n directory_prefix = \"./datasets/\"\n dataset_name = \"toy_binary\"\n train_data, _, dataset_info = FitHelper.load_dataset(name=dataset_name, directory_prefix=directory_prefix)\n label = dataset_info[\"label\"]\n init_args = dict(label=label, eval_metric=\"log_loss\", problem_type=dataset_info[\"problem_type\"])\n save_path = os.path.join(directory_prefix, dataset_name, f\"AutogluonOutput_{uuid.uuid4()}\")\n\n fit_args = dict(\n train_data=train_data,\n num_bag_folds=3,\n num_stack_levels=0,\n calibrate=False, # ensure calibration is also deterministic\n dynamic_stacking=False,\n fit_weighted_ensemble=False,\n )\n\n for fit_strategy, fold_fitting_strategy in [\n (\"sequential\", \"sequential_local\"),\n (\"sequential\", None),\n (\"parallel\", \"sequential_local\"),\n (\"parallel\", None),\n ]:\n if fold_fitting_strategy is not None:\n ag_args_ensemble = deepcopy(ag_args_ensemble)\n ag_args_ensemble[\"fold_fitting_strategy\"] = fold_fitting_strategy\n\n predictor = TabularPredictor(path=save_path, **init_args).fit(\n hyperparameters=hyperparameters,\n fit_strategy=fit_strategy,\n ag_args_ensemble=ag_args_ensemble,\n **fit_args,\n )\n\n for model_i, model_name in enumerate(predictor.model_names()):\n model = predictor._trainer.load_model(model_name)\n _expected_random_seeds = (\n expected_random_seeds[model_i] if isinstance(expected_random_seeds, tuple) else expected_random_seeds\n )\n for child_i, child_name in enumerate(model.models):\n child_model = model.load_child(child_name)\n expected_random_seed = _expected_random_seeds[child_i]\n assert child_model.random_seed == expected_random_seed, (\n f\"Random seed for bagged model should be {expected_random_seed}, but got {child_model.random_seed}\"\n )\n assert os.path.realpath(save_path) == os.path.realpath(predictor.path)\n shutil.rmtree(save_path, ignore_errors=True)\n"
},
{
"path": "tabular/tests/unittests/models/advanced/test_refit_full.py",
"content": "import shutil\n\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_refit_full_train_data_extra():\n \"\"\"\n Verifies that `refit_full(train_data_extra)` works.\n \"\"\"\n dataset = \"adult\" # Need a dataset with categorical features + NaNs in categories\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset)\n\n len_train = len(train_data)\n len_test = len(test_data)\n len_combined = len_train + len_test\n\n predictor = TabularPredictor(label=dataset_info[\"label\"], problem_type=dataset_info[\"problem_type\"])\n\n predictor.fit(\n train_data=train_data,\n hyperparameters={\"NN_TORCH\": {\"num_epochs\": 1}},\n raise_on_model_failure=True,\n fit_weighted_ensemble=False,\n )\n\n assert len(predictor.model_names()) == 1\n model_name = predictor.model_names()[0]\n refit_model_map = predictor.refit_full(train_data_extra=test_data)\n refit_model_name = refit_model_map[model_name]\n\n assert len(predictor.model_names()) == 2\n\n refit_model_info = predictor.model_info(refit_model_name)\n\n # Ensure refit uses all of train_data and all of train_data_extra\n assert refit_model_info[\"num_samples\"] == len_combined\n\n predictor.predict(test_data, model=refit_model_name)\n\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_refit_full_train_data_extra_bag():\n \"\"\"\n Verifies that `refit_full(train_data_extra)` works when bagging\n \"\"\"\n dataset = \"adult\" # Need a dataset with categorical features + NaNs in categories\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset)\n\n len_train = len(train_data)\n len_test = len(test_data)\n len_combined = len_train + len_test\n\n predictor = TabularPredictor(label=dataset_info[\"label\"], problem_type=dataset_info[\"problem_type\"])\n\n predictor.fit(\n train_data=train_data,\n hyperparameters={\"NN_TORCH\": {\"num_epochs\": 1}},\n raise_on_model_failure=True,\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n\n assert len(predictor.model_names()) == 1\n model_name = predictor.model_names()[0]\n refit_model_map = predictor.refit_full(train_data_extra=test_data)\n refit_model_name = refit_model_map[model_name]\n\n assert len(predictor.model_names()) == 2\n\n refit_model_info = predictor.model_info(refit_model_name)\n\n # Ensure refit uses all of train_data and all of train_data_extra\n assert refit_model_info[\"num_samples\"] == len_train\n assert refit_model_info[\"children_info\"][\"S1F1\"][\"num_samples\"] == len_combined\n\n predictor.predict(test_data, model=refit_model_name)\n\n shutil.rmtree(predictor.path, ignore_errors=True)\n"
},
{
"path": "tabular/tests/unittests/models/advanced/test_stack_feature_usage.py",
"content": "\"\"\"\nUnit tests to ensure correctness of internal stacking logic.\n\"\"\"\n\nimport shutil\n\nfrom autogluon.common.features.types import S_STACK\nfrom autogluon.core.models.ensemble.stacker_ensemble_model import StackerEnsembleModel\nfrom autogluon.tabular.predictor import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_stack_feature_usage_binary():\n \"\"\"Tests that stacker models use base model predictions as features correctly for binary\"\"\"\n dataset_name = \"adult\"\n expected_ancestors = {\"LightGBM_BAG_L1\", \"Dummy_BAG_L1\"}\n _fit_predictor_stack_feature_usage(\n dataset_name=dataset_name,\n max_base_models_per_type=1,\n max_base_models=2,\n sample_size=100,\n expected_ancestors=expected_ancestors,\n )\n\n\ndef test_stack_feature_usage_multiclass():\n \"\"\"Tests that stacker models use base model predictions as features correctly for multiclass\"\"\"\n dataset_name = \"covertype_small\"\n expected_ancestors = {\"LightGBM_BAG_L1\", \"KNeighbors_BAG_L1\"}\n _fit_predictor_stack_feature_usage(\n dataset_name=dataset_name,\n max_base_models_per_type=1,\n max_base_models=2,\n sample_size=100,\n expected_ancestors=expected_ancestors,\n )\n\n\ndef test_stack_feature_usage_regression():\n \"\"\"Tests that stacker models use base model predictions as features correctly for regression\"\"\"\n dataset_name = \"ames\"\n expected_ancestors = {\"LightGBM_BAG_L1\", \"KNeighbors_BAG_L1\"}\n _fit_predictor_stack_feature_usage(\n dataset_name=dataset_name,\n max_base_models_per_type=1,\n max_base_models=2,\n sample_size=100,\n expected_ancestors=expected_ancestors,\n )\n\n\ndef test_stack_feature_usage_binary_all():\n \"\"\"Tests that stacker models use base model predictions as features correctly for binary\"\"\"\n dataset_name = \"adult\"\n expected_ancestors = {\"LightGBM_BAG_L1\", \"LightGBM_2_BAG_L1\", \"KNeighbors_BAG_L1\", \"Dummy_BAG_L1\"}\n _fit_predictor_stack_feature_usage(\n dataset_name=dataset_name,\n max_base_models_per_type=0, # uncapped\n max_base_models=0, # uncapped\n sample_size=100,\n expected_ancestors=expected_ancestors,\n )\n\n\ndef test_stack_feature_usage_binary_only_max_models():\n \"\"\"Tests that stacker models use base model predictions as features correctly for binary\"\"\"\n dataset_name = \"adult\"\n expected_ancestors = {\"LightGBM_BAG_L1\", \"LightGBM_2_BAG_L1\"}\n _fit_predictor_stack_feature_usage(\n dataset_name=dataset_name,\n max_base_models_per_type=0, # uncapped\n max_base_models=2,\n sample_size=100,\n expected_ancestors=expected_ancestors,\n )\n\n\ndef _fit_predictor_stack_feature_usage(\n dataset_name: str,\n max_base_models_per_type: int,\n max_base_models: int,\n sample_size: int,\n expected_ancestors: set,\n):\n \"\"\"Tests that stacker models use base model predictions as features correctly\"\"\"\n fit_args = dict(\n hyperparameters={\n \"GBM\": [\n {\"num_iterations\": 200},\n {\"num_iterations\": 50},\n ],\n \"KNN\": {},\n \"DUMMY\": {},\n },\n ag_args_ensemble={\n \"max_base_models_per_type\": max_base_models_per_type,\n \"max_base_models\": max_base_models,\n \"fold_fitting_strategy\": \"sequential_local\",\n },\n num_bag_folds=2,\n num_stack_levels=1,\n fit_weighted_ensemble=False,\n )\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n refit_full=False,\n sample_size=sample_size,\n expected_model_count=7,\n delete_directory=False,\n )\n _assert_stack_features(predictor=predictor, expected_ancestors=expected_ancestors)\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef _assert_stack_features(predictor: TabularPredictor, expected_ancestors: set):\n \"\"\"\n Verifies that stack features are correctly set and used\n \"\"\"\n leaderboard = predictor.leaderboard(extra_info=True)\n leaderboard_l2 = leaderboard[leaderboard[\"stack_level\"] == 2]\n leaderboard_l2 = leaderboard_l2.set_index(\"model\")\n for m in leaderboard_l2.index:\n ancestors = set(leaderboard_l2.loc[m, \"ancestors\"])\n assert expected_ancestors == ancestors\n model = predictor._trainer.load_model(model_name=m)\n assert isinstance(model, StackerEnsembleModel)\n assert expected_ancestors == set(model.base_model_names)\n assert expected_ancestors == set(predictor._trainer.get_minimum_model_set(model=m, include_self=False))\n stack_columns = model.stack_columns\n stack_columns_fm = model.feature_metadata.get_features(required_special_types=[S_STACK])\n assert set(stack_columns) == set(stack_columns_fm)\n features = model.feature_metadata.get_features()\n assert set(features) == set(model.features)\n # Asserts that child models use the same features as parent models\n for child_model_name in model.models:\n child_model = model.load_child(model=child_model_name)\n child_stack_columns_fm = child_model.feature_metadata.get_features(required_special_types=[S_STACK])\n assert set(stack_columns) == set(child_stack_columns_fm)\n assert set(features) == set(child_model.features)\n assert model.feature_metadata == child_model.feature_metadata\n"
},
{
"path": "tabular/tests/unittests/models/test_advanced.py",
"content": "from autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.tabular.models.lgb.lgb_model import LGBModel\nfrom autogluon.tabular.testing import FitHelper, ModelFitHelper\n\n\ndef test_bagged_predict_children():\n fit_args = dict(k_fold=3)\n dataset_name = \"toy_binary\"\n model = BaggedEnsembleModel(\n model_base=LGBModel,\n model_base_kwargs=dict(hyperparameters=dict(num_boost_round=10)), # Speed up run\n hyperparameters={\"fold_fitting_strategy\": \"sequential_local\"}, # Speed up run\n )\n ModelFitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n model=model,\n fit_args=fit_args,\n check_predict_children=True,\n )\n\n\n# TODO: Test num_gpus>0\ndef test_resource_constraints():\n \"\"\"\n Verify that num_cpus and num_gpus are respected when specified in the fit call.\n Also verifies that constraints are respected for weighted ensemble models and during refit_full.\n Also verifies that specifying num_cpus > max_cpus will use max_cpus.\n \"\"\"\n num_gpus = 0\n\n max_cpus = ResourceManager.get_cpu_count()\n\n num_cpus_to_check = [1, 3]\n num_cpus_to_check = [c for c in num_cpus_to_check if c < max_cpus] + [max_cpus] + [max_cpus + 1000]\n\n for num_cpus in num_cpus_to_check:\n # Force DummyModel to raise an exception when fit.\n fit_args = dict(\n hyperparameters={\"GBM\": {\"num_boost_round\": 1}},\n num_cpus=num_cpus,\n num_gpus=num_gpus,\n refit_full=True,\n )\n\n dataset_name = \"toy_binary\"\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=4,\n refit_full=False,\n delete_directory=False,\n )\n\n info = predictor.info()\n\n for model in predictor.model_names():\n if num_cpus <= max_cpus:\n assert info[\"model_info\"][model][\"num_cpus\"] == num_cpus\n else:\n assert info[\"model_info\"][model][\"num_cpus\"] == max_cpus # Use max_cpus if num_cpus>max_cpus\n assert info[\"model_info\"][model][\"num_gpus\"] == num_gpus\n"
},
{
"path": "tabular/tests/unittests/models/test_automm.py",
"content": "import pytest\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper\n\n\n@pytest.mark.gpu\ndef test_automm_sts():\n if ResourceManager.get_gpu_count_torch() == 0:\n # Skip test if no GPU available\n pytest.skip(\"Skip, no GPU available.\")\n fit_args = dict(\n hyperparameters={\"AG_AUTOMM\": {\"env.num_workers\": 0, \"env.num_workers_inference\": 0}},\n time_limit=60,\n )\n dataset_name = \"sts\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n sample_size=100,\n refit_full=False,\n )\n\n\ndef test_handle_text_automm():\n hyperparameters = {\"AG_AUTOMM\": {}}\n\n assert TabularPredictor._check_if_hyperparameters_handle_text(hyperparameters)\n"
},
{
"path": "tabular/tests/unittests/models/test_catboost.py",
"content": "import sys\n\nimport pytest\n\nfrom autogluon.tabular.models.catboost.catboost_model import CatBoostModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"iterations\": 10}\n\n\n@pytest.mark.skipif(\n sys.version_info >= (3, 11) and sys.platform == \"darwin\", reason=\"catboost has no wheel for py311 darwin\"\n)\ndef test_catboost():\n model_cls = CatBoostModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\n@pytest.mark.parametrize(\"eval_metric\", [\"r2\", \"mean_absolute_error\"])\ndef test_catboost_can_train_with_nondefault_regression_eval_metrics(eval_metric):\n model_cls = CatBoostModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n init_args={\"eval_metric\": eval_metric},\n problem_types=[\"regression\"],\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_dummy.py",
"content": "import os\nfrom pathlib import Path\n\nimport pytest\n\nfrom autogluon.core.models.dummy.dummy_model import DummyModel\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.testing import FitHelper, ModelFitHelper\n\n\ndef test_no_models_will_raise():\n \"\"\"Tests that RuntimeError is raised when no models fit\"\"\"\n fit_args = dict(\n hyperparameters={},\n )\n\n dataset_name = \"toy_binary\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n\n with pytest.raises(RuntimeError):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n\ndef test_no_models():\n \"\"\"Tests that logic works properly when no models are trained and raise_on_no_models_fitted=False\"\"\"\n fit_args = dict(\n hyperparameters={},\n raise_on_no_models_fitted=False,\n )\n\n dataset_name = \"toy_binary\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n\n predictor = FitHelper.fit_dataset(\n train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args\n )\n\n assert not predictor.model_names()\n with pytest.raises(AssertionError):\n predictor.predict(test_data)\n assert len(predictor.leaderboard()) == 0\n assert len(predictor.leaderboard(test_data)) == 0\n assert len(predictor.model_failures()) == 0\n\n\ndef test_no_models_raise():\n \"\"\"Tests that logic works properly when no models are trained, and tests predictor.model_failures() and raise_on_no_models_fitted=False\"\"\"\n\n expected_exc_str = \"Test Error Message\"\n\n # Force DummyModel to raise an exception when fit.\n fit_args = dict(\n hyperparameters={DummyModel: {\"raise\": ValueError, \"raise_msg\": expected_exc_str}},\n raise_on_no_models_fitted=False,\n )\n\n dataset_name = \"toy_binary\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n\n predictor = FitHelper.fit_dataset(\n train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args\n )\n\n assert not predictor.model_names()\n with pytest.raises(AssertionError):\n predictor.predict(test_data)\n assert len(predictor.leaderboard()) == 0\n assert len(predictor.leaderboard(test_data)) == 0\n\n model_failures = predictor.model_failures()\n assert len(model_failures) == 1\n model_failures_dict = model_failures.iloc[0].to_dict()\n assert model_failures_dict[\"model\"] == \"Dummy\"\n assert model_failures_dict[\"model_type\"] == \"DummyModel\"\n assert model_failures_dict[\"exc_type\"] == \"ValueError\"\n assert model_failures_dict[\"exc_str\"] == expected_exc_str\n\n\ndef test_raise_on_model_failure():\n \"\"\"Tests that logic works properly when model raises exception and raise_on_model_failure=True\"\"\"\n\n expected_exc_str = \"Test Error Message\"\n\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=\"toy_binary\")\n\n # Force DummyModel to raise an exception when fit.\n fit_args = dict(\n hyperparameters={DummyModel: {\"raise\": ValueError, \"raise_msg\": expected_exc_str}},\n raise_on_model_failure=True,\n feature_generator=None,\n )\n\n with pytest.raises(ValueError) as excinfo:\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n assert str(excinfo.value) == \"Test Error Message\"\n\n\ndef test_raise_on_fit_args():\n \"\"\"Tests that ag.max_rows, ag.max_features, ag.max_classes, ag.problem_types work\"\"\"\n\n dataset_name = \"toy_binary\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_rows\": 1}]},\n raise_on_model_failure=True,\n )\n\n with pytest.raises(AssertionError, match=r\"ag.max_rows=1\"):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_rows\": 1, \"ag.ignore_constraints\": True}]},\n raise_on_model_failure=True,\n )\n\n # This works because ag.ignore_constraints is set to True, bypassing `ag.max_rows`\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n assert len(train_data) == 4\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_rows\": 3}]},\n raise_on_model_failure=True,\n )\n\n # This works because len(X) = 3 and len(X_val) = 1\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n # Check that bagging uses the full data for checking max rows\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_rows\": 3}]},\n raise_on_model_failure=True,\n num_bag_folds=4,\n )\n\n with pytest.raises(AssertionError, match=r\"ag.max_rows=3\"):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_features\": 0}]},\n raise_on_model_failure=True,\n )\n\n with pytest.raises(AssertionError, match=r\"ag.max_features=0\"):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_features\": 1}]},\n raise_on_model_failure=True,\n )\n\n # This works because len(X.columns) == 1\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_classes\": 1}]},\n raise_on_model_failure=True,\n )\n\n with pytest.raises(AssertionError, match=r\"ag.max_classes=1\"):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.max_classes\": 2}]},\n raise_on_model_failure=True,\n )\n\n # This works because self.num_classes = 2\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.problem_types\": [\"abc\", \"multiclass\", \"regression\"]}]},\n raise_on_model_failure=True,\n )\n\n with pytest.raises(AssertionError, match=r\"ag.problem_types=\\['abc', 'multiclass', 'regression'\\]\"):\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n fit_args = dict(\n hyperparameters={DummyModel: [{\"ag.problem_types\": [\"binary\", \"def\"]}]},\n raise_on_model_failure=True,\n )\n\n # This works because self.problem_type = \"binary\"\n FitHelper.fit_dataset(train_data=train_data, init_args=dict(label=dataset_info[\"label\"]), fit_args=fit_args)\n\n\ndef test_dummy():\n model_cls = DummyModel\n model_hyperparameters = {}\n\n \"\"\"Additionally tests that all metrics work\"\"\"\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters, extra_metrics=True)\n\n\ndef test_dummy_binary_model():\n fit_args = dict()\n dataset_name = \"toy_binary\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=DummyModel(), fit_args=fit_args)\n\n\ndef test_dummy_multiclass_model():\n fit_args = dict()\n dataset_name = \"toy_multiclass\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=DummyModel(), fit_args=fit_args)\n\n\ndef test_dummy_regression_model():\n fit_args = dict()\n dataset_name = \"toy_regression\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=DummyModel(), fit_args=fit_args)\n\n\ndef test_dummy_binary_absolute_path():\n \"\"\"Test that absolute path works\"\"\"\n fit_args = dict(\n hyperparameters={DummyModel: {}},\n )\n path = Path(\".\") / \"AG_test\"\n path = str(path.resolve())\n init_args = dict(path=path)\n\n dataset_name = \"toy_binary\"\n\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, init_args=init_args, fit_args=fit_args)\n\n\ndef test_dummy_binary_absolute_path_stack():\n \"\"\"Test that absolute path works\"\"\"\n fit_args = dict(\n hyperparameters={DummyModel: {}},\n num_bag_folds=2,\n num_bag_sets=2,\n num_stack_levels=1,\n )\n\n dataset_name = \"toy_binary\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, expected_model_count=4, path_as_absolute=True\n )\n\n\ndef test_dummy_binary_model_absolute_path():\n \"\"\"Test that absolute path works\"\"\"\n fit_args = dict()\n path = Path(\".\") / \"AG_test\"\n path = str(path.resolve())\n model = DummyModel(path=path)\n dataset_name = \"toy_binary\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=model, fit_args=fit_args)\n\n\ndef test_dummy_ag_ens_hyperparameter():\n \"\"\"\n Verifies that sending ag_args_ensemble arguments via the `ag.ens.` prefix works.\n \"\"\"\n hyperparameters = {\n \"ag.ens.fold_fitting_strategy\": \"sequential_local\",\n \"ag.ens.foo\": \"bar\",\n \"key1\": \"val1\",\n }\n fit_args = dict(\n hyperparameters={DummyModel: hyperparameters},\n num_bag_folds=2,\n )\n dataset_name = \"toy_binary\"\n\n predictor: TabularPredictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n delete_directory=False,\n refit_full=False,\n fit_weighted_ensemble=False,\n )\n assert len(predictor.model_names()) == 1\n model_name = predictor.model_names()[0]\n model_info = predictor.model_info(model=model_name)\n assert model_info[\"hyperparameters_user\"][\"fold_fitting_strategy\"] == \"sequential_local\"\n assert model_info[\"hyperparameters_user\"][\"foo\"] == \"bar\"\n assert model_info[\"hyperparameters\"][\"fold_fitting_strategy\"] == \"sequential_local\"\n assert model_info[\"hyperparameters\"][\"foo\"] == \"bar\"\n assert \"key1\" not in model_info[\"hyperparameters\"]\n assert model_info[\"bagged_info\"][\"child_hyperparameters\"] == {\"key1\": \"val1\"}\n"
},
{
"path": "tabular/tests/unittests/models/test_ebm.py",
"content": "from autogluon.tabular.models import EBMModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {}\n\n\ndef test_ebm():\n model_cls = EBMModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\ndef test_ebm_binary():\n fit_args = dict(\n hyperparameters={EBMModel: {}},\n )\n dataset_name = \"toy_binary\"\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, refit_full=False)\n\n\ndef test_ebm_multiclass():\n fit_args = dict(\n hyperparameters={EBMModel: {}},\n )\n dataset_name = \"covertype_small\"\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, refit_full=False)\n\n\ndef test_ebm_regression():\n fit_args = dict(\n hyperparameters={EBMModel: {}},\n )\n dataset_name = \"ames\"\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, refit_full=False)\n"
},
{
"path": "tabular/tests/unittests/models/test_image_predictor.py",
"content": "import pytest\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular import TabularPredictor\n\n\n@pytest.mark.gpu\ndef test_image_predictor_multiclass():\n if ResourceManager.get_gpu_count_torch() == 0:\n # Skip test if no GPU available\n pytest.skip(\"Skip, no GPU available.\")\n from autogluon.multimodal.utils.misc import shopee_dataset\n\n download_dir = \"./automm_shopee_data_multiclass\"\n train_data, test_data = shopee_dataset(download_dir)\n train_data = train_data.sample(100, random_state=0) # Subsample for faster test\n feature_metadata = FeatureMetadata.from_df(train_data).add_special_types({\"image\": [\"image_path\"]})\n predictor = TabularPredictor(label=\"label\").fit(\n train_data=train_data,\n hyperparameters={\"AG_IMAGE_NN\": {\"optim.max_epochs\": 1}},\n feature_metadata=feature_metadata,\n )\n leaderboard = predictor.leaderboard(test_data)\n assert len(leaderboard) > 0\n\n\n@pytest.mark.gpu\ndef test_image_predictor_regression():\n if ResourceManager.get_gpu_count_torch() == 0:\n # Skip test if no GPU available\n pytest.skip(\"Skip, no GPU available.\")\n from autogluon.multimodal.utils.misc import shopee_dataset\n\n download_dir = \"./automm_shopee_data_regression\"\n train_data, test_data = shopee_dataset(download_dir)\n train_data = train_data.sample(100, random_state=0) # Subsample for faster test\n feature_metadata = FeatureMetadata.from_df(train_data).add_special_types({\"image\": [\"image_path\"]})\n predictor = TabularPredictor(label=\"label\", problem_type=\"regression\").fit(\n train_data=train_data,\n hyperparameters={\"AG_IMAGE_NN\": {\"optim.max_epochs\": 1}},\n feature_metadata=feature_metadata,\n )\n leaderboard = predictor.leaderboard(test_data)\n assert len(leaderboard) > 0\n"
},
{
"path": "tabular/tests/unittests/models/test_knn.py",
"content": "from autogluon.tabular.models.knn.knn_model import KNNModel\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_knn():\n model_cls = KNNModel\n model_hyperparameters = {\"n_neighbors\": 2}\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n"
},
{
"path": "tabular/tests/unittests/models/test_lightgbm.py",
"content": "import numpy as np\nimport pytest\n\nfrom autogluon.tabular import TabularPredictor\nfrom autogluon.tabular.models.lgb.lgb_model import LGBModel\nfrom autogluon.tabular.testing import FitHelper, ModelFitHelper\n\n\ndef test_lightgbm():\n model_cls = LGBModel\n model_hyperparameters = {}\n\n \"\"\"Additionally tests that all metrics work\"\"\"\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters, extra_metrics=True)\n\n\ndef test_lightgbm_binary_model():\n fit_args = dict()\n dataset_name = \"toy_binary\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=LGBModel(), fit_args=fit_args)\n\n\ndef test_lightgbm_multiclass_model():\n fit_args = dict()\n dataset_name = \"toy_multiclass\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=LGBModel(), fit_args=fit_args)\n\n\ndef test_lightgbm_regression_model():\n fit_args = dict()\n dataset_name = \"toy_regression\"\n ModelFitHelper.fit_and_validate_dataset(dataset_name=dataset_name, model=LGBModel(), fit_args=fit_args)\n\n\ndef test_lightgbm_quantile_model():\n fit_args = dict()\n dataset_name = \"toy_quantile\"\n ModelFitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n model=LGBModel(\n problem_type=\"quantile\",\n hyperparameters={\"ag.quantile_levels\": [0.25, 0.5, 0.75]},\n ),\n fit_args=fit_args,\n )\n\n\ndef test_lightgbm_binary_with_calibrate_decision_threshold():\n \"\"\"Tests that calibrate_decision_threshold works and does not make the validation score worse on the given metric\"\"\"\n fit_args = dict(\n hyperparameters={LGBModel: {}},\n )\n dataset_name = \"toy_binary\"\n\n predictor: TabularPredictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, delete_directory=False, refit_full=False\n )\n\n for metric in [None, \"f1\", \"balanced_accuracy\", \"mcc\", \"recall\", \"precision\"]:\n decision_threshold = predictor.calibrate_decision_threshold(metric=metric)\n if metric is None:\n metric = predictor.eval_metric.name\n assert decision_threshold >= 0\n assert decision_threshold <= 1\n\n X_val, y_val = predictor.load_data_internal(data=\"val\", return_X=True, return_y=True)\n y_val = predictor.transform_labels(labels=y_val, inverse=True)\n\n y_pred_val = predictor.predict(data=X_val, transform_features=False)\n y_pred_val_w_decision_threshold = predictor.predict(\n data=X_val, decision_threshold=decision_threshold, transform_features=False\n )\n y_pred_multi_val_w_decision_threshold = predictor.predict_multi(\n data=X_val, decision_threshold=decision_threshold, transform_features=False\n )\n y_pred_multi_val_w_decision_threshold_cache = predictor.predict_multi(decision_threshold=decision_threshold)\n\n y_pred_proba_val = predictor.predict_proba(data=X_val, transform_features=False)\n y_pred_val_w_decision_threshold_from_proba = predictor.predict_from_proba(\n y_pred_proba=y_pred_proba_val, decision_threshold=decision_threshold\n )\n\n assert y_pred_val_w_decision_threshold.equals(y_pred_multi_val_w_decision_threshold[predictor.model_best])\n assert y_pred_val_w_decision_threshold.equals(\n y_pred_multi_val_w_decision_threshold_cache[predictor.model_best]\n )\n assert y_pred_val_w_decision_threshold.equals(y_pred_val_w_decision_threshold_from_proba)\n\n result = predictor.evaluate_predictions(y_true=y_val, y_pred=y_pred_val)\n result_calibrated = predictor.evaluate_predictions(y_true=y_val, y_pred=y_pred_val_w_decision_threshold)\n\n # Ensure validation score never becomes worse on the calibrated metric\n assert result[metric] <= result_calibrated[metric]\n if metric in [\"recall\"]:\n # recall should always be able to achieve a perfect validation score\n assert result_calibrated[metric] == 1.0\n\n assert predictor.calibrate_decision_threshold(metric=\"roc_auc\") == 0.5\n\n\ndef test_lightgbm_binary_with_calibrate_decision_threshold_bagged_refit():\n \"\"\"Tests that calibrate_decision_threshold works and does not make the validation score worse on the given metric\"\"\"\n fit_args = dict(\n hyperparameters={LGBModel: {}},\n num_bag_folds=2,\n calibrate_decision_threshold=True,\n )\n init_args = dict(eval_metric=\"f1\")\n dataset_name = \"toy_binary\"\n\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n label = dataset_info[\"label\"]\n predictor: TabularPredictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, init_args=init_args, fit_args=fit_args, delete_directory=False, refit_full=True\n )\n\n expected_decision_threshold = 0.499\n assert predictor._decision_threshold is not None\n assert np.isclose(predictor.decision_threshold, expected_decision_threshold)\n assert predictor.decision_threshold == predictor._decision_threshold\n optimal_decision_threshold = predictor.calibrate_decision_threshold()\n assert optimal_decision_threshold == predictor.decision_threshold\n og_threshold = predictor.decision_threshold\n\n y_pred_test = predictor.predict(test_data)\n\n scores_predictions = predictor.evaluate_predictions(y_true=test_data[label], y_pred=y_pred_test)\n scores = predictor.evaluate(test_data)\n scores_05 = predictor.evaluate(test_data, decision_threshold=0.5)\n\n for k in scores_predictions:\n assert scores[k] == scores_predictions[k]\n assert scores[\"f1\"] > scores_05[\"f1\"] # Calibration should help f1\n assert (\n scores[\"accuracy\"] == scores_05[\"accuracy\"]\n ) # Calibration should not change accuracy (for this specific dataset)\n\n predictor.set_decision_threshold(0.5)\n assert predictor.decision_threshold == 0.5\n assert predictor._decision_threshold == 0.5\n scores_05_native = predictor.evaluate(test_data)\n\n for k in scores_05:\n assert scores_05[k] == scores_05_native[k]\n\n leaderboard_05 = predictor.leaderboard(test_data)\n lb_score_05 = leaderboard_05[leaderboard_05[\"model\"] == predictor.model_best].iloc[0][\"score_test\"]\n assert lb_score_05 == scores_05[\"f1\"]\n\n predictor.set_decision_threshold(og_threshold)\n leaderboard = predictor.leaderboard(test_data)\n lb_score = leaderboard[leaderboard[\"model\"] == predictor.model_best].iloc[0][\"score_test\"]\n assert lb_score == scores[\"f1\"]\n\n\ndef test_clean_column_name_replaces_expected_symbols():\n assert LGBModel._clean_column_name_for_lgb('a\"b') == \"a_b\"\n assert LGBModel._clean_column_name_for_lgb(\"a,b\") == \"a_b\"\n assert LGBModel._clean_column_name_for_lgb(\"a:b\") == \"a_b\"\n assert LGBModel._clean_column_name_for_lgb(\"a{b}c\") == \"a_b_c\"\n assert LGBModel._clean_column_name_for_lgb(\"a[b]c\") == \"a_b_c\"\n\n\ndef test_rename_columns_outputs_are_unique_when_inputs_unique():\n features = ['a\"b', \"a,b\", \"x\", \"y\"]\n m = LGBModel._rename_columns(features)\n assert set(m.keys()) == set(features)\n assert len(set(m.values())) == len(features)\n\n\ndef test_rename_columns_resolves_cleaning_collisions_with_suffixes():\n features = ['a\"b', \"a,b\", \"a:b\"]\n m = LGBModel._rename_columns(features)\n # all clean to \"a_b\"\n assert m['a\"b'] == \"a_b\"\n assert m[\"a,b\"] == \"a_b_2\"\n assert m[\"a:b\"] == \"a_b_3\"\n assert len(set(m.values())) == 3\n\n\ndef test_rename_columns_avoids_clashing_with_existing_feature_names():\n # cleaned \"a\" for first, then second is literally \"a_2\"\n # ensure collision resolution doesn't produce duplicates\n features = [\"a\", \"a_2\", 'a\"2'] # 'a\"2' cleans to 'a_2'\n m = LGBModel._rename_columns(features)\n assert len(set(m.values())) == len(features)\n # expected behavior given algorithm/order:\n assert m[\"a\"] == \"a\"\n assert m[\"a_2\"] == \"a_2\"\n assert m['a\"2'] == \"a_2_2\" # because \"a_2\" already taken\n\n\ndef test_rename_columns_is_order_dependent_but_still_unique():\n f1 = ['a\"b', \"a,b\"]\n f2 = [\"a,b\", 'a\"b']\n m1 = LGBModel._rename_columns(f1)\n m2 = LGBModel._rename_columns(f2)\n\n assert len(set(m1.values())) == 2\n assert len(set(m2.values())) == 2\n # but the assignment of base vs suffixed swaps with order:\n assert m1['a\"b'] == \"a_b\"\n assert m1[\"a,b\"] == \"a_b_2\"\n assert m2[\"a,b\"] == \"a_b\"\n assert m2['a\"b'] == \"a_b_2\"\n\n\ndef test_non_string_features_are_returned_as_is_and_can_collide():\n # Demonstrates behavior: non-strings are not cleaned, but uniqueness is still enforced.\n features = [123, \"123\"]\n m = LGBModel._rename_columns(features)\n assert m[123] == 123\n assert m[\"123\"] == \"123\"\n assert len(set(m.values())) == 2\n\n\ndef test_duplicate_input_feature_names_raises_value_error():\n features = [\"dup\", \"dup\"]\n with pytest.raises(\n ValueError,\n match=\"features contains duplicates; cannot create 1-to-1 mapping with a dict.\",\n ):\n LGBModel._rename_columns(features)\n\n\ndef test_bool_int_key_collision_raises_value_error():\n # True == 1 in Python, so this is a duplicate under set/dict semantics\n features = [1, True]\n with pytest.raises(\n ValueError,\n match=\"features contains duplicates; cannot create 1-to-1 mapping with a dict.\",\n ):\n LGBModel._rename_columns(features)\n"
},
{
"path": "tabular/tests/unittests/models/test_lightgbm_prep.py",
"content": "from autogluon.tabular.models.tabprep.prep_lgb_model import PrepLGBModel\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_lightgbm():\n model_cls = PrepLGBModel\n model_hyperparameters = {\n \"ag.prep_params\": [\n [\n [\"ArithmeticFeatureGenerator\", {}],\n [\n [\"CategoricalInteractionFeatureGenerator\", {\"passthrough\": True}],\n [\"OOFTargetEncodingFeatureGenerator\", {}],\n ],\n ],\n ],\n \"ag.prep_params.passthrough_types\": {\"invalid_raw_types\": [\"category\", \"object\"]},\n }\n \"\"\"Additionally tests that all metrics work\"\"\"\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters, extra_metrics=True)\n"
},
{
"path": "tabular/tests/unittests/models/test_linear.py",
"content": "from autogluon.tabular.models.lr.lr_model import LinearModel\nfrom autogluon.tabular.testing import FitHelper\n\n\ndef test_linear():\n model_cls = LinearModel\n model_hyperparameters = {}\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n"
},
{
"path": "tabular/tests/unittests/models/test_mitra.py",
"content": "import pytest\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.tabular.models.mitra.mitra_model import MitraModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"fine_tune_steps\": 2}\n\n\n@pytest.mark.gpu\ndef test_mitra():\n if ResourceManager.get_gpu_count_torch() == 0:\n # Skip test if no GPU available\n pytest.skip(\"Skip, no GPU available.\")\n\n model_cls = MitraModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n # Mitra returns different predictions when predicting on an individual sample\n verify_single_prediction_equivalent_to_multi=False,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_realmlp.py",
"content": "from autogluon.tabular.models.realmlp.realmlp_model import RealMLPModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_epochs\": 2}\n\n\ndef test_realmlp():\n model_cls = RealMLPModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_rf.py",
"content": "import copy\n\nfrom autogluon.tabular.models.rf.rf_model import RFModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_estimators\": 10}\n\n\ndef test_rf():\n model_cls = RFModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\ndef test_rf_compile_onnx():\n model_cls = RFModel\n model_hyperparameters = toy_model_params\n compiler_configs = {RFModel: {\"compiler\": \"onnx\"}}\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n compile=True,\n compiler_configs=compiler_configs,\n # RandomForest ONNX compilation not supported for quantile\n problem_types=[\"binary\", \"multiclass\", \"regression\"],\n bag=False, # ONNX compilation fails in bagging with refit_full\n )\n\n\ndef test_rf_binary_compile_onnx_as_ag_arg():\n model_params = copy.deepcopy(toy_model_params)\n model_params[\"ag.compile\"] = {\"compiler\": \"onnx\"}\n\n fit_args = dict(\n hyperparameters={RFModel: model_params},\n )\n dataset_name = \"toy_binary\"\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args)\n\n\ndef test_rf_binary_compile_onnx_no_config_bagging():\n # FIXME: The below code will crash because:\n # 1. We train with a bag, specifically RandomForest that has efficient OOB and thus only trains 1 fold model\n # 2. We compile RandomForest bag to onnx\n # 3. We call refit_full, performing efficient cloning when fitting refit_full for RandomForest, avoiding fitting again\n # 4. The save of the fold 1 model of RandomForest_BAG_L1 into new location in RandomForest_BAG_L1_FULL does not carry over the model.onnx file\n # 5. Crashes when trying to load the model.onnx file because it doesn't exist.\n # FIXME: This bug only appears if the user calls refit_full on compiled models.\n # Solution: Either,\n # 1. force copy the entire directory of the fold 1 model when cloning instead of calling model.save() -> reuse logic in predictor.clone()\n # 2. model._compiler stores context of files it has created / depends on, and then detects if saving to a new location,\n # then copies the files to that location or computes them again.\n run_test = False\n if run_test:\n fit_args = dict(\n hyperparameters={RFModel: toy_model_params},\n num_bag_folds=2,\n )\n dataset_name = \"toy_binary\"\n compiler_configs = \"auto\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabdpt.py",
"content": "from autogluon.tabular.models.tabdpt.tabdpt_model import TabDPTModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {}\n\n\ndef test_tabdpt():\n model_cls = TabDPTModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n # TabDPT returns different predictions when predicting on an individual sample\n verify_single_prediction_equivalent_to_multi=False,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabicl.py",
"content": "from autogluon.tabular.models.tabicl.tabicl_model import TabICLModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {}\n\n\ndef test_tabicl():\n model_cls = TabICLModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n verify_single_prediction_equivalent_to_multi=True,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabm.py",
"content": "from autogluon.tabular.models.tabm.tabm_model import TabMModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_epochs\": 5}\n\n\ndef test_tabm():\n model_cls = TabMModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabpfnv2.py",
"content": "from autogluon.tabular.models.tabpfnv2.tabpfnv2_5_model import RealTabPFNv2Model\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_estimators\": 1}\n\n\ndef test_tabpfnv2():\n model_cls = RealTabPFNv2Model\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n # TabPFN returns different predictions when predicting on an individual sample\n verify_single_prediction_equivalent_to_multi=False,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabular_nn.py",
"content": "import copy\nimport shutil\n\nfrom autogluon.tabular.models.tabular_nn.torch.tabular_nn_torch import TabularNeuralNetTorchModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"num_epochs\": 3}\n\n\ndef test_tabular_nn():\n model_cls = TabularNeuralNetTorchModel\n model_hyperparameters = copy.deepcopy(toy_model_params)\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\ndef test_tabular_nn_binary_compile_onnx():\n fit_args = dict(\n hyperparameters={TabularNeuralNetTorchModel: toy_model_params},\n )\n dataset_name = \"toy_binary\"\n compiler_configs = {TabularNeuralNetTorchModel: {\"compiler\": \"onnx\"}}\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n from autogluon.tabular.models.tabular_nn.compilers.onnx import TabularNeuralNetTorchOnnxTransformer\n\n assert isinstance(\n predictor._learner.trainer.models[\"NeuralNetTorch\"].processor, TabularNeuralNetTorchOnnxTransformer\n )\n\n\ndef test_tabular_nn_binary_compile_onnx_as_ag_arg():\n model_params = {\"ag.compile\": {\"compiler\": \"onnx\"}}\n model_params.update(toy_model_params)\n fit_args = dict(\n hyperparameters={TabularNeuralNetTorchModel: model_params},\n )\n dataset_name = \"toy_binary\"\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, refit_full=True, delete_directory=False\n )\n from autogluon.tabular.models.tabular_nn.compilers.onnx import TabularNeuralNetTorchOnnxTransformer\n\n assert isinstance(\n predictor._learner.trainer.load_model(\"NeuralNetTorch\").processor, TabularNeuralNetTorchOnnxTransformer\n )\n assert isinstance(\n predictor._learner.trainer.load_model(\"NeuralNetTorch_FULL\").processor, TabularNeuralNetTorchOnnxTransformer\n )\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_tabular_nn_multiclass_compile_onnx():\n fit_args = dict(\n hyperparameters={TabularNeuralNetTorchModel: toy_model_params},\n )\n dataset_name = \"toy_multiclass\"\n compiler_configs = {TabularNeuralNetTorchModel: {\"compiler\": \"onnx\"}}\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n from autogluon.tabular.models.tabular_nn.compilers.onnx import TabularNeuralNetTorchOnnxTransformer\n\n assert isinstance(\n predictor._learner.trainer.models[\"NeuralNetTorch\"].processor, TabularNeuralNetTorchOnnxTransformer\n )\n\n\ndef test_tabular_nn_regression_compile_onnx():\n fit_args = dict(\n hyperparameters={TabularNeuralNetTorchModel: toy_model_params},\n )\n dataset_name = \"toy_regression\"\n compiler_configs = {TabularNeuralNetTorchModel: {\"compiler\": \"onnx\"}}\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n from autogluon.tabular.models.tabular_nn.compilers.onnx import TabularNeuralNetTorchOnnxTransformer\n\n assert isinstance(\n predictor._learner.trainer.models[\"NeuralNetTorch\"].processor, TabularNeuralNetTorchOnnxTransformer\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_tabular_nn_fastai.py",
"content": "from unittest import mock\n\nimport numpy as np\nimport pytest\nfrom fastai.callback.core import CancelFitException\n\nfrom autogluon.tabular.models.fastainn.callbacks import BatchTimeTracker\nfrom autogluon.tabular.models.fastainn.tabular_nn_fastai import NNFastAiTabularModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"epochs\": 3}\n\n\ndef test_tabular_nn_fastai():\n model_cls = NNFastAiTabularModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(\n model_cls=model_cls,\n model_hyperparameters=model_hyperparameters,\n verify_load_wo_cuda=True,\n )\n\n\n__GET_EPOCHS_NUMBER_CASES = {\n \"happy_path\": [dict(time_left=45, batch_size=256, epochs=\"auto\"), 2],\n \"given negative time return 0 epochs\": [dict(time_left=-45, batch_size=256, epochs=\"auto\"), 0],\n \"given time for more than default_epochs epochs, return default_epochs\": [\n dict(time_left=21 * 31, epochs=\"auto\", batch_size=256, default_epochs=12),\n 12,\n ],\n \"given time for less than 1 epoch, return 0 epoch\": [dict(time_left=10, batch_size=256, epochs=\"auto\"), 0],\n \"given no time_left, return default_epochs\": [dict(epochs=\"auto\", batch_size=256, default_epochs=14), 14],\n \"given there is not enough batches to get min_batches_count, return default_epochs\": [\n dict(epochs=\"auto\", time_left=45, batch_size=4096, default_epochs=14),\n 14,\n ],\n \"given explicit epoch count return specified value\": [dict(time_left=45, batch_size=256, epochs=100), 100],\n}\n\n\n@pytest.mark.parametrize(\"test_input\", __GET_EPOCHS_NUMBER_CASES.values(), ids=__GET_EPOCHS_NUMBER_CASES.keys())\ndef test_get_epochs_number(test_input):\n args, epochs_expected = test_input\n with mock.patch.object(NNFastAiTabularModel, \"_measure_batch_times\", return_value=1.2732) as mock_method:\n # batches = (4000/256) + 1 = 16\n # est_epoch_time = 16 * 1.2732 = 20.371\n # time_left:45/est_epoch_time:20.371 = 2\n model = NNFastAiTabularModel(path=\"\", name=\"\")\n assert epochs_expected == model._get_epochs_number(4000, min_batches_count=4, **args)\n if \"time_left\" in args:\n if args.get(\"epochs\", None) == \"auto\" and args[\"batch_size\"] * 4 <= 4000:\n mock_method.assert_called_with(4)\n\n\n__GET_BATCH_SIZE_CASES = {\n \"given batch size provided return specified value\": [dict(bs=111, input_size=400), 111],\n \"given batch size larger than dataset use default_batch_size_for_small_inputs\": [dict(bs=111, input_size=100), 32],\n \"given batch size auto return default_batch_size_for_small_inputs for small datasets\": [\n dict(bs=\"auto\", input_size=100),\n 32,\n ],\n \"given batch size auto return value for small datasets\": [dict(bs=\"auto\", input_size=2048), 256],\n \"given batch size auto return value for large datasets\": [dict(bs=\"auto\", input_size=200000), 512],\n}\n\n\n@pytest.mark.parametrize(\"test_input\", __GET_BATCH_SIZE_CASES.values(), ids=__GET_BATCH_SIZE_CASES.keys())\ndef test_get_batch_size_with_bs_provided(test_input):\n args, expected_bs = test_input\n bs, input_size = args[\"bs\"], args[\"input_size\"]\n model = NNFastAiTabularModel(path=\"\", name=\"\")\n model.params[\"bs\"] = bs\n x = np.arange(input_size)\n assert expected_bs == model._get_batch_size(x)\n\n\ndef test_BatchTimeTracker():\n with mock.patch.object(BatchTimeTracker, \"_time_now\") as mock_method:\n mock_method.side_effect = [10, 40]\n iterations_to_complete = 3\n tracker = BatchTimeTracker(batches_to_measure=iterations_to_complete)\n stopped = False\n for i in range(iterations_to_complete + 1):\n try:\n tracker.after_batch()\n if i == iterations_to_complete:\n raise ValueError(\"CancelFitException should be raised, but was not\")\n except CancelFitException:\n stopped = True\n assert stopped\n assert tracker.batch_measured_time == 10.0\n"
},
{
"path": "tabular/tests/unittests/models/test_text_prediction_v1_model.py",
"content": "import pytest\n\nfrom autogluon.tabular.testing import FitHelper\n\n\n# FIXME: Post 0.1, refit_full by storing the iteration idx for reaching the best performance.\n@pytest.mark.gpu\ndef test_text_prediction_v1_sts():\n pytest.skip(\"Temporary skip the unittest\")\n fit_args = dict(\n hyperparameters={\n \"AG_TEXT_NN\": {\n \"optim.max_epochs\": 1,\n \"model.hf_text.checkpoint_name\": \"google/electra-small-discriminator\",\n }\n },\n )\n dataset_name = \"sts\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n sample_size=100,\n refit_full=True,\n )\n"
},
{
"path": "tabular/tests/unittests/models/test_xgboost.py",
"content": "from autogluon.tabular.models.xgboost.xgboost_model import XGBoostModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_estimators\": 10}\n\n\ndef test_xgboost():\n model_cls = XGBoostModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\ndef test_xgboost_binary_enable_categorical():\n fit_args = dict(\n hyperparameters={XGBoostModel: {\"enable_categorical\": True}},\n )\n dataset_name = \"toy_binary\"\n FitHelper.fit_and_validate_dataset(dataset_name=dataset_name, fit_args=fit_args, refit_full=False)\n"
},
{
"path": "tabular/tests/unittests/models/test_xt.py",
"content": "from autogluon.tabular.models.xt.xt_model import XTModel\nfrom autogluon.tabular.testing import FitHelper\n\ntoy_model_params = {\"n_estimators\": 10}\n\n\ndef test_xt():\n model_cls = XTModel\n model_hyperparameters = toy_model_params\n\n FitHelper.verify_model(model_cls=model_cls, model_hyperparameters=model_hyperparameters)\n\n\ndef test_xt_binary_compile_onnx():\n fit_args = dict(\n hyperparameters={XTModel: toy_model_params},\n )\n dataset_name = \"toy_binary\"\n compiler_configs = {XTModel: {\"compiler\": \"onnx\"}}\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n\n\ndef test_xt_multiclass_compile_onnx():\n fit_args = dict(\n hyperparameters={XTModel: toy_model_params},\n )\n dataset_name = \"toy_multiclass\"\n compiler_configs = {XTModel: {\"compiler\": \"onnx\"}}\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n\n\ndef test_xt_regression_compile_onnx():\n fit_args = dict(\n hyperparameters={XTModel: toy_model_params},\n )\n compiler_configs = {XTModel: {\"compiler\": \"onnx\"}}\n dataset_name = \"toy_regression\"\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name, fit_args=fit_args, compile=True, compiler_configs=compiler_configs\n )\n"
},
{
"path": "tabular/tests/unittests/pseudolabel/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/pseudolabel/pseudo_filter.py",
"content": "import inspect\n\nimport numpy as np\nimport pandas\nimport pandas as pd\n\nfrom autogluon.core.pseudolabeling.pseudolabeling import filter_pseudo\n\n\ndef get_default_args(func):\n signature = inspect.signature(func)\n return {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty}\n\n\ndef get_default_pseudo_test_args():\n default_args = get_default_args(filter_pseudo)\n sample_percent = default_args[\"percent_sample\"]\n min_percent = default_args[\"min_proportion_prob\"]\n max_percent = default_args[\"max_proportion_prob\"]\n threshold = default_args[\"threshold\"]\n\n return sample_percent, min_percent, max_percent, threshold\n\n\ndef test_regression_pseudofilter():\n y_reg_fake = np.random.rand(200)\n y_reg_fake = pandas.Series(data=y_reg_fake)\n\n sample_percent, _, _, _ = get_default_pseudo_test_args()\n pseudo_idxes = filter_pseudo(y_reg_fake, problem_type=\"regression\")\n assert len(pseudo_idxes) == int(sample_percent * len(y_reg_fake))\n\n\ndef test_classification_pseudofilter():\n _, min_percent, max_percent, threshold = get_default_pseudo_test_args()\n middle_percent = (max_percent + min_percent) / 2\n num_rows = 100\n num_above_threshold = int(middle_percent * num_rows)\n num_below_threshold = num_rows - num_above_threshold\n y_reg_fake = np.ones((num_above_threshold))\n\n # Test if percent preds is below min threshold\n y_reg_fake_below_min = np.zeros((num_rows, 2))\n y_reg_fake_below_min = pd.DataFrame(data=y_reg_fake_below_min)\n pseudo_indices_ans = filter_pseudo(y_reg_fake_below_min, \"binary\")\n assert num_rows == len(pseudo_indices_ans)\n\n # Test if percent preds is above max threshold\n y_reg_fake_above_max = pandas.DataFrame(data=y_reg_fake)\n pseudo_indices_ans = filter_pseudo(y_reg_fake_above_max, \"binary\")\n num_rows_threshold = max(np.ceil(max_percent * len(y_reg_fake_above_max)), 1)\n curr_threshold = y_reg_fake_above_max.loc[num_rows_threshold - 1]\n answer = len(y_reg_fake_above_max >= curr_threshold)\n assert answer == len(pseudo_indices_ans)\n\n # Test if normal functionality beginning\n y_reg_fake = np.column_stack((y_reg_fake, np.zeros((num_above_threshold))))\n y_reg_fake = np.row_stack((y_reg_fake, np.zeros((num_below_threshold, 2))))\n\n y_reg_fake = pandas.DataFrame(data=y_reg_fake)\n pseudo_flag = y_reg_fake.max(axis=1) > threshold\n pseudo_indices_ans = pseudo_flag[pseudo_flag == True]\n\n pseudo_idxes = filter_pseudo(y_reg_fake, \"binary\")\n assert len(pseudo_idxes) == len(pseudo_indices_ans)\n"
},
{
"path": "tabular/tests/unittests/registry/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/registry/test_model_registry.py",
"content": "from __future__ import annotations\n\nfrom types import MappingProxyType\nfrom typing import Type\n\nimport pytest\n\nfrom autogluon.core.models import (\n AbstractModel,\n DummyModel,\n GreedyWeightedEnsembleModel,\n SimpleWeightedEnsembleModel,\n)\nfrom autogluon.tabular.models import (\n BoostedRulesModel,\n CatBoostModel,\n EBMModel,\n FigsModel,\n FTTransformerModel,\n GreedyTreeModel,\n HSTreeModel,\n ImagePredictorModel,\n KNNModel,\n LGBModel,\n LinearModel,\n MitraModel,\n MultiModalPredictorModel,\n NNFastAiTabularModel,\n PrepLGBModel,\n RealMLPModel,\n RealTabPFNv2Model,\n RealTabPFNv25Model,\n RFModel,\n RuleFitModel,\n TabDPTModel,\n TabICLModel,\n TabMModel,\n TabPFNMixModel,\n TabularNeuralNetTorchModel,\n TextPredictorModel,\n XGBoostModel,\n XTModel,\n)\nfrom autogluon.tabular.registry import ModelRegistry, ag_model_registry\n\nEXPECTED_MODEL_KEYS = {\n RFModel: \"RF\",\n XTModel: \"XT\",\n KNNModel: \"KNN\",\n LGBModel: \"GBM\",\n PrepLGBModel: \"GBM_PREP\",\n CatBoostModel: \"CAT\",\n XGBoostModel: \"XGB\",\n RealMLPModel: \"REALMLP\",\n TabularNeuralNetTorchModel: \"NN_TORCH\",\n LinearModel: \"LR\",\n NNFastAiTabularModel: \"FASTAI\",\n TextPredictorModel: \"AG_TEXT_NN\",\n ImagePredictorModel: \"AG_IMAGE_NN\",\n MultiModalPredictorModel: \"AG_AUTOMM\",\n FTTransformerModel: \"FT_TRANSFORMER\",\n TabDPTModel: \"TABDPT\",\n TabICLModel: \"TABICL\",\n TabMModel: \"TABM\",\n TabPFNMixModel: \"TABPFNMIX\",\n MitraModel: \"MITRA\",\n GreedyWeightedEnsembleModel: \"ENS_WEIGHTED\",\n SimpleWeightedEnsembleModel: \"SIMPLE_ENS_WEIGHTED\",\n RuleFitModel: \"IM_RULEFIT\",\n GreedyTreeModel: \"IM_GREEDYTREE\",\n FigsModel: \"IM_FIGS\",\n HSTreeModel: \"IM_HSTREE\",\n BoostedRulesModel: \"IM_BOOSTEDRULES\",\n DummyModel: \"DUMMY\",\n EBMModel: \"EBM\",\n RealTabPFNv25Model: \"REALTABPFN-V2.5\",\n RealTabPFNv2Model: \"REALTABPFN-V2\",\n}\n\nEXPECTED_MODEL_NAMES = {\n RFModel: \"RandomForest\",\n XTModel: \"ExtraTrees\",\n KNNModel: \"KNeighbors\",\n LGBModel: \"LightGBM\",\n PrepLGBModel: \"LightGBMPrep\",\n CatBoostModel: \"CatBoost\",\n XGBoostModel: \"XGBoost\",\n RealMLPModel: \"RealMLP\",\n TabularNeuralNetTorchModel: \"NeuralNetTorch\",\n LinearModel: \"LinearModel\",\n NNFastAiTabularModel: \"NeuralNetFastAI\",\n TextPredictorModel: \"TextPredictor\",\n ImagePredictorModel: \"ImagePredictor\",\n MultiModalPredictorModel: \"MultiModalPredictor\",\n FTTransformerModel: \"FTTransformer\",\n TabDPTModel: \"TabDPT\",\n TabICLModel: \"TabICL\",\n TabMModel: \"TabM\",\n TabPFNMixModel: \"TabPFNMix\",\n MitraModel: \"Mitra\",\n GreedyWeightedEnsembleModel: \"WeightedEnsemble\",\n SimpleWeightedEnsembleModel: \"WeightedEnsemble\",\n RuleFitModel: \"RuleFit\",\n GreedyTreeModel: \"GreedyTree\",\n FigsModel: \"Figs\",\n HSTreeModel: \"HierarchicalShrinkageTree\",\n BoostedRulesModel: \"BoostedRules\",\n DummyModel: \"Dummy\",\n EBMModel: \"EBM\",\n RealTabPFNv25Model: \"RealTabPFN-v2.5\",\n RealTabPFNv2Model: \"RealTabPFN-v2\",\n}\n\n# Higher values indicate higher priority, priority dictates the order models are trained for a given level.\nEXPECTED_MODEL_PRIORITY = {\n RFModel: 80,\n XTModel: 60,\n KNNModel: 100,\n LGBModel: 90,\n PrepLGBModel: 90,\n CatBoostModel: 70,\n XGBoostModel: 40,\n RealMLPModel: 75,\n TabularNeuralNetTorchModel: 25,\n LinearModel: 30,\n EBMModel: 35,\n NNFastAiTabularModel: 50,\n TextPredictorModel: 0,\n ImagePredictorModel: 0,\n MultiModalPredictorModel: 0,\n FTTransformerModel: 0,\n TabDPTModel: 50,\n TabICLModel: 65,\n TabMModel: 85,\n TabPFNMixModel: 45,\n MitraModel: 55,\n GreedyWeightedEnsembleModel: 0,\n SimpleWeightedEnsembleModel: 0,\n RuleFitModel: 0,\n GreedyTreeModel: 0,\n FigsModel: 0,\n HSTreeModel: 0,\n BoostedRulesModel: 0,\n DummyModel: 0,\n RealTabPFNv25Model: 40,\n RealTabPFNv2Model: 40,\n}\n\nEXPECTED_MODEL_PRIORITY_BY_PROBLEM_TYPE = {\n LGBModel: {\n \"softclass\": 100,\n },\n PrepLGBModel: {\n \"softclass\": 100,\n },\n CatBoostModel: {\n \"softclass\": 60,\n },\n NNFastAiTabularModel: {\n \"multiclass\": 95,\n },\n}\n\nEXPECTED_REGISTERED_MODEL_CLS_LST = list(EXPECTED_MODEL_NAMES.keys())\nREGISTERED_MODEL_CLS_LST = ag_model_registry.model_cls_list\n\n\n@pytest.mark.parametrize(\n \"model_cls\",\n REGISTERED_MODEL_CLS_LST,\n ids=[c.__name__ for c in REGISTERED_MODEL_CLS_LST],\n) # noqa\ndef test_model_cls_key(model_cls: Type[AbstractModel]):\n expected_model_key = EXPECTED_MODEL_KEYS[model_cls]\n assert expected_model_key == model_cls.ag_key\n\n\ndef verify_registry(model_cls: Type[AbstractModel], model_registry: ModelRegistry):\n \"\"\"\n Verifies that all methods work as intended in ModelRegistry, assuming `model_cls` is already registered.\n \"\"\"\n assert model_registry.exists(model_cls)\n assert model_cls.ag_key == model_registry.key(model_cls)\n assert model_cls.ag_priority == model_registry.priority(model_cls)\n\n assert model_cls in model_registry.model_cls_list\n assert model_cls.ag_key in model_registry.keys\n\n assert model_cls == model_registry.key_to_cls(model_registry.key(model_cls))\n key_to_cls_map = model_registry.key_to_cls_map()\n assert model_cls == key_to_cls_map[model_cls.ag_key]\n\n name_map = model_registry.name_map()\n assert model_cls.ag_name == name_map[model_cls]\n assert model_cls.ag_name == model_registry.name(model_cls)\n\n priority_map = model_registry.priority_map()\n assert model_cls.ag_priority == priority_map[model_cls]\n for problem_type in model_cls.ag_priority_by_problem_type:\n priority_map_problem_type = model_registry.priority_map(problem_type=problem_type)\n assert model_cls.get_ag_priority(problem_type) == model_registry.priority(model_cls, problem_type=problem_type)\n assert model_cls.get_ag_priority(problem_type) == priority_map_problem_type[model_cls]\n\n\n@pytest.mark.parametrize(\n \"model_cls\",\n REGISTERED_MODEL_CLS_LST,\n ids=[c.__name__ for c in REGISTERED_MODEL_CLS_LST],\n) # noqa\ndef test_model_registry(model_cls: Type[AbstractModel]):\n \"\"\"\n Verifies that all methods work as intended in ModelRegistry.\n \"\"\"\n verify_registry(model_cls=model_cls, model_registry=ag_model_registry)\n\n\ndef test_model_registry_new():\n \"\"\"\n Verifies that all methods work as intended in a new ModelRegistry.\n \"\"\"\n model_registry_new = ModelRegistry()\n\n assert not model_registry_new.exists(RFModel)\n assert model_registry_new.model_cls_list == []\n assert model_registry_new.keys == []\n assert model_registry_new.name_map() == {}\n assert model_registry_new.priority_map() == {}\n\n model_registry_new.add(RFModel)\n assert model_registry_new.model_cls_list == [RFModel]\n verify_registry(model_cls=RFModel, model_registry=model_registry_new)\n\n model_registry_new.remove(model_cls=RFModel)\n assert not model_registry_new.exists(RFModel)\n assert model_registry_new.model_cls_list == []\n assert model_registry_new.keys == []\n assert model_registry_new.name_map() == {}\n assert model_registry_new.priority_map() == {}\n\n\ndef test_no_unknown_model_cls_registered():\n assert set(ag_model_registry.model_cls_list) == set(EXPECTED_REGISTERED_MODEL_CLS_LST)\n\n\n@pytest.mark.parametrize(\n \"model_cls\",\n REGISTERED_MODEL_CLS_LST,\n ids=[c.__name__ for c in REGISTERED_MODEL_CLS_LST],\n) # noqa\ndef test_model_cls_name(model_cls: Type[AbstractModel]):\n expected_model_name = EXPECTED_MODEL_NAMES[model_cls]\n assert expected_model_name == model_cls.ag_name\n\n\n@pytest.mark.parametrize(\n \"model_cls\",\n REGISTERED_MODEL_CLS_LST,\n ids=[c.__name__ for c in REGISTERED_MODEL_CLS_LST],\n) # noqa\ndef test_model_cls_priority(model_cls: Type[AbstractModel]):\n expected_model_priority = EXPECTED_MODEL_PRIORITY[model_cls]\n assert expected_model_priority == model_cls.ag_priority\n\n\n@pytest.mark.parametrize(\n \"model_cls\",\n REGISTERED_MODEL_CLS_LST,\n ids=[c.__name__ for c in REGISTERED_MODEL_CLS_LST],\n) # noqa\ndef test_model_cls_priority_by_problem_type(model_cls: Type[AbstractModel]):\n expected_model_priority_by_problem_type = EXPECTED_MODEL_PRIORITY_BY_PROBLEM_TYPE.get(model_cls, {})\n expected_model_priority_default = EXPECTED_MODEL_PRIORITY[model_cls]\n assert expected_model_priority_by_problem_type == model_cls.ag_priority_by_problem_type\n assert isinstance(model_cls.ag_priority_by_problem_type, MappingProxyType)\n for problem_type in [\"binary\", \"multiclass\", \"regression\", \"quantile\", \"softclass\"]:\n expected_model_priority = expected_model_priority_by_problem_type.get(\n problem_type, expected_model_priority_default\n )\n model_priority = model_cls.get_ag_priority(problem_type=problem_type)\n assert expected_model_priority == model_priority\n assert expected_model_priority_default == model_cls.get_ag_priority()\n\n\ndef test_model_cls_all_present():\n assert len(REGISTERED_MODEL_CLS_LST) == len(set(REGISTERED_MODEL_CLS_LST))\n assert set(EXPECTED_REGISTERED_MODEL_CLS_LST) == set(REGISTERED_MODEL_CLS_LST)\n\n\ndef test_model_cls_no_duplicate_keys():\n keys = set()\n for c in REGISTERED_MODEL_CLS_LST:\n if c.ag_key in keys:\n raise AssertionError(f\"Two model classes cannot share the same key: {c.ag_key}\")\n keys.add(c.ag_key)\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/__init__.py",
"content": ""
},
{
"path": "tabular/tests/unittests/resource_allocation/test_bagging_resource_allocation.py",
"content": "import time\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.core.models.ensemble.fold_fitting_strategy import (\n ParallelLocalFoldFittingStrategy,\n SequentialLocalFoldFittingStrategy,\n)\nfrom autogluon.tabular.models import AbstractModel\n\n\nclass DummyBaseModel(AbstractModel):\n def __init__(self, minimum_resources=None, default_resources=None, **kwargs):\n self._minimum_resources = minimum_resources\n self._default_resources = default_resources\n super().__init__(**kwargs)\n\n def get_minimum_resources(self, **kwargs):\n return self._minimum_resources\n\n def _get_default_resources(self):\n num_cpus = self._default_resources.get(\"num_cpus\")\n num_gpus = self._default_resources.get(\"num_gpus\")\n return num_cpus, num_gpus\n\n\nclass DummyModel(DummyBaseModel):\n pass\n\n\nclass DummyBaggedModel(BaggedEnsembleModel):\n pass\n\n\ndef _prepare_data():\n # prepare an all numeric data so that we don't need to clean labels and features\n data = [[1, 10], [2, 20], [3, 30]]\n df = pd.DataFrame(data, columns=[\"Number\", \"Age\"])\n label = \"Age\"\n X = df.drop(columns=[label])\n y = df[label]\n return X, y\n\n\ndef _construct_dummy_fold_strategy(\n fold_strategy_cls,\n num_jobs,\n num_folds_parallel,\n bagged_resources=None,\n model_base_resources=None,\n model_base_minimum_resources=None,\n model_base_default_resources=None,\n):\n if bagged_resources is None:\n bagged_resources = {}\n if model_base_resources is None:\n model_base_resources = {}\n if model_base_minimum_resources is None:\n model_base_minimum_resources = {}\n dummy_model_base = DummyModel(\n minimum_resources=model_base_minimum_resources,\n default_resources=model_base_default_resources,\n hyperparameters={\"ag_args_fit\": model_base_resources},\n )\n dummy_bagged_ensemble_model = DummyBaggedModel(dummy_model_base, hyperparameters={\"ag_args_fit\": bagged_resources})\n train_data, test_data = _prepare_data()\n args = dict(\n model_base=dummy_model_base,\n model_base_kwargs=dict(),\n bagged_ensemble_model=dummy_bagged_ensemble_model,\n X=train_data,\n y=test_data,\n X_pseudo=None,\n y_pseudo=None,\n sample_weight=None,\n time_start=time.time(),\n time_limit=0,\n models=[],\n oof_pred_proba=np.array([]),\n oof_pred_model_repeats=np.array([]),\n save_folds=True,\n time_limit_fold_ratio=1,\n **bagged_resources, # These will be processed at abstract model level in a real tabular predictor\n )\n if fold_strategy_cls == ParallelLocalFoldFittingStrategy:\n args[\"num_jobs\"] = num_jobs\n args[\"num_folds_parallel\"] = num_folds_parallel\n return fold_strategy_cls(**args)\n\n\n@pytest.mark.parametrize(\"fold_strategy_cls\", [ParallelLocalFoldFittingStrategy, SequentialLocalFoldFittingStrategy])\ndef test_bagging_invalid_resources_per_fold(fold_strategy_cls):\n # resources per fold more than resources to ensemble model\n with pytest.raises(AssertionError) as e:\n _construct_dummy_fold_strategy(\n fold_strategy_cls=fold_strategy_cls,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 1, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 2, \"num_gpus\": 1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n )\n # resources per fold less than minimum resources\n with pytest.raises(AssertionError) as e:\n _construct_dummy_fold_strategy(\n fold_strategy_cls=fold_strategy_cls,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.5},\n )\n\n\ndef test_parallel_bagging_resources_per_fold():\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=ParallelLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 4, \"num_gpus\": 1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 4, \"num_gpus\": 1}\n assert fold_fitting_strategy.resources_model == {\"num_cpus\": 4, \"num_gpus\": 1}\n assert fold_fitting_strategy.num_parallel_jobs == 1\n assert fold_fitting_strategy.batches == 8\n\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=ParallelLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 1, \"num_gpus\": 0.1}\n assert fold_fitting_strategy.resources_model == {\"num_cpus\": 1, \"num_gpus\": 0.1}\n assert fold_fitting_strategy.num_parallel_jobs == 8\n assert fold_fitting_strategy.batches == 1\n\n\ndef test_parallel_bagging_no_resources_per_fold():\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=ParallelLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=4,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 2, \"num_gpus\": 0.25}\n assert fold_fitting_strategy.resources_model == {\"num_cpus\": 2, \"num_gpus\": 0.25}\n assert fold_fitting_strategy.num_parallel_jobs == 4\n assert fold_fitting_strategy.batches == 2\n\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=ParallelLocalFoldFittingStrategy,\n num_jobs=4,\n num_folds_parallel=2,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 2, \"num_gpus\": 0.2},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 4, \"num_gpus\": 0.5}\n assert fold_fitting_strategy.resources_model == {\"num_cpus\": 2, \"num_gpus\": 0.2}\n assert fold_fitting_strategy.num_parallel_jobs == 2\n assert fold_fitting_strategy.batches == 2\n\n\ndef test_sequential_bagging_resources_per_fold():\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=SequentialLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 4, \"num_gpus\": 1},\n model_base_default_resources={\"num_cpus\": 1, \"num_gpus\": 0},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 4, \"num_gpus\": 1}\n assert fold_fitting_strategy.user_resources_per_job == {\"num_cpus\": 4, \"num_gpus\": 1}\n\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=SequentialLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=8,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 1, \"num_gpus\": 0},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n )\n assert fold_fitting_strategy.resources == {\"num_cpus\": 1, \"num_gpus\": 0.1}\n assert fold_fitting_strategy.user_resources_per_job == {\"num_cpus\": 1, \"num_gpus\": 0.1}\n\n\ndef test_sequential_bagging_no_resources_per_fold():\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=SequentialLocalFoldFittingStrategy,\n num_jobs=8,\n num_folds_parallel=4,\n bagged_resources={\"num_cpus\": 8, \"num_gpus\": 1},\n model_base_minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n model_base_default_resources={\"num_cpus\": 4, \"num_gpus\": 0.5},\n )\n assert fold_fitting_strategy.num_cpus == 8\n assert fold_fitting_strategy.num_gpus == 1\n assert fold_fitting_strategy.resources == {\"num_cpus\": 4, \"num_gpus\": 0.5}\n assert fold_fitting_strategy.user_resources_per_job == None\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_custom_memory_limit.py",
"content": "import pytest\n\nfrom autogluon.tabular.testing import FitHelper\n\n\n@pytest.fixture()\ndef get_and_assert_max_memory():\n import os\n\n import psutil\n\n # Mock patch to guarantee that test does not fail\n # due to memory changing between calls to psutil.\n p = psutil.Process()\n allocated_memory = p.memory_info()\n psutil.Process.memory_info = lambda _: allocated_memory\n\n # Import after mock patch above.\n from autogluon.common.utils.resource_utils import ResourceManager\n\n max_memory = 48.0\n yield max_memory # Wait for test to set up custom memory limit\n\n # Check custom memory limit\n try:\n assert ResourceManager.get_memory_size(format=\"GB\") == max_memory\n assert (max_memory * (1024.0**3)) - allocated_memory.rss == ResourceManager.get_available_virtual_mem(\n format=\"B\",\n )\n finally:\n del os.environ[\"AG_MEMORY_LIMIT_IN_GB\"]\n\n\ndef test_custom_memory_soft_limit_tabular_fit(get_and_assert_max_memory):\n fit_args = dict(\n hyperparameters={\"DUMMY\": {}},\n memory_limit=get_and_assert_max_memory,\n dynamic_stacking=False,\n fit_weighted_ensemble=False,\n num_bag_folds=2,\n num_bag_sets=1,\n num_stack_levels=0,\n ag_args_ensemble={\"fold_fitting_strategy\": \"sequential_local\"},\n )\n dataset_name = \"adult\"\n\n FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n expected_model_count=1,\n refit_full=False,\n )\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_gpu_assignment.py",
"content": "\"\"\"\nThis module contains integration tests that verify the _calculate_gpu_assignment() method specifically for\nParallelFoldFittingStrategy works correctly by submitting actual Ray remote tasks and verifying that tasks see the\ncorrect GPU assignments via CUDA_VISIBLE_DEVICES and torch.cuda.\n\"\"\"\n\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.core.models.ensemble.fold_fitting_strategy import ParallelLocalFoldFittingStrategy\nfrom autogluon.tabular.models import AbstractModel\n\n\nclass DummyBaseModel(AbstractModel):\n def __init__(self, minimum_resources=None, default_resources=None, **kwargs):\n self._minimum_resources = minimum_resources\n self._default_resources = default_resources\n super().__init__(**kwargs)\n\n def get_minimum_resources(self, **kwargs):\n return self._minimum_resources\n\n def _get_default_resources(self):\n num_cpus = self._default_resources.get(\"num_cpus\")\n num_gpus = self._default_resources.get(\"num_gpus\")\n return num_cpus, num_gpus\n\n\nclass DummyModel(DummyBaseModel):\n pass\n\n\nclass DummyBaggedModel(BaggedEnsembleModel):\n pass\n\n\ndef _prepare_data():\n data = [[1, 10], [2, 20], [3, 30]]\n df = pd.DataFrame(data, columns=[\"Number\", \"Age\"])\n label = \"Age\"\n X = df.drop(columns=[label])\n y = df[label]\n return X, y\n\n\ndef _construct_parallel_fold_strategy(\n num_cpus=8,\n num_gpus=4,\n num_jobs=8,\n num_folds_parallel=None,\n model_base_minimum_resources=None,\n model_base_default_resources=None,\n X_pseudo=None,\n y_pseudo=None,\n sample_weight=None,\n time_start=None,\n time_limit=None,\n save_folds=True,\n):\n \"\"\"\n Construct a ParallelLocalFoldFittingStrategy instance for testing.\n\n Parameters\n ----------\n num_cpus : int, default=8\n Total CPUs available\n num_gpus : int or float, default=4\n Total GPUs available\n num_jobs : int, default=8\n Total number of fold jobs\n num_folds_parallel : int, optional\n Number of folds to train in parallel. Defaults to num_jobs if not specified.\n model_base_minimum_resources : dict, optional\n Minimum resources required by the model. Defaults to {\"num_cpus\": 1, \"num_gpus\": 0}\n model_base_default_resources : dict, optional\n Default resources for the model. Defaults to {\"num_cpus\": 4, \"num_gpus\": 0}\n X_pseudo : DataFrame, optional\n Pseudo-labeled data features. Defaults to None\n y_pseudo : Series, optional\n Pseudo-labeled data labels. Defaults to None\n sample_weight : array-like, optional\n Sample weights for training. Defaults to None\n time_start : float, optional\n Time when training started. Defaults to current time.time()\n time_limit : float, optional\n Time limit for training in seconds. Defaults to None\n save_folds : bool, default=True\n Whether to save the fold models to disk\n\n Returns\n -------\n ParallelLocalFoldFittingStrategy\n An instance of the fold fitting strategy for testing\n \"\"\"\n if num_folds_parallel is None:\n num_folds_parallel = num_jobs\n if model_base_minimum_resources is None:\n model_base_minimum_resources = {\"num_cpus\": 1, \"num_gpus\": 0}\n if model_base_default_resources is None:\n model_base_default_resources = {\"num_cpus\": 4, \"num_gpus\": 0}\n if time_start is None:\n time_start = time.time()\n\n dummy_model_base = DummyModel(\n minimum_resources=model_base_minimum_resources,\n default_resources=model_base_default_resources,\n hyperparameters={},\n )\n dummy_bagged_ensemble_model = DummyBaggedModel(dummy_model_base, hyperparameters={})\n train_data, test_data = _prepare_data()\n\n args = dict(\n num_jobs=num_jobs,\n num_folds_parallel=num_folds_parallel,\n model_base=dummy_model_base,\n model_base_kwargs=dict(),\n bagged_ensemble_model=dummy_bagged_ensemble_model,\n X=train_data,\n y=test_data,\n X_pseudo=X_pseudo,\n y_pseudo=y_pseudo,\n sample_weight=sample_weight,\n time_start=time_start,\n time_limit=time_limit,\n models=[],\n oof_pred_proba=np.array([]),\n oof_pred_model_repeats=np.array([]),\n save_folds=save_folds,\n num_cpus=num_cpus,\n num_gpus=num_gpus,\n )\n return ParallelLocalFoldFittingStrategy(**args)\n\n\ndef _create_ray_gpu_reporter():\n \"\"\"\n Create a Ray remote function that reports GPU information.\n\n This function is created outside the test class so it can be serialized\n and sent to Ray remote workers.\n \"\"\"\n import ray\n\n @ray.remote\n def ray_task_report_gpu_info(task_id, assigned_gpu_ids):\n \"\"\"\n Ray remote task that reports GPU information.\n\n This simulates what happens in _ray_fit() in fold_fitting_strategy.py\n where GPU assignments are used to set CUDA_VISIBLE_DEVICES and torch reports GPU info.\n\n Parameters\n ----------\n task_id : int\n The task ID for identification\n assigned_gpu_ids : list\n The list of GPU IDs assigned to this task\n\n Returns\n -------\n dict\n Dictionary containing GPU info reported by the task\n \"\"\"\n import os\n\n # Simulate the assignment logic in _ray_fit()\n if assigned_gpu_ids:\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, assigned_gpu_ids))\n\n report = {\n \"task_id\": task_id,\n \"assigned_gpu_ids\": assigned_gpu_ids,\n \"CUDA_VISIBLE_DEVICES\": os.environ.get(\"CUDA_VISIBLE_DEVICES\", \"not set\"),\n }\n\n # Try to get torch GPU info if available\n try:\n import torch\n\n report[\"torch_gpu_count\"] = torch.cuda.device_count()\n if torch.cuda.is_available():\n report[\"torch_current_device\"] = torch.cuda.current_device()\n else:\n report[\"torch_current_device\"] = None\n except ImportError:\n report[\"torch_gpu_count\"] = \"torch not available\"\n report[\"torch_current_device\"] = \"torch not available\"\n\n return report\n\n return ray_task_report_gpu_info\n\n\nclass TestRayGpuAssignmentIntegration:\n \"\"\"Integration tests that verify GPU assignments work correctly with Ray.\"\"\"\n\n @staticmethod\n def _get_system_gpu_count():\n \"\"\"Get actual GPU count on the system.\"\"\"\n try:\n import torch\n\n return torch.cuda.device_count()\n except Exception:\n return 0\n\n def _check_ray_init(self, num_cpus, num_gpus):\n \"\"\"Ensure Ray is initialized with the specified resources.\"\"\"\n import ray\n\n if not ray.is_initialized():\n ray.init(num_cpus=num_cpus, num_gpus=num_gpus)\n\n def _calculate_assignments(self, strategy, num_tasks, gpus_per_task, total_gpus):\n \"\"\"Calculate GPU assignments for multiple tasks.\"\"\"\n gpu_assignments = {}\n for task_id in range(num_tasks):\n gpu_assignments[task_id] = strategy._calculate_gpu_assignment(\n task_id=task_id, gpus_per_task=gpus_per_task, total_gpus=total_gpus\n )\n return gpu_assignments\n\n def _submit_and_collect_tasks(self, ray_task_report_gpu_info, gpu_assignments):\n \"\"\"Submit Ray tasks and collect their results.\"\"\"\n import ray\n\n task_refs = []\n for task_id, assigned_gpus in gpu_assignments.items():\n ref = ray_task_report_gpu_info.remote(task_id, assigned_gpus)\n task_refs.append(ref)\n results = ray.get(task_refs)\n return results\n\n def _verify_assignment_structure(self, gpu_assignments, num_tasks, expected_gpus_per_task):\n \"\"\"Verify the structure of GPU assignments.\"\"\"\n assert len(gpu_assignments) == num_tasks, f\"Expected {num_tasks} task assignments, got {len(gpu_assignments)}\"\n for task_id in range(num_tasks):\n assert len(gpu_assignments[task_id]) == expected_gpus_per_task, (\n f\"Expected {expected_gpus_per_task} GPUs for task {task_id}, got {len(gpu_assignments[task_id])}\"\n )\n\n def _verify_cuda_visible_devices(self, results, expected_cuda_visible):\n \"\"\"Verify CUDA_VISIBLE_DEVICES for each task result.\"\"\"\n for result in results:\n task_id = result[\"task_id\"]\n expected = expected_cuda_visible[task_id]\n assert result[\"CUDA_VISIBLE_DEVICES\"] == expected, (\n f\"Task {task_id} expected CUDA_VISIBLE_DEVICES='{expected}', got '{result['CUDA_VISIBLE_DEVICES']}'\"\n )\n\n def _verify_torch_gpu_count(self, results, expected_gpu_counts):\n \"\"\"Verify torch reports correct GPU count for each task.\"\"\"\n for result in results:\n task_id = result[\"task_id\"]\n if result[\"torch_gpu_count\"] != \"torch not available\":\n expected_count = expected_gpu_counts.get(task_id, expected_gpu_counts.get(\"default\"))\n assert result[\"torch_gpu_count\"] == expected_count, (\n f\"Task {task_id} expected {expected_count} GPU(s), saw {result['torch_gpu_count']}\"\n )\n\n def test_ray_gpu_assignment_no_gpu_integration(self):\n \"\"\"\n Integration Test: Verify GPU assignment with no GPUs\n\n Scenario: 0 GPUs, 1 task\n Expected: Task runs on CPU, CUDA_VISIBLE_DEVICES = 'not set'\n \"\"\"\n import ray # Ray is required for these tests\n\n self._check_ray_init(num_cpus=4, num_gpus=0)\n\n strategy = _construct_parallel_fold_strategy(num_cpus=4, num_gpus=0, num_jobs=1)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=1, gpus_per_task=0, total_gpus=0)\n\n assert gpu_assignments[0] == [], f\"Expected empty GPU list for task 0, got {gpu_assignments[0]}\"\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_single_gpu_task_execution(self):\n \"\"\"\n Integration Test: Verify Ray tasks see correct GPU assignment with single GPU\n\n Scenario: 1 GPU available, 2 tasks, each task requests 0.5 GPU\n num_gpus_per_task = 0.5 (fractional, round-robin)\n Expected: Both tasks assigned to GPU 0, both see GPU 0 in CUDA_VISIBLE_DEVICES\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus == 0:\n pytest.skip(\"No GPUs available on system\")\n\n self._check_ray_init(num_cpus=4, num_gpus=1)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=4, num_gpus=1, num_jobs=2)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=2, gpus_per_task=0.5, total_gpus=1)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=2, expected_gpus_per_task=1)\n assert gpu_assignments[0] == [0], f\"Expected [0] for task 0, got {gpu_assignments[0]}\"\n assert gpu_assignments[1] == [0], f\"Expected [0] for task 1, got {gpu_assignments[1]}\"\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0\", 1: \"0\"})\n self._verify_torch_gpu_count(results, {\"default\": 1})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_insufficient_gpus_multiple_tasks(self):\n \"\"\"\n Integration Test: Multiple tasks with insufficient GPUs (oversubscription)\n\n Scenario: 1 GPU available, 2 tasks, each task requests 1 GPU\n num_gpus_per_task = 1 (whole number)\n Expected:\n - Task 0 sees CUDA_VISIBLE_DEVICES='0'\n - Task 1 sees CUDA_VISIBLE_DEVICES='0' (wraps around due to oversubscription)\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus == 0:\n pytest.skip(\"No GPUs available on system\")\n\n self._check_ray_init(num_cpus=4, num_gpus=1)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=4, num_gpus=1, num_jobs=2)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=2, gpus_per_task=1, total_gpus=1)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=2, expected_gpus_per_task=1)\n assert gpu_assignments[0] == [0], f\"Expected [0] for task 0, got {gpu_assignments[0]}\"\n assert gpu_assignments[1] == [0], f\"Expected [0] for task 1, got {gpu_assignments[1]}\"\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0\", 1: \"0\"})\n self._verify_torch_gpu_count(results, {\"default\": 1})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_multiple_gpus_consecutive_task_execution(self):\n \"\"\"\n Integration Test: Ray tasks verify consecutive GPU assignment\n\n Scenario: 4 GPUs, 2 tasks, each task requests 2 GPUs\n num_gpus_per_task = 2 (whole number, consecutive assignment)\n Expected:\n - Task 0 sees CUDA_VISIBLE_DEVICES='0,1'\n - Task 1 sees CUDA_VISIBLE_DEVICES='2,3'\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus < 4:\n pytest.skip(f\"Need at least 4 GPUs, only {system_gpus} available\")\n\n self._check_ray_init(num_cpus=8, num_gpus=4)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=8, num_gpus=4, num_jobs=2)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=2, gpus_per_task=2, total_gpus=4)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=2, expected_gpus_per_task=2)\n assert gpu_assignments[0] == [0, 1], f\"Expected [0, 1] for task 0, got {gpu_assignments[0]}\"\n assert gpu_assignments[1] == [2, 3], f\"Expected [2, 3] for task 1, got {gpu_assignments[1]}\"\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0,1\", 1: \"2,3\"})\n self._verify_torch_gpu_count(results, {0: 2, 1: 2})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_multiple_gpus_round_robin_task_execution(self):\n \"\"\"\n Integration Test: Ray tasks verify round-robin GPU assignment\n\n Scenario: 2 GPUs, 4 tasks, each task requests 0.5 GPUs\n num_gpus_per_task = 0.5 (fractional, round-robin)\n Expected:\n - Task 0 sees CUDA_VISIBLE_DEVICES='0'\n - Task 1 sees CUDA_VISIBLE_DEVICES='1'\n - Task 2 sees CUDA_VISIBLE_DEVICES='0'\n - Task 3 sees CUDA_VISIBLE_DEVICES='1'\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus < 2:\n pytest.skip(f\"Need at least 2 GPUs, only {system_gpus} available\")\n\n self._check_ray_init(num_cpus=8, num_gpus=2)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=8, num_gpus=2, num_jobs=4)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=4, gpus_per_task=0.5, total_gpus=2)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=4, expected_gpus_per_task=1)\n expected_assignments = [0, 1, 0, 1]\n for task_id, expected_gpu in enumerate(expected_assignments):\n assert gpu_assignments[task_id] == [expected_gpu], (\n f\"Expected [{expected_gpu}] for task {task_id}, got {gpu_assignments[task_id]}\"\n )\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0\", 1: \"1\", 2: \"0\", 3: \"1\"})\n self._verify_torch_gpu_count(results, {\"default\": 1})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_concurrent_task_execution(self):\n \"\"\"\n Integration Test: Multiple Ray tasks execute concurrently with correct GPU assignments\n\n Scenario: 4 GPUs, 4 tasks (one-to-one mapping), each task requests 1 GPU\n num_gpus_per_task = 1 (whole number)\n Expected: Each task sees its assigned GPU (0, 1, 2, or 3)\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus < 4:\n pytest.skip(f\"Need at least 4 GPUs, only {system_gpus} available\")\n\n self._check_ray_init(num_cpus=16, num_gpus=4)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=16, num_gpus=4, num_jobs=4)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=4, gpus_per_task=1, total_gpus=4)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=4, expected_gpus_per_task=1)\n for task_id in range(4):\n assert gpu_assignments[task_id] == [task_id], (\n f\"Expected [{task_id}] for task {task_id}, got {gpu_assignments[task_id]}\"\n )\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n expected_cuda = {i: str(i) for i in range(4)}\n self._verify_cuda_visible_devices(results, expected_cuda)\n self._verify_torch_gpu_count(results, {\"default\": 1})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_edge_case_fractional_round_robin(self):\n \"\"\"\n Edge Case Test: Fractional GPU allocation with round-robin\n\n Scenario: 2 GPUs, 3 tasks, each task requests 2/3 GPUs\n num_gpus_per_task = 0.666... (fractional, round-robin)\n Expected: Tasks assigned in round-robin: Task 0->GPU 0, Task 1->GPU 1, Task 2->GPU 0\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus == 0:\n pytest.skip(\"No GPUs available on system\")\n\n self._check_ray_init(num_cpus=8, num_gpus=2)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=8, num_gpus=2, num_jobs=3)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=3, gpus_per_task=2 / 3, total_gpus=2)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=3, expected_gpus_per_task=1)\n assert gpu_assignments[0] == [0], f\"Task 0 should get GPU [0], got {gpu_assignments[0]}\"\n assert gpu_assignments[1] == [1], f\"Task 1 should get GPU [1], got {gpu_assignments[1]}\"\n assert gpu_assignments[2] == [0], f\"Task 2 should get GPU [0], got {gpu_assignments[2]}\"\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0\", 1: \"1\", 2: \"0\"})\n self._verify_torch_gpu_count(results, {\"default\": 1})\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_edge_case_float_gpu_per_task_raises(self):\n \"\"\"\n Edge Case Test: Float GPU allocation when >= 1 should raise AssertionError\n\n Scenario: 3 GPUs, 2 tasks, each task requests 1.5 GPUs\n num_gpus_per_task = 1.5 (float, >= 1)\n Expected: Should raise AssertionError because range() requires an integer\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus < 3:\n pytest.skip(f\"Need at least 3 GPUs, only {system_gpus} available\")\n\n self._check_ray_init(num_cpus=8, num_gpus=3)\n\n strategy = _construct_parallel_fold_strategy(num_cpus=8, num_gpus=3, num_jobs=2)\n\n # Should raise AssertionError because gpus_per_task must be int when >= 1\n with pytest.raises(AssertionError, match=\"When gpus_per_task >= 1, it must be an int\"):\n self._calculate_assignments(strategy, num_tasks=2, gpus_per_task=1.5, total_gpus=3)\n\n ray.shutdown()\n\n @pytest.mark.multi_gpu\n def test_ray_gpu_assignment_gpus_per_task_greater_than_total_task(self):\n \"\"\"\n Integration Test: Ray tasks where each task requires all available GPUs\n\n Scenario: 4 GPUs available, 2 tasks, each task requires 4 GPUs\n num_gpus_per_task = 4 (whole number, requires oversubscription)\n Expected:\n - Task 0 sees CUDA_VISIBLE_DEVICES='0,1,2,3'\n - Task 1 sees CUDA_VISIBLE_DEVICES='0,1,2,3'\n \"\"\"\n import ray\n\n system_gpus = self._get_system_gpu_count()\n if system_gpus < 4:\n pytest.skip(f\"Need at least 4 GPUs, only {system_gpus} available\")\n\n self._check_ray_init(num_cpus=8, num_gpus=4)\n ray_task_report_gpu_info = _create_ray_gpu_reporter()\n\n strategy = _construct_parallel_fold_strategy(num_cpus=8, num_gpus=4, num_jobs=2)\n gpu_assignments = self._calculate_assignments(strategy, num_tasks=2, gpus_per_task=4, total_gpus=4)\n\n self._verify_assignment_structure(gpu_assignments, num_tasks=2, expected_gpus_per_task=4)\n assert gpu_assignments[0] == [0, 1, 2, 3], f\"Expected [0, 1, 2, 3] for task 0, got {gpu_assignments[0]}\"\n assert gpu_assignments[1] == [0, 1, 2, 3], f\"Expected [0, 1, 2, 3] for task 1, got {gpu_assignments[1]}\"\n\n results = self._submit_and_collect_tasks(ray_task_report_gpu_info, gpu_assignments)\n self._verify_cuda_visible_devices(results, {0: \"0,1,2,3\", 1: \"0,1,2,3\"})\n self._verify_torch_gpu_count(results, {0: 4, 1: 4})\n\n ray.shutdown()\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_hpo_resource_allocation.py",
"content": "import pandas as pd\nimport pytest\n\nfrom autogluon.core.hpo.executors import CustomHpoExecutor, RayHpoExecutor\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.tabular.models import AbstractModel\n\n\nclass DummyBaseModel(AbstractModel):\n def __init__(self, minimum_resources=None, default_resources=None, **kwargs):\n self._minimum_resources = minimum_resources\n self._default_resources = default_resources\n super().__init__(**kwargs)\n\n def get_minimum_resources(self, **kwargs):\n return self._minimum_resources\n\n def _get_default_resources(self):\n num_cpus = self._default_resources.get(\"num_cpus\")\n num_gpus = self._default_resources.get(\"num_gpus\")\n return num_cpus, num_gpus\n\n\nclass DummyModel(DummyBaseModel):\n pass\n\n\nclass DummyBaggedModel(BaggedEnsembleModel):\n pass\n\n\ndummy_x = pd.DataFrame([1, 2, 3], columns=[\"Dummy\"])\n\n\ndef _initialize_executor(executor_cls, hyperparameter_tune_kwargs):\n executor = executor_cls()\n executor.initialize(hyperparameter_tune_kwargs)\n return executor\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_invalid_resources_per_fold_more_than_total(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test invalid resources per fold larger than total resources\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 10, \"num_gpus\": 1}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 2}}\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_invalid_resources_per_fold_less_than_minimum(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test invalid resources per fold less than minimum resources required\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 2}}\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_invalid_resources_per_trial_more_than_total(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test invalid resources per trial larger than total resources\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 1}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 10, \"num_gpus\": 1}}\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_invalid_resources_per_trial_less_than_minimum(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test invalid resources per trial less than minimum resources required\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.1}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0}}\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_valid_resources_per_fold_and_valid_resources_per_trial(\n mock_system_resources_ctx_mgr, executor_cls\n):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test valid resources per fold and resources per trial\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.1}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 0.5}}\n )\n bagged_model.initialize()\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n # 1 bag in parallel, 4 folds in parallel per bagged ensemble, each using 1 cpu and 0.1 gpus\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 4, \"num_gpus\": 0.4}\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_valid_resources_per_fold_and_no_resources_per_trial(\n mock_system_resources_ctx_mgr, executor_cls\n):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n # Test valid resources per fold and no resources per trial\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.01},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.1}},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model,\n )\n bagged_model.initialize()\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n # 1 bag in parallel, 8 folds in parallel per bagged ensemble, each using 1 cpu and 0.1 gpus\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 8, \"num_gpus\": 0.8}\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_valid_resources_per_trial_and_no_resources_per_fold(\n mock_system_resources_ctx_mgr, executor_cls\n):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n # Test valid resources per trial and no resources per fold\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(\n model_base=base_model, hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.1}}\n )\n bagged_model.initialize()\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n # 1 bag in parallel, 1 fold in parallel per bagged ensemble, using 1 cpu and 0.1 gpus\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 1, \"num_gpus\": 0.1}\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_and_bagging_no_resources_per_trial_and_no_resources_per_fold(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 2}\n # Test valid resources per trial and no resources per fold\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n base_model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.5},\n )\n base_model.initialize()\n bagged_model = DummyBaggedModel(model_base=base_model, hyperparameters={})\n bagged_model.initialize()\n executor.register_resources(bagged_model, k_fold=8, X=dummy_x, **total_resources)\n # Only 1 trial can run at a time. Give full resources to it\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 8, \"num_gpus\": 1}\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_without_bagging_invalid_resources_per_trial_more_than_total_resources(\n mock_system_resources_ctx_mgr, executor_cls\n):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 4}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test valid resources per fold and resources per trial\n model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 10, \"num_gpus\": 0.2}},\n )\n model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(model, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_without_bagging_invalid_resources_per_trial_less_than_minimum_resources(\n mock_system_resources_ctx_mgr, executor_cls\n):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 4}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test valid resources per fold and resources per trial\n model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.01}},\n )\n model.initialize()\n with pytest.raises(AssertionError) as e:\n executor.register_resources(model, X=dummy_x, **total_resources)\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_without_bagging_valid_resources_per_trial(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 4}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test valid resources per fold and resources per trial\n model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n hyperparameters={\"ag_args_fit\": {\"num_cpus\": 1, \"num_gpus\": 0.2}},\n )\n model.initialize()\n executor.register_resources(model, X=dummy_x, **total_resources)\n # 4 trials in parallel, each using 1 cpu and 0.2 gpus\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 1, \"num_gpus\": 0.2}\n\n\n@pytest.mark.parametrize(\"executor_cls\", [RayHpoExecutor, CustomHpoExecutor])\ndef test_hpo_without_bagging_no_resources_per_trial(mock_system_resources_ctx_mgr, executor_cls):\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": 4}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n total_resources = {\"num_cpus\": 8, \"num_gpus\": 1}\n with mock_system_resources_ctx_mgr(num_cpus=total_resources[\"num_cpus\"], num_gpus=total_resources[\"num_gpus\"]):\n # Test valid resources per fold and resources per trial\n model = DummyModel(\n minimum_resources={\"num_cpus\": 1, \"num_gpus\": 0.1},\n )\n model.initialize()\n executor.register_resources(model, X=dummy_x, **total_resources)\n if executor_cls == RayHpoExecutor:\n # 4 trials in parallel, each using 1 cpu and 0.25 gpus(the maximum possible)\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 2, \"num_gpus\": 0.25}\n elif executor_cls == CustomHpoExecutor:\n # custom backend use all resources for one trial\n assert executor.hyperparameter_tune_kwargs[\"resources_per_trial\"] == {\"num_cpus\": 8, \"num_gpus\": 1}\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_resource_allocation_combined.py",
"content": "import time\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.core.hpo.executors import CustomHpoExecutor, RayHpoExecutor\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.core.models.ensemble.fold_fitting_strategy import (\n ParallelLocalFoldFittingStrategy,\n SequentialLocalFoldFittingStrategy,\n)\nfrom autogluon.tabular.models import AbstractModel\n\n\nclass DummyBaseModel(AbstractModel):\n def __init__(self, minimum_resources=None, default_resources=None, **kwargs):\n self._minimum_resources = minimum_resources\n self._default_resources = default_resources\n super().__init__(**kwargs)\n\n def get_minimum_resources(self, **kwargs):\n return self._minimum_resources\n\n def _get_default_resources(self):\n num_cpus = self._default_resources.get(\"num_cpus\")\n num_gpus = self._default_resources.get(\"num_gpus\")\n return num_cpus, num_gpus\n\n\nclass DummyModel(DummyBaseModel):\n pass\n\n\nclass DummyBaggedModel(BaggedEnsembleModel):\n pass\n\n\ndummy_x = pd.DataFrame([1, 2, 3], columns=[\"Dummy\"])\n\n\ndef _initialize_executor(executor_cls, hyperparameter_tune_kwargs):\n executor = executor_cls()\n executor.initialize(hyperparameter_tune_kwargs)\n return executor\n\n\ndef _prepare_data():\n # prepare an all numeric data so that we don't need to clean labels and features\n data = [[1, 10], [2, 20], [3, 30]]\n df = pd.DataFrame(data, columns=[\"Number\", \"Age\"])\n label = \"Age\"\n X = df.drop(columns=[label])\n y = df[label]\n return X, y\n\n\ndef _construct_dummy_fold_strategy(\n fold_strategy_cls, num_jobs, num_folds_parallel, resource_granted, model_base, bagged_model\n):\n train_data, test_data = _prepare_data()\n args = dict(\n model_base=model_base,\n model_base_kwargs=dict(),\n bagged_ensemble_model=bagged_model,\n X=train_data,\n y=test_data,\n X_pseudo=None,\n y_pseudo=None,\n sample_weight=None,\n time_start=time.time(),\n time_limit=0,\n models=[],\n oof_pred_proba=np.array([]),\n oof_pred_model_repeats=np.array([]),\n save_folds=True,\n time_limit_fold_ratio=1,\n **resource_granted, # These will be processed at abstract model level in a real tabular predictor\n )\n if fold_strategy_cls == ParallelLocalFoldFittingStrategy:\n args[\"num_jobs\"] = num_jobs\n args[\"num_folds_parallel\"] = num_folds_parallel\n return fold_strategy_cls(**args)\n\n\ndef _test_bagging(\n fold_strategy_cls,\n num_jobs,\n num_folds_parallel,\n model_base,\n bagged_model,\n resources,\n expected_answer,\n):\n fold_fitting_strategy = _construct_dummy_fold_strategy(\n fold_strategy_cls=fold_strategy_cls,\n num_jobs=num_jobs,\n num_folds_parallel=num_folds_parallel,\n resource_granted=resources,\n model_base=model_base,\n bagged_model=bagged_model,\n )\n expected_resources_per_model = expected_answer[\"resources_per_model\"]\n assert fold_fitting_strategy.resources == expected_resources_per_model\n if fold_strategy_cls == ParallelLocalFoldFittingStrategy:\n expected_model_in_parallel = expected_answer[\"model_in_parallel\"]\n assert fold_fitting_strategy.num_parallel_jobs == expected_model_in_parallel\n\n\ndef _test_functionality(mock_system_resources_ctx_mgr, test_args):\n system_resources = test_args.get(\"system_resources\", {\"num_cpus\": 16, \"num_gpus\": 4})\n total_resources = test_args.get(\"total_resources\", {\"num_cpus\": \"auto\", \"num_gpus\": \"auto\"})\n model_default_resources = test_args.get(\"model_default_resources\", {\"num_cpus\": 1, \"num_gpus\": 0})\n model_minimum_resources = test_args.get(\"model_minimum_resources\", {\"num_cpus\": 1, \"num_gpus\": 0})\n ag_args_ensemble = test_args.get(\"ag_args_ensemble\", {})\n ag_args_fit = test_args.get(\"ag_args_fit\", {})\n num_bag_folds = test_args.get(\"num_bag_folds\", 0)\n fold_strategy_cls = test_args.get(\"fold_strategy_cls\", ParallelLocalFoldFittingStrategy)\n num_trials = test_args.get(\"num_trials\", 0)\n executor_cls = test_args.get(\"executor_cls\", RayHpoExecutor)\n expected_answer = test_args.get(\"expected_answer\")\n hpo = num_trials > 0\n parallel_hpo = hpo and executor_cls == RayHpoExecutor\n with mock_system_resources_ctx_mgr(\n num_cpus=system_resources.get(\"num_cpus\"), num_gpus=system_resources.get(\"num_gpus\")\n ):\n model = DummyModel(\n minimum_resources=model_minimum_resources,\n default_resources=model_default_resources,\n hyperparameters={\"ag_args_fit\": ag_args_fit},\n )\n model.initialize()\n if num_bag_folds > 0:\n model = DummyBaggedModel(model, hyperparameters={\"ag_args_fit\": ag_args_ensemble})\n model.initialize()\n resources = model._preprocess_fit_resources(\n total_resources=total_resources, k_fold=num_bag_folds, parallel_hpo=parallel_hpo\n )\n resources.pop(\"k_fold\")\n if hpo:\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"random\", \"num_trials\": num_trials}\n executor = _initialize_executor(executor_cls, hyperparameter_tune_kwargs)\n executor.register_resources(model, k_fold=num_bag_folds, X=dummy_x, **resources)\n resources_per_trial = executor.hyperparameter_tune_kwargs[\"resources_per_trial\"]\n assert resources_per_trial == expected_answer[\"resources_per_trial\"]\n if num_bag_folds > 0:\n _test_bagging(\n fold_strategy_cls=fold_strategy_cls,\n num_jobs=num_bag_folds,\n num_folds_parallel=num_bag_folds,\n model_base=model.model_base,\n bagged_model=model,\n resources=resources_per_trial,\n expected_answer=expected_answer,\n )\n else:\n if num_bag_folds > 0:\n _test_bagging(\n fold_strategy_cls=fold_strategy_cls,\n num_jobs=num_bag_folds,\n num_folds_parallel=num_bag_folds,\n model_base=model.model_base,\n bagged_model=model,\n resources=resources,\n expected_answer=expected_answer,\n )\n else:\n assert resources == expected_answer[\"resources_per_model\"]\n\n\ntests_dict = {\n \"valid_ag_args_fit\": (\n {\n \"ag_args_fit\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 8, \"num_gpus\": 2}},\n }\n ),\n \"valid_ag_args_fit_without_gpu_default_no_gpu\": (\n {\n \"ag_args_fit\": {\"num_cpus\": 8},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 0},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 8, \"num_gpus\": 0}},\n }\n ),\n \"valid_ag_args_fit_without_gpu_default_gpu\": (\n {\n \"ag_args_fit\": {\"num_cpus\": 8},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 8, \"num_gpus\": 1}},\n }\n ),\n \"valid_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"ag_args_ensemble\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1}},\n }\n ),\n \"valid_ag_args_ensemble\": (\n {\n \"ag_args_ensemble\": {\"num_cpus\": 8, \"num_gpus\": 2}, # should be ignored\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"total_resources_with_valid_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5}},\n }\n ),\n \"total_resources_with_valid_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1}, # should be ignored\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 8, \"num_gpus\": 2}},\n }\n ),\n \"total_resources_with_valid_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1}},\n }\n ),\n \"total_resources\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 8, \"num_gpus\": 2}},\n }\n ),\n \"without_anything\": (\n {\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"bagging_with_total_resources_and_valid_ag_args_ensemble_and_valid_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 2},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}, \"model_in_parallel\": 2},\n }\n ),\n \"bagging_with_valid_ag_args_ensemble_and_valid_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 2},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}, \"model_in_parallel\": 2},\n }\n ),\n \"bagging_with_valid_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 8, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}, \"model_in_parallel\": 4},\n }\n ),\n \"bagging_with_valid_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1}, \"model_in_parallel\": 4},\n }\n ),\n \"bagging_without_anything\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 0},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0}, \"model_in_parallel\": 8},\n }\n ),\n \"bagging_without_anything_with_gpu\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"expected_answer\": {\"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5}, \"model_in_parallel\": 8},\n }\n ),\n \"sequential_bagging_with_total_resources_and_valid_ag_args_ensemble_and_valid_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_bag_folds\": 8,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n },\n }\n ),\n \"sequential_bagging_with_valid_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1},\n },\n }\n ),\n \"sequential_bagging_with_valid_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n # \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1},\n # FIXME: Above is commented out in v1.4 to fix bug in sequential.\n # But this creates inconsistency with Parallel logic. Resolve in v1.5.\n # To me, the below seems correct,\n # as we don't want to use GPUs for a model that doesn't default to using them.\n \"resources_per_model\": {\"num_cpus\": 1, \"num_gpus\": 0},\n },\n }\n ),\n \"sequential_bagging_with_valid_ag_args_fit_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1},\n },\n }\n ),\n \"sequential_bagging_without_anything\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_bag_folds\": 8,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 1},\n },\n }\n ),\n \"hpo_with_total_resources_and_ag_args_ensemble_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"hpo_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1}\n }, # ag_args_ensemble shouldn't affect hpo without bagging\n }\n ),\n \"hpo_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"hpo_with_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2}\n }, # ag_args_ensemble shouldn't affect hpo without bagging\n }\n ),\n \"hpo_with_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"hpo_without_anything\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 0},\n \"num_trials\": 2,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 0}},\n }\n ),\n \"hpo_without_anything_with_gpu\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2}},\n }\n ),\n \"custom_hpo_with_total_resources_and_ag_args_ensemble_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"custom_hpo_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2}\n }, # ag_args_ensemble shouldn't affect hpo without bagging\n }\n ),\n \"custom_hpo_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"custom_hpo_with_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 1, \"num_gpus\": 1}\n }, # ag_args_ensemble shouldn't affect hpo without bagging\n }\n ),\n \"custom_hpo_with_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 2, \"num_gpus\": 1}},\n }\n ),\n \"custom_hpo_without_anything\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 0},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 1, \"num_gpus\": 0}},\n }\n ),\n \"custom_hpo_without_anything_with_gpu\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"expected_answer\": {\"resources_per_trial\": {\"num_cpus\": 1, \"num_gpus\": 1}},\n }\n ),\n \"hpo_and_bagging_with_total_resources\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 2,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"model_in_parallel\": 2, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_total_resources_and_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 1, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 1, \"num_gpus\": 0.5},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 4, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_with_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 4, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_without_anything\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 0},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 0},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0},\n \"model_in_parallel\": 4, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_bagging_without_anything_with_gpu\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_default_resources\": {\"num_cpus\": 1, \"num_gpus\": 1},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"model_in_parallel\": 4, # This is models running in parallel in a bagged model\n },\n }\n ),\n \"hpo_and_sequential_bagging_with_total_resources_and_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n },\n }\n ),\n \"hpo_and_sequential_bagging_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n # \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n # FIXME: Above is commented out in v1.4 to fix bug in sequential.\n # But this creates inconsistency with Parallel logic. Resolve in v1.5.\n \"resources_per_model\": {\"num_cpus\": 1, \"num_gpus\": 0},\n },\n }\n ),\n \"hpo_and_sequential_bagging_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n # TODO: This is incorrect but doesn't cause big enough issue...\n # hpo resource calculator needs to know which folding strategy will be used, which is only being inferred later...\n # we calculate parallel folding for now...\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n },\n }\n ),\n \"hpo_and_sequential_bagging_with_total_resources\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n # TODO: This is incorrect but doesn't cause big enough issue...\n # hpo resource calculator needs to know which folding strategy will be used, which is only being inferred later...\n # we calculate parallel folding for now...\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n },\n }\n ),\n \"custom_hpo_and_bagging_with_total_resources_and_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2,\n },\n }\n ),\n \"custom_hpo_and_bagging_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 2,\n },\n }\n ),\n \"custom_hpo_and_bagging_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"model_in_parallel\": 2,\n },\n }\n ),\n \"custom_hpo_and_bagging_with_total_resources\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_in_parallel\": 4,\n },\n }\n ),\n \"custom_hpo_and_sequential_bagging_with_total_resources_and_ag_args_ensemble_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n },\n }\n ),\n \"custom_hpo_and_sequential_bagging_with_total_resources_and_ag_args_ensemble\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n \"resources_per_trial\": {\"num_cpus\": 4, \"num_gpus\": 1},\n # \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n # FIXME: Above is commented out in v1.4 to fix bug in sequential.\n # But this creates inconsistency with Parallel logic. Resolve in v1.5.\n \"resources_per_model\": {\"num_cpus\": 1, \"num_gpus\": 0},\n },\n }\n ),\n \"custom_hpo_and_sequential_bagging_with_total_resources_and_ag_args_fit\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n # TODO: This is incorrect but doesn't cause big enough issue...\n # hpo resource calculator needs to know which folding strategy will be used to determine if need to consider num_folds\n # but folding strategy is only being decided later...\n # we calculate parallel folding for now...\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 4, \"num_gpus\": 1},\n },\n }\n ),\n \"custom_hpo_and_sequential_bagging_with_total_resources\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"model_minimum_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"model_default_resources\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"num_bag_folds\": 4,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"expected_answer\": {\n # TODO: This is incorrect but doesn't cause big enough issue...\n # hpo resource calculator needs to know which folding strategy will be used to determine if need to consider num_folds\n # but folding strategy is only being decided later...\n # we calculate parallel folding for now...\n \"resources_per_trial\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"resources_per_model\": {\"num_cpus\": 2, \"num_gpus\": 0.5},\n },\n }\n ),\n # Should raise staring now\n \"hpo_and_bagging_invalid_ag_args_ensemble_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"num_trials\": 2,\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"hpo_and_bagging_invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"num_trials\": 2,\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"bagging_invalid_ag_args_ensemble_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"bagging_invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 1},\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"hpo_invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"should_raise\": True,\n }\n ),\n \"custom_hpo_invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"should_raise\": True,\n }\n ),\n \"sequential_bagging_invalid_ag_args_ensemble_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_ensemble\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"num_bag_folds\": 2,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"should_raise\": True,\n }\n ),\n \"sequential_bagging_invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"num_bag_folds\": 2,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"should_raise\": True,\n }\n ),\n \"invalid_ag_args_fit_more_than_total\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"total_resources\": {\"num_cpus\": 8, \"num_gpus\": 2},\n \"ag_args_fit\": {\"num_cpus\": 99, \"num_gpus\": 99},\n \"should_raise\": True,\n }\n ),\n \"hpo_and_bagging_invalid_ag_args_ensemble_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"hpo_and_bagging_invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"bagging_invalid_ag_args_ensemble_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"bagging_invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_bag_folds\": 2,\n \"should_raise\": True,\n }\n ),\n \"sequential_bagging_invalid_ag_args_ensemble_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_bag_folds\": 2,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"should_raise\": True,\n }\n ),\n \"sequential_bagging_invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_bag_folds\": 2,\n \"fold_strategy_cls\": SequentialLocalFoldFittingStrategy,\n \"should_raise\": True,\n }\n ),\n \"hpo_invalid_ag_args_ensemble_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"should_raise\": True,\n }\n ),\n \"hpo_invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"should_raise\": True,\n }\n ),\n \"custom_hpo_invalid_ag_args_ensemble_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"should_raise\": True,\n }\n ),\n \"custom_hpo_invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_ensemble\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"num_trials\": 2,\n \"executor_cls\": CustomHpoExecutor,\n \"should_raise\": True,\n }\n ),\n \"invalid_ag_args_fit_less_than_min\": (\n {\n \"system_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"model_minimum_resources\": {\"num_cpus\": 16, \"num_gpus\": 4},\n \"ag_args_fit\": {\"num_cpus\": 4, \"num_gpus\": 1},\n \"should_raise\": True,\n }\n ),\n}\n\nids = sorted(list(tests_dict.keys()))\ntests = [tests_dict[id] for id in ids]\n\n\n@pytest.mark.parametrize(\"test_args\", tests, ids=ids)\ndef test_resource_allocation_combined_valid(mock_system_resources_ctx_mgr, test_args):\n should_raise = test_args.get(\"should_raise\", False)\n if should_raise:\n with pytest.raises(AssertionError) as e:\n _test_functionality(mock_system_resources_ctx_mgr=mock_system_resources_ctx_mgr, test_args=test_args)\n else:\n _test_functionality(mock_system_resources_ctx_mgr=mock_system_resources_ctx_mgr, test_args=test_args)\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_resources_mocking.py",
"content": "from autogluon.common.utils.resource_utils import ResourceManager\n\n\ndef test_resources_mocking(mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus):\n real_num_cpus = ResourceManager.get_cpu_count()\n real_num_gpus = ResourceManager.get_gpu_count()\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n assert ResourceManager.get_cpu_count() == mock_num_cpus\n assert ResourceManager.get_gpu_count() == mock_num_gpus\n assert ResourceManager.get_cpu_count() == real_num_cpus\n assert ResourceManager.get_gpu_count() == real_num_gpus\n"
},
{
"path": "tabular/tests/unittests/resource_allocation/test_total_resource_allocation.py",
"content": "import pytest\n\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.models.ensemble.bagged_ensemble_model import BaggedEnsembleModel\nfrom autogluon.tabular.models import AbstractModel\n\n\nclass DummyBaseModel(AbstractModel):\n def __init__(self, minimum_resources={}, **kwargs):\n self._minimum_resources = minimum_resources\n super().__init__(**kwargs)\n\n def get_minimum_resources(self, **kwargs):\n return self._minimum_resources\n\n def _get_default_resources(self):\n num_cpus = 1\n num_gpus = 1\n return num_cpus, num_gpus\n\n\nclass DummyModel(DummyBaseModel):\n pass\n\n\nclass DummyBaggedModel(BaggedEnsembleModel):\n pass\n\n\ndef test_bagged_model_with_total_resources(mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus, k_fold):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n model_base.initialize()\n bagged_model = DummyBaggedModel(model_base)\n total_resources = {\n \"num_cpus\": 1,\n \"num_gpus\": 0,\n }\n bagged_model.initialize()\n resources = bagged_model._preprocess_fit_resources(total_resources=total_resources, k_fold=k_fold)\n resources.pop(\"k_fold\")\n assert resources == total_resources\n\n # Given total resources more than what the system has\n total_resources = {\n \"num_cpus\": 99999,\n \"num_gpus\": 99999,\n }\n resources = bagged_model._preprocess_fit_resources(total_resources=total_resources, k_fold=k_fold)\n resources.pop(\"k_fold\")\n assert resources == {\"num_cpus\": ResourceManager.get_cpu_count(), \"num_gpus\": ResourceManager.get_gpu_count()}\n\n\ndef test_bagged_model_with_total_resources_and_ensemble_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus, k_fold\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n total_resources = {\n \"num_cpus\": 8,\n \"num_gpus\": 1,\n }\n model_base = DummyModel()\n model_base.initialize()\n bagged_model = DummyBaggedModel(\n model_base,\n hyperparameters={\n \"ag_args_fit\": {\n \"num_cpus\": 10,\n \"num_gpus\": 1,\n }\n },\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n bagged_model._preprocess_fit_resources(total_resources=total_resources, k_fold=k_fold)\n\n total_resources = {\n \"num_cpus\": 8,\n \"num_gpus\": 1,\n }\n model_base = DummyModel()\n ensemble_ag_args_fit = {\n \"num_cpus\": 4,\n \"num_gpus\": 1,\n }\n model_base.initialize()\n bagged_model = DummyBaggedModel(model_base, hyperparameters={\"ag_args_fit\": ensemble_ag_args_fit})\n bagged_model.initialize()\n resources = bagged_model._preprocess_fit_resources(total_resources=total_resources, k_fold=k_fold)\n resources.pop(\"k_fold\")\n assert resources == ensemble_ag_args_fit\n\n\ndef test_bagged_model_with_total_resources_but_no_gpu_specified(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus, k_fold\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n model_base.initialize()\n total_resources = {\n \"num_cpus\": 2,\n }\n bagged_model = DummyBaggedModel(model_base)\n bagged_model.initialize()\n resources = bagged_model._preprocess_fit_resources(total_resources=total_resources, k_fold=k_fold)\n resources.pop(\"k_fold\")\n default_model_resources = {\n \"num_cpus\": 2,\n \"num_gpus\": ResourceManager.get_gpu_count(),\n } # return all gpu resources as default needs gpu\n assert resources == default_model_resources\n\n\ndef test_bagged_model_without_total_resources_but_with_ensemble_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus, k_fold\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n model_base.initialize()\n bagged_model = DummyBaggedModel(\n model_base,\n hyperparameters={\n \"ag_args_fit\": {\n \"num_cpus\": 99999,\n \"num_gpus\": 99999,\n }\n },\n )\n bagged_model.initialize()\n with pytest.raises(AssertionError) as e:\n bagged_model._preprocess_fit_resources(k_fold=k_fold)\n\n model_base = DummyModel()\n model_base.initialize()\n ensemble_ag_args_fit = {\n \"num_cpus\": 1,\n \"num_gpus\": 0,\n }\n bagged_model = DummyBaggedModel(model_base, hyperparameters={\"ag_args_fit\": ensemble_ag_args_fit})\n bagged_model.initialize()\n resources = bagged_model._preprocess_fit_resources(k_fold=k_fold)\n resources.pop(\"k_fold\")\n assert resources == ensemble_ag_args_fit\n\n\ndef test_bagged_model_without_total_resources_and_without_model_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus, k_fold\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n model_base.initialize()\n bagged_model = DummyBaggedModel(model_base)\n bagged_model.initialize()\n resources = bagged_model._preprocess_fit_resources(k_fold=k_fold)\n resources.pop(\"k_fold\")\n # Bagged model should take all resources and internally calculate correct resources given ag_args_ensemble and ag_args_fit\n expected_model_resources = {\n \"num_cpus\": ResourceManager.get_cpu_count(),\n \"num_gpus\": ResourceManager.get_gpu_count(),\n }\n assert resources == expected_model_resources\n\n\ndef test_nonbagged_model_with_total_resources(mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n total_resources = {\n \"num_cpus\": 1,\n \"num_gpus\": 0,\n }\n resources = model_base._preprocess_fit_resources(total_resources=total_resources)\n assert resources == total_resources\n\n\ndef test_nonbagged_model_with_total_resources_but_no_gpu_specified(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n # If model by default needs gpu, we use it even if user didn't specify it\n model_base = DummyModel()\n total_resources = {\n \"num_cpus\": 2,\n }\n resources = model_base._preprocess_fit_resources(total_resources=total_resources)\n _, default_model_num_gpus = model_base._get_default_resources()\n default_model_resources = {\"num_cpus\": 2, \"num_gpus\": default_model_num_gpus}\n assert resources == default_model_resources\n\n\ndef test_nonbagged_model_with_total_resources_and_model_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel(hyperparameters={\"ag_args_fit\": {\"num_cpus\": 2, \"num_gpus\": 1}})\n total_resources = {\n \"num_cpus\": 1,\n \"num_gpus\": 1,\n }\n with pytest.raises(AssertionError) as e:\n model_base._preprocess_fit_resources(total_resources=total_resources)\n\n ag_args_fit = {\"num_cpus\": 1, \"num_gpus\": 0}\n model_base = DummyModel(hyperparameters={\"ag_args_fit\": ag_args_fit})\n total_resources = {\n \"num_cpus\": 8,\n \"num_gpus\": 1,\n }\n resources = model_base._preprocess_fit_resources(total_resources=total_resources)\n # Here both total_resources and ag_args_fit are specified, respect ag_args_fit\n assert resources == ag_args_fit\n\n\ndef test_nonbagged_model_without_total_resources(mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel()\n resources = model_base._preprocess_fit_resources()\n default_model_num_cpus, default_model_num_gpus = model_base._get_default_resources()\n default_model_resources = {\"num_cpus\": default_model_num_cpus, \"num_gpus\": default_model_num_gpus}\n assert resources == default_model_resources\n\n\ndef test_nonbagged_model_without_total_resources_but_with_model_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel(hyperparameters={\"ag_args_fit\": {\"num_cpus\": 99999, \"num_gpus\": 99999}})\n with pytest.raises(AssertionError) as e:\n model_base._preprocess_fit_resources()\n\n ag_args_fit = {\"num_cpus\": 2, \"num_gpus\": 2}\n model_base = DummyModel(hyperparameters={\"ag_args_fit\": ag_args_fit})\n resources = model_base._preprocess_fit_resources()\n assert resources == ag_args_fit\n\n\ndef test_nonbagged_model_without_total_resources_and_without_model_resources(\n mock_system_resources_ctx_mgr, mock_num_cpus, mock_num_gpus\n):\n with mock_system_resources_ctx_mgr(num_cpus=mock_num_cpus, num_gpus=mock_num_gpus):\n model_base = DummyModel(hyperparameters={})\n resources = model_base._preprocess_fit_resources()\n default_model_num_cpus, default_model_num_gpus = model_base._get_default_resources()\n default_model_resources = {\n \"num_cpus\": min(default_model_num_cpus, ResourceManager.get_cpu_count()),\n \"num_gpus\": min(default_model_num_gpus, ResourceManager.get_gpu_count()),\n }\n assert resources == default_model_resources\n"
},
{
"path": "tabular/tests/unittests/test_tabular.py",
"content": "\"\"\"Runs autogluon.tabular on multiple benchmark datasets.\n\n# TODO: assess that Autogluon correctly inferred the type of each feature (continuous vs categorical vs text)\n\n# TODO: may want to take allowed run-time of AutoGluon into account? Eg. can produce performance vs training time curves for each dataset.\n\n# TODO: We'd like to add extra benchmark datasets with the following properties:\n- parquet file format\n- poker hand data: https://archive.ics.uci.edu/ml/datasets/Poker+Hand\n- test dataset with just one data point\n- test dataset where order of columns different than in training data (same column names)\n- extreme-multiclass classification (500+ classes)\n- high-dimensional features + low-sample size\n- high levels of missingness in test data only, no missingness in train data\n- classification w severe class imbalance\n- regression with severely skewed Y-values (eg. predicting count data)\n- text features in dataset\n\"\"\"\n\nfrom __future__ import annotations\n\nimport os\nimport shutil\n\nimport numpy as np\nimport pandas as pd\nimport pytest\n\nfrom autogluon.common import space\nfrom autogluon.common.utils.simulation_utils import convert_simulation_artifacts_to_tabular_predictions_dict\nfrom autogluon.core.constants import BINARY, MULTICLASS, PROBLEM_TYPES_CLASSIFICATION, QUANTILE\nfrom autogluon.tabular import TabularDataset, TabularPredictor, __version__\nfrom autogluon.tabular.testing import FitHelper\n\nPARALLEL_LOCAL_BAGGING = \"parallel_local\"\nSEQUENTIAL_LOCAL_BAGGING = \"sequential_local\"\non_windows = os.name == \"nt\"\n\n\ndef test_tabular():\n \"\"\"\n Verifies that default parameter TabularPredictor works on binary, multiclass, regression and quantile tasks.\n \"\"\"\n fit_args = {\"time_limit\": 60}\n run_tabular_benchmarks(fit_args=fit_args)\n\n\ndef _assert_predict_dict_identical_to_predict(predictor: TabularPredictor, data: pd.DataFrame):\n \"\"\"Assert that predict_multi is identical to looping calls to predict\"\"\"\n for as_pandas in [True, False]:\n for inverse_transform in [True, False]:\n predict_dict = predictor.predict_multi(data=data, as_pandas=as_pandas, inverse_transform=inverse_transform)\n assert set(predictor.model_names()) == set(predict_dict.keys())\n for m in predictor.model_names():\n if not inverse_transform:\n model_pred = predictor._learner.predict(\n data, model=m, as_pandas=as_pandas, inverse_transform=inverse_transform\n )\n else:\n model_pred = predictor.predict(data, model=m, as_pandas=as_pandas)\n if as_pandas:\n # pandas default int type on Windows is int64, while on Linux it is int32\n if model_pred.dtype in [\"int64\", \"int32\"]:\n assert predict_dict[m].dtype in [\"int64\", \"int32\"]\n assert model_pred.astype(\"int64\").equals(predict_dict[m].astype(\"int64\"))\n else:\n assert model_pred.equals(predict_dict[m])\n else:\n assert np.array_equal(model_pred, predict_dict[m])\n\n\ndef _assert_predict_proba_dict_identical_to_predict_proba(predictor: TabularPredictor, data: pd.DataFrame):\n \"\"\"Assert that predict_proba_multi is identical to looping calls to predict_proba\"\"\"\n for as_pandas in [True, False]:\n for inverse_transform in [True, False]:\n for as_multiclass in [True, False]:\n predict_proba_dict = predictor.predict_proba_multi(\n data=data, as_pandas=as_pandas, as_multiclass=as_multiclass, inverse_transform=inverse_transform\n )\n assert set(predictor.model_names()) == set(predict_proba_dict.keys())\n for m in predictor.model_names():\n if not inverse_transform:\n model_pred_proba = predictor._learner.predict_proba(\n data,\n model=m,\n as_pandas=as_pandas,\n as_multiclass=as_multiclass,\n inverse_transform=inverse_transform,\n )\n else:\n model_pred_proba = predictor.predict_proba(\n data, model=m, as_pandas=as_pandas, as_multiclass=as_multiclass\n )\n if as_pandas:\n assert model_pred_proba.equals(predict_proba_dict[m])\n else:\n assert np.array_equal(model_pred_proba, predict_proba_dict[m])\n\n\ndef test_advanced_functionality():\n \"\"\"\n Tests a bunch of advanced functionality, including when used in combination.\n The idea is that if this test passes, we are in good shape.\n Simpler to test all of this within one test as it avoids repeating redundant setup such as fitting a predictor.\n \"\"\"\n directory_prefix = \"./datasets/\"\n dataset_name = \"toy_binary_10\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset_name)\n problem_type = dataset_info[\"problem_type\"]\n label = dataset_info[\"label\"]\n\n print(f\"Evaluating Advanced Functionality on Benchmark Dataset {dataset_name}\")\n directory = directory_prefix + \"advanced/\" + dataset_name + \"/\"\n savedir = directory + \"AutogluonOutput/\"\n shutil.rmtree(\n savedir, ignore_errors=True\n ) # Delete AutoGluon output directory to ensure previous runs' information has been removed.\n savedir_predictor_original = savedir + \"predictor/\"\n predictor: TabularPredictor = TabularPredictor(\n label=label, problem_type=problem_type, path=savedir_predictor_original\n ).fit(train_data)\n\n version_in_file = predictor._load_version_file(path=predictor.path)\n assert version_in_file == __version__\n\n leaderboard = predictor.leaderboard(data=test_data)\n\n # test metric_error leaderboard\n leaderboard_error = predictor.leaderboard(data=test_data, score_format=\"error\")\n assert sorted(leaderboard[\"model\"].to_list()) == sorted(leaderboard_error[\"model\"].to_list())\n leaderboard_combined = pd.merge(\n leaderboard, leaderboard_error[[\"model\", \"metric_error_test\", \"metric_error_val\"]], on=[\"model\"]\n )\n score_test = leaderboard_combined[\"score_test\"].to_list()\n score_val = leaderboard_combined[\"score_val\"].to_list()\n metric_error_test = leaderboard_combined[\"metric_error_test\"].to_list()\n metric_error_val = leaderboard_combined[\"metric_error_val\"].to_list()\n for score, error in zip(score_test, metric_error_test):\n assert predictor.eval_metric.convert_score_to_error(score) == error\n for score, error in zip(score_val, metric_error_val):\n assert predictor.eval_metric.convert_score_to_error(score) == error\n\n if not on_windows:\n predictor.plot_ensemble_model()\n\n # Test get_simulation_artifact\n simulation_artifact_no_test = predictor.simulation_artifact()\n assert \"pred_proba_dict_test\" not in simulation_artifact_no_test\n assert \"y_test\" not in simulation_artifact_no_test\n simulation_artifact = predictor.simulation_artifact(test_data=test_data)\n assert sorted(list(simulation_artifact[\"pred_proba_dict_test\"].keys())) == sorted(\n predictor.model_names(can_infer=True)\n )\n assert simulation_artifact[\"y_test\"].equals(predictor.transform_labels(test_data[label]))\n for sim_artifact in [simulation_artifact, simulation_artifact_no_test]:\n assert sim_artifact[\"label\"] == predictor.label\n assert sorted(list(sim_artifact[\"pred_proba_dict_val\"].keys())) == sorted(\n predictor.model_names(can_infer=True)\n )\n assert sim_artifact[\"eval_metric\"] == predictor.eval_metric.name\n assert sim_artifact[\"problem_type\"] == predictor.problem_type\n simulation_artifacts = {dataset_name: {0: simulation_artifact}}\n\n # Test convert_simulation_artifacts_to_tabular_predictions_dict\n aggregated_pred_proba, aggregated_ground_truth = convert_simulation_artifacts_to_tabular_predictions_dict(\n simulation_artifacts=simulation_artifacts\n )\n assert set(aggregated_pred_proba[dataset_name][0][\"pred_proba_dict_val\"].keys()) == set(\n predictor.model_names(can_infer=True)\n )\n assert set(aggregated_pred_proba[dataset_name][0][\"pred_proba_dict_test\"].keys()) == set(\n predictor.model_names(can_infer=True)\n )\n ground_truth_keys_expected = set(simulation_artifact.keys())\n ground_truth_keys_expected.remove(\"pred_proba_dict_val\")\n ground_truth_keys_expected.remove(\"pred_proba_dict_test\")\n assert set(aggregated_ground_truth[dataset_name][0].keys()) == ground_truth_keys_expected\n\n extra_metrics = [\"accuracy\", \"roc_auc\", \"log_loss\"]\n test_data_no_label = test_data.drop(columns=[label])\n with pytest.raises(ValueError):\n # Error because skip_score is False and label not present\n predictor.leaderboard(data=test_data_no_label)\n with pytest.raises(ValueError):\n # Error because extra_metrics != None and label not present\n predictor.leaderboard(data=test_data_no_label, skip_score=True, extra_metrics=extra_metrics)\n # Valid because skip_score=True\n leaderboard_no_score = predictor.leaderboard(data=test_data.drop(columns=[label]), skip_score=True)\n assert len(leaderboard) == len(leaderboard_no_score)\n assert \"pred_time_test\" in leaderboard_no_score\n assert \"pred_time_test_marginal\" in leaderboard_no_score\n assert \"score_test\" in leaderboard_no_score\n for i in range(len(leaderboard_no_score)):\n # Assert that score_test is NaN for all models\n assert leaderboard_no_score[\"score_test\"].isnull().iloc[i]\n leaderboard_extra = predictor.leaderboard(data=test_data, extra_info=True, extra_metrics=extra_metrics)\n _assert_predict_dict_identical_to_predict(predictor=predictor, data=test_data)\n _assert_predict_proba_dict_identical_to_predict_proba(predictor=predictor, data=test_data)\n assert set(predictor.model_names()) == set(leaderboard[\"model\"])\n assert set(predictor.model_names()) == set(leaderboard_extra[\"model\"])\n assert set(leaderboard_extra.columns).issuperset(set(leaderboard.columns))\n assert len(leaderboard) == len(leaderboard_extra)\n assert set(leaderboard_extra.columns).issuperset(\n set(extra_metrics)\n ) # Assert that extra_metrics are present in output\n num_models = len(predictor.model_names())\n feature_importances = predictor.feature_importance(data=test_data)\n original_features = set(train_data.columns)\n original_features.remove(label)\n assert set(feature_importances.index) == original_features\n assert set(feature_importances.columns) == {\"importance\", \"stddev\", \"p_value\", \"n\", \"p99_high\", \"p99_low\"}\n predictor.transform_features()\n test_data_transformed = predictor.transform_features(data=test_data)\n info = predictor.info()\n for model in predictor.model_names():\n model_info = predictor.model_info(model=model)\n model_info_2 = info[\"model_info\"][model]\n assert model_info[\"name\"] == model_info_2[\"name\"]\n assert model_info[\"name\"] == model\n assert set(model_info.keys()) == set(model_info_2.keys())\n model_hyperparameters = predictor.model_hyperparameters(model=model)\n assert isinstance(model_hyperparameters, dict)\n\n # Assert that transform_features=False works correctly\n y_pred = predictor.predict(test_data)\n y_pred_from_transform = predictor.predict(test_data_transformed, transform_features=False)\n assert y_pred.equals(y_pred_from_transform)\n\n y_pred_proba = None\n if predictor.can_predict_proba:\n y_pred_proba = predictor.predict_proba(test_data)\n y_pred_proba_from_transform = predictor.predict_proba(test_data_transformed, transform_features=False)\n assert y_pred_proba.equals(y_pred_proba_from_transform)\n\n assert predictor.model_names(persisted=True) == [] # Assert that no models were persisted during training\n assert predictor.unpersist() == [] # Assert that no models were unpersisted\n\n persisted_models = predictor.persist(models=\"all\", max_memory=None)\n assert set(predictor.model_names(persisted=True)) == set(persisted_models) # Ensure all models are persisted\n assert (\n predictor.persist(models=\"all\", max_memory=None) == []\n ) # Ensure that no additional models are persisted on repeated calls\n unpersised_models = predictor.unpersist()\n assert set(unpersised_models) == set(persisted_models)\n assert predictor.model_names(persisted=True) == [] # Assert that all models were unpersisted\n\n # Raise exception\n with pytest.raises(ValueError):\n predictor.persist(models=[\"UNKNOWN_MODEL_1\", \"UNKNOWN_MODEL_2\"])\n\n assert predictor.model_names(persisted=True) == []\n\n assert predictor.unpersist(models=[\"UNKNOWN_MODEL_1\", \"UNKNOWN_MODEL_2\"]) == []\n\n predictor.persist(models=\"all\", max_memory=None)\n predictor.save() # Save predictor while models are persisted: Intended functionality is that they won't be persisted when loaded.\n predictor_loaded = TabularPredictor.load(predictor.path) # Assert that predictor loading works\n leaderboard_loaded = predictor_loaded.leaderboard(data=test_data)\n assert len(leaderboard) == len(leaderboard_loaded)\n assert (\n predictor_loaded.model_names(persisted=True) == []\n ) # Assert that models were not still persisted after loading predictor\n\n _assert_predictor_size(predictor=predictor)\n # Test cloning logic\n with pytest.raises(AssertionError):\n # Ensure don't overwrite existing predictor\n predictor.clone(path=predictor.path)\n path_clone = predictor.clone(path=predictor.path + \"_clone\")\n predictor_clone = TabularPredictor.load(path_clone)\n assert predictor.path != predictor_clone.path\n if predictor_clone.can_predict_proba:\n y_pred_proba_clone = predictor_clone.predict_proba(test_data)\n assert y_pred_proba.equals(y_pred_proba_clone)\n y_pred_clone = predictor_clone.predict(test_data)\n assert y_pred.equals(y_pred_clone)\n leaderboard_clone = predictor_clone.leaderboard(data=test_data)\n assert len(leaderboard) == len(leaderboard_clone)\n\n # Test cloning for deployment logic\n path_clone_for_deployment_og = predictor.path + \"_clone_for_deployment\"\n with pytest.raises(FileNotFoundError):\n # Assert that predictor does not exist originally\n TabularPredictor.load(path_clone_for_deployment_og)\n path_clone_for_deployment = predictor.clone_for_deployment(path=path_clone_for_deployment_og)\n assert path_clone_for_deployment == path_clone_for_deployment_og\n predictor_clone_for_deployment = TabularPredictor.load(path_clone_for_deployment)\n assert predictor.path != predictor_clone_for_deployment.path\n if predictor_clone_for_deployment.can_predict_proba:\n y_pred_proba_clone_for_deployment = predictor_clone_for_deployment.predict_proba(test_data)\n assert y_pred_proba.equals(y_pred_proba_clone_for_deployment)\n y_pred_clone_for_deployment = predictor_clone_for_deployment.predict(test_data)\n assert y_pred.equals(y_pred_clone_for_deployment)\n leaderboard_clone_for_deployment = predictor_clone_for_deployment.leaderboard(data=test_data)\n assert len(leaderboard) >= len(leaderboard_clone_for_deployment)\n # Raise exception due to lacking fit artifacts\n with pytest.raises(FileNotFoundError):\n predictor_clone_for_deployment.refit_full()\n\n assert predictor.model_refit_map() == dict()\n predictor.refit_full()\n if not on_windows:\n predictor.plot_ensemble_model()\n assert len(predictor.model_refit_map()) == num_models\n assert len(predictor.model_names()) == num_models * 2\n for model in predictor.model_names():\n predictor.predict(data=test_data, model=model)\n predictor.refit_full() # Confirm that refit_models aren't further refit.\n assert len(predictor.model_refit_map()) == num_models\n assert len(predictor.model_names()) == num_models * 2\n predictor.delete_models(models_to_keep=[], dry_run=True) # Test that dry-run doesn't delete models\n assert len(predictor.model_names()) == num_models * 2\n predictor.predict(data=test_data)\n\n # Test refit_full with train_data_extra argument\n refit_full_models = list(predictor.model_refit_map().values())\n predictor.delete_models(models_to_delete=refit_full_models)\n assert len(predictor.model_names()) == num_models\n assert predictor.model_refit_map() == dict()\n predictor.refit_full(train_data_extra=test_data) # train_data_extra argument\n assert len(predictor.model_names()) == num_models * 2\n assert len(predictor.model_refit_map()) == num_models\n predictor.predict(data=test_data)\n\n predictor.delete_models(models_to_keep=[], dry_run=False) # Test that dry_run=False deletes models\n assert len(predictor.model_names()) == 0\n assert len(predictor.leaderboard()) == 0\n assert len(predictor.leaderboard(extra_info=True)) == 0\n # Assert that predictor can no longer predict\n try:\n predictor.predict(data=test_data)\n except:\n pass\n else:\n raise AssertionError(\"predictor.predict should raise exception after all models are deleted\")\n # Assert that clone is not impacted by changes to original\n assert len(predictor_clone.leaderboard(data=test_data)) == len(leaderboard_clone)\n if predictor_clone.can_predict_proba:\n y_pred_proba_clone_2 = predictor_clone.predict_proba(data=test_data)\n assert y_pred_proba.equals(y_pred_proba_clone_2)\n y_pred_clone_2 = predictor_clone.predict(data=test_data)\n assert y_pred.equals(y_pred_clone_2)\n print(\"Tabular Advanced Functionality Test Succeeded.\")\n\n\ndef _assert_predictor_size(predictor: TabularPredictor):\n predictor_size_disk = predictor.disk_usage()\n predictor_size_disk_per_file = predictor.disk_usage_per_file()\n assert predictor_size_disk > 0 # Assert that .disk_usage() produces a >0 result and doesn't crash\n assert len(predictor_size_disk_per_file) > 0\n assert predictor_size_disk == predictor_size_disk_per_file.sum()\n\n\ndef test_advanced_functionality_bagging():\n directory_prefix = \"./datasets/\"\n dataset_name = \"toy_binary_10\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(\"toy_binary_10\")\n problem_type = dataset_info[\"problem_type\"]\n label = dataset_info[\"label\"]\n\n print(f\"Evaluating Advanced Functionality (Bagging) on Benchmark Dataset {dataset_name}\")\n directory = directory_prefix + \"advanced/\" + dataset_name + \"/\"\n savedir = directory + \"AutogluonOutput/\"\n shutil.rmtree(\n savedir, ignore_errors=True\n ) # Delete AutoGluon output directory to ensure previous runs' information has been removed.\n gbm_hyperparameters = {\"ag_args_fit\": {\"foo\": 5}}\n predictor = TabularPredictor(label=label, problem_type=problem_type, path=savedir).fit(\n train_data,\n num_bag_folds=2,\n hyperparameters={\"GBM\": gbm_hyperparameters},\n )\n\n expected_num_models = 2\n assert len(predictor.model_names()) == expected_num_models\n\n _assert_predict_dict_identical_to_predict(predictor=predictor, data=test_data)\n _assert_predict_proba_dict_identical_to_predict_proba(predictor=predictor, data=test_data)\n\n oof_pred_proba = predictor.predict_proba_oof()\n assert len(oof_pred_proba) == len(train_data)\n\n predict_proba_dict_oof = predictor.predict_proba_multi()\n for m in predictor.model_names():\n predict_proba_oof = predictor.predict_proba_oof(model=m)\n assert predict_proba_oof.equals(predict_proba_dict_oof[m])\n\n predict_dict_oof = predictor.predict_multi()\n for m in predictor.model_names():\n predict_oof = predictor.predict_oof(model=m)\n assert predict_oof.equals(predict_dict_oof[m])\n\n score_oof = predictor.evaluate_predictions(train_data[label], oof_pred_proba)\n model_best = predictor.model_best\n\n predictor.refit_full()\n assert len(predictor.model_refit_map()) == expected_num_models\n assert len(predictor.model_names()) == expected_num_models * 2\n\n model_best_refit = predictor.model_best\n assert model_best != model_best_refit\n\n # assert that refit model uses original model's OOF predictions\n oof_pred_proba_refit = predictor.predict_proba_oof()\n assert oof_pred_proba.equals(oof_pred_proba_refit)\n\n # check predict_proba_multi after refit does not raise an exception\n predict_proba_dict_oof = predictor.predict_proba_multi()\n for m in predictor.model_names():\n predict_proba_oof = predictor.predict_proba_oof(model=m)\n assert predict_proba_oof.equals(predict_proba_dict_oof[m])\n\n # check predict_multi after refit does not raise an exception\n predict_dict_oof = predictor.predict_multi()\n for m in predictor.model_names():\n predict_oof = predictor.predict_oof(model=m)\n assert predict_oof.equals(predict_dict_oof[m])\n\n info = predictor.info()\n for model in predictor.model_names():\n model_info = predictor.model_info(model=model)\n model_info_2 = info[\"model_info\"][model]\n assert model_info[\"name\"] == model_info_2[\"name\"]\n assert model_info[\"name\"] == model\n assert set(model_info.keys()) == set(model_info_2.keys())\n model_hyperparameters = predictor.model_hyperparameters(model=model)\n assert isinstance(model_hyperparameters, dict)\n assert predictor.model_hyperparameters(model=\"LightGBM_BAG_L1\") == gbm_hyperparameters\n lightgbm_full_params = predictor.model_hyperparameters(\n model=\"LightGBM_BAG_L1_FULL\", include_ag_args_ensemble=False\n )\n assert lightgbm_full_params != gbm_hyperparameters\n lightgbm_full_params.pop(\"num_boost_round\")\n assert lightgbm_full_params == gbm_hyperparameters\n\n\ndef verify_predictor(\n predictor: TabularPredictor,\n train_data: pd.DataFrame,\n test_data: pd.DataFrame,\n crash_in_oof: bool,\n run_distill: bool,\n):\n label = predictor.label\n y_test = test_data[label]\n assert len(predictor._trainer._models_failed_to_train_errors.keys()) == 0\n results = predictor.fit_summary(verbosity=4)\n original_features = list(train_data)\n original_features.remove(label)\n assert original_features == predictor.original_features\n y_pred_empty = predictor.predict(test_data[0:0])\n assert len(y_pred_empty) == 0\n y_pred = predictor.predict(test_data)\n perf_dict = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)\n if predictor._trainer.bagged_mode and not crash_in_oof:\n # TODO: Test index alignment with original training data (first handle duplicated rows / dropped rows edge cases)\n y_pred_oof = predictor.predict_oof()\n y_pred_proba_oof = predictor.predict_proba_oof(as_multiclass=False)\n y_pred_oof_transformed = predictor.predict_oof(transformed=True)\n y_pred_proba_oof_transformed = predictor.predict_proba_oof(as_multiclass=False, transformed=True)\n\n # Assert expected type output\n if predictor.problem_type == QUANTILE:\n assert isinstance(y_pred_oof, pd.DataFrame)\n assert isinstance(y_pred_oof_transformed, pd.DataFrame)\n else:\n assert isinstance(y_pred_oof, pd.Series)\n assert isinstance(y_pred_oof_transformed, pd.Series)\n if predictor.problem_type in [MULTICLASS, QUANTILE]:\n assert isinstance(y_pred_proba_oof, pd.DataFrame)\n assert isinstance(y_pred_proba_oof_transformed, pd.DataFrame)\n else:\n if predictor.problem_type == BINARY:\n assert isinstance(predictor.predict_proba_oof(), pd.DataFrame)\n assert isinstance(y_pred_proba_oof, pd.Series)\n assert isinstance(y_pred_proba_oof_transformed, pd.Series)\n\n assert y_pred_oof_transformed.equals(predictor.transform_labels(y_pred_oof, proba=False))\n\n # Test that the transform_labels method is capable of reproducing the same output when converting back and forth, and test that oof 'transform' parameter works properly.\n y_pred_proba_oof_inverse = predictor.transform_labels(y_pred_proba_oof, proba=True)\n y_pred_proba_oof_inverse_inverse = predictor.transform_labels(\n y_pred_proba_oof_inverse, proba=True, inverse=True\n )\n y_pred_oof_inverse = predictor.transform_labels(y_pred_oof)\n y_pred_oof_inverse_inverse = predictor.transform_labels(y_pred_oof_inverse, inverse=True)\n\n if isinstance(y_pred_proba_oof_transformed, pd.DataFrame):\n pd.testing.assert_frame_equal(y_pred_proba_oof_transformed, y_pred_proba_oof_inverse)\n pd.testing.assert_frame_equal(y_pred_proba_oof, y_pred_proba_oof_inverse_inverse)\n else:\n pd.testing.assert_series_equal(y_pred_proba_oof_transformed, y_pred_proba_oof_inverse)\n pd.testing.assert_series_equal(y_pred_proba_oof, y_pred_proba_oof_inverse_inverse)\n if isinstance(y_pred_oof_transformed, pd.DataFrame):\n pd.testing.assert_frame_equal(y_pred_oof_transformed, y_pred_oof_inverse)\n pd.testing.assert_frame_equal(y_pred_oof, y_pred_oof_inverse_inverse)\n else:\n pd.testing.assert_series_equal(y_pred_oof_transformed, y_pred_oof_inverse)\n pd.testing.assert_series_equal(y_pred_oof, y_pred_oof_inverse_inverse)\n\n # Test that index of both the internal training data and the oof outputs are consistent in their index values.\n X_internal, y_internal = predictor.load_data_internal()\n y_internal_index = list(y_internal.index)\n assert list(X_internal.index) == y_internal_index\n assert list(y_pred_oof.index) == y_internal_index\n assert list(y_pred_proba_oof.index) == y_internal_index\n assert list(y_pred_oof_transformed.index) == y_internal_index\n assert list(y_pred_proba_oof_transformed.index) == y_internal_index\n else:\n # Raise exception\n with pytest.raises(AssertionError):\n predictor.predict_oof()\n with pytest.raises(AssertionError):\n predictor.predict_proba_oof()\n if run_distill:\n predictor.distill(time_limit=60, augment_args={\"size_factor\": 0.5})\n\n\ndef run_tabular_benchmarks(\n fit_args: dict,\n subsample_size: int | None = None,\n datasets: list[str] | None = None,\n run_distill: bool = False,\n crash_in_oof: bool = False,\n):\n print(\"Running fit with args:\")\n print(fit_args)\n\n if datasets is None:\n datasets = [\n \"toy_binary_10\",\n \"toy_multiclass_10\",\n \"toy_regression_10\",\n \"toy_quantile_10\",\n ]\n for dataset_name in datasets:\n predictor = FitHelper.fit_and_validate_dataset(\n dataset_name=dataset_name,\n fit_args=fit_args,\n sample_size=subsample_size,\n refit_full=False,\n expected_model_count=None,\n raise_on_model_failure=True,\n delete_directory=False,\n )\n train_data, test_data, dataset_info = FitHelper.load_dataset(name=dataset_name)\n verify_predictor(\n predictor=predictor,\n train_data=train_data,\n test_data=test_data,\n crash_in_oof=crash_in_oof,\n run_distill=run_distill,\n )\n shutil.rmtree(predictor.path, ignore_errors=True)\n\n\ndef test_pseudolabeling():\n datasets = [\n \"toy_binary\",\n \"toy_multiclass\",\n \"toy_regression\",\n ]\n\n hyperparam_setting = {\n \"GBM\": {\"num_boost_round\": 10},\n \"XGB\": {\"n_estimators\": 10},\n }\n\n fit_args = dict(\n hyperparameters=hyperparam_setting,\n time_limit=20,\n )\n\n fit_args_best = dict(\n presets=\"best_quality\",\n num_bag_folds=2,\n num_bag_sets=1,\n ag_args_ensemble=dict(fold_fitting_strategy=\"sequential_local\"),\n dynamic_stacking=False,\n )\n for idx in range(len(datasets)):\n dataset = datasets[idx]\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset)\n label = dataset_info[\"label\"]\n problem_type = dataset_info[\"problem_type\"]\n name = dataset\n\n print(f\"Testing dataset with name: {name}, problem type: {problem_type}\")\n\n if problem_type in PROBLEM_TYPES_CLASSIFICATION:\n valid_class_idxes = test_data[label].isin(train_data[label].unique())\n test_data = test_data[valid_class_idxes]\n\n error_msg_og = (\n f\"pseudolabel threw an exception during fit, it should have \"\n f\"succeeded on problem type:{problem_type} with dataset name:{name}, \"\n f\"with problem_type: {problem_type}. Under settings:\"\n )\n\n # Test label already given. If test label already given doesn't use pseudo labeling filter.\n try:\n print(\"Pseudolabel Testing: Pre-labeled data 'fit_pseudolabel'\")\n _, y_pred_proba = TabularPredictor(label=label, problem_type=problem_type).fit_pseudolabel(\n pseudo_data=test_data,\n return_pred_prob=True,\n train_data=train_data,\n **fit_args,\n )\n except Exception as e:\n assert False, error_msg_og + \"labeled test data\"\n\n try:\n print(\"Pseudolabel Testing: Pre-labeled data, best quality 'fit_pseudolabel'\")\n _, y_pred_proba = TabularPredictor(label=label, problem_type=problem_type).fit_pseudolabel(\n pseudo_data=test_data,\n return_pred_prob=True,\n train_data=train_data,\n **fit_args_best,\n **fit_args,\n )\n except Exception as e:\n assert False, error_msg_og + \"labeled test data, best quality\"\n\n # Test unlabeled pseudo data\n unlabeled_test_data = test_data.drop(columns=label)\n for flag_ensemble in [True, False]:\n error_prefix = \"ensemble \" if flag_ensemble else \"\"\n error_msg = error_prefix + error_msg_og\n for is_weighted_ensemble in [True, False]:\n error_suffix = \" with pseudo label model weighted ensembling\" if is_weighted_ensemble else \"\"\n\n try:\n print(\"Pseudolabel Testing: Unlabeled data 'fit_pseudolabel'\")\n _, y_pred_proba = TabularPredictor(label=label, problem_type=problem_type).fit_pseudolabel(\n pseudo_data=unlabeled_test_data,\n return_pred_prob=True,\n train_data=train_data,\n use_ensemble=flag_ensemble,\n fit_ensemble=is_weighted_ensemble,\n **fit_args,\n )\n except Exception as e:\n assert False, error_msg + \"unlabeled test data\" + error_suffix\n\n try:\n print(\"Pseudolabel Testing: Unlabeled data, best quality 'fit_pseudolabel'\")\n _, y_pred_proba = TabularPredictor(label=label, problem_type=problem_type).fit_pseudolabel(\n pseudo_data=unlabeled_test_data,\n return_pred_prob=True,\n train_data=train_data,\n use_ensemble=flag_ensemble,\n fit_ensemble=is_weighted_ensemble,\n **fit_args_best,\n **fit_args,\n )\n except Exception as e:\n assert False, error_msg + \"unlabeled test data, best quality\" + error_suffix\n\n\ndef test_tabular_bag_stack_hpo():\n num_bag_folds = 2\n num_bag_sets = 2\n num_stack_levels = 1\n time_limit = 50\n hyperparameters = {\n \"GBM\": {\"num_boost_round\": 20, \"learning_rate\": space.Real(0.01, 0.1)},\n \"NN_TORCH\": {\"num_epochs\": 1, \"learning_rate\": space.Real(0.001, 0.01)},\n }\n hyperparameter_tune_kwargs = {\"scheduler\": \"local\", \"searcher\": \"auto\"}\n\n datasets = [\"toy_binary_10\"]\n subsample_size = 100\n\n fit_args = {\n \"num_bag_folds\": num_bag_folds,\n \"num_bag_sets\": num_bag_sets,\n \"num_stack_levels\": num_stack_levels,\n \"time_limit\": time_limit,\n \"hyperparameter_tune_kwargs\": hyperparameter_tune_kwargs,\n \"hyperparameters\": hyperparameters,\n }\n run_tabular_benchmarks(\n subsample_size=subsample_size,\n datasets=datasets,\n fit_args=fit_args,\n )\n\n\ndef test_tabular_hpo():\n hyperparameter_tune_kwargs = {\n \"scheduler\": \"local\",\n \"searcher\": \"auto\",\n \"num_trials\": 3,\n }\n subsample_size = 100\n fit_args = {\n \"verbosity\": 2, # how much output to print\n \"time_limit\": 240,\n \"hyperparameter_tune_kwargs\": hyperparameter_tune_kwargs,\n }\n run_tabular_benchmarks(subsample_size=subsample_size, fit_args=fit_args)\n\n\ndef test_tabular_feature_prune():\n feature_prune_kwargs = {\n \"stop_threshold\": 3,\n \"prune_ratio\": 0.05,\n \"prune_threshold\": \"none\",\n \"n_train_subsample\": 1000,\n \"n_fi_subsample\": 5000,\n \"min_fi_samples\": 5000,\n \"feature_prune_time_limit\": 10,\n \"raise_exception\": True,\n }\n datasets = [\"adult\"]\n\n subsample_size = 1000\n gbm_options = {\"num_boost_round\": 20}\n hyperparameters = {\"GBM\": gbm_options}\n\n fit_args = {\n \"hyperparameters\": hyperparameters,\n \"feature_prune_kwargs\": feature_prune_kwargs,\n \"time_limit\": 60,\n }\n run_tabular_benchmarks(subsample_size=subsample_size, datasets=datasets, fit_args=fit_args)\n\n\ndef _construct_tabular_bag_test_config(fold_fitting_strategy) -> dict:\n num_bag_folds = 3\n num_bag_sets = 2\n num_stack_levels = 0\n\n nn_options = {\"num_epochs\": 1}\n gbm_options = {\"num_boost_round\": 30}\n hyperparameters = {\"GBM\": gbm_options, \"NN_TORCH\": nn_options}\n\n fit_args = {\n \"num_bag_folds\": num_bag_folds,\n \"num_bag_sets\": num_bag_sets,\n \"num_stack_levels\": num_stack_levels,\n \"hyperparameters\": hyperparameters,\n \"ag_args_ensemble\": {\n \"fold_fitting_strategy\": fold_fitting_strategy,\n },\n \"time_limit\": 60,\n }\n ###################################################################\n return fit_args\n\n\ndef test_tabular_parallel_local_bagging():\n fit_args = _construct_tabular_bag_test_config(PARALLEL_LOCAL_BAGGING)\n run_tabular_benchmarks(fit_args=fit_args)\n\n\ndef test_tabular_sequential_local_bagging():\n fit_args = _construct_tabular_bag_test_config(SEQUENTIAL_LOCAL_BAGGING)\n run_tabular_benchmarks(fit_args=fit_args)\n\n\ndef test_sample_weight():\n dataset_name = \"toy_regression_10\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset_name)\n label = dataset_info[\"label\"]\n problem_type = dataset_info[\"problem_type\"]\n\n directory_prefix = \"./datasets/\"\n print(f\"Evaluating Benchmark Dataset {dataset_name}\")\n directory = os.path.join(directory_prefix, dataset_name)\n savedir = os.path.join(directory, \"AutogluonOutput\")\n shutil.rmtree(\n savedir, ignore_errors=True\n ) # Delete AutoGluon output directory to ensure previous runs' information has been removed.\n sample_weight = \"sample_weights\"\n weights = np.abs(\n np.random.rand(\n len(train_data),\n )\n )\n test_weights = np.abs(\n np.random.rand(\n len(test_data),\n )\n )\n train_data[sample_weight] = weights\n test_data_weighted = test_data.copy()\n test_data_weighted[sample_weight] = test_weights\n fit_args = {\"raise_on_model_failure\": True}\n predictor = TabularPredictor(\n label=label, path=savedir, problem_type=problem_type, sample_weight=sample_weight\n ).fit(train_data, **fit_args)\n ldr = predictor.leaderboard(test_data)\n perf = predictor.evaluate(test_data)\n # Run again with weight_evaluation:\n # FIXME: RMSE doesn't support sample_weight, this entire call doesn't make sense\n predictor = TabularPredictor(\n label=label, path=savedir, problem_type=problem_type, sample_weight=sample_weight, weight_evaluation=True\n ).fit(train_data, **fit_args)\n # perf = predictor.evaluate(test_data_weighted) # TODO: Doesn't work without implementing sample_weight in evaluate\n predictor.distill(time_limit=10)\n ldr = predictor.leaderboard(test_data_weighted)\n\n\ndef test_tabular_bag_stack():\n num_bag_folds = 2\n num_bag_sets = 1\n num_stack_levels = 1\n\n datasets = [\"toy_binary_10\"]\n\n nn_options = {\"num_epochs\": 2}\n gbm_options = [\n {\"num_boost_round\": 40},\n {\n \"num_boost_round\": 100,\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ]\n hyperparameters = {\"GBM\": gbm_options, \"NN_TORCH\": nn_options}\n time_limit = 240\n\n fit_args = {\n \"num_bag_folds\": num_bag_folds,\n \"num_bag_sets\": num_bag_sets,\n \"num_stack_levels\": num_stack_levels,\n \"hyperparameters\": hyperparameters,\n \"ag_args_ensemble\": dict(fold_fitting_strategy=\"sequential_local\"),\n \"time_limit\": time_limit,\n }\n run_tabular_benchmarks(fit_args=fit_args, run_distill=True, datasets=datasets)\n\n\ndef test_tabular_bag_stack_use_bag_holdout():\n num_bag_folds = 2\n num_bag_sets = 1\n num_stack_levels = 1\n\n datasets = [\"toy_binary_10\"]\n\n nn_options = {\"num_epochs\": 2}\n gbm_options = [\n {\"num_boost_round\": 40},\n {\n \"num_boost_round\": 100,\n \"learning_rate\": 0.03,\n \"num_leaves\": 128,\n \"feature_fraction\": 0.9,\n \"min_data_in_leaf\": 3,\n \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n },\n ]\n hyperparameters = {\"GBM\": gbm_options, \"NN_TORCH\": nn_options}\n time_limit = 240\n\n fit_args = {\n \"num_bag_folds\": num_bag_folds,\n \"num_bag_sets\": num_bag_sets,\n \"num_stack_levels\": num_stack_levels,\n \"use_bag_holdout\": True,\n \"hyperparameters\": hyperparameters,\n \"time_limit\": time_limit,\n \"ag_args_ensemble\": dict(fold_fitting_strategy=\"sequential_local\"),\n }\n run_tabular_benchmarks(\n fit_args=fit_args,\n run_distill=True,\n crash_in_oof=True,\n datasets=datasets,\n )\n\n\ndef test_tabular_raise_on_nonfinite_float_labels():\n predictor = TabularPredictor(label=\"y\")\n nonfinite_values = [np.nan, np.inf, -np.inf]\n\n for idx, nonfinite_value in enumerate(nonfinite_values):\n train_data = TabularDataset({\"x\": [0.0, 1.0, 2.0, 3.0, 4.0], \"y\": [0.0, 1.0, 2.0, 3.0, 4.0]})\n train_data.loc[idx, \"y\"] = nonfinite_value\n\n with pytest.raises(ValueError) as ex_info:\n predictor.fit(train_data)\n assert str(ex_info.value).split()[-1] == str(idx)\n\n\ndef test_tabular_raise_on_nonfinite_class_labels():\n predictor = TabularPredictor(label=\"y\")\n nonfinite_values = [np.nan, np.inf, -np.inf]\n\n for idx, nonfinite_value in enumerate(nonfinite_values):\n train_data = TabularDataset({\"x\": [0.0, 1.0, 2.0, 3.0, 4.0], \"y\": [\"a\", \"b\", \"c\", \"d\", \"e\"]})\n train_data.loc[idx, \"y\"] = nonfinite_value\n\n with pytest.raises(ValueError) as ex_info:\n predictor.fit(train_data)\n assert str(ex_info.value).split()[-1] == str(idx)\n\n\ndef test_tabular_log_to_file():\n dataset = \"toy_binary_10\"\n train_data, test_data, dataset_info = FitHelper.load_dataset(dataset)\n label = dataset_info[\"label\"]\n\n predictor = TabularPredictor(label=label, log_to_file=True).fit(\n train_data=train_data, hyperparameters={\"DUMMY\": {}}\n )\n log = TabularPredictor.load_log(predictor_path=predictor.path)\n assert \"TabularPredictor saved.\" in log[-1]\n\n log_file = os.path.join(\".\", \"temp.log\")\n predictor = TabularPredictor(label=label, log_to_file=True, log_file_path=log_file).fit(\n train_data=train_data, hyperparameters={\"DUMMY\": {}}\n )\n log = TabularPredictor.load_log(log_file_path=log_file)\n assert \"TabularPredictor saved.\" in log[-1]\n if not on_windows:\n os.remove(log_file)\n\n with pytest.raises(AssertionError):\n TabularPredictor.load_log()\n"
},
{
"path": "timeseries/setup.py",
"content": "#!/usr/bin/env python\nimport importlib.util\n\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport os\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec)\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag)\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = \"timeseries\"\ninstall_requires = [\n # version ranges added in ag.get_dependency_version_ranges()\n \"joblib\", # version range defined in `core/_setup_utils.py`\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"scipy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"torch\", # version range defined in `core/_setup_utils.py`\n \"lightning\", # version range defined in `core/_setup_utils.py`\n \"transformers[sentencepiece]\", # version range defined in `core/_setup_utils.py`\n \"accelerate\", # version range defined in `core/_setup_utils.py`\n \"gluonts>=0.15.0,<0.17\",\n \"networkx\", # version range defined in `core/_setup_utils.py`\n \"statsforecast>=1.7.0,<2.0.2\",\n \"mlforecast>=0.14.0,<0.15.0\", # cannot upgrade since v0.15.0 introduced a breaking change to DirectTabular\n \"utilsforecast>=0.2.3,<0.2.12\", # to prevent breaking changes that propagate through mlforecast's dependency\n \"coreforecast>=0.0.12,<0.0.17\", # to prevent breaking changes that propagate through mlforecast's dependency\n \"fugue>=0.9.0\", # prevent dependency clash with omegaconf\n \"tqdm\", # version range defined in `core/_setup_utils.py`\n \"orjson~=3.9\", # use faster JSON implementation in GluonTS\n \"einops>=0.7,<1\", # required by Chronos-2 and Toto\n \"chronos-forecasting>=2.2.2,<2.4\",\n \"peft>=0.13.0,<0.18\", # version range same as in chronos-forecasting[extras]\n \"tensorboard>=2.9,<3\", # fixes https://github.com/autogluon/autogluon/issues/3612\n f\"autogluon.core=={version}\",\n f\"autogluon.common=={version}\",\n f\"autogluon.features=={version}\",\n f\"autogluon.tabular[catboost,lightgbm,xgboost]=={version}\",\n]\n\nextras_require = {\n \"tests\": [\n \"pytest\",\n \"ruff>=0.0.285\",\n \"flaky>=3.7,<4\",\n \"pytest-timeout>=2.1,<3\",\n ],\n \"ray\": [\n f\"autogluon.core[raytune]=={version}\",\n ],\n}\n\nextras_require[\"all\"] = list(set.union(*(set(extras_require[extra]) for extra in [\"ray\"])))\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n extras_require=extras_require,\n **setup_args,\n )\n"
},
{
"path": "timeseries/src/autogluon/timeseries/__init__.py",
"content": "from autogluon.common.utils.log_utils import _add_stream_handler\n\ntry:\n from .version import __version__\nexcept ImportError:\n pass\n\nfrom .dataset import TimeSeriesDataFrame\nfrom .predictor import TimeSeriesPredictor\n\n_add_stream_handler()\n\n\n__all__ = [\"TimeSeriesDataFrame\", \"TimeSeriesPredictor\"]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/configs/__init__.py",
"content": "from .hyperparameter_presets import get_hyperparameter_presets\nfrom .predictor_presets import get_predictor_presets\n\n__all__ = [\"get_hyperparameter_presets\", \"get_predictor_presets\"]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/configs/hyperparameter_presets.py",
"content": "from typing import Any\n\n\ndef get_hyperparameter_presets() -> dict[str, dict[str, dict[str, Any] | list[dict[str, Any]]]]:\n return {\n \"very_light\": {\n \"Naive\": {},\n \"SeasonalNaive\": {},\n \"ETS\": {},\n \"Theta\": {},\n \"RecursiveTabular\": {\"max_num_samples\": 100_000},\n \"DirectTabular\": {\"max_num_samples\": 100_000},\n },\n \"light\": {\n \"SeasonalNaive\": {},\n \"ETS\": {},\n \"Theta\": {},\n \"RecursiveTabular\": {},\n \"DirectTabular\": {},\n \"TemporalFusionTransformer\": {},\n \"Chronos2\": {\"model_path\": \"autogluon/chronos-2-small\"},\n },\n \"default\": {\n \"SeasonalNaive\": {},\n \"AutoETS\": {},\n \"DynamicOptimizedTheta\": {},\n \"RecursiveTabular\": {},\n \"DirectTabular\": {},\n \"TemporalFusionTransformer\": {},\n \"DeepAR\": {},\n \"Chronos2\": [\n {},\n {\n \"ag_args\": {\"name_suffix\": \"SmallFineTuned\"},\n \"model_path\": \"autogluon/chronos-2-small\",\n \"fine_tune\": True,\n \"eval_during_fine_tune\": True,\n },\n ],\n \"Chronos\": {\n \"ag_args\": {\"name_suffix\": \"WithRegressor\"},\n \"model_path\": \"bolt_small\",\n \"target_scaler\": \"standard\",\n \"covariate_regressor\": {\"model_name\": \"CAT\", \"model_hyperparameters\": {\"iterations\": 1000}},\n },\n },\n }\n"
},
{
"path": "timeseries/src/autogluon/timeseries/configs/predictor_presets.py",
"content": "\"\"\"Preset configurations for autogluon.timeseries Predictors\"\"\"\n\nfrom typing import Any\n\nTIMESERIES_PRESETS_ALIASES = dict(\n best=\"best_quality\",\n high=\"high_quality\",\n medium=\"medium_quality\",\n bq=\"best_quality\",\n hq=\"high_quality\",\n mq=\"medium_quality\",\n)\n\n\ndef get_predictor_presets() -> dict[str, Any]:\n predictor_presets = dict(\n best_quality={\"hyperparameters\": \"default\", \"num_val_windows\": \"auto\", \"refit_every_n_windows\": \"auto\"},\n high_quality={\"hyperparameters\": \"default\"},\n medium_quality={\"hyperparameters\": \"light\"},\n fast_training={\"hyperparameters\": \"very_light\"},\n # Chronos-2 models\n chronos2={\n \"hyperparameters\": {\"Chronos2\": {\"model_path\": \"autogluon/chronos-2\"}},\n \"skip_model_selection\": True,\n },\n chronos2_small={\n \"hyperparameters\": {\"Chronos2\": {\"model_path\": \"autogluon/chronos-2-small\"}},\n \"skip_model_selection\": True,\n },\n chronos2_ensemble={\n \"hyperparameters\": {\n \"Chronos2\": [\n {\"model_path\": \"autogluon/chronos-2\", \"ag_args\": {\"name_suffix\": \"ZeroShot\"}},\n {\n \"model_path\": \"autogluon/chronos-2-small\",\n \"fine_tune\": True,\n \"eval_during_fine_tune\": True,\n \"ag_args\": {\"name_suffix\": \"SmallFineTuned\"},\n },\n ]\n },\n },\n # Chronos-Bolt models\n bolt_tiny={\n \"hyperparameters\": {\"Chronos\": {\"model_path\": \"bolt_tiny\"}},\n \"skip_model_selection\": True,\n },\n bolt_mini={\n \"hyperparameters\": {\"Chronos\": {\"model_path\": \"bolt_mini\"}},\n \"skip_model_selection\": True,\n },\n bolt_small={\n \"hyperparameters\": {\"Chronos\": {\"model_path\": \"bolt_small\"}},\n \"skip_model_selection\": True,\n },\n bolt_base={\n \"hyperparameters\": {\"Chronos\": {\"model_path\": \"bolt_base\"}},\n \"skip_model_selection\": True,\n },\n )\n\n # update with aliases\n predictor_presets = {\n **predictor_presets,\n **{k: predictor_presets[v].copy() for k, v in TIMESERIES_PRESETS_ALIASES.items()},\n }\n\n return predictor_presets\n"
},
{
"path": "timeseries/src/autogluon/timeseries/dataset/__init__.py",
"content": "from .ts_dataframe import TimeSeriesDataFrame\n\n__all__ = [\"TimeSeriesDataFrame\"]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/dataset/ts_dataframe.py",
"content": "from __future__ import annotations\n\nimport copy\nimport itertools\nimport logging\nimport reprlib\nfrom collections.abc import Iterable\nfrom itertools import islice\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Any, Final, Type, overload\n\nimport numpy as np\nimport pandas as pd\nfrom joblib.parallel import Parallel, delayed\nfrom pandas.core.internals import ArrayManager, BlockManager # type: ignore\nfrom typing_extensions import Self\n\nfrom autogluon.common.loaders import load_pd\n\nlogger = logging.getLogger(__name__)\n\n\nclass TimeSeriesDataFrame(pd.DataFrame):\n \"\"\"A collection of univariate time series, where each row is identified by an (``item_id``, ``timestamp``) pair.\n\n For example, a time series dataframe could represent the daily sales of a collection of products, where each\n ``item_id`` corresponds to a product and ``timestamp`` corresponds to the day of the record.\n\n Parameters\n ----------\n data : pd.DataFrame, str, pathlib.Path or Iterable\n Time series data to construct a ``TimeSeriesDataFrame``. The class currently supports four input formats.\n\n 1. Time series data in a pandas DataFrame format without multi-index. For example::\n\n item_id timestamp target\n 0 0 2019-01-01 0\n 1 0 2019-01-02 1\n 2 0 2019-01-03 2\n 3 1 2019-01-01 3\n 4 1 2019-01-02 4\n 5 1 2019-01-03 5\n 6 2 2019-01-01 6\n 7 2 2019-01-02 7\n 8 2 2019-01-03 8\n\n You can also use :meth:`~autogluon.timeseries.TimeSeriesDataFrame.from_data_frame` for loading data in such format.\n\n 2. Path to a data file in CSV or Parquet format. The file must contain columns ``item_id`` and ``timestamp``, as well as columns with time series values. This is similar to Option 1 above (pandas DataFrame format without multi-index). Both remote (e.g., S3) and local paths are accepted. You can also use :meth:`~autogluon.timeseries.TimeSeriesDataFrame.from_path` for loading data in such format.\n\n 3. Time series data in pandas DataFrame format with multi-index on ``item_id`` and ``timestamp``. For example::\n\n target\n item_id timestamp\n 0 2019-01-01 0\n 2019-01-02 1\n 2019-01-03 2\n 1 2019-01-01 3\n 2019-01-02 4\n 2019-01-03 5\n 2 2019-01-01 6\n 2019-01-02 7\n 2019-01-03 8\n\n 4. Time series data in Iterable format. For example::\n\n iterable_dataset = [\n {\"target\": [0, 1, 2], \"start\": pd.Period(\"01-01-2019\", freq='D')},\n {\"target\": [3, 4, 5], \"start\": pd.Period(\"01-01-2019\", freq='D')},\n {\"target\": [6, 7, 8], \"start\": pd.Period(\"01-01-2019\", freq='D')}\n ]\n\n You can also use :meth:`~autogluon.timeseries.TimeSeriesDataFrame.from_iterable_dataset` for loading data in such format.\n\n static_features : pd.DataFrame, str or pathlib.Path, optional\n An optional dataframe describing the metadata of each individual time series that does not change with time.\n Can take real-valued or categorical values. For example, if ``TimeSeriesDataFrame`` contains sales of various\n products, static features may refer to time-independent features like color or brand.\n\n The index of the ``static_features`` index must contain a single entry for each item present in the respective\n ``TimeSeriesDataFrame``. For example, the following ``TimeSeriesDataFrame``::\n\n target\n item_id timestamp\n A 2019-01-01 0\n 2019-01-02 1\n 2019-01-03 2\n B 2019-01-01 3\n 2019-01-02 4\n 2019-01-03 5\n\n is compatible with the following ``static_features``::\n\n feat_1 feat_2\n item_id\n A 2.0 bar\n B 5.0 foo\n\n ``TimeSeriesDataFrame`` will ensure consistency of static features during serialization/deserialization, copy\n and slice operations.\n\n If ``static_features`` are provided during ``fit``, the ``TimeSeriesPredictor`` expects the same metadata to be\n available during prediction time.\n id_column : str, optional\n Name of the ``item_id`` column, if it's different from the default. This argument is only used when\n constructing a TimeSeriesDataFrame using format 1 (DataFrame without multi-index) or 2 (path to a file).\n timestamp_column : str, optional\n Name of the ``timestamp`` column, if it's different from the default. This argument is only used when\n constructing a TimeSeriesDataFrame using format 1 (DataFrame without multi-index) or 2 (path to a file).\n num_cpus : int, default = -1\n Number of CPU cores used to process the iterable dataset in parallel. Set to -1 to use all cores. This argument\n is only used when constructing a TimeSeriesDataFrame using format 4 (iterable dataset).\n\n \"\"\"\n\n index: pd.MultiIndex # type: ignore\n _metadata = [\"_static_features\"]\n\n IRREGULAR_TIME_INDEX_FREQSTR: Final[str] = \"IRREG\"\n ITEMID: Final[str] = \"item_id\"\n TIMESTAMP: Final[str] = \"timestamp\"\n\n def __init__(\n self,\n data: pd.DataFrame | str | Path | Iterable,\n static_features: pd.DataFrame | str | Path | None = None,\n id_column: str | None = None,\n timestamp_column: str | None = None,\n num_cpus: int = -1,\n *args,\n **kwargs,\n ):\n if isinstance(data, (BlockManager, ArrayManager)):\n # necessary for copy constructor to work in pandas <= 2.0.x. In >= 2.1.x this is replaced by _constructor_from_mgr\n pass\n elif isinstance(data, pd.DataFrame):\n if isinstance(data.index, pd.MultiIndex):\n self._validate_multi_index_data_frame(data)\n else:\n data = self._construct_tsdf_from_data_frame(\n data, id_column=id_column, timestamp_column=timestamp_column\n )\n elif isinstance(data, (str, Path)):\n data = self._construct_tsdf_from_data_frame(\n load_pd.load(str(data)), id_column=id_column, timestamp_column=timestamp_column\n )\n elif isinstance(data, Iterable):\n data = self._construct_tsdf_from_iterable_dataset(data, num_cpus=num_cpus)\n else:\n raise ValueError(f\"data must be a pd.DataFrame, Iterable, string or Path (received {type(data)}).\")\n super().__init__(data=data, *args, **kwargs) # type: ignore\n self._static_features: pd.DataFrame | None = None\n if static_features is not None:\n self.static_features = self._construct_static_features(static_features, id_column=id_column)\n\n @property\n def _constructor(self) -> Type[TimeSeriesDataFrame]:\n return TimeSeriesDataFrame\n\n def _constructor_from_mgr(self, mgr, axes):\n # Use the default constructor when constructing from _mgr. Otherwise pandas enters an infinite recursion by\n # repeatedly calling TimeSeriesDataFrame constructor\n df = self._from_mgr(mgr, axes=axes)\n df._static_features = self._static_features\n return df\n\n @classmethod\n def _construct_tsdf_from_data_frame(\n cls,\n df: pd.DataFrame,\n id_column: str | None = None,\n timestamp_column: str | None = None,\n ) -> pd.DataFrame:\n df = df.copy()\n if id_column is not None:\n assert id_column in df.columns, f\"Column '{id_column}' not found!\"\n if id_column != cls.ITEMID and cls.ITEMID in df.columns:\n logger.warning(\n f\"Renaming existing column '{cls.ITEMID}' -> '__{cls.ITEMID}' to avoid name collisions.\"\n )\n df.rename(columns={cls.ITEMID: \"__\" + cls.ITEMID}, inplace=True)\n df.rename(columns={id_column: cls.ITEMID}, inplace=True)\n\n if timestamp_column is not None:\n assert timestamp_column in df.columns, f\"Column '{timestamp_column}' not found!\"\n if timestamp_column != cls.TIMESTAMP and cls.TIMESTAMP in df.columns:\n logger.warning(\n f\"Renaming existing column '{cls.TIMESTAMP}' -> '__{cls.TIMESTAMP}' to avoid name collisions.\"\n )\n df.rename(columns={cls.TIMESTAMP: \"__\" + cls.TIMESTAMP}, inplace=True)\n df.rename(columns={timestamp_column: cls.TIMESTAMP}, inplace=True)\n\n if cls.TIMESTAMP in df.columns:\n df[cls.TIMESTAMP] = pd.to_datetime(df[cls.TIMESTAMP])\n\n cls._validate_data_frame(df)\n return df.set_index([cls.ITEMID, cls.TIMESTAMP])\n\n @classmethod\n def _construct_tsdf_from_iterable_dataset(cls, iterable_dataset: Iterable, num_cpus: int = -1) -> pd.DataFrame:\n def load_single_item(item_id: int, ts: dict) -> pd.DataFrame:\n start_timestamp = ts[\"start\"]\n freq = start_timestamp.freq\n if isinstance(start_timestamp, pd.Period):\n start_timestamp = start_timestamp.to_timestamp(how=\"S\")\n target = ts[\"target\"]\n datetime_index = tuple(pd.date_range(start_timestamp, periods=len(target), freq=freq))\n idx = pd.MultiIndex.from_product([(item_id,), datetime_index], names=[cls.ITEMID, cls.TIMESTAMP])\n return pd.Series(target, name=\"target\", index=idx).to_frame()\n\n cls._validate_iterable(iterable_dataset)\n all_ts = Parallel(n_jobs=num_cpus)(\n delayed(load_single_item)(item_id, ts) for item_id, ts in enumerate(iterable_dataset)\n )\n return pd.concat(all_ts)\n\n @classmethod\n def _validate_multi_index_data_frame(cls, data: pd.DataFrame):\n \"\"\"Validate a multi-index pd.DataFrame can be converted to TimeSeriesDataFrame\"\"\"\n\n if not isinstance(data, pd.DataFrame):\n raise ValueError(f\"data must be a pd.DataFrame, got {type(data)}\")\n if not isinstance(data.index, pd.MultiIndex):\n raise ValueError(f\"data must have pd.MultiIndex, got {type(data.index)}\")\n if not pd.api.types.is_datetime64_dtype(data.index.dtypes[cls.TIMESTAMP]):\n raise ValueError(f\"for {cls.TIMESTAMP}, the only pandas dtype allowed is `datetime64`.\")\n if not data.index.names == (f\"{cls.ITEMID}\", f\"{cls.TIMESTAMP}\"):\n raise ValueError(\n f\"data must have index names as ('{cls.ITEMID}', '{cls.TIMESTAMP}'), got {data.index.names}\"\n )\n item_id_index = data.index.levels[0]\n if not (pd.api.types.is_integer_dtype(item_id_index) or pd.api.types.is_string_dtype(item_id_index)):\n raise ValueError(f\"all entries in index `{cls.ITEMID}` must be of integer or string dtype\")\n\n @classmethod\n def _validate_data_frame(cls, df: pd.DataFrame):\n \"\"\"Validate that a pd.DataFrame with ITEMID and TIMESTAMP columns can be converted to TimeSeriesDataFrame\"\"\"\n if not isinstance(df, pd.DataFrame):\n raise ValueError(f\"data must be a pd.DataFrame, got {type(df)}\")\n if cls.ITEMID not in df.columns:\n raise ValueError(f\"data must have a `{cls.ITEMID}` column\")\n if cls.TIMESTAMP not in df.columns:\n raise ValueError(f\"data must have a `{cls.TIMESTAMP}` column\")\n if df[cls.ITEMID].isnull().any():\n raise ValueError(f\"`{cls.ITEMID}` column can not have nan\")\n if df[cls.TIMESTAMP].isnull().any():\n raise ValueError(f\"`{cls.TIMESTAMP}` column can not have nan\")\n if not pd.api.types.is_datetime64_dtype(df[cls.TIMESTAMP]):\n raise ValueError(f\"for {cls.TIMESTAMP}, the only pandas dtype allowed is `datetime64`.\")\n item_id_column = df[cls.ITEMID]\n if not (pd.api.types.is_integer_dtype(item_id_column) or pd.api.types.is_string_dtype(item_id_column)):\n raise ValueError(f\"all entries in column `{cls.ITEMID}` must be of integer or string dtype\")\n\n @classmethod\n def _validate_iterable(cls, data: Iterable):\n if not isinstance(data, Iterable):\n raise ValueError(\"data must be of type Iterable.\")\n\n first = next(iter(data), None)\n if first is None:\n raise ValueError(\"data has no time-series.\")\n\n for i, ts in enumerate(itertools.chain([first], data)):\n if not isinstance(ts, dict):\n raise ValueError(f\"{i}'th time-series in data must be a dict, got{type(ts)}\")\n if not (\"target\" in ts and \"start\" in ts):\n raise ValueError(f\"{i}'th time-series in data must have 'target' and 'start', got{ts.keys()}\")\n if not isinstance(ts[\"start\"], pd.Period):\n raise ValueError(f\"{i}'th time-series must have a pandas Period as 'start', got {ts['start']}\")\n\n @classmethod\n def from_data_frame(\n cls,\n df: pd.DataFrame,\n id_column: str | None = None,\n timestamp_column: str | None = None,\n static_features_df: pd.DataFrame | None = None,\n ) -> TimeSeriesDataFrame:\n \"\"\"Construct a ``TimeSeriesDataFrame`` from a pandas DataFrame.\n\n Parameters\n ----------\n df : pd.DataFrame\n A pd.DataFrame with 'item_id' and 'timestamp' as columns. For example::\n\n item_id timestamp target\n 0 0 2019-01-01 0\n 1 0 2019-01-02 1\n 2 0 2019-01-03 2\n 3 1 2019-01-01 3\n 4 1 2019-01-02 4\n 5 1 2019-01-03 5\n 6 2 2019-01-01 6\n 7 2 2019-01-02 7\n 8 2 2019-01-03 8\n id_column : str, optional\n Name of the 'item_id' column if column name is different\n timestamp_column : str, optional\n Name of the 'timestamp' column if column name is different\n static_features_df : pd.DataFrame, optional\n A pd.DataFrame with 'item_id' column that contains the static features for each time series. For example::\n\n item_id feat_1 feat_2\n 0 0 foo 0.5\n 1 1 foo 2.2\n 2 2 bar 0.1\n\n Returns\n -------\n ts_df: TimeSeriesDataFrame\n A dataframe in TimeSeriesDataFrame format.\n \"\"\"\n return cls(df, static_features=static_features_df, id_column=id_column, timestamp_column=timestamp_column)\n\n @classmethod\n def from_path(\n cls,\n path: str | Path,\n id_column: str | None = None,\n timestamp_column: str | None = None,\n static_features_path: str | Path | None = None,\n ) -> TimeSeriesDataFrame:\n \"\"\"Construct a ``TimeSeriesDataFrame`` from a CSV or Parquet file.\n\n Parameters\n ----------\n path : str or pathlib.Path\n Path to a local or remote (e.g., S3) file containing the time series data in CSV or Parquet format.\n Example file contents::\n\n item_id,timestamp,target\n 0,2019-01-01,0\n 0,2019-01-02,1\n 0,2019-01-03,2\n 1,2019-01-01,3\n 1,2019-01-02,4\n 1,2019-01-03,5\n 2,2019-01-01,6\n 2,2019-01-02,7\n 2,2019-01-03,8\n\n id_column : str, optional\n Name of the 'item_id' column if column name is different\n timestamp_column : str, optional\n Name of the 'timestamp' column if column name is different\n static_features_path : str or pathlib.Path, optional\n Path to a local or remote (e.g., S3) file containing static features in CSV or Parquet format.\n Example file contents::\n\n item_id,feat_1,feat_2\n 0,foo,0.5\n 1,foo,2.2\n 2,bar,0.1\n\n Returns\n -------\n ts_df: TimeSeriesDataFrame\n A dataframe in TimeSeriesDataFrame format.\n \"\"\"\n return cls(path, static_features=static_features_path, id_column=id_column, timestamp_column=timestamp_column)\n\n @classmethod\n def from_iterable_dataset(cls, iterable_dataset: Iterable, num_cpus: int = -1) -> TimeSeriesDataFrame:\n \"\"\"Construct a ``TimeSeriesDataFrame`` from an Iterable of dictionaries each of which\n represent a single time series.\n\n This function also offers compatibility with GluonTS `ListDataset format `_.\n\n Parameters\n ----------\n iterable_dataset: Iterable\n An iterator over dictionaries, each with a ``target`` field specifying the value of the\n (univariate) time series, and a ``start`` field with the starting time as a pandas Period .\n Example::\n\n iterable_dataset = [\n {\"target\": [0, 1, 2], \"start\": pd.Period(\"01-01-2019\", freq='D')},\n {\"target\": [3, 4, 5], \"start\": pd.Period(\"01-01-2019\", freq='D')},\n {\"target\": [6, 7, 8], \"start\": pd.Period(\"01-01-2019\", freq='D')}\n ]\n num_cpus : int, default = -1\n Number of CPU cores used to process the iterable dataset in parallel. Set to -1 to use all cores.\n\n Returns\n -------\n ts_df: TimeSeriesDataFrame\n A dataframe in TimeSeriesDataFrame format.\n \"\"\"\n return cls(iterable_dataset, num_cpus=num_cpus)\n\n @property\n def item_ids(self) -> pd.Index:\n \"\"\"List of unique time series IDs contained in the data set.\"\"\"\n return self.index.unique(level=self.ITEMID)\n\n @classmethod\n def _construct_static_features(\n cls,\n static_features: pd.DataFrame | str | Path,\n id_column: str | None = None,\n ) -> pd.DataFrame:\n if isinstance(static_features, (str, Path)):\n static_features = load_pd.load(str(static_features))\n if not isinstance(static_features, pd.DataFrame):\n raise ValueError(\n f\"static_features must be a pd.DataFrame, string or Path (received {type(static_features)})\"\n )\n\n if id_column is not None:\n assert id_column in static_features.columns, f\"Column '{id_column}' not found in static_features!\"\n if id_column != cls.ITEMID and cls.ITEMID in static_features.columns:\n logger.warning(\n f\"Renaming existing column '{cls.ITEMID}' -> '__{cls.ITEMID}' to avoid name collisions.\"\n )\n static_features.rename(columns={cls.ITEMID: \"__\" + cls.ITEMID}, inplace=True)\n static_features.rename(columns={id_column: cls.ITEMID}, inplace=True)\n return static_features\n\n @property\n def static_features(self):\n return self._static_features\n\n @static_features.setter\n def static_features(self, value: pd.DataFrame | None):\n # if the current item index is not a multiindex, then we are dealing with a single\n # item slice. this should only happen when the user explicitly requests only a\n # single item or during `slice_by_timestep`. In this case we do not set static features\n if not isinstance(self.index, pd.MultiIndex):\n return\n\n if value is not None:\n if isinstance(value, pd.Series):\n value = value.to_frame()\n if not isinstance(value, pd.DataFrame):\n raise ValueError(f\"static_features must be a pandas DataFrame (received object of type {type(value)})\")\n if isinstance(value.index, pd.MultiIndex):\n raise ValueError(\"static_features cannot have a MultiIndex\")\n\n # Avoid modifying static features inplace\n value = value.copy()\n if self.ITEMID in value.columns and value.index.name != self.ITEMID:\n value = value.set_index(self.ITEMID)\n if value.index.name != self.ITEMID:\n value.index.rename(self.ITEMID, inplace=True)\n missing_item_ids = self.item_ids.difference(value.index)\n if len(missing_item_ids) > 0:\n raise ValueError(\n \"Following item_ids are missing from the index of static_features: \"\n f\"{reprlib.repr(missing_item_ids.to_list())}\"\n )\n # if provided static features are a strict superset of the item index, we take a subset to ensure consistency\n if len(value.index.difference(self.item_ids)) > 0:\n value = value.reindex(self.item_ids)\n\n self._static_features = value\n\n def infer_frequency(self, num_items: int | None = None, raise_if_irregular: bool = False) -> str:\n \"\"\"Infer the time series frequency based on the timestamps of the observations.\n\n Parameters\n ----------\n num_items : int or None, default = None\n Number of items (individual time series) randomly selected to infer the frequency. Lower values speed up\n the method, but increase the chance that some items with invalid frequency are missed by subsampling.\n\n If set to ``None``, all items will be used for inferring the frequency.\n raise_if_irregular : bool, default = False\n If True, an exception will be raised if some items have an irregular frequency, or if different items have\n different frequencies.\n\n Returns\n -------\n freq : str\n If all time series have a regular frequency, returns a pandas-compatible `frequency alias `_.\n\n If some items have an irregular frequency or if different items have different frequencies, returns string\n ``IRREG``.\n \"\"\"\n ts_df = self\n if num_items is not None and ts_df.num_items > num_items:\n items_subset = ts_df.item_ids.to_series().sample(n=num_items, random_state=123)\n ts_df = ts_df.loc[items_subset]\n\n if not ts_df.index.is_monotonic_increasing:\n ts_df = ts_df.sort_index()\n\n indptr = ts_df.get_indptr()\n item_ids = ts_df.item_ids\n timestamps = ts_df.index.get_level_values(level=1)\n candidate_freq = ts_df.index.levels[1].freq\n\n frequencies = []\n irregular_items = []\n for i in range(len(indptr) - 1):\n start, end = indptr[i], indptr[i + 1]\n item_timestamps = timestamps[start:end]\n inferred_freq = item_timestamps.inferred_freq\n\n # Fallback option: maybe original index has a `freq` attribute that pandas fails to infer (e.g., 'SME')\n if inferred_freq is None and candidate_freq is not None:\n try:\n # If this line does not raise an exception, then candidate_freq is a compatible frequency\n item_timestamps.freq = candidate_freq\n except ValueError:\n inferred_freq = None\n else:\n inferred_freq = candidate_freq.freqstr\n\n if inferred_freq is None:\n irregular_items.append(item_ids[i])\n else:\n frequencies.append(inferred_freq)\n\n unique_freqs = list(set(frequencies))\n if len(unique_freqs) != 1 or len(irregular_items) > 0:\n if raise_if_irregular:\n if irregular_items:\n raise ValueError(\n f\"Cannot infer frequency. Items with irregular frequency: {reprlib.repr(irregular_items)}\"\n )\n else:\n raise ValueError(f\"Cannot infer frequency. Multiple frequencies detected: {unique_freqs}\")\n else:\n return self.IRREGULAR_TIME_INDEX_FREQSTR\n else:\n return pd.tseries.frequencies.to_offset(unique_freqs[0]).freqstr\n\n @property\n def freq(self):\n \"\"\"Inferred pandas-compatible frequency of the timestamps in the dataframe.\n\n Computed using a random subset of the time series for speed. This may sometimes result in incorrectly inferred\n values. For reliable results, use :meth:`~autogluon.timeseries.TimeSeriesDataFrame.infer_frequency`.\n \"\"\"\n inferred_freq = self.infer_frequency(num_items=50)\n return None if inferred_freq == self.IRREGULAR_TIME_INDEX_FREQSTR else inferred_freq\n\n @property\n def num_items(self):\n \"\"\"Number of items (time series) in the data set.\"\"\"\n return len(self.item_ids)\n\n def num_timesteps_per_item(self) -> pd.Series:\n \"\"\"Number of observations in each time series in the dataframe.\n\n Returns a ``pandas.Series`` with ``item_id`` as index and number of observations per item as values.\n \"\"\"\n counts = pd.Series(self.index.codes[0]).value_counts(sort=False)\n counts.index = self.index.levels[0][counts.index]\n return counts\n\n def copy(self: TimeSeriesDataFrame, deep: bool = True) -> TimeSeriesDataFrame:\n \"\"\"Make a copy of the TimeSeriesDataFrame.\n\n When ``deep=True`` (default), a new object will be created with a copy of the calling object's data and\n indices. Modifications to the data or indices of the copy will not be reflected in the original object.\n\n When ``deep=False``, a new object will be created without copying the calling object's data or index (only\n references to the data and index are copied). Any changes to the data of the original will be reflected in the\n shallow copy (and vice versa).\n\n For more details, see `pandas documentation `_.\n \"\"\"\n obj = super().copy(deep=deep)\n\n # also perform a deep copy for static features\n if deep:\n for k in obj._metadata:\n setattr(obj, k, copy.deepcopy(getattr(obj, k)))\n return obj\n\n def __finalize__( # noqa\n self: TimeSeriesDataFrame, other, method: str | None = None, **kwargs\n ) -> TimeSeriesDataFrame:\n super().__finalize__(other=other, method=method, **kwargs)\n # when finalizing the copy/slice operation, we use the property setter to stay consistent\n # with the item index\n if hasattr(other, \"_static_features\"):\n self.static_features = other._static_features\n return self\n\n def split_by_time(self, cutoff_time: pd.Timestamp) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame]:\n \"\"\"Split dataframe to two different ``TimeSeriesDataFrame`` s before and after a certain ``cutoff_time``.\n\n Parameters\n ----------\n cutoff_time: pd.Timestamp\n The time to split the current dataframe into two dataframes.\n\n Returns\n -------\n data_before: TimeSeriesDataFrame\n Data frame containing time series before the ``cutoff_time`` (exclude ``cutoff_time``).\n data_after: TimeSeriesDataFrame\n Data frame containing time series after the ``cutoff_time`` (include ``cutoff_time``).\n \"\"\"\n\n nanosecond_before_cutoff = cutoff_time - pd.Timedelta(nanoseconds=1)\n data_before = self.loc[(slice(None), slice(None, nanosecond_before_cutoff)), :]\n data_after = self.loc[(slice(None), slice(cutoff_time, None)), :]\n before = TimeSeriesDataFrame(data_before, static_features=self.static_features)\n after = TimeSeriesDataFrame(data_after, static_features=self.static_features)\n return before, after\n\n def slice_by_timestep(self, start_index: int | None = None, end_index: int | None = None) -> TimeSeriesDataFrame:\n \"\"\"Select a subsequence from each time series between start (inclusive) and end (exclusive) indices.\n\n This operation is equivalent to selecting a slice ``[start_index : end_index]`` from each time series, and then\n combining these slices into a new ``TimeSeriesDataFrame``. See examples below.\n\n It is recommended to sort the index with ``ts_df.sort_index()`` before calling this method to take advantage of\n a fast optimized algorithm.\n\n Parameters\n ----------\n start_index : int or None\n Start index (inclusive) of the slice for each time series.\n Negative values are counted from the end of each time series.\n When set to None, the slice starts from the beginning of each time series.\n end_index : int or None\n End index (exclusive) of the slice for each time series.\n Negative values are counted from the end of each time series.\n When set to None, the slice includes the end of each time series.\n\n Returns\n -------\n ts_df : TimeSeriesDataFrame\n A new time series dataframe containing entries of the original time series between start and end indices.\n\n Examples\n --------\n >>> ts_df\n target\n item_id timestamp\n 0 2019-01-01 0\n 2019-01-02 1\n 2019-01-03 2\n 1 2019-01-02 3\n 2019-01-03 4\n 2019-01-04 5\n 2 2019-01-03 6\n 2019-01-04 7\n 2019-01-05 8\n\n Select the first entry of each time series\n\n >>> df.slice_by_timestep(0, 1)\n target\n item_id timestamp\n 0 2019-01-01 0\n 1 2019-01-02 3\n 2 2019-01-03 6\n\n Select the last 2 entries of each time series\n\n >>> df.slice_by_timestep(-2, None)\n target\n item_id timestamp\n 0 2019-01-02 1\n 2019-01-03 2\n 1 2019-01-03 4\n 2019-01-04 5\n 2 2019-01-04 7\n 2019-01-05 8\n\n Select all except the last entry of each time series\n\n >>> df.slice_by_timestep(None, -1)\n target\n item_id timestamp\n 0 2019-01-01 0\n 2019-01-02 1\n 1 2019-01-02 3\n 2019-01-03 4\n 2 2019-01-03 6\n 2019-01-04 7\n\n Copy the entire dataframe\n\n >>> df.slice_by_timestep(None, None)\n target\n item_id timestamp\n 0 2019-01-01 0\n 2019-01-02 1\n 2019-01-03 2\n 1 2019-01-02 3\n 2019-01-03 4\n 2019-01-04 5\n 2 2019-01-03 6\n 2019-01-04 7\n 2019-01-05 8\n\n \"\"\"\n if start_index is not None and not isinstance(start_index, int):\n raise ValueError(f\"start_index must be of type int or None (got {type(start_index)})\")\n if end_index is not None and not isinstance(end_index, int):\n raise ValueError(f\"end_index must be of type int or None (got {type(end_index)})\")\n\n if start_index is None and end_index is None:\n # Return a copy to avoid in-place modification.\n # self.copy() is much faster than self.loc[ones(len(self), dtype=bool)]\n return self.copy()\n\n if self.index.is_monotonic_increasing:\n # Use a fast optimized algorithm if the index is sorted\n indptr = self.get_indptr()\n lengths = np.diff(indptr)\n starts = indptr[:-1]\n\n slice_start = (\n np.zeros_like(lengths)\n if start_index is None\n else np.clip(np.where(start_index >= 0, start_index, lengths + start_index), 0, lengths)\n )\n slice_end = (\n lengths.copy()\n if end_index is None\n else np.clip(np.where(end_index >= 0, end_index, lengths + end_index), 0, lengths)\n )\n\n # Filter out invalid slices where start >= end\n valid_slices = slice_start < slice_end\n if not np.any(valid_slices):\n # Return empty dataframe with same structure\n return self.loc[np.zeros(len(self), dtype=bool)]\n\n starts = starts[valid_slices]\n slice_start = slice_start[valid_slices]\n slice_end = slice_end[valid_slices]\n\n # We put 1 at the slice_start index for each item and -1 at the slice_end index for each item.\n # After we apply cumsum we get the indicator mask selecting values between slice_start and slice_end\n # cumsum([0, 0, 1, 0, 0, -1, 0]) -> [0, 0, 1, 1, 1, 0, 0]\n # We need array of size len(self) + 1 in case events[starts + slice_end] tries to access position len(self)\n events = np.zeros(len(self) + 1, dtype=np.int8)\n events[starts + slice_start] += 1\n events[starts + slice_end] -= 1\n mask = np.cumsum(events)[:-1].astype(bool)\n # loc[mask] returns a view of the original data - modifying it will produce a SettingWithCopyWarning\n return self.loc[mask]\n else:\n # Fall back to a slow groupby operation\n result = self.groupby(level=self.ITEMID, sort=False, as_index=False).nth(slice(start_index, end_index))\n result.static_features = self.static_features\n return result\n\n def slice_by_time(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> TimeSeriesDataFrame:\n \"\"\"Select a subsequence from each time series between start (inclusive) and end (exclusive) timestamps.\n\n Parameters\n ----------\n start_time: pd.Timestamp\n Start time (inclusive) of the slice for each time series.\n end_time: pd.Timestamp\n End time (exclusive) of the slice for each time series.\n\n Returns\n -------\n ts_df : TimeSeriesDataFrame\n A new time series dataframe containing entries of the original time series between start and end timestamps.\n \"\"\"\n\n if end_time < start_time:\n raise ValueError(f\"end_time {end_time} is earlier than start_time {start_time}\")\n\n nanosecond_before_end_time = end_time - pd.Timedelta(nanoseconds=1)\n return TimeSeriesDataFrame(\n self.loc[(slice(None), slice(start_time, nanosecond_before_end_time)), :],\n static_features=self.static_features,\n )\n\n @classmethod\n def from_pickle(cls, filepath_or_buffer: Any) -> TimeSeriesDataFrame:\n \"\"\"Convenience method to read pickled time series dataframes. If the read pickle\n file refers to a plain pandas DataFrame, it will be cast to a TimeSeriesDataFrame.\n\n Parameters\n ----------\n filepath_or_buffer: Any\n Filename provided as a string or an ``IOBuffer`` containing the pickled object.\n\n Returns\n -------\n ts_df : TimeSeriesDataFrame\n The pickled time series dataframe.\n \"\"\"\n try:\n data = pd.read_pickle(filepath_or_buffer)\n return data if isinstance(data, cls) else cls(data)\n except Exception as err: # noqa\n raise IOError(f\"Could not load pickled data set due to error: {str(err)}\")\n\n def fill_missing_values(self, method: str = \"auto\", value: float = 0.0) -> TimeSeriesDataFrame:\n \"\"\"Fill missing values represented by NaN.\n\n .. note::\n This method assumes that the index of the TimeSeriesDataFrame is sorted by [item_id, timestamp].\n\n If the index is not sorted, this method will log a warning and may produce an incorrect result.\n\n Parameters\n ----------\n method : str, default = \"auto\"\n Method used to impute missing values.\n\n - ``\"auto\"`` - first forward fill (to fill the in-between and trailing NaNs), then backward fill (to fill the leading NaNs)\n - ``\"ffill\"`` or ``\"pad\"`` - propagate last valid observation forward. Note: missing values at the start of the time series are not filled.\n - ``\"bfill\"`` or ``\"backfill\"`` - use next valid observation to fill gap. Note: this may result in information leakage; missing values at the end of the time series are not filled.\n - ``\"constant\"`` - replace NaNs with the given constant ``value``.\n - ``\"interpolate\"`` - fill NaN values using linear interpolation. Note: this may result in information leakage.\n value : float, default = 0.0\n Value used by the \"constant\" imputation method.\n\n Examples\n --------\n >>> ts_df\n target\n item_id timestamp\n 0 2019-01-01 NaN\n 2019-01-02 NaN\n 2019-01-03 1.0\n 2019-01-04 NaN\n 2019-01-05 NaN\n 2019-01-06 2.0\n 2019-01-07 NaN\n 1 2019-02-04 NaN\n 2019-02-05 3.0\n 2019-02-06 NaN\n 2019-02-07 4.0\n\n >>> ts_df.fill_missing_values(method=\"auto\")\n target\n item_id timestamp\n 0 2019-01-01 1.0\n 2019-01-02 1.0\n 2019-01-03 1.0\n 2019-01-04 1.0\n 2019-01-05 1.0\n 2019-01-06 2.0\n 2019-01-07 2.0\n 1 2019-02-04 3.0\n 2019-02-05 3.0\n 2019-02-06 3.0\n 2019-02-07 4.0\n\n \"\"\"\n # Convert to pd.DataFrame for faster processing\n df = pd.DataFrame(self)\n\n # Skip filling if there are no NaNs\n if not df.isna().any(axis=None):\n return self\n\n if not self.index.is_monotonic_increasing:\n logger.warning(\n \"Trying to fill missing values in an unsorted dataframe. \"\n \"It is highly recommended to call `ts_df.sort_index()` before calling `ts_df.fill_missing_values()`\"\n )\n\n grouped_df = df.groupby(level=self.ITEMID, sort=False, group_keys=False)\n if method == \"auto\":\n filled_df = grouped_df.ffill()\n # If necessary, fill missing values at the start of each time series with bfill\n if filled_df.isna().any(axis=None):\n filled_df = filled_df.groupby(level=self.ITEMID, sort=False, group_keys=False).bfill()\n elif method in [\"ffill\", \"pad\"]:\n filled_df = grouped_df.ffill()\n elif method in [\"bfill\", \"backfill\"]:\n filled_df = grouped_df.bfill()\n elif method == \"constant\":\n filled_df = self.fillna(value=value)\n elif method == \"interpolate\":\n filled_df = grouped_df.apply(lambda ts: ts.interpolate())\n else:\n raise ValueError(\n \"Invalid fill method. Expecting one of \"\n \"{'auto', 'ffill', 'pad', 'bfill', 'backfill', 'constant', 'interpolate'}. \"\n f\"Got {method}\"\n )\n return TimeSeriesDataFrame(filled_df, static_features=self.static_features)\n\n def dropna(self, how: str = \"any\") -> TimeSeriesDataFrame: # type: ignore[override]\n \"\"\"Drop rows containing NaNs.\n\n Parameters\n ----------\n how : {\"any\", \"all\"}, default = \"any\"\n Determine if row or column is removed from TimeSeriesDataFrame, when we have at least one NaN or all NaN.\n\n - \"any\" : If any NaN values are present, drop that row or column.\n - \"all\" : If all values are NaN, drop that row or column.\n \"\"\"\n # We need to cast to a DataFrame first. Calling self.dropna() results in an exception because self.T\n # (used inside dropna) is not supported for TimeSeriesDataFrame\n dropped_df = pd.DataFrame(self).dropna(how=how)\n return TimeSeriesDataFrame(dropped_df, static_features=self.static_features)\n\n # added for static type checker compatibility\n def assign(self, **kwargs) -> TimeSeriesDataFrame:\n \"\"\"Assign new columns to the time series dataframe. See :meth:`pandas.DataFrame.assign` for details.\"\"\"\n return super().assign(**kwargs) # type: ignore\n\n # added for static type checker compatibility\n def sort_index(self, *args, **kwargs) -> TimeSeriesDataFrame:\n return super().sort_index(*args, **kwargs) # type: ignore\n\n def get_model_inputs_for_scoring(\n self, prediction_length: int, known_covariates_names: list[str] | None = None\n ) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame | None]:\n \"\"\"Prepare model inputs necessary to predict the last ``prediction_length`` time steps of each time series in the dataset.\n\n Parameters\n ----------\n prediction_length : int\n The forecast horizon, i.e., How many time steps into the future must be predicted.\n known_covariates_names : list[str], optional\n Names of the dataframe columns that contain covariates known in the future.\n See ``known_covariates_names`` of :class:`~autogluon.timeseries.TimeSeriesPredictor` for more details.\n\n Returns\n -------\n past_data : TimeSeriesDataFrame\n Data, where the last ``prediction_length`` time steps have been removed from the end of each time series.\n known_covariates : TimeSeriesDataFrame or None\n If ``known_covariates_names`` was provided, dataframe with the values of the known covariates during the\n forecast horizon. Otherwise, ``None``.\n \"\"\"\n past_data = self.slice_by_timestep(None, -prediction_length)\n if known_covariates_names is not None and len(known_covariates_names) > 0:\n future_data = self.slice_by_timestep(-prediction_length, None)\n known_covariates = future_data[known_covariates_names]\n else:\n known_covariates = None\n return past_data, known_covariates\n\n def train_test_split(\n self,\n prediction_length: int,\n end_index: int | None = None,\n suffix: str | None = None,\n ) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame]:\n \"\"\"Generate a train/test split from the given dataset.\n\n This method can be used to generate splits for multi-window backtesting.\n\n .. note::\n This method automatically sorts the TimeSeriesDataFrame by [item_id, timestamp].\n\n Parameters\n ----------\n prediction_length : int\n Number of time steps in a single evaluation window.\n end_index : int, optional\n If given, all time series will be shortened up to ``end_idx`` before the train/test splitting. In other\n words, test data will include the slice ``[:end_index]`` of each time series, and train data will include\n the slice ``[:end_index - prediction_length]``.\n suffix : str, optional\n Suffix appended to all entries in the ``item_id`` index level.\n\n Returns\n -------\n train_data : TimeSeriesDataFrame\n Train portion of the data. Contains the slice ``[:-prediction_length]`` of each time series in ``test_data``.\n test_data : TimeSeriesDataFrame\n Test portion of the data. Contains the slice ``[:end_idx]`` of each time series in the original dataset.\n \"\"\"\n df = self\n if not df.index.is_monotonic_increasing:\n logger.warning(\"Sorting the dataframe index before generating the train/test split.\")\n df = df.sort_index()\n test_data = df.slice_by_timestep(None, end_index)\n train_data = test_data.slice_by_timestep(None, -prediction_length)\n\n if suffix is not None:\n for data in [train_data, test_data]:\n new_item_id = data.index.levels[0].astype(str) + suffix\n data.index = data.index.set_levels(levels=new_item_id, level=0)\n if data.static_features is not None:\n data.static_features.index = data.static_features.index.astype(str)\n data.static_features.index += suffix\n return train_data, test_data\n\n def convert_frequency(\n self,\n freq: str | pd.DateOffset,\n agg_numeric: str = \"mean\",\n agg_categorical: str = \"first\",\n num_cpus: int = -1,\n chunk_size: int = 100,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n \"\"\"Convert each time series in the dataframe to the given frequency.\n\n This method is useful for two purposes:\n\n 1. Converting an irregularly-sampled time series to a regular time index.\n 2. Aggregating time series data by downsampling (e.g., convert daily sales into weekly sales)\n\n Standard ``df.groupby(...).resample(...)`` can be extremely slow for large datasets, so we parallelize this\n operation across multiple CPU cores.\n\n Parameters\n ----------\n freq : str | pd.DateOffset\n Frequency to which the data should be converted. See `pandas frequency aliases `_\n for supported values.\n agg_numeric : {\"max\", \"min\", \"sum\", \"mean\", \"median\", \"first\", \"last\"}, default = \"mean\"\n Aggregation method applied to numeric columns.\n agg_categorical : {\"first\", \"last\"}, default = \"first\"\n Aggregation method applied to categorical columns.\n num_cpus : int, default = -1\n Number of CPU cores used when resampling in parallel. Set to -1 to use all cores.\n chunk_size : int, default = 100\n Number of time series in a chunk assigned to each parallel worker.\n **kwargs\n Additional keywords arguments that will be passed to ``pandas.DataFrameGroupBy.resample``.\n\n Returns\n -------\n ts_df : TimeSeriesDataFrame\n A new time series dataframe with time series resampled at the new frequency. Output may contain missing\n values represented by ``NaN`` if original data does not have information for the given period.\n\n Examples\n --------\n Convert irregularly-sampled time series data to a regular index\n\n >>> ts_df\n target\n item_id timestamp\n 0 2019-01-01 NaN\n 2019-01-03 1.0\n 2019-01-06 2.0\n 2019-01-07 NaN\n 1 2019-02-04 3.0\n 2019-02-07 4.0\n >>> ts_df.convert_frequency(freq=\"D\")\n target\n item_id timestamp\n 0 2019-01-01 NaN\n 2019-01-02 NaN\n 2019-01-03 1.0\n 2019-01-04 NaN\n 2019-01-05 NaN\n 2019-01-06 2.0\n 2019-01-07 NaN\n 1 2019-02-04 3.0\n 2019-02-05 NaN\n 2019-02-06 NaN\n 2019-02-07 4.0\n\n Downsample quarterly data to yearly frequency\n\n >>> ts_df\n target\n item_id timestamp\n 0 2020-03-31 1.0\n 2020-06-30 2.0\n 2020-09-30 3.0\n 2020-12-31 4.0\n 2021-03-31 5.0\n 2021-06-30 6.0\n 2021-09-30 7.0\n 2021-12-31 8.0\n >>> ts_df.convert_frequency(\"YE\")\n target\n item_id timestamp\n 0 2020-12-31 2.5\n 2021-12-31 6.5\n >>> ts_df.convert_frequency(\"YE\", agg_numeric=\"sum\")\n target\n item_id timestamp\n 0 2020-12-31 10.0\n 2021-12-31 26.0\n \"\"\"\n offset = pd.tseries.frequencies.to_offset(freq)\n\n # We need to aggregate categorical columns separately because .agg(\"mean\") deletes all non-numeric columns\n aggregation = {}\n for col in self.columns:\n if pd.api.types.is_numeric_dtype(self.dtypes[col]):\n aggregation[col] = agg_numeric\n else:\n aggregation[col] = agg_categorical\n\n def split_into_chunks(iterable: Iterable, size: int) -> Iterable[Iterable]:\n # Based on https://stackoverflow.com/a/22045226/5497447\n iterable = iter(iterable)\n return iter(lambda: tuple(islice(iterable, size)), ())\n\n def resample_chunk(chunk: Iterable[tuple[str, pd.DataFrame]]) -> pd.DataFrame:\n resampled_dfs = []\n for item_id, df in chunk:\n resampled_df = df.resample(offset, level=self.TIMESTAMP, **kwargs).agg(aggregation)\n resampled_dfs.append(pd.concat({item_id: resampled_df}, names=[self.ITEMID]))\n return pd.concat(resampled_dfs)\n\n # Resampling time for 1 item < overhead time for a single parallel job. Therefore, we group items into chunks\n # so that the speedup from parallelization isn't dominated by the communication costs.\n df = pd.DataFrame(self)\n # Make sure that timestamp index has dtype 'datetime64[ns]', otherwise index may contain NaT values.\n # See https://github.com/autogluon/autogluon/issues/4917\n df.index = df.index.set_levels(df.index.levels[1].astype(\"datetime64[ns]\"), level=self.TIMESTAMP)\n chunks = split_into_chunks(df.groupby(level=self.ITEMID, sort=False), chunk_size)\n resampled_chunks = Parallel(n_jobs=num_cpus)(delayed(resample_chunk)(chunk) for chunk in chunks)\n resampled_df = TimeSeriesDataFrame(pd.concat(resampled_chunks))\n resampled_df.static_features = self.static_features\n return resampled_df\n\n def to_data_frame(self) -> pd.DataFrame:\n \"\"\"Convert ``TimeSeriesDataFrame`` to a ``pandas.DataFrame``\"\"\"\n return pd.DataFrame(self)\n\n def get_indptr(self) -> np.ndarray:\n \"\"\"[Advanced] Get a numpy array of shape [num_items + 1] that points to the start and end of each time series.\n\n This method assumes that the TimeSeriesDataFrame is sorted by [item_id, timestamp].\n \"\"\"\n return np.concatenate([[0], np.cumsum(self.num_timesteps_per_item().to_numpy())]).astype(np.int32)\n\n # inline typing stubs for various overridden methods\n if TYPE_CHECKING:\n\n def query( # type: ignore\n self, expr: str, *, inplace: bool = False, **kwargs\n ) -> Self: ...\n\n def reindex(*args, **kwargs) -> Self: ... # type: ignore\n\n @overload\n def __new__(cls, data: pd.DataFrame, static_features: pd.DataFrame | None = None) -> Self: ... # type: ignore\n @overload\n def __new__(\n cls,\n data: pd.DataFrame | str | Path | Iterable,\n static_features: pd.DataFrame | str | Path | None = None,\n id_column: str | None = None,\n timestamp_column: str | None = None,\n num_cpus: int = -1,\n *args,\n **kwargs,\n ) -> Self:\n \"\"\"This overload is needed since in pandas, during type checking, the default constructor resolves to __new__\"\"\"\n ...\n\n @overload\n def __getitem__(self, items: list[str]) -> Self: ... # type: ignore\n @overload\n def __getitem__(self, item: str) -> pd.Series: ... # type: ignore\n\n\n# TODO: remove with v2.0\n# module-level constants kept for backward compatibility.\nITEMID = TimeSeriesDataFrame.ITEMID\nTIMESTAMP = TimeSeriesDataFrame.TIMESTAMP\n"
},
{
"path": "timeseries/src/autogluon/timeseries/learner.py",
"content": "import logging\nimport reprlib\nimport time\nfrom typing import Any, Literal, Type\n\nimport pandas as pd\n\nfrom autogluon.core.learner import AbstractLearner\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric\nfrom autogluon.timeseries.models.abstract import AbstractTimeSeriesModel\nfrom autogluon.timeseries.trainer import TimeSeriesTrainer\nfrom autogluon.timeseries.utils.features import TimeSeriesFeatureGenerator\nfrom autogluon.timeseries.utils.forecast import make_future_data_frame\n\nlogger = logging.getLogger(__name__)\n\n\nclass TimeSeriesLearner(AbstractLearner):\n \"\"\"TimeSeriesLearner encompasses a full time series learning problem for a\n training run, and keeps track of datasets, features, and the trainer object.\n \"\"\"\n\n def __init__(\n self,\n path_context: str,\n target: str = \"target\",\n known_covariates_names: list[str] | None = None,\n trainer_type: Type[TimeSeriesTrainer] = TimeSeriesTrainer,\n eval_metric: str | TimeSeriesScorer | None = None,\n prediction_length: int = 1,\n cache_predictions: bool = True,\n ensemble_model_type: Type | None = None,\n **kwargs,\n ):\n super().__init__(path_context=path_context)\n self.eval_metric = check_get_evaluation_metric(eval_metric, prediction_length=prediction_length)\n self.trainer_type = trainer_type\n self.target = target\n self.known_covariates_names = [] if known_covariates_names is None else known_covariates_names\n self.prediction_length = prediction_length\n self.quantile_levels = kwargs.get(\"quantile_levels\", [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])\n self.cache_predictions = cache_predictions\n self.freq: str | None = None\n self.ensemble_model_type = ensemble_model_type\n\n self.feature_generator = TimeSeriesFeatureGenerator(\n target=self.target, known_covariates_names=self.known_covariates_names\n )\n\n def load_trainer(self) -> TimeSeriesTrainer: # type: ignore\n \"\"\"Return the trainer object corresponding to the learner.\"\"\"\n return super().load_trainer() # type: ignore\n\n def fit(\n self,\n train_data: TimeSeriesDataFrame,\n hyperparameters: str | dict,\n val_data: TimeSeriesDataFrame | None = None,\n hyperparameter_tune_kwargs: str | dict | None = None,\n ensemble_hyperparameters: dict[str, Any] | list[dict[str, Any]] | None = None,\n time_limit: float | None = None,\n num_val_windows: tuple[int, ...] = (1,),\n val_step_size: int | None = None,\n refit_every_n_windows: int | None = 1,\n random_seed: int | None = None,\n **kwargs,\n ) -> None:\n self._time_limit = time_limit\n time_start = time.time()\n\n train_data = self.feature_generator.fit_transform(train_data)\n if val_data is not None:\n val_data = self.feature_generator.transform(val_data, data_frame_name=\"tuning_data\")\n\n self.freq = train_data.freq\n\n trainer_init_kwargs = kwargs.copy()\n trainer_init_kwargs.update(\n dict(\n path=self.model_context,\n prediction_length=self.prediction_length,\n eval_metric=self.eval_metric,\n target=self.target,\n quantile_levels=self.quantile_levels,\n verbosity=kwargs.get(\"verbosity\", 2),\n skip_model_selection=kwargs.get(\"skip_model_selection\", False),\n enable_ensemble=kwargs.get(\"enable_ensemble\", True),\n covariate_metadata=self.feature_generator.covariate_metadata,\n num_val_windows=num_val_windows,\n val_step_size=val_step_size,\n refit_every_n_windows=refit_every_n_windows,\n cache_predictions=self.cache_predictions,\n ensemble_model_type=self.ensemble_model_type,\n )\n )\n\n assert issubclass(self.trainer_type, TimeSeriesTrainer)\n self.trainer: TimeSeriesTrainer | None = self.trainer_type(**trainer_init_kwargs)\n self.trainer_path = self.trainer.path\n self.save()\n\n logger.info(f\"\\nAutoGluon will gauge predictive performance using evaluation metric: '{self.eval_metric}'\")\n if not self.eval_metric.greater_is_better_internal:\n logger.info(\n \"\\tThis metric's sign has been flipped to adhere to being higher_is_better. The metric score can be multiplied by -1 to get the metric value.\"\n )\n\n logger.info(\"===================================================\")\n\n self.trainer.fit(\n train_data=train_data,\n val_data=val_data,\n hyperparameters=hyperparameters,\n hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n ensemble_hyperparameters=ensemble_hyperparameters,\n excluded_model_types=kwargs.get(\"excluded_model_types\"),\n time_limit=time_limit,\n random_seed=random_seed,\n )\n\n self._time_fit_training = time.time() - time_start\n self.save()\n\n def _align_covariates_with_forecast_index(\n self,\n known_covariates: TimeSeriesDataFrame | None,\n data: TimeSeriesDataFrame,\n ) -> TimeSeriesDataFrame | None:\n \"\"\"Select the relevant item_ids and timestamps from the known_covariates dataframe.\n\n If some of the item_ids or timestamps are missing, an exception is raised.\n \"\"\"\n if len(self.known_covariates_names) == 0:\n return None\n if len(self.known_covariates_names) > 0 and known_covariates is None:\n raise ValueError(\n f\"known_covariates {self.known_covariates_names} for the forecast horizon should be provided at prediction time.\"\n )\n assert known_covariates is not None\n\n if self.target in known_covariates.columns:\n known_covariates = known_covariates.drop(self.target, axis=1)\n\n missing_item_ids = data.item_ids.difference(known_covariates.item_ids)\n if len(missing_item_ids) > 0:\n raise ValueError(\n f\"known_covariates are missing information for the following item_ids: {reprlib.repr(missing_item_ids.to_list())}.\"\n )\n\n forecast_index = pd.MultiIndex.from_frame(\n make_future_data_frame(data, prediction_length=self.prediction_length, freq=self.freq)\n )\n try:\n known_covariates = known_covariates.loc[forecast_index] # type: ignore\n except KeyError:\n raise ValueError(\n \"`known_covariates` should include the `item_id` and `timestamp` values covering the forecast horizon \"\n \"(i.e., the next `prediction_length` time steps following the end of each time series in the input \"\n \"data). Use `TimeSeriesPredictor.make_future_data_frame` to generate the required `item_id` and \"\n \"`timestamp` combinations for the `known_covariates`.\"\n )\n return known_covariates\n\n def predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n model: str | AbstractTimeSeriesModel | None = None,\n use_cache: bool = True,\n random_seed: int | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n data = self.feature_generator.transform(data)\n known_covariates = self.feature_generator.transform_future_known_covariates(known_covariates)\n known_covariates = self._align_covariates_with_forecast_index(known_covariates=known_covariates, data=data)\n return self.load_trainer().predict(\n data=data,\n known_covariates=known_covariates,\n model=model,\n use_cache=use_cache,\n random_seed=random_seed,\n **kwargs,\n )\n\n def score(\n self,\n data: TimeSeriesDataFrame,\n model: str | AbstractTimeSeriesModel | None = None,\n metric: str | TimeSeriesScorer | None = None,\n use_cache: bool = True,\n ) -> float:\n data = self.feature_generator.transform(data)\n return self.load_trainer().score(data=data, model=model, metric=metric, use_cache=use_cache)\n\n def evaluate(\n self,\n data: TimeSeriesDataFrame,\n model: str | None = None,\n metrics: str | TimeSeriesScorer | list[str | TimeSeriesScorer] | None = None,\n use_cache: bool = True,\n ) -> dict[str, float]:\n data = self.feature_generator.transform(data)\n return self.load_trainer().evaluate(data=data, model=model, metrics=metrics, use_cache=use_cache)\n\n def get_feature_importance(\n self,\n data: TimeSeriesDataFrame | None = None,\n model: str | None = None,\n metric: str | TimeSeriesScorer | None = None,\n features: list[str] | None = None,\n time_limit: float | None = None,\n method: Literal[\"naive\", \"permutation\"] = \"permutation\",\n subsample_size: int = 50,\n num_iterations: int | None = None,\n random_seed: int | None = None,\n relative_scores: bool = False,\n include_confidence_band: bool = True,\n confidence_level: float = 0.99,\n ) -> pd.DataFrame:\n trainer = self.load_trainer()\n if data is None:\n data = trainer.load_val_data() or trainer.load_train_data()\n\n # if features are provided in the dataframe, check that they are valid features in the covariate metadata\n provided_static_columns = [] if data.static_features is None else data.static_features.columns\n unused_features = [\n f\n for f in set(provided_static_columns).union(set(data.columns) - {self.target})\n if f not in self.feature_generator.covariate_metadata.all_features\n ]\n\n if features is None:\n features = self.feature_generator.covariate_metadata.all_features\n else:\n if len(features) == 0:\n raise ValueError(\n \"No features provided to compute feature importance. At least some valid features should be provided.\"\n )\n for fn in features:\n if fn not in self.feature_generator.covariate_metadata.all_features and fn not in unused_features:\n raise ValueError(f\"Feature {fn} not found in covariate metadata or the dataset.\")\n\n if len(set(features)) < len(features):\n raise ValueError(\n \"Duplicate feature names provided to compute feature importance. \"\n \"Please provide unique feature names across both static features and covariates.\"\n )\n\n data = self.feature_generator.transform(data)\n\n importance_df = trainer.get_feature_importance(\n data=data,\n features=features,\n model=model,\n metric=metric,\n time_limit=time_limit,\n method=method,\n subsample_size=subsample_size,\n num_iterations=num_iterations,\n random_seed=random_seed,\n relative_scores=relative_scores,\n include_confidence_band=include_confidence_band,\n confidence_level=confidence_level,\n )\n\n for feature in set(features).union(unused_features):\n if feature not in importance_df.index:\n importance_df.loc[feature] = (\n [0, 0, 0] if not include_confidence_band else [0, 0, 0, float(\"nan\"), float(\"nan\")]\n )\n\n return importance_df\n\n def leaderboard(\n self,\n data: TimeSeriesDataFrame | None = None,\n extra_info: bool = False,\n extra_metrics: list[str | TimeSeriesScorer] | None = None,\n use_cache: bool = True,\n ) -> pd.DataFrame:\n if data is not None:\n data = self.feature_generator.transform(data)\n return self.load_trainer().leaderboard(\n data, extra_info=extra_info, extra_metrics=extra_metrics, use_cache=use_cache\n )\n\n def get_info(self, include_model_info: bool = False, **kwargs) -> dict[str, Any]:\n learner_info = super().get_info(include_model_info=include_model_info)\n trainer = self.load_trainer()\n trainer_info = trainer.get_info(include_model_info=include_model_info)\n learner_info.update(\n {\n \"time_fit_training\": self._time_fit_training,\n \"time_limit\": self._time_limit,\n }\n )\n\n learner_info.update(trainer_info)\n # self.random_state not used during fitting, so we don't include it in the summary\n # TODO: Report random seed passed to predictor.fit?\n learner_info.pop(\"random_state\", None)\n return learner_info\n\n def persist_trainer(\n self, models: Literal[\"all\", \"best\"] | list[str] = \"all\", with_ancestors: bool = False\n ) -> list[str]:\n \"\"\"Loads models and trainer in memory so that they don't have to be\n loaded during predictions\n\n Returns\n -------\n list_of_models\n List of models persisted in memory\n \"\"\"\n self.trainer = self.load_trainer()\n return self.trainer.persist(models, with_ancestors=with_ancestors)\n\n def unpersist_trainer(self) -> list[str]:\n \"\"\"Unloads models and trainer from memory. Models will have to be reloaded from disk\n when predicting.\n\n Returns\n -------\n list_of_models\n List of models removed from memory\n \"\"\"\n unpersisted_models = self.load_trainer().unpersist()\n self.trainer = None # type: ignore\n return unpersisted_models\n\n def refit_full(self, model: str = \"all\") -> dict[str, str]:\n return self.load_trainer().refit_full(model=model)\n\n def backtest_predictions(\n self,\n data: TimeSeriesDataFrame | None,\n model_names: list[str],\n num_val_windows: int | None = None,\n val_step_size: int | None = None,\n use_cache: bool = True,\n ) -> dict[str, list[TimeSeriesDataFrame]]:\n if data is not None:\n data = self.feature_generator.transform(data)\n return self.load_trainer().backtest_predictions(\n model_names=model_names,\n data=data,\n num_val_windows=num_val_windows,\n val_step_size=val_step_size,\n use_cache=use_cache,\n )\n\n def backtest_targets(\n self,\n data: TimeSeriesDataFrame | None,\n num_val_windows: int | None = None,\n val_step_size: int | None = None,\n ) -> list[TimeSeriesDataFrame]:\n if data is not None:\n data = self.feature_generator.transform(data)\n return self.load_trainer().backtest_targets(\n data=data,\n num_val_windows=num_val_windows,\n val_step_size=val_step_size,\n )\n"
},
{
"path": "timeseries/src/autogluon/timeseries/metrics/__init__.py",
"content": "from __future__ import annotations\n\nfrom pprint import pformat\nfrom typing import Any, Sequence, Type\n\nimport numpy as np\n\nfrom .abstract import TimeSeriesScorer\nfrom .point import MAE, MAPE, MASE, MSE, RMSE, RMSLE, RMSSE, SMAPE, WAPE, WCD\nfrom .quantile import SQL, WQL\n\n__all__ = [\n \"TimeSeriesScorer\",\n \"check_get_evaluation_metric\",\n \"MAE\",\n \"MAPE\",\n \"MASE\",\n \"SMAPE\",\n \"MSE\",\n \"RMSE\",\n \"RMSLE\",\n \"RMSSE\",\n \"SQL\",\n \"WAPE\",\n \"WCD\",\n \"WQL\",\n]\n\nDEFAULT_METRIC_NAME = \"WQL\"\n\nAVAILABLE_METRICS: dict[str, Type[TimeSeriesScorer]] = {\n \"MASE\": MASE,\n \"MAPE\": MAPE,\n \"SMAPE\": SMAPE,\n \"RMSE\": RMSE,\n \"RMSLE\": RMSLE,\n \"RMSSE\": RMSSE,\n \"WAPE\": WAPE,\n \"SQL\": SQL,\n \"WQL\": WQL,\n \"MSE\": MSE,\n \"MAE\": MAE,\n}\n\n# For backward compatibility\nDEPRECATED_METRICS = {\n \"mean_wQuantileLoss\": \"WQL\",\n}\n\n# Experimental metrics that are not yet user facing\nEXPERIMENTAL_METRICS: dict[str, Type[TimeSeriesScorer]] = {\n \"WCD\": WCD,\n}\n\n\ndef check_get_evaluation_metric(\n eval_metric: str | TimeSeriesScorer | Type[TimeSeriesScorer] | None,\n prediction_length: int,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | np.ndarray | None = None,\n) -> TimeSeriesScorer:\n \"\"\"Factory method for TimeSeriesScorer objects.\n\n Returns\n -------\n scorer\n A `TimeSeriesScorer` object based on the provided `eval_metric`.\n\n `scorer.prediction_length` is always set to the `prediction_length` provided to this method.\n\n If `seasonal_period` is not `None`, then `scorer.seasonal_period` is set to this value. Otherwise the original\n value of `seasonal_period` is kept.\n\n If `horizon_weight` is not `None`, then `scorer.horizon_weight` is set to this value. Otherwise the original\n value of `horizon_weight` is kept.\n \"\"\"\n scorer: TimeSeriesScorer\n metric_kwargs: dict[str, Any] = dict(\n prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight\n )\n if isinstance(eval_metric, TimeSeriesScorer):\n scorer = eval_metric\n scorer.prediction_length = prediction_length\n if seasonal_period is not None:\n scorer.seasonal_period = seasonal_period\n if horizon_weight is not None:\n scorer.horizon_weight = scorer.check_get_horizon_weight(\n horizon_weight, prediction_length=prediction_length\n )\n elif isinstance(eval_metric, type) and issubclass(eval_metric, TimeSeriesScorer):\n # e.g., user passed `eval_metric=CustomMetric` instead of `eval_metric=CustomMetric()`\n scorer = eval_metric(**metric_kwargs)\n elif isinstance(eval_metric, str):\n metric_name = DEPRECATED_METRICS.get(eval_metric, eval_metric).upper()\n if metric_name in AVAILABLE_METRICS:\n scorer = AVAILABLE_METRICS[metric_name](**metric_kwargs)\n elif metric_name in EXPERIMENTAL_METRICS:\n scorer = EXPERIMENTAL_METRICS[metric_name](**metric_kwargs)\n else:\n raise ValueError(\n f\"Time series metric {eval_metric} not supported. Available metrics are:\\n\"\n f\"{pformat(sorted(AVAILABLE_METRICS.keys()))}\"\n )\n elif eval_metric is None:\n scorer = AVAILABLE_METRICS[DEFAULT_METRIC_NAME](**metric_kwargs)\n else:\n raise ValueError(\n f\"eval_metric must be of type str, TimeSeriesScorer or None \"\n f\"(received eval_metric = {eval_metric} of type {type(eval_metric)})\"\n )\n return scorer\n"
},
{
"path": "timeseries/src/autogluon/timeseries/metrics/abstract.py",
"content": "import warnings\nfrom typing import Sequence, overload\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.timeseries import TimeSeriesDataFrame\nfrom autogluon.timeseries.utils.datetime import get_seasonality\nfrom autogluon.timeseries.utils.warning_filters import warning_filter\n\n\nclass TimeSeriesScorer:\n \"\"\"Base class for all evaluation metrics used in AutoGluon-TimeSeries.\n\n This object always returns the metric in greater-is-better format.\n\n Follows the design of ``autogluon.core.metrics.Scorer``.\n\n Parameters\n ----------\n prediction_length : int, default = 1\n The length of the forecast horizon. The predictions provided to the ``TimeSeriesScorer`` are expected to contain\n a forecast for this many time steps for each time series.\n seasonal_period : int or None, default = None\n Seasonal period used to compute some evaluation metrics such as mean absolute scaled error (MASE). Defaults to\n ``None``, in which case the seasonal period is computed based on the data frequency.\n horizon_weight : Sequence[float], np.ndarray or None, default = None\n Weight assigned to each time step in the forecast horizon when computing the metric. If provided, the\n ``horizon_weight`` will be stored as a numpy array of shape ``[1, prediction_length]``.\n\n Attributes\n ----------\n greater_is_better_internal : bool, default = False\n Whether internal method :meth:`~autogluon.timeseries.metrics.TimeSeriesScorer.compute_metric` is\n a loss function (default), meaning low is good, or a score function, meaning high is good.\n optimum : float, default = 0.0\n The best score achievable by the score function, i.e. maximum in case of scorer function and minimum in case of\n loss function.\n optimized_by_median : bool, default = False\n Whether given point forecast metric is optimized by the median (if True) or expected value (if False). If True,\n all models in AutoGluon-TimeSeries will attempt to paste median forecast into the \"mean\" column.\n needs_quantile : bool, default = False\n Whether the given metric uses the quantile predictions. Some models will modify the training procedure if they\n are trained to optimize a quantile metric.\n equivalent_tabular_regression_metric : str\n Name of an equivalent metric used by AutoGluon-Tabular with ``problem_type=\"regression\"``. Used by forecasting\n models that train tabular regression models under the hood. This attribute should only be specified by point\n forecast metrics.\n \"\"\"\n\n greater_is_better_internal: bool = False\n optimum: float = 0.0\n optimized_by_median: bool = False\n needs_quantile: bool = False\n equivalent_tabular_regression_metric: str | None = None\n\n def __init__(\n self,\n prediction_length: int = 1,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | None = None,\n ):\n self.prediction_length = int(prediction_length)\n if self.prediction_length < 1:\n raise ValueError(f\"prediction_length must be >= 1 (received {prediction_length})\")\n self.seasonal_period = seasonal_period\n self.horizon_weight = self.check_get_horizon_weight(horizon_weight, prediction_length=prediction_length)\n\n @property\n def sign(self) -> int:\n return 1 if self.greater_is_better_internal else -1\n\n @property\n def name(self) -> str:\n return f\"{self.__class__.__name__}\"\n\n def __repr__(self) -> str:\n return self.name\n\n def __str__(self) -> str:\n return self.name\n\n @property\n def name_with_sign(self) -> str:\n if self.greater_is_better_internal:\n prefix = \"\"\n else:\n prefix = \"-\"\n return f\"{prefix}{self.name}\"\n\n def __call__(\n self,\n data: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n seasonal_period = get_seasonality(data.freq) if self.seasonal_period is None else self.seasonal_period\n\n if \"prediction_length\" in kwargs:\n warnings.warn(\n \"Passing `prediction_length` to `TimeSeriesScorer.__call__` is deprecated and will be removed in v2.0. \"\n \"Please set the `eval_metric.prediction_length` attribute instead.\",\n category=FutureWarning,\n )\n self.prediction_length = kwargs[\"prediction_length\"]\n self.horizon_weight = self.check_get_horizon_weight(self.horizon_weight, self.prediction_length)\n\n data_past = data.slice_by_timestep(None, -self.prediction_length)\n data_future = data.slice_by_timestep(-self.prediction_length, None)\n\n assert not predictions.isna().any().any(), \"Predictions contain NaN values.\"\n assert (predictions.num_timesteps_per_item() == self.prediction_length).all()\n assert data_future.index.equals(predictions.index), \"Prediction and data indices do not match.\"\n\n try:\n with warning_filter():\n self.save_past_metrics(\n data_past=data_past,\n target=target,\n seasonal_period=seasonal_period,\n **kwargs,\n )\n metric_value = self.compute_metric(\n data_future=data_future,\n predictions=predictions,\n target=target,\n **kwargs,\n )\n finally:\n self.clear_past_metrics()\n return metric_value * self.sign\n\n score = __call__\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n \"\"\"Internal method that computes the metric for given forecast & actual data.\n\n This method should be implemented by all custom metrics.\n\n Parameters\n ----------\n data_future : TimeSeriesDataFrame\n Actual values of the time series during the forecast horizon (``prediction_length`` values for each time\n series in the dataset). Must have the same index as ``predictions``.\n predictions : TimeSeriesDataFrame\n Data frame with predictions for the forecast horizon. Contain columns \"mean\" (point forecast) and the\n columns corresponding to each of the quantile levels. Must have the same index as ``data_future``.\n target : str, default = \"target\"\n Name of the column in ``data_future`` that contains the target time series.\n\n Returns\n -------\n score : float\n Value of the metric for given forecast and data. If self.greater_is_better_internal is True, returns score\n in greater-is-better format, otherwise in lower-is-better format.\n\n \"\"\"\n raise NotImplementedError\n\n def save_past_metrics(\n self,\n data_past: TimeSeriesDataFrame,\n target: str = \"target\",\n seasonal_period: int = 1,\n **kwargs,\n ) -> None:\n \"\"\"Compute auxiliary metrics on past data (before forecast horizon), if the chosen metric requires it.\n\n This method should only be implemented by metrics that rely on historical (in-sample) data, such as Mean Absolute\n Scaled Error (MASE) https://en.wikipedia.org/wiki/Mean_absolute_scaled_error.\n\n We keep this method separate from :meth:`compute_metric` to avoid redundant computations when fitting ensemble.\n \"\"\"\n pass\n\n def clear_past_metrics(self) -> None:\n \"\"\"Clear auxiliary metrics saved in :meth:`save_past_metrics`.\n\n This method should only be implemented if :meth:`save_past_metrics` has been implemented.\n \"\"\"\n pass\n\n def error(self, *args, **kwargs):\n \"\"\"Return error in lower-is-better format.\"\"\"\n return self.optimum - self.score(*args, **kwargs)\n\n @staticmethod\n def _safemean(array: np.ndarray | pd.Series) -> float:\n \"\"\"Compute mean of a numpy array-like object, ignoring inf, -inf and nan values.\"\"\"\n return float(np.mean(array[np.isfinite(array)]))\n\n @staticmethod\n def _get_point_forecast_score_inputs(\n data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = \"target\"\n ) -> tuple[pd.Series, pd.Series]:\n \"\"\"Get inputs necessary to compute point forecast metrics.\n\n Returns\n -------\n y_true\n Target time series values during the forecast horizon.\n y_pred\n Predicted time series values during the forecast horizon.\n \"\"\"\n y_true = data_future[target]\n y_pred = predictions[\"mean\"]\n return y_true, y_pred\n\n @staticmethod\n def _get_quantile_forecast_score_inputs(\n data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = \"target\"\n ) -> tuple[pd.Series, pd.DataFrame, np.ndarray]:\n \"\"\"Get inputs necessary to compute quantile forecast metrics.\n\n Returns\n -------\n y_true\n Target time series values during the forecast horizon.\n q_pred\n Quantile forecast for each predicted quantile level. Column order corresponds to ``quantile_levels``.\n quantile_levels\n Quantile levels for which the forecasts are generated (as floats).\n \"\"\"\n quantile_columns = [col for col in predictions.columns if col != \"mean\"]\n y_true = data_future[target]\n q_pred = pd.DataFrame(predictions[quantile_columns])\n quantile_levels = np.array(quantile_columns, dtype=float)\n return y_true, q_pred, quantile_levels\n\n @overload\n @staticmethod\n def check_get_horizon_weight(horizon_weight: None, prediction_length: int) -> None: ...\n @overload\n @staticmethod\n def check_get_horizon_weight(\n horizon_weight: Sequence[float] | np.ndarray, prediction_length: int\n ) -> np.ndarray: ...\n\n @staticmethod\n def check_get_horizon_weight(\n horizon_weight: Sequence[float] | np.ndarray | None, prediction_length: int\n ) -> np.ndarray | None:\n \"\"\"Convert horizon_weight to a non-negative numpy array that sums up to prediction_length.\n Raises an exception if horizon_weight has an invalid shape or contains invalid values.\n\n Returns\n -------\n horizon_weight\n None if the input is None, otherwise a numpy array of shape [1, prediction_length].\n \"\"\"\n if horizon_weight is None:\n return None\n horizon_weight_np = np.ravel(horizon_weight).astype(np.float64)\n if horizon_weight_np.shape != (prediction_length,):\n raise ValueError(\n f\"horizon_weight must have length equal to {prediction_length=} (got {len(horizon_weight)=})\"\n )\n if not (horizon_weight_np >= 0).all():\n raise ValueError(f\"All values in horizon_weight must be >= 0 (got {horizon_weight})\")\n if not horizon_weight_np.sum() > 0:\n raise ValueError(f\"At least some values in horizon_weight must be > 0 (got {horizon_weight})\")\n if not np.isfinite(horizon_weight_np).all():\n raise ValueError(f\"All horizon_weight values must be finite (got {horizon_weight})\")\n horizon_weight_np = horizon_weight_np * prediction_length / horizon_weight_np.sum()\n return horizon_weight_np.reshape([1, prediction_length])\n"
},
{
"path": "timeseries/src/autogluon/timeseries/metrics/point.py",
"content": "import logging\nimport warnings\nfrom typing import Sequence\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.timeseries import TimeSeriesDataFrame\n\nfrom .abstract import TimeSeriesScorer\nfrom .utils import in_sample_abs_seasonal_error, in_sample_squared_seasonal_error\n\nlogger = logging.getLogger(__name__)\n\n\nclass RMSE(TimeSeriesScorer):\n r\"\"\"Root mean squared error.\n\n .. math::\n\n \\operatorname{RMSE} = \\sqrt{\\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N}\\sum_{t=T+1}^{T+H} (y_{i,t} - f_{i,t})^2}\n\n\n Properties:\n\n - scale-dependent (time series with large absolute value contribute more to the loss)\n - heavily penalizes models that cannot quickly adapt to abrupt changes in the time series\n - sensitive to outliers\n - prefers models that accurately estimate the mean (expected value)\n\n\n References\n ----------\n - `Wikipedia `_\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n equivalent_tabular_regression_metric = \"root_mean_squared_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = ((y_true - y_pred) ** 2).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return np.sqrt(self._safemean(errors))\n\n\nclass MSE(TimeSeriesScorer):\n r\"\"\"Mean squared error.\n\n Using this metric will lead to forecast of the mean.\n\n .. math::\n\n \\operatorname{MSE} = \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N}\\sum_{t=T+1}^{T+H} (y_{i,t} - f_{i,t})^2\n\n Properties:\n\n - scale-dependent (time series with large absolute value contribute more to the loss)\n - heavily penalizes models that cannot quickly adapt to abrupt changes in the time series\n - sensitive to outliers\n - prefers models that accurately estimate the mean (expected value)\n\n References\n ----------\n - `Wikipedia `_\n\n \"\"\"\n\n equivalent_tabular_regression_metric = \"mean_squared_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = ((y_true - y_pred) ** 2).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return self._safemean(errors)\n\n\nclass MAE(TimeSeriesScorer):\n r\"\"\"Mean absolute error.\n\n .. math::\n\n \\operatorname{MAE} = \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N}\\sum_{t=T+1}^{T+H} |y_{i,t} - f_{i,t}|\n\n Properties:\n\n - scale-dependent (time series with large absolute value contribute more to the loss)\n - not sensitive to outliers\n - prefers models that accurately estimate the median\n\n References\n ----------\n - `Wikipedia `_\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"mean_absolute_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = np.abs(y_true - y_pred).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return self._safemean(errors)\n\n\nclass WAPE(TimeSeriesScorer):\n r\"\"\"Weighted absolute percentage error.\n\n Defined as sum of absolute errors divided by the sum of absolute time series values in the forecast horizon.\n\n .. math::\n\n \\operatorname{WAPE} = \\frac{1}{\\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} |y_{i, t}|} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} |y_{i,t} - f_{i,t}|\n\n Properties:\n\n - scale-dependent (time series with large absolute value contribute more to the loss)\n - not sensitive to outliers\n - prefers models that accurately estimate the median\n\n If ``self.horizon_weight`` is provided, both the errors and the target time series in the denominator will be re-weighted.\n\n References\n ----------\n - `Wikipedia `_\n \"\"\"\n\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"mean_absolute_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = np.abs(y_true - y_pred).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n y_true = y_true.reshape([-1, self.prediction_length]) * self.horizon_weight\n return np.nansum(errors) / np.nansum(np.abs(y_true))\n\n\nclass SMAPE(TimeSeriesScorer):\n r\"\"\"Symmetric mean absolute percentage error.\n\n .. math::\n\n \\operatorname{SMAPE} = 2 \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} \\frac{ |y_{i,t} - f_{i,t}|}{|y_{i,t}| + |f_{i,t}|}\n\n Properties:\n\n - should only be used if all time series have positive values\n - poorly suited for sparse & intermittent time series that contain zero values\n - penalizes overprediction more heavily than underprediction\n\n References\n ----------\n - `Wikipedia `_\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"symmetric_mean_absolute_percentage_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = (np.abs(y_true - y_pred) / (np.abs(y_true) + np.abs(y_pred))).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return 2 * self._safemean(errors)\n\n\nclass MAPE(TimeSeriesScorer):\n r\"\"\"Mean absolute percentage error.\n\n .. math::\n\n \\operatorname{MAPE} = \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} \\frac{ |y_{i,t} - f_{i,t}|}{|y_{i,t}|}\n\n Properties:\n\n - should only be used if all time series have positive values\n - undefined for time series that contain zero values\n - penalizes overprediction more heavily than underprediction\n\n References\n ----------\n - `Wikipedia `_\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"mean_absolute_percentage_error\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = (np.abs(y_true - y_pred) / np.abs(y_true)).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return self._safemean(errors)\n\n\nclass MASE(TimeSeriesScorer):\n r\"\"\"Mean absolute scaled error.\n\n Normalizes the absolute error for each time series by the historical seasonal error of this time series.\n\n .. math::\n\n \\operatorname{MASE} = \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\frac{1}{a_i} \\sum_{t=T+1}^{T+H} |y_{i,t} - f_{i,t}|\n\n where :math:`a_i` is the historical absolute seasonal error defined as\n\n .. math::\n\n a_i = \\frac{1}{T-m} \\sum_{t=m+1}^T |y_{i,t} - y_{i,t-m}|\n\n and :math:`m` is the seasonal period of the time series (``eval_metric_seasonal_period``).\n\n Properties:\n\n - scaled metric (normalizes the error for each time series by the scale of that time series)\n - undefined for constant time series\n - not sensitive to outliers\n - prefers models that accurately estimate the median\n\n References\n ----------\n - `Wikipedia `_\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n optimized_by_median = True\n equivalent_tabular_regression_metric = \"mean_absolute_error\"\n\n def __init__(\n self,\n prediction_length: int = 1,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | None = None,\n ):\n super().__init__(\n prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight\n )\n self._past_abs_seasonal_error: pd.Series | None = None\n\n def save_past_metrics(\n self, data_past: TimeSeriesDataFrame, target: str = \"target\", seasonal_period: int = 1, **kwargs\n ) -> None:\n self._past_abs_seasonal_error = in_sample_abs_seasonal_error(\n y_past=data_past[target], seasonal_period=seasonal_period\n )\n\n def clear_past_metrics(self) -> None:\n self._past_abs_seasonal_error = None\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n if self._past_abs_seasonal_error is None:\n raise AssertionError(\"Call `save_past_metrics` before `compute_metric`\")\n\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n\n errors = np.abs(y_true - y_pred).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return self._safemean(errors / self._past_abs_seasonal_error.to_numpy()[:, None])\n\n\nclass RMSSE(TimeSeriesScorer):\n r\"\"\"Root mean squared scaled error.\n\n Normalizes the absolute error for each time series by the historical seasonal error of this time series.\n\n .. math::\n\n \\operatorname{RMSSE} = \\sqrt{\\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\frac{1}{s_i} \\sum_{t=T+1}^{T+H} (y_{i,t} - f_{i,t})^2}\n\n where :math:`s_i` is the historical squared seasonal error defined as\n\n .. math::\n\n s_i = \\frac{1}{T-m} \\sum_{t=m+1}^T (y_{i,t} - y_{i,t-m})^2\n\n and :math:`m` is the seasonal period of the time series (``eval_metric_seasonal_period``).\n\n\n Properties:\n\n - scaled metric (normalizes the error for each time series by the scale of that time series)\n - undefined for constant time series\n - heavily penalizes models that cannot quickly adapt to abrupt changes in the time series\n - sensitive to outliers\n - prefers models that accurately estimate the mean (expected value)\n\n\n References\n ----------\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n equivalent_tabular_regression_metric = \"root_mean_squared_error\"\n\n def __init__(\n self,\n prediction_length: int = 1,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | None = None,\n ):\n super().__init__(\n prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight\n )\n self._past_squared_seasonal_error: pd.Series | None = None\n\n def save_past_metrics(\n self, data_past: TimeSeriesDataFrame, target: str = \"target\", seasonal_period: int = 1, **kwargs\n ) -> None:\n self._past_squared_seasonal_error = in_sample_squared_seasonal_error(\n y_past=data_past[target], seasonal_period=seasonal_period\n )\n\n def clear_past_metrics(self) -> None:\n self._past_squared_seasonal_error = None\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n if self._past_squared_seasonal_error is None:\n raise AssertionError(\"Call `save_past_metrics` before `compute_metric`\")\n\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n errors = ((y_true - y_pred) ** 2).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return np.sqrt(self._safemean(errors / self._past_squared_seasonal_error.to_numpy()[:, None]))\n\n\nclass RMSLE(TimeSeriesScorer):\n r\"\"\"Root mean squared logarithmic error.\n\n Applies a logarithmic transformation to the predictions before computing the root mean squared error. Assumes\n both the ground truth and predictions are positive. If negative predictions are given, they will be clipped to zero.\n\n .. math::\n\n \\operatorname{RMSLE} = \\sqrt{\\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} (\\ln(1 + y_{i,t}) - \\ln(1 + f_{i,t}))^2}\n\n\n Properties:\n\n - undefined for time series with negative values\n - penalizes models that underpredict more than models that overpredict\n - insensitive to effects of outliers and scale, best when targets can vary or trend exponentially\n\n\n References\n ----------\n - `Scikit-learn: `_\n \"\"\"\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n y_true, y_pred = y_true.to_numpy(), y_pred.to_numpy()\n y_pred = np.clip(y_pred, a_min=0.0, a_max=None)\n\n errors = np.power(np.log1p(y_pred) - np.log1p(y_true), 2).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return np.sqrt(self._safemean(errors))\n\n def __call__(\n self,\n data: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n if (data[target] < 0).any():\n raise ValueError(f\"{self.name} cannot be used if target time series contains negative values!\")\n return super().__call__(\n data=data,\n predictions=predictions,\n target=target,\n **kwargs,\n )\n\n\nclass WCD(TimeSeriesScorer):\n r\"\"\"Weighted cumulative discrepancy.\n\n Measures the discrepancy between the cumulative sum of the forecast and the cumulative sum of the actual values.\n\n .. math::\n\n \\operatorname{WCD} = 2 \\cdot \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} \\alpha \\cdot \\max(0, -d_{i, t}) + (1 - \\alpha) \\cdot \\max(0, d_{i, t})\n\n where :math:`d_{i, t}` is the difference between the cumulative predicted value and the cumulative actual value\n\n .. math::\n\n d_{i, t} = \\left(\\sum_{s=T+1}^t f_{i, s}) - \\left(\\sum_{s=T+1}^t y_{i, s})\n\n Parameters\n ----------\n alpha : float, default = 0.5\n Values > 0.5 put a stronger penalty on underpredictions (when cumulative forecast is below the\n cumulative actual value). Values < 0.5 put a stronger penalty on overpredictions.\n \"\"\"\n\n def __init__(\n self,\n alpha: float = 0.5,\n prediction_length: int = 1,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | None = None,\n ):\n super().__init__(\n prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight\n )\n assert 0 < alpha < 1, \"alpha must be in (0, 1)\"\n self.alpha = alpha\n warnings.warn(\n f\"{self.name} is an experimental metric. Its behavior may change in the future version of AutoGluon.\"\n )\n\n def _fast_cumsum(self, y: np.ndarray) -> np.ndarray:\n \"\"\"Compute the cumulative sum for each consecutive `self.prediction_length` items in the array.\"\"\"\n y = y.reshape(-1, self.prediction_length)\n return np.nancumsum(y, axis=1).ravel()\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)\n cumsum_true = self._fast_cumsum(y_true.to_numpy())\n cumsum_pred = self._fast_cumsum(y_pred.to_numpy())\n diffs = cumsum_pred - cumsum_true\n errors = (diffs * np.where(diffs < 0, -self.alpha, (1 - self.alpha))).reshape([-1, self.prediction_length])\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return 2 * self._safemean(errors)\n"
},
{
"path": "timeseries/src/autogluon/timeseries/metrics/quantile.py",
"content": "from typing import Sequence\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\n\nfrom .abstract import TimeSeriesScorer\nfrom .utils import in_sample_abs_seasonal_error\n\n\nclass WQL(TimeSeriesScorer):\n r\"\"\"Weighted quantile loss.\n\n Also known as weighted pinball loss.\n\n Defined as total quantile loss divided by the sum of absolute time series values in the forecast horizon.\n\n .. math::\n\n \\operatorname{WQL} = \\frac{1}{\\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} |y_{i, t}|} \\sum_{i=1}^{N} \\sum_{t=T+1}^{T+H} \\sum_{q} \\rho_q(y_{i,t}, f^q_{i,t})\n\n Properties:\n\n - scale-dependent (time series with large absolute value contribute more to the loss)\n - equivalent to WAPE if ``quantile_levels = [0.5]``\n\n If ``horizon_weight`` is provided, both the errors and the target time series in the denominator will be re-weighted.\n\n References\n ----------\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n needs_quantile = True\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n y_true, q_pred, quantile_levels = self._get_quantile_forecast_score_inputs(data_future, predictions, target)\n y_true = y_true.to_numpy()[:, None] # shape [N, 1]\n q_pred = q_pred.to_numpy() # shape [N, len(quantile_levels)]\n\n errors = (\n np.abs((q_pred - y_true) * ((y_true <= q_pred) - quantile_levels))\n .mean(axis=1)\n .reshape([-1, self.prediction_length])\n )\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n y_true = y_true.reshape([-1, self.prediction_length]) * self.horizon_weight\n return 2 * np.nansum(errors) / np.nansum(np.abs(y_true))\n\n\nclass SQL(TimeSeriesScorer):\n r\"\"\"Scaled quantile loss.\n\n Also known as scaled pinball loss.\n\n Normalizes the quantile loss for each time series by the historical seasonal error of this time series.\n\n .. math::\n\n \\operatorname{SQL} = \\frac{1}{N} \\frac{1}{H} \\sum_{i=1}^{N} \\frac{1}{a_i} \\sum_{t=T+1}^{T+H} \\sum_{q} \\rho_q(y_{i,t}, f^q_{i,t})\n\n where :math:`a_i` is the historical absolute seasonal error defined as\n\n .. math::\n\n a_i = \\frac{1}{T-m} \\sum_{t=m+1}^T |y_{i,t} - y_{i,t-m}|\n\n and :math:`m` is the seasonal period of the time series (``eval_metric_seasonal_period``).\n\n\n Properties:\n\n - scaled metric (normalizes the error for each time series by the scale of that time series)\n - undefined for constant time series\n - equivalent to MASE if ``quantile_levels = [0.5]``\n\n References\n ----------\n - `Forecasting: Principles and Practice `_\n \"\"\"\n\n needs_quantile = True\n\n def __init__(\n self,\n prediction_length: int = 1,\n seasonal_period: int | None = None,\n horizon_weight: Sequence[float] | None = None,\n ):\n super().__init__(\n prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight\n )\n self._past_abs_seasonal_error: pd.Series | None = None\n\n def save_past_metrics(\n self, data_past: TimeSeriesDataFrame, target: str = \"target\", seasonal_period: int = 1, **kwargs\n ) -> None:\n self._past_abs_seasonal_error = in_sample_abs_seasonal_error(\n y_past=data_past[target], seasonal_period=seasonal_period\n )\n\n def clear_past_metrics(self) -> None:\n self._past_abs_seasonal_error = None\n\n def compute_metric(\n self,\n data_future: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n target: str = \"target\",\n **kwargs,\n ) -> float:\n if self._past_abs_seasonal_error is None:\n raise AssertionError(\"Call `save_past_metrics` before `compute_metric`\")\n\n y_true, q_pred, quantile_levels = self._get_quantile_forecast_score_inputs(data_future, predictions, target)\n q_pred = q_pred.to_numpy()\n y_true = y_true.to_numpy()[:, None] # shape [N, 1]\n\n errors = (\n np.abs((q_pred - y_true) * ((y_true <= q_pred) - quantile_levels))\n .mean(axis=1)\n .reshape([-1, self.prediction_length])\n )\n if self.horizon_weight is not None:\n errors *= self.horizon_weight\n return 2 * self._safemean(errors / self._past_abs_seasonal_error.to_numpy()[:, None])\n"
},
{
"path": "timeseries/src/autogluon/timeseries/metrics/utils.py",
"content": "import pandas as pd\n\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\n\n\ndef _get_seasonal_diffs(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:\n return y_past.groupby(level=TimeSeriesDataFrame.ITEMID, sort=False).diff(seasonal_period).abs()\n\n\ndef in_sample_abs_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:\n \"\"\"Compute seasonal naive forecast error (predict value from seasonal_period steps ago) for each time series.\"\"\"\n seasonal_diffs = _get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)\n return seasonal_diffs.groupby(level=TimeSeriesDataFrame.ITEMID, sort=False).mean().fillna(1.0)\n\n\ndef in_sample_squared_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:\n seasonal_diffs = _get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)\n return seasonal_diffs.pow(2.0).groupby(level=TimeSeriesDataFrame.ITEMID, sort=False).mean().fillna(1.0)\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/__init__.py",
"content": "from .autogluon_tabular import DirectTabularModel, PerStepTabularModel, RecursiveTabularModel\nfrom .chronos import Chronos2Model, ChronosModel\nfrom .gluonts import (\n DeepARModel,\n DLinearModel,\n PatchTSTModel,\n SimpleFeedForwardModel,\n TemporalFusionTransformerModel,\n TiDEModel,\n WaveNetModel,\n)\nfrom .local import (\n ADIDAModel,\n ARIMAModel,\n AutoARIMAModel,\n AutoCESModel,\n AutoETSModel,\n AverageModel,\n CrostonModel,\n DynamicOptimizedThetaModel,\n ETSModel,\n IMAPAModel,\n NaiveModel,\n NPTSModel,\n SeasonalAverageModel,\n SeasonalNaiveModel,\n ThetaModel,\n ZeroModel,\n)\nfrom .registry import ModelRegistry\nfrom .toto import TotoModel\n\n__all__ = [\n \"ADIDAModel\",\n \"ARIMAModel\",\n \"AutoARIMAModel\",\n \"AutoCESModel\",\n \"AutoETSModel\",\n \"AverageModel\",\n \"CrostonModel\",\n \"DLinearModel\",\n \"DeepARModel\",\n \"DirectTabularModel\",\n \"DynamicOptimizedThetaModel\",\n \"ETSModel\",\n \"IMAPAModel\",\n \"ChronosModel\",\n \"Chronos2Model\",\n \"ModelRegistry\",\n \"NPTSModel\",\n \"NaiveModel\",\n \"PatchTSTModel\",\n \"PerStepTabularModel\",\n \"RecursiveTabularModel\",\n \"SeasonalAverageModel\",\n \"SeasonalNaiveModel\",\n \"SimpleFeedForwardModel\",\n \"TemporalFusionTransformerModel\",\n \"ThetaModel\",\n \"TiDEModel\",\n \"TotoModel\",\n \"WaveNetModel\",\n \"ZeroModel\",\n]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/abstract/__init__.py",
"content": "from .abstract_timeseries_model import AbstractTimeSeriesModel, TimeSeriesModelBase\n\n__all__ = [\"AbstractTimeSeriesModel\", \"TimeSeriesModelBase\"]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/abstract/abstract_timeseries_model.py",
"content": "import copy\nimport logging\nimport os\nimport re\nimport time\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Sequence\n\nimport pandas as pd\nfrom typing_extensions import Self\n\nfrom autogluon.common import space\nfrom autogluon.common.loaders import load_pkl\nfrom autogluon.common.savers import save_pkl\nfrom autogluon.common.utils.resource_utils import get_resource_manager\nfrom autogluon.common.utils.utils import setup_outputdir\nfrom autogluon.core.constants import AG_ARGS_FIT, REFIT_FULL_SUFFIX\nfrom autogluon.core.models import ModelBase\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric\nfrom autogluon.timeseries.models.registry import ModelRegistry\nfrom autogluon.timeseries.regressor import CovariateRegressor, get_covariate_regressor\nfrom autogluon.timeseries.transforms import CovariateScaler, TargetScaler, get_covariate_scaler, get_target_scaler\nfrom autogluon.timeseries.utils.features import CovariateMetadata\nfrom autogluon.timeseries.utils.forecast import make_future_data_frame\n\nfrom .tunable import TimeSeriesTunable\n\nlogger = logging.getLogger(__name__)\n\n\nclass TimeSeriesModelBase(ModelBase, ABC):\n \"\"\"Abstract base class for all `Model` objects in autogluon.timeseries, including both\n forecasting models and forecast combination/ensemble models.\n\n Parameters\n ----------\n path\n Directory location to store all outputs.\n If None, a new unique time-stamped directory is chosen.\n freq\n Frequency string (cf. gluonts frequency strings) describing the frequency\n of the time series data. For example, \"h\" for hourly or \"D\" for daily data.\n prediction_length\n Length of the prediction horizon, i.e., the number of time steps the model\n is fit to forecast.\n name\n Name of the subdirectory inside path where model will be saved.\n The final model directory will be os.path.join(path, name)\n If None, defaults to the model's class name: self.__class__.__name__\n covariate_metadata\n A mapping of different covariate types known to autogluon.timeseries to column names\n in the data set.\n eval_metric\n Metric by which predictions will be ultimately evaluated on future test data. This only impacts\n ``model.score()``, as eval_metric is not used during training. Available metrics can be found in\n ``autogluon.timeseries.metrics``.\n hyperparameters\n Hyperparameters that will be used by the model (can be search spaces instead of fixed values).\n If None, model defaults are used. This is identical to passing an empty dictionary.\n \"\"\"\n\n model_file_name = \"model.pkl\"\n model_info_name = \"info.pkl\"\n _oof_filename = \"oof.pkl\"\n\n # TODO: For which models should we override this parameter?\n _covariate_regressor_fit_time_fraction: float = 0.5\n default_max_time_limit_ratio: float = 0.9\n\n _supports_known_covariates: bool = False\n _supports_past_covariates: bool = False\n _supports_static_features: bool = False\n\n def __init__(\n self,\n path: str | None = None,\n name: str | None = None,\n hyperparameters: dict[str, Any] | None = None,\n freq: str | None = None,\n prediction_length: int = 1,\n covariate_metadata: CovariateMetadata | None = None,\n target: str = \"target\",\n quantile_levels: Sequence[float] = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9),\n eval_metric: str | TimeSeriesScorer | None = None,\n ):\n self.name = name or re.sub(r\"Model$\", \"\", self.__class__.__name__)\n\n self.path_root = path\n if self.path_root is None:\n path_suffix = self.name\n # TODO: Would be ideal to not create dir, but still track that it is unique. However, this isn't possible\n # to do without a global list of used dirs or using UUID.\n path_cur = setup_outputdir(path=None, create_dir=True, path_suffix=path_suffix)\n self.path_root = path_cur.rsplit(self.name, 1)[0]\n logger.log(20, f\"Warning: No path was specified for model, defaulting to: {self.path_root}\")\n\n self.path = os.path.join(self.path_root, self.name)\n\n self.eval_metric = check_get_evaluation_metric(eval_metric, prediction_length=prediction_length)\n self.target: str = target\n self.covariate_metadata = covariate_metadata or CovariateMetadata()\n\n self.freq: str | None = freq\n self.prediction_length: int = prediction_length\n self.quantile_levels: list[float] = list(quantile_levels)\n\n if not all(0 < q < 1 for q in self.quantile_levels):\n raise ValueError(\"Invalid quantile_levels specified. Quantiles must be between 0 and 1 (exclusive).\")\n\n # We ensure that P50 forecast is always among the \"raw\" predictions generated by _predict.\n # We remove P50 from the final predictions if P50 wasn't present among the specified quantile_levels.\n if 0.5 not in self.quantile_levels:\n self.must_drop_median = True\n self.quantile_levels = sorted(set([0.5] + self.quantile_levels))\n else:\n self.must_drop_median = False\n\n self._oof_predictions: list[TimeSeriesDataFrame] | None = None\n\n # user provided hyperparameters and extra arguments that are used during model training\n self._hyperparameters, self._extra_ag_args = self._check_and_split_hyperparameters(hyperparameters)\n\n # Time taken to fit in seconds (Training data)\n self.fit_time: float | None = None\n # Time taken to predict in seconds, for a single prediction horizon on validation data\n self.predict_time: float | None = None\n # Time taken to predict 1 row of data in seconds (with batch size `predict_1_batch_size`)\n self.predict_1_time: float | None = None\n # Useful for ensembles, additional prediction time excluding base models. None for base models.\n self.predict_time_marginal: float | None = None\n # Score with eval_metric on validation data\n self.val_score: float | None = None\n\n def __repr__(self) -> str:\n return self.name\n\n def rename(self, name: str) -> None:\n if self.name is not None and len(self.name) > 0:\n self.path = os.path.join(os.path.dirname(self.path), name)\n else:\n self.path = os.path.join(self.path, name)\n self.name = name\n\n def set_contexts(self, path_context):\n self.path = path_context\n self.path_root = self.path.rsplit(self.name, 1)[0]\n\n def cache_oof_predictions(self, predictions: TimeSeriesDataFrame | list[TimeSeriesDataFrame]) -> None:\n if isinstance(predictions, TimeSeriesDataFrame):\n predictions = [predictions]\n self._oof_predictions = predictions\n\n @classmethod\n def _check_and_split_hyperparameters(\n cls, hyperparameters: dict[str, Any] | None = None\n ) -> tuple[dict[str, Any], dict[str, Any]]:\n \"\"\"Given the user-specified hyperparameters, split into `hyperparameters` and `extra_ag_args`, intended\n to be used during model initialization.\n\n Parameters\n ----------\n hyperparameters\n The model hyperparameters dictionary provided to the model constructor.\n\n Returns\n -------\n hyperparameters\n Native model hyperparameters that are passed into the \"inner model\" AutoGluon wraps\n extra_ag_args\n Special auxiliary parameters that modify the model training process used by AutoGluon\n \"\"\"\n hyperparameters = copy.deepcopy(hyperparameters) if hyperparameters is not None else dict()\n assert isinstance(hyperparameters, dict), (\n f\"Invalid dtype for hyperparameters. Expected dict, but got {type(hyperparameters)}\"\n )\n for k in hyperparameters.keys():\n if not isinstance(k, str):\n logger.warning(\n f\"Warning: Specified hyperparameter key is not of type str: {k} (type={type(k)}). \"\n f\"There might be a bug in your configuration.\"\n )\n\n extra_ag_args = hyperparameters.pop(AG_ARGS_FIT, {})\n if not isinstance(extra_ag_args, dict):\n raise ValueError(\n f\"Invalid hyperparameter type for `{AG_ARGS_FIT}`. Expected dict, but got {type(extra_ag_args)}\"\n )\n return hyperparameters, extra_ag_args\n\n def save(self, path: str | None = None, verbose: bool = True) -> str:\n if path is None:\n path = self.path\n\n # Save self._oof_predictions as a separate file, not model attribute\n if self._oof_predictions is not None:\n save_pkl.save(\n path=os.path.join(path, \"utils\", self._oof_filename),\n object=self._oof_predictions,\n verbose=verbose,\n )\n oof_predictions = self._oof_predictions\n self._oof_predictions = None\n\n file_path = os.path.join(path, self.model_file_name)\n save_pkl.save(path=file_path, object=self, verbose=verbose)\n\n self._oof_predictions = oof_predictions\n return path\n\n @classmethod\n def load(cls, path: str, reset_paths: bool = True, load_oof: bool = False, verbose: bool = True) -> Self:\n file_path = os.path.join(path, cls.model_file_name)\n model = load_pkl.load(path=file_path, verbose=verbose)\n if reset_paths:\n model.set_contexts(path)\n if load_oof and model._oof_predictions is None:\n model._oof_predictions = cls.load_oof_predictions(path=path, verbose=verbose)\n return model\n\n @classmethod\n def load_oof_predictions(cls, path: str, verbose: bool = True) -> list[TimeSeriesDataFrame]:\n \"\"\"Load the cached OOF predictions from disk.\"\"\"\n return load_pkl.load(path=os.path.join(path, \"utils\", cls._oof_filename), verbose=verbose)\n\n @property\n def supports_known_covariates(self) -> bool:\n return (\n self.get_hyperparameters().get(\"covariate_regressor\") is not None\n or self.__class__._supports_known_covariates\n )\n\n @property\n def supports_past_covariates(self) -> bool:\n return self.__class__._supports_past_covariates\n\n @property\n def supports_static_features(self) -> bool:\n return (\n self.get_hyperparameters().get(\"covariate_regressor\") is not None\n or self.__class__._supports_static_features\n )\n\n def get_oof_predictions(self):\n if self._oof_predictions is None:\n self._oof_predictions = self.load_oof_predictions(self.path)\n return self._oof_predictions\n\n def _get_default_hyperparameters(self) -> dict:\n return {}\n\n def get_hyperparameters(self) -> dict:\n \"\"\"Get dictionary of hyperparameters that will be passed to the \"inner model\" that AutoGluon wraps.\"\"\"\n return {**self._get_default_hyperparameters(), **self._hyperparameters}\n\n def get_hyperparameter(self, key: str) -> Any:\n \"\"\"Get a single hyperparameter value for the \"inner model\".\"\"\"\n return self.get_hyperparameters()[key]\n\n def get_info(self) -> dict:\n \"\"\"\n Returns a dictionary of numerous fields describing the model.\n \"\"\"\n info = {\n \"name\": self.name,\n \"model_type\": type(self).__name__,\n \"eval_metric\": self.eval_metric,\n \"fit_time\": self.fit_time,\n \"predict_time\": self.predict_time,\n \"freq\": self.freq,\n \"prediction_length\": self.prediction_length,\n \"quantile_levels\": self.quantile_levels,\n \"val_score\": self.val_score,\n \"hyperparameters\": self.get_hyperparameters(),\n \"covariate_metadata\": self.covariate_metadata.to_dict(),\n }\n return info\n\n @classmethod\n def load_info(cls, path: str, load_model_if_required: bool = True) -> dict:\n # TODO: remove?\n load_path = os.path.join(path, cls.model_info_name)\n try:\n return load_pkl.load(path=load_path)\n except:\n if load_model_if_required:\n model = cls.load(path=path, reset_paths=True)\n return model.get_info()\n else:\n raise\n\n def _is_gpu_available(self) -> bool:\n return False\n\n @staticmethod\n def _get_system_resources() -> dict[str, Any]:\n resource_manager = get_resource_manager()\n system_num_cpus = resource_manager.get_cpu_count()\n system_num_gpus = resource_manager.get_gpu_count()\n return {\n \"num_cpus\": system_num_cpus,\n \"num_gpus\": system_num_gpus,\n }\n\n def _get_model_base(self) -> Self:\n return self\n\n def persist(self) -> Self:\n \"\"\"Ask the model to persist its assets in memory, i.e., to predict with low latency. In practice\n this is used for pretrained models that have to lazy-load model parameters to device memory at\n prediction time.\n \"\"\"\n return self\n\n def _more_tags(self) -> dict:\n \"\"\"Encode model properties using tags, similar to sklearn & autogluon.tabular.\n\n For more details, see `autogluon.core.models.abstract.AbstractModel._get_tags()` and\n https://scikit-learn.org/stable/_sources/developers/develop.rst.txt.\n\n List of currently supported tags:\n - allow_nan: Can the model handle data with missing values represented by np.nan?\n - can_refit_full: Does it make sense to retrain the model without validation data?\n See `autogluon.core.models.abstract._tags._DEFAULT_TAGS` for more details.\n - can_use_train_data: Can the model use train_data if it's provided to model.fit()?\n - can_use_val_data: Can the model use val_data if it's provided to model.fit()?\n \"\"\"\n return {\n \"allow_nan\": False,\n \"can_refit_full\": False,\n \"can_use_train_data\": True,\n \"can_use_val_data\": False,\n }\n\n def get_params(self) -> dict:\n \"\"\"Get the constructor parameters required for cloning this model object\"\"\"\n # We only use the user-provided hyperparameters for cloning. We cannot use the output of get_hyperparameters()\n # since it may contain search spaces that won't be converted to concrete values during HPO\n hyperparameters = self._hyperparameters.copy()\n if self._extra_ag_args:\n hyperparameters[AG_ARGS_FIT] = self._extra_ag_args.copy()\n\n return dict(\n path=self.path_root,\n name=self.name,\n eval_metric=self.eval_metric,\n hyperparameters=hyperparameters,\n freq=self.freq,\n prediction_length=self.prediction_length,\n quantile_levels=self.quantile_levels,\n covariate_metadata=self.covariate_metadata,\n target=self.target,\n )\n\n def convert_to_refit_full_via_copy(self) -> Self:\n # save the model as a new model on disk\n previous_name = self.name\n self.rename(self.name + REFIT_FULL_SUFFIX)\n refit_model_path = self.path\n self.save(path=self.path, verbose=False)\n\n self.rename(previous_name)\n\n refit_model = self.load(path=refit_model_path, verbose=False)\n refit_model.val_score = None\n refit_model.predict_time = None\n\n return refit_model\n\n def convert_to_refit_full_template(self) -> Self:\n \"\"\"After calling this function, returned model should be able to be fit without `val_data`.\"\"\"\n params = copy.deepcopy(self.get_params())\n\n # Remove 0.5 from quantile_levels so that the cloned model sets its must_drop_median correctly\n if self.must_drop_median:\n params[\"quantile_levels\"].remove(0.5)\n\n if \"hyperparameters\" not in params:\n params[\"hyperparameters\"] = dict()\n\n if AG_ARGS_FIT not in params[\"hyperparameters\"]:\n params[\"hyperparameters\"][AG_ARGS_FIT] = dict()\n\n params[\"name\"] = params[\"name\"] + REFIT_FULL_SUFFIX\n template = self.__class__(**params)\n\n return template\n\n\nclass AbstractTimeSeriesModel(TimeSeriesModelBase, TimeSeriesTunable, metaclass=ModelRegistry):\n \"\"\"Abstract base class for all time series models that take historical data as input and\n make predictions for the forecast horizon.\n \"\"\"\n\n ag_priority: int = 0\n\n def __init__(\n self,\n path: str | None = None,\n name: str | None = None,\n hyperparameters: dict[str, Any] | None = None,\n freq: str | None = None,\n prediction_length: int = 1,\n covariate_metadata: CovariateMetadata | None = None,\n target: str = \"target\",\n quantile_levels: Sequence[float] = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9),\n eval_metric: str | TimeSeriesScorer | None = None,\n ):\n # TODO: make freq a required argument in AbstractTimeSeriesModel\n super().__init__(\n path=path,\n name=name,\n hyperparameters=hyperparameters,\n freq=freq,\n prediction_length=prediction_length,\n covariate_metadata=covariate_metadata,\n target=target,\n quantile_levels=quantile_levels,\n eval_metric=eval_metric,\n )\n self.target_scaler: TargetScaler | None\n self.covariate_scaler: CovariateScaler | None\n self.covariate_regressor: CovariateRegressor | None\n\n def _initialize_transforms_and_regressor(self) -> None:\n self.target_scaler = get_target_scaler(self.get_hyperparameters().get(\"target_scaler\"), target=self.target)\n self.covariate_scaler = get_covariate_scaler(\n self.get_hyperparameters().get(\"covariate_scaler\"),\n covariate_metadata=self.covariate_metadata,\n use_static_features=self.supports_static_features,\n use_known_covariates=self.supports_known_covariates,\n use_past_covariates=self.supports_past_covariates,\n )\n self.covariate_regressor = get_covariate_regressor(\n self.get_hyperparameters().get(\"covariate_regressor\"),\n target=self.target,\n covariate_metadata=self.covariate_metadata,\n )\n\n @property\n def allowed_hyperparameters(self) -> list[str]:\n \"\"\"List of hyperparameters allowed by the model.\"\"\"\n return [\"target_scaler\", \"covariate_regressor\", \"covariate_scaler\"]\n\n def fit(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None = None,\n time_limit: float | None = None,\n verbosity: int = 2,\n **kwargs,\n ) -> Self:\n \"\"\"Fit timeseries model.\n\n Models should not override the `fit` method, but instead override the `_fit` method which\n has the same arguments.\n\n Parameters\n ----------\n train_data\n The training data provided in the library's `autogluon.timeseries.dataset.TimeSeriesDataFrame`\n format.\n val_data\n The validation data set in the same format as training data.\n time_limit\n Time limit in seconds to adhere to when fitting model.\n Ideally, model should early stop during fit to avoid going over the time limit if specified.\n num_cpus\n How many CPUs to use during fit.\n This is counted in virtual cores, not in physical cores.\n If 'auto', model decides.\n num_gpus\n How many GPUs to use during fit.\n If 'auto', model decides.\n verbosity\n Verbosity levels range from 0 to 4 and control how much information is printed.\n Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings).\n **kwargs\n Any additional fit arguments a model supports.\n\n Returns\n -------\n model\n The fitted model object\n \"\"\"\n start_time = time.monotonic()\n self._initialize_transforms_and_regressor()\n\n if self.target_scaler is not None:\n train_data = self.target_scaler.fit_transform(train_data)\n\n if self.covariate_scaler is not None:\n train_data = self.covariate_scaler.fit_transform(train_data)\n\n if self.covariate_regressor is not None:\n covariate_regressor_time_limit = (\n self._covariate_regressor_fit_time_fraction * time_limit if time_limit is not None else None\n )\n self.covariate_regressor.fit(\n train_data,\n time_limit=covariate_regressor_time_limit,\n verbosity=verbosity - 1,\n )\n\n if self._get_tags()[\"can_use_train_data\"]:\n if self.covariate_regressor is not None:\n train_data = self.covariate_regressor.transform(train_data)\n train_data, _ = self.preprocess(train_data, is_train=True)\n\n if self._get_tags()[\"can_use_val_data\"] and val_data is not None:\n if self.target_scaler is not None:\n val_data = self.target_scaler.transform(val_data)\n if self.covariate_scaler is not None:\n val_data = self.covariate_scaler.transform(val_data)\n if self.covariate_regressor is not None:\n val_data = self.covariate_regressor.transform(val_data)\n val_data, _ = self.preprocess(val_data, is_train=False)\n\n if time_limit is not None:\n time_limit = time_limit - (time.monotonic() - start_time)\n time_limit = self._preprocess_time_limit(time_limit=time_limit)\n if time_limit <= 0:\n logger.warning(\n f\"\\tWarning: Model has no time left to train, skipping model... (Time Left = {time_limit:.1f}s)\"\n )\n raise TimeLimitExceeded\n\n self._fit(\n train_data=train_data,\n val_data=val_data,\n time_limit=time_limit,\n verbosity=verbosity,\n **(self._get_system_resources() | kwargs),\n )\n\n return self\n\n @abstractmethod\n def _fit(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None = None,\n time_limit: float | None = None,\n num_cpus: int | None = None,\n num_gpus: int | None = None,\n verbosity: int = 2,\n **kwargs,\n ) -> None:\n \"\"\"Private method for `fit`. See `fit` for documentation of arguments. Apart from\n the model training logic, `fit` additionally implements other logic such as keeping\n track of the time limit, etc.\n \"\"\"\n pass\n\n # TODO: this check cannot be moved inside fit because of the complex way in which\n # MultiWindowBacktestingModel handles hyperparameter spaces during initialization.\n # Move inside fit() after refactoring MultiWindowBacktestingModel\n def _check_fit_params(self):\n # gracefully handle hyperparameter specifications if they are provided to fit instead\n if any(isinstance(v, space.Space) for v in self.get_hyperparameters().values()):\n raise ValueError(\n \"Hyperparameter spaces provided to `fit`. Please provide concrete values \"\n \"as hyperparameters when initializing or use `hyperparameter_tune` instead.\"\n )\n\n def _log_unused_hyperparameters(self, extra_allowed_hyperparameters: list[str] | None = None) -> None:\n \"\"\"Log a warning if unused hyperparameters were provided to the model.\"\"\"\n allowed_hyperparameters = self.allowed_hyperparameters\n if extra_allowed_hyperparameters is not None:\n allowed_hyperparameters = allowed_hyperparameters + extra_allowed_hyperparameters\n\n unused_hyperparameters = [key for key in self.get_hyperparameters() if key not in allowed_hyperparameters]\n if len(unused_hyperparameters) > 0:\n logger.warning(\n f\"{self.name} ignores following hyperparameters: {unused_hyperparameters}. \"\n f\"See the documentation for {self.name} for the list of supported hyperparameters.\"\n )\n\n def predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n \"\"\"Given a dataset, predict the next `self.prediction_length` time steps.\n This method produces predictions for the forecast horizon *after* the individual time series.\n\n For example, if the data set includes 24 hour time series, of hourly data, starting from\n 00:00 on day 1, and forecast horizon is set to 5. The forecasts are five time steps 00:00-04:00\n on day 2.\n\n Parameters\n ----------\n data\n The dataset where each time series is the \"context\" for predictions. For ensemble models that depend on\n the predictions of other models, this method may accept a dictionary of previous models' predictions.\n known_covariates\n A TimeSeriesDataFrame containing the values of the known covariates during the forecast horizon.\n\n Returns\n -------\n predictions\n pandas dataframes with a timestamp index, where each input item from the input\n data is given as a separate forecast item in the dictionary, keyed by the `item_id`s\n of input items.\n \"\"\"\n if self.target_scaler is not None:\n data = self.target_scaler.fit_transform(data)\n if self.covariate_scaler is not None:\n data = self.covariate_scaler.fit_transform(data)\n known_covariates = self.covariate_scaler.transform_known_covariates(known_covariates)\n if self.covariate_regressor is not None:\n data = self.covariate_regressor.fit_transform(data)\n\n data, known_covariates = self.preprocess(data, known_covariates, is_train=False)\n\n # FIXME: Set self.covariate_regressor=None so to avoid copying it across processes during _predict\n # FIXME: The clean solution is to convert all methods executed in parallel to @classmethod\n covariate_regressor = self.covariate_regressor\n self.covariate_regressor = None\n predictions = self._predict(data=data, known_covariates=known_covariates, **kwargs)\n self.covariate_regressor = covariate_regressor\n\n # Ensure that 'mean' is the leading column. Trailing columns might not match quantile_levels if self is\n # a MultiWindowBacktestingModel and base_model.must_drop_median=True\n column_order = pd.Index([\"mean\"] + [col for col in predictions.columns if col != \"mean\"])\n if not predictions.columns.equals(column_order):\n predictions = predictions.reindex(columns=column_order)\n\n # \"0.5\" might be missing from the quantiles if self is a MultiWindowBacktestingModel\n if \"0.5\" in predictions.columns:\n if self.eval_metric.optimized_by_median:\n predictions[\"mean\"] = predictions[\"0.5\"]\n if self.must_drop_median:\n predictions = predictions.drop(\"0.5\", axis=1)\n\n if self.covariate_regressor is not None:\n if known_covariates is None:\n known_covariates = TimeSeriesDataFrame.from_data_frame(\n pd.DataFrame(index=self.get_forecast_horizon_index(data), dtype=\"float32\")\n )\n\n predictions = self.covariate_regressor.inverse_transform(\n predictions,\n known_covariates=known_covariates,\n static_features=data.static_features,\n )\n\n if self.target_scaler is not None:\n predictions = self.target_scaler.inverse_transform(predictions)\n return predictions\n\n def get_forecast_horizon_index(self, data: TimeSeriesDataFrame) -> pd.MultiIndex:\n \"\"\"For each item in the dataframe, get timestamps for the next `prediction_length` time steps into the future.\"\"\"\n return pd.MultiIndex.from_frame(\n make_future_data_frame(data, prediction_length=self.prediction_length, freq=self.freq)\n )\n\n @abstractmethod\n def _predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n \"\"\"Private method for `predict`. See `predict` for documentation of arguments.\"\"\"\n pass\n\n def _preprocess_time_limit(self, time_limit: float) -> float:\n max_time_limit_ratio = self._extra_ag_args.get(\"max_time_limit_ratio\", self.default_max_time_limit_ratio)\n max_time_limit = self._extra_ag_args.get(\"max_time_limit\")\n\n time_limit *= max_time_limit_ratio\n\n if max_time_limit is not None:\n time_limit = min(time_limit, max_time_limit)\n\n return time_limit\n\n def _get_search_space(self):\n \"\"\"Sets up default search space for HPO. Each hyperparameter which user did not specify is converted from\n default fixed value to default search space.\n \"\"\"\n params = self._hyperparameters.copy()\n return params\n\n def _score_with_predictions(\n self,\n data: TimeSeriesDataFrame,\n predictions: TimeSeriesDataFrame,\n ) -> float:\n \"\"\"Compute the score measuring how well the predictions align with the data.\"\"\"\n return self.eval_metric.score(\n data=data,\n predictions=predictions,\n target=self.target,\n )\n\n def score(self, data: TimeSeriesDataFrame) -> float:\n \"\"\"Return the evaluation scores for given metric and dataset. The last\n `self.prediction_length` time steps of each time series in the input data set\n will be held out and used for computing the evaluation score. Time series\n models always return higher-is-better type scores.\n\n Parameters\n ----------\n data\n Dataset used for scoring.\n\n Returns\n -------\n score\n The computed forecast evaluation score on the last `self.prediction_length`\n time steps of each time series.\n \"\"\"\n past_data, known_covariates = data.get_model_inputs_for_scoring(\n prediction_length=self.prediction_length, known_covariates_names=self.covariate_metadata.known_covariates\n )\n predictions = self.predict(past_data, known_covariates=known_covariates)\n return self._score_with_predictions(data=data, predictions=predictions)\n\n def score_and_cache_oof(\n self,\n val_data: TimeSeriesDataFrame,\n store_val_score: bool = False,\n store_predict_time: bool = False,\n **predict_kwargs,\n ) -> None:\n \"\"\"Compute val_score, predict_time and cache out-of-fold (OOF) predictions.\"\"\"\n past_data, known_covariates = val_data.get_model_inputs_for_scoring(\n prediction_length=self.prediction_length, known_covariates_names=self.covariate_metadata.known_covariates\n )\n predict_start_time = time.time()\n oof_predictions = self.predict(past_data, known_covariates=known_covariates, **predict_kwargs)\n self.cache_oof_predictions(oof_predictions)\n if store_predict_time:\n self.predict_time = time.time() - predict_start_time\n if store_val_score:\n self.val_score = self._score_with_predictions(val_data, oof_predictions)\n\n def preprocess(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n is_train: bool = False,\n **kwargs,\n ) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame | None]:\n \"\"\"Method that implements model-specific preprocessing logic.\"\"\"\n return data, known_covariates\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/abstract/model_trial.py",
"content": "import logging\nimport os\nimport time\nimport traceback\n\nfrom autogluon.core.models.abstract.model_trial import init_model\nfrom autogluon.core.utils.exceptions import TimeLimitExceeded\nfrom autogluon.core.utils.loaders import load_pkl\n\nlogger = logging.getLogger(\"autogluon.timeseries.trainer\")\n\n\ndef model_trial(\n args,\n model_cls,\n init_params,\n train_path,\n val_path,\n time_start,\n hpo_executor,\n is_bagged_model=False,\n reporter=None, # reporter only used by custom strategy, hence optional\n time_limit=None,\n fit_kwargs=None,\n):\n \"\"\"Runs a single trial of a hyperparameter tuning. Replaces\n `core.models.abstract.model_trial.model_trial` for timeseries models.\n \"\"\"\n model = init_model(\n args, model_cls, init_params, backend=hpo_executor.executor_type, is_bagged_model=is_bagged_model\n )\n model.set_contexts(path_context=os.path.join(model.path_root, model.name))\n\n train_data = load_pkl.load(train_path)\n val_data = load_pkl.load(val_path)\n\n eval_metric = model.eval_metric\n\n try:\n model = fit_and_save_model(\n model,\n fit_kwargs,\n train_data,\n val_data,\n eval_metric,\n time_start=time_start,\n time_limit=time_limit,\n )\n except Exception as err:\n if not isinstance(err, TimeLimitExceeded):\n logger.error(f\"\\tWarning: Exception caused {model.name} to fail during training... Skipping this model.\")\n logger.error(f\"\\t{err}\")\n logger.debug(traceback.format_exc())\n # In case of TimeLimitExceed, val_score could be None\n hpo_executor.report(\n reporter=reporter,\n epoch=1,\n validation_performance=model.val_score if model.val_score is not None else float(\"-inf\"),\n )\n if reporter is not None:\n reporter.terminate()\n else:\n hpo_executor.report(reporter=reporter, epoch=1, validation_performance=model.val_score)\n\n\ndef fit_and_save_model(model, fit_kwargs, train_data, val_data, eval_metric, time_start, time_limit=None):\n time_current = time.time()\n time_elapsed = time_current - time_start\n if time_limit is not None:\n time_left = time_limit - time_elapsed\n if time_left <= 0:\n raise TimeLimitExceeded\n else:\n time_left = None\n\n time_fit_start = time.time()\n model.fit(train_data=train_data, val_data=val_data, time_limit=time_left, **fit_kwargs)\n model.fit_time = time.time() - time_fit_start\n if val_data is not None:\n model.score_and_cache_oof(val_data, store_val_score=True, store_predict_time=True)\n\n logger.debug(f\"\\tHyperparameter tune run: {model.name}\")\n logger.debug(f\"\\t\\t{model.val_score:<7.4f}\".ljust(15) + f\"= Validation score ({eval_metric.name_with_sign})\")\n logger.debug(f\"\\t\\t{model.fit_time:<7.3f} s\".ljust(15) + \"= Training runtime\")\n logger.debug(f\"\\t\\t{model.predict_time:<7.3f} s\".ljust(15) + \"= Training runtime\")\n model.save()\n return model\n\n\ndef skip_hpo(model, train_data, val_data, time_limit=None):\n \"\"\"Skip hyperparameter optimization and train model with given parameters.\n Replaces `core.models.abstract.model_trial.skip_hpo` for timeseries\n models.\n \"\"\"\n fit_and_save_model(\n model=model,\n train_data=train_data,\n val_data=val_data,\n fit_kwargs={},\n eval_metric=model.eval_metric,\n time_start=time.time(),\n time_limit=time_limit,\n )\n hpo_results = {\"total_time\": model.fit_time}\n hpo_model_performances = {model.name: model.val_score}\n hpo_results[\"hpo_model_performances\"] = hpo_model_performances\n hpo_models = {model.name: model.path}\n return hpo_models, hpo_results\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/abstract/tunable.py",
"content": "from __future__ import annotations\n\nimport logging\nimport os\nimport time\nfrom abc import ABC, abstractmethod\nfrom contextlib import nullcontext\nfrom typing import Any\n\nfrom typing_extensions import Self\n\nfrom autogluon.common.savers import save_pkl\nfrom autogluon.common.utils.distribute_utils import DistributedContext\nfrom autogluon.common.utils.log_utils import DuplicateFilter\nfrom autogluon.common.utils.try_import import try_import_ray\nfrom autogluon.core.hpo.constants import CUSTOM_BACKEND, RAY_BACKEND\nfrom autogluon.core.hpo.exceptions import EmptySearchSpace\nfrom autogluon.core.hpo.executors import HpoExecutor, HpoExecutorFactory, RayHpoExecutor\nfrom autogluon.core.models import Tunable\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.utils.warning_filters import disable_stdout, warning_filter\n\nfrom .model_trial import model_trial, skip_hpo\n\nlogger = logging.getLogger(__name__)\ndup_filter = DuplicateFilter()\nlogger.addFilter(dup_filter)\n\n\nclass TimeSeriesTunable(Tunable, ABC):\n @abstractmethod\n def __init__(self) -> None:\n self.name: str\n self.path: str\n self.path_root: str\n\n def hyperparameter_tune(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None,\n val_splitter: Any = None,\n default_num_trials: int | None = 1,\n refit_every_n_windows: int | None = 1,\n hyperparameter_tune_kwargs: str | dict = \"auto\",\n time_limit: float | None = None,\n ) -> tuple[dict[str, Any], Any]:\n hpo_executor = self._get_default_hpo_executor()\n hpo_executor.initialize(\n hyperparameter_tune_kwargs, default_num_trials=default_num_trials, time_limit=time_limit\n )\n\n # we use k_fold=1 to circumvent autogluon.core logic to manage resources during parallelization\n # of different folds\n # FIXME: we pass in self which currently does not inherit from AbstractModel\n hpo_executor.register_resources(self, k_fold=1, **self._get_system_resources()) # type: ignore\n\n time_start = time.time()\n logger.debug(f\"\\tStarting hyperparameter tuning for {self.name}\")\n search_space = self._get_search_space()\n\n try:\n hpo_executor.validate_search_space(search_space, self.name)\n except EmptySearchSpace:\n return skip_hpo(self, train_data, val_data, time_limit=hpo_executor.time_limit)\n\n train_path, val_path = self._save_with_data(train_data, val_data)\n\n train_fn_kwargs = self._get_hpo_train_fn_kwargs(\n model_cls=self.__class__,\n init_params=self.get_params(),\n time_start=time_start,\n time_limit=hpo_executor.time_limit,\n fit_kwargs=dict(\n val_splitter=val_splitter,\n refit_every_n_windows=refit_every_n_windows,\n ),\n train_path=train_path,\n val_path=val_path,\n hpo_executor=hpo_executor,\n )\n\n minimum_resources = self.get_minimum_resources(is_gpu_available=self._is_gpu_available())\n hpo_context = disable_stdout if isinstance(hpo_executor, RayHpoExecutor) else nullcontext\n\n minimum_cpu_per_trial = minimum_resources.get(\"num_cpus\", 1)\n if not isinstance(minimum_cpu_per_trial, int):\n logger.warning(\n f\"Minimum number of CPUs per trial for {self.name} is not an integer. \"\n f\"Setting to 1. Minimum number of CPUs per trial: {minimum_cpu_per_trial}\"\n )\n minimum_cpu_per_trial = 1\n\n with hpo_context(), warning_filter(): # prevent Ray from outputting its results to stdout with print\n hpo_executor.execute(\n model_trial=model_trial,\n train_fn_kwargs=train_fn_kwargs,\n directory=self.path,\n minimum_cpu_per_trial=minimum_cpu_per_trial,\n minimum_gpu_per_trial=minimum_resources.get(\"num_gpus\", 0),\n model_estimate_memory_usage=None, # type: ignore\n adapter_type=\"timeseries\",\n )\n\n assert self.path_root is not None\n hpo_models, analysis = hpo_executor.get_hpo_results(\n model_name=self.name,\n model_path_root=self.path_root,\n time_start=time_start,\n )\n\n return hpo_models, analysis\n\n def _get_default_hpo_executor(self) -> HpoExecutor:\n backend = (\n self._get_model_base()._get_hpo_backend()\n ) # If ensemble, will use the base model to determine backend\n if backend == RAY_BACKEND:\n try:\n try_import_ray()\n except Exception as e:\n warning_msg = f\"Will use custom hpo logic because ray import failed. Reason: {str(e)}\"\n dup_filter.attach_filter_targets(warning_msg)\n logger.warning(warning_msg)\n backend = CUSTOM_BACKEND\n hpo_executor = HpoExecutorFactory.get_hpo_executor(backend)() # type: ignore\n return hpo_executor\n\n def _get_hpo_backend(self) -> str:\n \"\"\"Choose which backend(\"ray\" or \"custom\") to use for hpo\"\"\"\n if DistributedContext.is_distributed_mode():\n return RAY_BACKEND\n return CUSTOM_BACKEND\n\n def _get_hpo_train_fn_kwargs(self, **train_fn_kwargs) -> dict:\n \"\"\"Update kwargs passed to model_trial depending on the model configuration.\n\n These kwargs need to be updated, for example, by MultiWindowBacktestingModel.\n \"\"\"\n return train_fn_kwargs\n\n def estimate_memory_usage(self, *args, **kwargs) -> float | None:\n \"\"\"Return the estimated memory usage of the model. None if memory usage cannot be\n estimated.\n \"\"\"\n return None\n\n def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]:\n return {\n \"num_cpus\": 1,\n }\n\n def _save_with_data(\n self, train_data: TimeSeriesDataFrame, val_data: TimeSeriesDataFrame | None\n ) -> tuple[str, str]:\n self.path = os.path.abspath(self.path)\n self.path_root = self.path.rsplit(self.name, 1)[0]\n\n dataset_train_filename = \"dataset_train.pkl\"\n train_path = os.path.join(self.path, dataset_train_filename)\n save_pkl.save(path=train_path, object=train_data)\n\n dataset_val_filename = \"dataset_val.pkl\"\n val_path = os.path.join(self.path, dataset_val_filename)\n save_pkl.save(path=val_path, object=val_data)\n return train_path, val_path\n\n @abstractmethod\n def _get_model_base(self) -> Self:\n pass\n\n @abstractmethod\n def _is_gpu_available(self) -> bool:\n pass\n\n @abstractmethod\n def _get_search_space(self) -> dict[str, Any]:\n pass\n\n @abstractmethod\n def get_params(self) -> dict:\n \"\"\"Return a clean copy of constructor parameters that can be used to\n clone the current model.\n \"\"\"\n pass\n\n @staticmethod\n @abstractmethod\n def _get_system_resources() -> dict[str, Any]:\n pass\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/autogluon_tabular/__init__.py",
"content": "from .mlforecast import DirectTabularModel, RecursiveTabularModel\nfrom .per_step import PerStepTabularModel\n\n__all__ = [\n \"DirectTabularModel\",\n \"RecursiveTabularModel\",\n \"PerStepTabularModel\",\n]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/autogluon_tabular/mlforecast.py",
"content": "import copy\nimport logging\nimport math\nimport time\nimport warnings\nfrom typing import Any, Callable, Collection, Type\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.base import BaseEstimator\n\nimport autogluon.core as ag\nfrom autogluon.core.models import AbstractModel as AbstractTabularModel\nfrom autogluon.features import AutoMLPipelineFeatureGenerator\nfrom autogluon.tabular.registry import ag_model_registry\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.metrics.abstract import TimeSeriesScorer\nfrom autogluon.timeseries.metrics.utils import in_sample_squared_seasonal_error\nfrom autogluon.timeseries.models.abstract import AbstractTimeSeriesModel\nfrom autogluon.timeseries.models.local import SeasonalNaiveModel\nfrom autogluon.timeseries.utils.datetime import (\n get_lags_for_frequency,\n get_seasonality,\n get_time_features_for_frequency,\n)\nfrom autogluon.timeseries.utils.warning_filters import set_loggers_level, warning_filter\n\nfrom .utils import MLF_ITEMID, MLF_TARGET, MLF_TIMESTAMP\n\nlogger = logging.getLogger(__name__)\n\n\nclass TabularModel(BaseEstimator):\n \"\"\"A scikit-learn compatible wrapper for arbitrary autogluon.tabular models\"\"\"\n\n def __init__(self, model_class: Type[AbstractTabularModel], model_kwargs: dict | None = None):\n self.model_class = model_class\n self.model_kwargs = {} if model_kwargs is None else model_kwargs\n self.feature_pipeline = AutoMLPipelineFeatureGenerator(verbosity=0)\n\n def fit(self, X: pd.DataFrame, y: pd.Series, X_val: pd.DataFrame, y_val: pd.Series, **kwargs):\n self.model = self.model_class(**self.model_kwargs)\n X = self.feature_pipeline.fit_transform(X=X)\n X_val = self.feature_pipeline.transform(X=X_val)\n self.model.fit(X=X, y=y, X_val=X_val, y_val=y_val, **kwargs)\n return self\n\n def predict(self, X: pd.DataFrame, **kwargs):\n X = self.feature_pipeline.transform(X=X)\n return self.model.predict(X=X, **kwargs)\n\n def get_params(self, deep=True):\n params = {\"model_class\": self.model_class, \"model_kwargs\": self.model_kwargs}\n if deep:\n return copy.deepcopy(params)\n else:\n return params\n\n\nclass AbstractMLForecastModel(AbstractTimeSeriesModel):\n _supports_known_covariates = True\n _supports_static_features = True\n\n def __init__(\n self,\n freq: str | None = None,\n prediction_length: int = 1,\n path: str | None = None,\n name: str | None = None,\n eval_metric: str | TimeSeriesScorer | None = None,\n hyperparameters: dict[str, Any] | None = None,\n **kwargs,\n ):\n super().__init__(\n path=path,\n freq=freq,\n prediction_length=prediction_length,\n name=name,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n from mlforecast import MLForecast\n from mlforecast.target_transforms import BaseTargetTransform\n\n self._sum_of_differences: int = 0 # number of time steps removed from each series by differencing\n self._max_ts_length: int | None = None\n self._target_lags: np.ndarray\n self._date_features: list[Callable]\n self._mlf: MLForecast\n self._scaler: BaseTargetTransform | None = None\n self._residuals_std_per_item: pd.Series\n self._train_target_median: float | None = None\n self._non_boolean_real_covariates: list[str] = []\n\n def _initialize_transforms_and_regressor(self):\n super()._initialize_transforms_and_regressor()\n # Do not create a scaler in the model, scaler will be passed to MLForecast\n self.target_scaler = None\n\n @property\n def allowed_hyperparameters(self) -> list[str]:\n return super().allowed_hyperparameters + [\n \"lags\",\n \"date_features\",\n \"differences\",\n \"model_name\",\n \"model_hyperparameters\",\n \"max_num_items\",\n \"max_num_samples\",\n \"lag_transforms\",\n ]\n\n def preprocess(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n is_train: bool = False,\n **kwargs,\n ) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame | None]:\n if is_train:\n # All-NaN series are removed; partially-NaN series in train_data are handled inside _generate_train_val_dfs\n all_nan_items = data.item_ids[\n data[self.target].isna().groupby(TimeSeriesDataFrame.ITEMID, sort=False).all()\n ]\n if len(all_nan_items):\n data = data.query(\"item_id not in @all_nan_items\")\n else:\n data = data.fill_missing_values()\n # Fill time series consisting of all NaNs with the median of target in train_data\n if data.isna().any(axis=None):\n data[self.target] = data[self.target].fillna(value=self._train_target_median)\n return data, known_covariates\n\n def _get_default_hyperparameters(self) -> dict[str, Any]:\n return {\n \"max_num_items\": 20_000,\n \"max_num_samples\": 1_000_000,\n \"model_name\": \"GBM\",\n \"model_hyperparameters\": {},\n }\n\n def _create_tabular_model(self, model_name: str, model_hyperparameters: dict[str, Any]) -> TabularModel:\n raise NotImplementedError\n\n def _get_mlforecast_init_args(\n self, train_data: TimeSeriesDataFrame, model_params: dict[str, Any]\n ) -> dict[str, Any]:\n from mlforecast.target_transforms import Differences\n\n from .transforms import MLForecastScaler\n\n lags = model_params.get(\"lags\")\n if lags is None:\n assert self.freq is not None\n lags = get_lags_for_frequency(self.freq)\n self._target_lags = np.array(sorted(set(lags)), dtype=np.int64)\n\n date_features = model_params.get(\"date_features\")\n if date_features is None:\n date_features = get_time_features_for_frequency(self.freq)\n known_covariates = self.covariate_metadata.known_covariates\n conflicting = [f.__name__ for f in date_features if f.__name__ in known_covariates]\n if conflicting:\n logger.info(f\"\\tRemoved automatic date_features {conflicting} since they clash with known_covariates\")\n self._date_features = [f for f in date_features if f.__name__ not in known_covariates]\n\n target_transforms = []\n differences = model_params.get(\"differences\")\n assert isinstance(differences, Collection)\n\n ts_lengths = train_data.num_timesteps_per_item()\n required_ts_length = sum(differences) + 1\n all_train_ts_are_long_enough = ts_lengths.min() >= required_ts_length\n some_ts_available_for_validation = ts_lengths.max() >= required_ts_length + self.prediction_length\n if not (all_train_ts_are_long_enough and some_ts_available_for_validation):\n logger.warning(\n f\"\\tTime series in the dataset are too short for chosen differences {differences}. \"\n f\"Setting differences to [1].\"\n )\n differences = [1]\n\n if len(differences) > 0:\n target_transforms.append(Differences(differences))\n self._sum_of_differences = sum(differences)\n\n if \"target_scaler\" in model_params and \"scaler\" in model_params:\n warnings.warn(\n f\"Both 'target_scaler' and 'scaler' hyperparameters are provided to {self.__class__.__name__}. \"\n \"Please only set the 'target_scaler' parameter.\"\n )\n # Support \"scaler\" for backward compatibility\n scaler_type = model_params.get(\"target_scaler\", model_params.get(\"scaler\"))\n if scaler_type is not None:\n self._scaler = MLForecastScaler(scaler_type=scaler_type)\n target_transforms.append(self._scaler)\n\n return {\n \"lags\": self._target_lags.tolist(),\n \"date_features\": self._date_features,\n \"target_transforms\": target_transforms,\n \"lag_transforms\": model_params.get(\"lag_transforms\"),\n }\n\n def _mask_df(self, df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Apply a mask that mimics the situation at prediction time when target/covariates are unknown during the\n forecast horizon.\n\n This method is overridden by DirectTabularModel.\n \"\"\"\n return df\n\n @staticmethod\n def _shorten_all_series(mlforecast_df: pd.DataFrame, max_length: int) -> pd.DataFrame:\n logger.debug(f\"Shortening all series to at most {max_length}\")\n return mlforecast_df.groupby(MLF_ITEMID, as_index=False, sort=False).tail(max_length)\n\n def _generate_train_val_dfs(\n self, data: TimeSeriesDataFrame, max_num_items: int | None = None, max_num_samples: int | None = None\n ) -> tuple[pd.DataFrame, pd.DataFrame]:\n # Exclude items that are too short for chosen differences - otherwise exception will be raised\n if self._sum_of_differences > 0:\n ts_lengths = data.num_timesteps_per_item()\n items_to_exclude = ts_lengths.index[ts_lengths <= self._sum_of_differences]\n if len(items_to_exclude) > 0:\n logger.debug(f\"Removing {len(items_to_exclude)} items that are too short for chosen differences\")\n data = data.query(\"item_id not in @items_to_exclude\")\n\n if max_num_items is not None and data.num_items > max_num_items:\n items_to_keep = data.item_ids.to_series().sample(n=int(max_num_items)) # noqa: F841\n data = data.query(\"item_id in @items_to_keep\")\n\n # MLForecast.preprocess does not support missing values, but we will exclude them later from the training set\n missing_entries = data.index[data[self.target].isna()]\n data = data.fill_missing_values()\n\n num_items = data.num_items\n mlforecast_df = self._to_mlforecast_df(data, data.static_features)\n\n # Shorten time series before calling preprocess to avoid high memory usage\n if max_num_samples is not None:\n max_samples_per_ts = max(200, math.ceil(max_num_samples / num_items))\n self._max_ts_length = max_samples_per_ts + self.prediction_length + self._sum_of_differences\n mlforecast_df = self._shorten_all_series(mlforecast_df, self._max_ts_length)\n\n # Unless we set static_features=[], MLForecast interprets all known covariates as static features\n df = self._mlf.preprocess(mlforecast_df, dropna=False, static_features=[])\n # df.query results in 2x memory saving compared to df.dropna(subset=\"y\")\n df = df.query(\"y.notnull()\") # type: ignore\n\n df = self._mask_df(df)\n\n # We remove originally missing values filled via imputation from the training set\n if len(missing_entries):\n df = df.set_index([\"unique_id\", \"ds\"]).drop(missing_entries, errors=\"ignore\").reset_index()\n\n if max_num_samples is not None and len(df) > max_num_samples:\n df = df.sample(n=max_num_samples)\n\n grouped_df = df.groupby(MLF_ITEMID, sort=False)\n\n # Use up to `prediction_length` last rows as validation set (but no more than 50% of the rows)\n val_rows_per_item = min(self.prediction_length, math.ceil(0.5 * len(df) / num_items))\n train_df = grouped_df.nth(slice(None, -val_rows_per_item))\n val_df = grouped_df.tail(val_rows_per_item)\n logger.debug(f\"train_df shape: {train_df.shape}, val_df shape: {val_df.shape}\")\n\n return train_df.drop(columns=[MLF_TIMESTAMP]), val_df.drop(columns=[MLF_TIMESTAMP]) # type: ignore\n\n def _to_mlforecast_df(\n self,\n data: TimeSeriesDataFrame,\n static_features: pd.DataFrame | None,\n include_target: bool = True,\n ) -> pd.DataFrame:\n \"\"\"Convert TimeSeriesDataFrame to a format expected by MLForecast methods `predict` and `preprocess`.\n\n Each row contains unique_id, ds, y, and (optionally) known covariates & static features.\n \"\"\"\n # TODO: Add support for past_covariates\n selected_columns = self.covariate_metadata.known_covariates.copy()\n column_name_mapping = {TimeSeriesDataFrame.ITEMID: MLF_ITEMID, TimeSeriesDataFrame.TIMESTAMP: MLF_TIMESTAMP}\n if include_target:\n selected_columns += [self.target]\n column_name_mapping[self.target] = MLF_TARGET\n\n df = pd.DataFrame(data)[selected_columns].reset_index()\n if static_features is not None:\n df = pd.merge(\n df, static_features, how=\"left\", on=TimeSeriesDataFrame.ITEMID, suffixes=(None, \"_static_feat\")\n )\n\n for col in self._non_boolean_real_covariates:\n # Normalize non-boolean features using mean_abs scaling\n df[f\"__scaled_{col}\"] = (\n df[col]\n / df[col]\n .abs()\n .groupby(df[TimeSeriesDataFrame.ITEMID])\n .mean()\n .reindex(df[TimeSeriesDataFrame.ITEMID])\n .values\n )\n\n # Convert float64 to float32 to reduce memory usage\n float64_cols = list(df.select_dtypes(include=\"float64\"))\n df[float64_cols] = df[float64_cols].astype(\"float32\")\n\n # We assume that df is sorted by 'unique_id' inside `TimeSeriesPredictor._check_and_prepare_data_frame`\n return df.rename(columns=column_name_mapping)\n\n def _fit(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None = None,\n time_limit: float | None = None,\n num_cpus: int | None = None,\n num_gpus: int | None = None,\n verbosity: int = 2,\n **kwargs,\n ) -> None:\n from mlforecast import MLForecast\n\n self._check_fit_params()\n self._log_unused_hyperparameters()\n fit_start_time = time.time()\n self._train_target_median = train_data[self.target].median()\n for col in self.covariate_metadata.known_covariates_real:\n if not set(train_data[col].unique()) == set([0, 1]):\n self._non_boolean_real_covariates.append(col)\n model_params = self.get_hyperparameters()\n\n mlforecast_init_args = self._get_mlforecast_init_args(train_data, model_params)\n assert self.freq is not None\n self._mlf = MLForecast(models={}, freq=self.freq, **mlforecast_init_args)\n\n # We generate train/val splits from train_data and ignore val_data to avoid overfitting\n train_df, val_df = self._generate_train_val_dfs(\n train_data,\n max_num_items=model_params[\"max_num_items\"],\n max_num_samples=model_params[\"max_num_samples\"],\n )\n\n with set_loggers_level(regex=r\"^autogluon\\.(tabular|features).*\", level=logging.ERROR):\n tabular_model = self._create_tabular_model(\n model_name=model_params[\"model_name\"], model_hyperparameters=model_params[\"model_hyperparameters\"]\n )\n tabular_model.fit(\n X=train_df.drop(columns=[MLF_TARGET, MLF_ITEMID]),\n y=train_df[MLF_TARGET],\n X_val=val_df.drop(columns=[MLF_TARGET, MLF_ITEMID]),\n y_val=val_df[MLF_TARGET],\n time_limit=(None if time_limit is None else time_limit - (time.time() - fit_start_time)),\n verbosity=verbosity - 1,\n )\n\n # We directly insert the trained model into models_ since calling _mlf.fit_models does not support X_val, y_val\n self._mlf.models_ = {\"mean\": tabular_model}\n\n self._save_residuals_std(val_df)\n\n def get_tabular_model(self) -> TabularModel:\n \"\"\"Get the underlying tabular regression model.\"\"\"\n assert \"mean\" in self._mlf.models_, \"Call `fit` before calling `get_tabular_model`\"\n mean_estimator = self._mlf.models_[\"mean\"]\n assert isinstance(mean_estimator, TabularModel)\n return mean_estimator\n\n def _save_residuals_std(self, val_df: pd.DataFrame) -> None:\n \"\"\"Compute standard deviation of residuals for each item using the validation set.\n\n Saves per-item residuals to `self.residuals_std_per_item`.\n \"\"\"\n residuals_df = val_df[[MLF_ITEMID, MLF_TARGET]]\n mean_estimator = self.get_tabular_model()\n\n residuals_df = residuals_df.assign(y_pred=mean_estimator.predict(val_df))\n if self._scaler is not None:\n # Scaler expects to find column MLF_TIMESTAMP even though it's not used - fill with dummy\n residuals_df = residuals_df.assign(**{MLF_TIMESTAMP: np.datetime64(\"2010-01-01\")})\n residuals_df = self._scaler.inverse_transform(residuals_df)\n\n assert isinstance(residuals_df, pd.DataFrame)\n residuals = residuals_df[MLF_TARGET] - residuals_df[\"y_pred\"]\n self._residuals_std_per_item = (\n residuals.pow(2.0).groupby(val_df[MLF_ITEMID].values, sort=False).mean().pow(0.5) # type: ignore\n )\n\n def _remove_short_ts_and_generate_fallback_forecast(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n ) -> tuple[TimeSeriesDataFrame, TimeSeriesDataFrame, TimeSeriesDataFrame | None]:\n \"\"\"Remove series that are too short for chosen differencing from data and generate naive forecast for them.\n\n Returns\n -------\n data_long\n Data containing only time series that are long enough for the model to predict.\n known_covariates_long\n Future known covariates containing only time series that are long enough for the model to predict.\n forecast_for_short_series\n Seasonal naive forecast for short series, if there are any in the dataset.\n \"\"\"\n ts_lengths = data.num_timesteps_per_item()\n short_series = ts_lengths.index[ts_lengths <= self._sum_of_differences]\n if len(short_series) > 0:\n logger.warning(\n f\"Warning: {len(short_series)} time series ({len(short_series) / len(ts_lengths):.1%}) are shorter \"\n f\"than {self._sum_of_differences} and cannot be predicted by {self.name}. \"\n \"Fallback model SeasonalNaive is used for these time series.\"\n )\n data_short = data.query(\"item_id in @short_series\")\n seasonal_naive = SeasonalNaiveModel(\n freq=self.freq,\n prediction_length=self.prediction_length,\n target=self.target,\n quantile_levels=self.quantile_levels,\n )\n seasonal_naive.fit(train_data=data_short)\n forecast_for_short_series = seasonal_naive.predict(data_short)\n\n data_long = data.query(\"item_id not in @short_series\")\n if known_covariates is not None:\n known_covariates_long = known_covariates.query(\"item_id not in @short_series\")\n else:\n known_covariates_long = None\n else:\n data_long = data\n known_covariates_long = known_covariates\n forecast_for_short_series = None\n return data_long, known_covariates_long, forecast_for_short_series\n\n def _add_gaussian_quantiles(\n self, predictions: pd.DataFrame, repeated_item_ids: pd.Series, past_target: pd.Series\n ) -> pd.DataFrame:\n \"\"\"\n Add quantile levels assuming that residuals follow normal distribution\n \"\"\"\n from scipy.stats import norm\n\n num_items = int(len(predictions) / self.prediction_length)\n sqrt_h = np.sqrt(np.arange(1, self.prediction_length + 1))\n # Series where normal_scale_per_timestep.loc[item_id].loc[N] = sqrt(1 + N) for N in range(prediction_length)\n normal_scale_per_timestep = pd.Series(np.tile(sqrt_h, num_items), index=repeated_item_ids)\n\n residuals_std_per_timestep = self._residuals_std_per_item.reindex(repeated_item_ids)\n # Use in-sample seasonal error in for items not seen during fit\n items_not_seen_during_fit = residuals_std_per_timestep.index[residuals_std_per_timestep.isna()].unique()\n if len(items_not_seen_during_fit) > 0:\n scale_for_new_items: pd.Series = in_sample_squared_seasonal_error(\n y_past=past_target.loc[items_not_seen_during_fit]\n ).pow(0.5)\n residuals_std_per_timestep = residuals_std_per_timestep.fillna(scale_for_new_items)\n\n std_per_timestep = residuals_std_per_timestep * normal_scale_per_timestep\n for q in self.quantile_levels:\n predictions[str(q)] = predictions[\"mean\"] + norm.ppf(q) * std_per_timestep.to_numpy()\n return predictions\n\n def _more_tags(self) -> dict[str, Any]:\n return {\"allow_nan\": True, \"can_refit_full\": True}\n\n\nclass DirectTabularModel(AbstractMLForecastModel):\n \"\"\"Predict all future time series values simultaneously using a regression model from AutoGluon-Tabular.\n\n A single tabular model is used to forecast all future time series values using the following features:\n\n - lag features (observed time series values) based on ``freq`` of the data\n - time features (e.g., day of the week) based on the timestamp of the measurement\n - known covariates (if available)\n - static features of each item (if available)\n\n Features not known during the forecast horizon (e.g., future target values) are replaced by NaNs.\n\n If ``eval_metric.needs_quantile``, the tabular regression model will be trained with ``\"quantile\"`` problem type.\n Otherwise, the model will be trained with ``\"regression\"`` problem type, and dummy quantiles will be\n obtained by assuming that the residuals follow zero-mean normal distribution.\n\n Based on the `mlforecast `_ library.\n\n\n Other Parameters\n ----------------\n lags : list[int], default = None\n Lags of the target that will be used as features for predictions. If None, will be determined automatically\n based on the frequency of the data.\n date_features : list[str | Callable], default = None\n Features computed from the dates. Can be pandas date attributes or functions that will take the dates as input.\n If None, will be determined automatically based on the frequency of the data.\n differences : list[int], default = []\n Differences to take of the target before computing the features. These are restored at the forecasting step.\n Defaults to no differencing.\n target_scaler : {\"standard\", \"mean_abs\", \"min_max\", \"robust\", None}, default = \"mean_abs\"\n Scaling applied to each time series. Scaling is applied after differencing.\n model_name : str, default = \"GBM\"\n Name of the tabular regression model. See ``autogluon.tabular.registry.ag_model_registry`` or\n `the documentation `_ for the list of available\n tabular models.\n model_hyperparameters : dict[str, Any], optional\n Hyperparameters passed to the tabular regression model.\n max_num_items : int or None, default = 20_000\n If not None, the model will randomly select this many time series for training and validation.\n max_num_samples : int or None, default = 1_000_000\n If not None, training dataset passed to the tabular regression model will contain at most this many rows\n (starting from the end of each time series).\n \"\"\"\n\n ag_priority = 85\n\n @property\n def is_quantile_model(self) -> bool:\n return self.eval_metric.needs_quantile\n\n def get_hyperparameters(self) -> dict[str, Any]:\n model_params = super().get_hyperparameters()\n # We don't set 'target_scaler' if user already provided 'scaler' to avoid overriding the user-provided value\n if \"scaler\" not in model_params:\n model_params.setdefault(\"target_scaler\", \"mean_abs\")\n if \"differences\" not in model_params or model_params[\"differences\"] is None:\n model_params[\"differences\"] = []\n if \"lag_transforms\" in model_params:\n model_params.pop(\"lag_transforms\")\n logger.warning(f\"{self.name} does not support the 'lag_transforms' hyperparameter.\")\n return model_params\n\n def _mask_df(self, df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Apply a mask that mimics the situation at prediction time when target/covariates are unknown during the\n forecast horizon.\n \"\"\"\n # Fix seed to make the model deterministic\n rng = np.random.default_rng(seed=123)\n num_hidden = rng.integers(0, self.prediction_length, size=len(df))\n lag_cols = [f\"lag{lag}\" for lag in self._target_lags]\n mask = num_hidden[:, None] < self._target_lags[None] # shape [len(num_hidden), len(_target_lags)]\n # use df.loc[:, lag_cols] instead of df[lag_cols] to avoid SettingWithCopyWarning\n df.loc[:, lag_cols] = df[lag_cols].where(mask, other=np.nan)\n return df\n\n def _save_residuals_std(self, val_df: pd.DataFrame) -> None:\n if self.is_quantile_model:\n # Quantile model does not require residuals to produce prediction intervals\n self._residuals_std_per_item = pd.Series(1.0, index=val_df[MLF_ITEMID].unique())\n else:\n super()._save_residuals_std(val_df=val_df)\n\n def _predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n from .transforms import apply_inverse_transform\n\n original_item_id_order = data.item_ids\n data, known_covariates, forecast_for_short_series = self._remove_short_ts_and_generate_fallback_forecast(\n data=data, known_covariates=known_covariates\n )\n if len(data) == 0:\n # All time series are too short for chosen differences\n assert forecast_for_short_series is not None\n return forecast_for_short_series\n\n if known_covariates is not None:\n data_future = known_covariates.copy()\n else:\n future_index = self.get_forecast_horizon_index(data)\n data_future = pd.DataFrame(columns=[self.target], index=future_index, dtype=\"float32\")\n # MLForecast raises exception of target contains NaN. We use inf as placeholder, replace them by NaN afterwards\n data_future[self.target] = float(\"inf\")\n data_extended = pd.concat([data, data_future])\n mlforecast_df = self._to_mlforecast_df(data_extended, data.static_features) # type: ignore\n if self._max_ts_length is not None:\n # We appended `prediction_length` time steps to each series, so increase length\n mlforecast_df = self._shorten_all_series(mlforecast_df, self._max_ts_length + self.prediction_length)\n df = self._mlf.preprocess(mlforecast_df, dropna=False, static_features=[])\n assert isinstance(df, pd.DataFrame)\n\n df = df.groupby(MLF_ITEMID, sort=False).tail(self.prediction_length)\n df = df.replace(float(\"inf\"), float(\"nan\"))\n\n mean_estimator = self.get_tabular_model()\n raw_predictions = mean_estimator.predict(df)\n predictions = self._postprocess_predictions(raw_predictions, repeated_item_ids=df[MLF_ITEMID])\n # Paste columns one by one to preserve dtypes\n predictions[MLF_ITEMID] = df[MLF_ITEMID].values\n predictions[MLF_TIMESTAMP] = df[MLF_TIMESTAMP].values\n\n if hasattr(self._mlf.ts, \"target_transforms\"):\n # Ensure that transforms are fitted only on past data\n mlforecast_df_past = self._to_mlforecast_df(data, None)\n if self._max_ts_length is not None:\n mlforecast_df_past = self._shorten_all_series(mlforecast_df_past, self._max_ts_length)\n self._mlf.preprocess(mlforecast_df_past, static_features=[], dropna=False)\n assert self._mlf.ts.target_transforms is not None\n for tfm in self._mlf.ts.target_transforms[::-1]:\n predictions = apply_inverse_transform(predictions, transform=tfm)\n\n if not self.is_quantile_model:\n predictions = self._add_gaussian_quantiles(\n predictions, repeated_item_ids=predictions[MLF_ITEMID], past_target=data[self.target]\n )\n predictions_tsdf: TimeSeriesDataFrame = TimeSeriesDataFrame(\n predictions.rename(\n columns={MLF_ITEMID: TimeSeriesDataFrame.ITEMID, MLF_TIMESTAMP: TimeSeriesDataFrame.TIMESTAMP}\n )\n )\n\n if forecast_for_short_series is not None:\n predictions_tsdf = pd.concat([predictions_tsdf, forecast_for_short_series]) # type: ignore\n predictions_tsdf = predictions_tsdf.reindex(original_item_id_order, level=TimeSeriesDataFrame.ITEMID)\n\n return predictions_tsdf\n\n def _postprocess_predictions(\n self, predictions: np.ndarray | pd.Series, repeated_item_ids: pd.Series\n ) -> pd.DataFrame:\n if self.is_quantile_model:\n predictions_df = pd.DataFrame(predictions, columns=[str(q) for q in self.quantile_levels])\n predictions_df.values.sort(axis=1)\n predictions_df[\"mean\"] = predictions_df[\"0.5\"]\n else:\n predictions_df = pd.DataFrame(predictions, columns=[\"mean\"])\n\n column_order = [\"mean\"] + [col for col in predictions_df.columns if col != \"mean\"]\n return predictions_df[column_order]\n\n def _create_tabular_model(self, model_name: str, model_hyperparameters: dict[str, Any]) -> TabularModel:\n model_class = ag_model_registry.key_to_cls(model_name)\n if self.is_quantile_model:\n problem_type = ag.constants.QUANTILE\n eval_metric = \"pinball_loss\"\n model_hyperparameters[\"ag.quantile_levels\"] = self.quantile_levels\n else:\n problem_type = ag.constants.REGRESSION\n eval_metric = self.eval_metric.equivalent_tabular_regression_metric or \"mean_absolute_error\"\n return TabularModel(\n model_class=model_class,\n model_kwargs={\n \"path\": \"\",\n \"name\": model_class.__name__,\n \"hyperparameters\": model_hyperparameters,\n \"problem_type\": problem_type,\n \"eval_metric\": eval_metric,\n },\n )\n\n\nclass RecursiveTabularModel(AbstractMLForecastModel):\n \"\"\"Predict future time series values one by one using a regression model from AutoGluon-Tabular.\n\n A single tabular regression model is used to forecast the future time series values using the following features:\n\n - lag features (observed time series values) based on ``freq`` of the data\n - time features (e.g., day of the week) based on the timestamp of the measurement\n - known covariates (if available)\n - static features of each item (if available)\n\n The tabular model will always be trained with ``\"regression\"`` problem type, and dummy quantiles will be\n obtained by assuming that the residuals follow zero-mean normal distribution.\n\n Based on the `mlforecast `_ library.\n\n\n Other Parameters\n ----------------\n lags : list[int], default = None\n Lags of the target that will be used as features for predictions. If None, will be determined automatically\n based on the frequency of the data.\n date_features : list[str | Callable], default = None\n Features computed from the dates. Can be pandas date attributes or functions that will take the dates as input.\n If None, will be determined automatically based on the frequency of the data.\n differences : list[int], default = None\n Differences to take of the target before computing the features. These are restored at the forecasting step.\n If None, will be set to ``[seasonal_period]``, where seasonal_period is determined based on the data frequency.\n target_scaler : {\"standard\", \"mean_abs\", \"min_max\", \"robust\", None}, default = \"standard\"\n Scaling applied to each time series. Scaling is applied after differencing.\n lag_transforms : dict[int, list[Callable]], default = None\n Dictionary mapping lag periods to transformation functions applied to lagged target values (e.g., rolling mean).\n See `MLForecast documentation `_ for more details.\n model_name : str, default = \"GBM\"\n Name of the tabular regression model. See ``autogluon.tabular.registry.ag_model_registry`` or\n `the documentation `_ for the list of available\n tabular models.\n model_hyperparameters : dict[str, Any], optional\n Hyperparameters passed to the tabular regression model.\n max_num_items : int or None, default = 20_000\n If not None, the model will randomly select this many time series for training and validation.\n max_num_samples : int or None, default = 1_000_000\n If not None, training dataset passed to the tabular regression model will contain at most this many rows\n (starting from the end of each time series).\n \"\"\"\n\n ag_priority = 90\n\n def get_hyperparameters(self) -> dict[str, Any]:\n model_params = super().get_hyperparameters()\n # We don't set 'target_scaler' if user already provided 'scaler' to avoid overriding the user-provided value\n if \"scaler\" not in model_params:\n model_params.setdefault(\"target_scaler\", \"standard\")\n if \"differences\" not in model_params or model_params[\"differences\"] is None:\n model_params[\"differences\"] = [get_seasonality(self.freq)]\n return model_params\n\n def _predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n original_item_id_order = data.item_ids\n data, known_covariates, forecast_for_short_series = self._remove_short_ts_and_generate_fallback_forecast(\n data=data, known_covariates=known_covariates\n )\n if len(data) == 0:\n # All time series are too short for chosen differences\n assert forecast_for_short_series is not None\n return forecast_for_short_series\n\n new_df = self._to_mlforecast_df(data, data.static_features)\n if self._max_ts_length is not None:\n new_df = self._shorten_all_series(new_df, self._max_ts_length)\n if known_covariates is None:\n future_index = self.get_forecast_horizon_index(data)\n known_covariates = TimeSeriesDataFrame(\n pd.DataFrame(columns=[self.target], index=future_index, dtype=\"float32\")\n )\n X_df = self._to_mlforecast_df(known_covariates, data.static_features, include_target=False)\n # If both covariates & static features are missing, set X_df = None to avoid exception from MLForecast\n if len(X_df.columns.difference([MLF_ITEMID, MLF_TIMESTAMP])) == 0:\n X_df = None\n with warning_filter():\n raw_predictions = self._mlf.predict(\n h=self.prediction_length,\n new_df=new_df,\n X_df=X_df,\n )\n assert isinstance(raw_predictions, pd.DataFrame)\n raw_predictions = raw_predictions.rename(\n columns={MLF_ITEMID: TimeSeriesDataFrame.ITEMID, MLF_TIMESTAMP: TimeSeriesDataFrame.TIMESTAMP}\n )\n\n predictions: TimeSeriesDataFrame = TimeSeriesDataFrame(\n self._add_gaussian_quantiles(\n raw_predictions,\n repeated_item_ids=raw_predictions[TimeSeriesDataFrame.ITEMID],\n past_target=data[self.target],\n )\n )\n if forecast_for_short_series is not None:\n predictions = pd.concat([predictions, forecast_for_short_series]) # type: ignore\n return predictions.reindex(original_item_id_order, level=TimeSeriesDataFrame.ITEMID)\n\n def _create_tabular_model(self, model_name: str, model_hyperparameters: dict[str, Any]) -> TabularModel:\n model_class = ag_model_registry.key_to_cls(model_name)\n return TabularModel(\n model_class=model_class,\n model_kwargs={\n \"path\": \"\",\n \"name\": model_class.__name__,\n \"hyperparameters\": model_hyperparameters,\n \"problem_type\": ag.constants.REGRESSION,\n \"eval_metric\": self.eval_metric.equivalent_tabular_regression_metric or \"mean_absolute_error\",\n },\n )\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/autogluon_tabular/per_step.py",
"content": "import logging\nimport math\nimport os\nimport time\nfrom typing import Any, Callable, Literal, Type\n\nimport numpy as np\nimport pandas as pd\nimport scipy.stats\nfrom joblib import Parallel, cpu_count, delayed\n\nfrom autogluon.common.loaders import load_pkl\nfrom autogluon.common.savers import save_pkl\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\nfrom autogluon.core.constants import QUANTILE, REGRESSION\nfrom autogluon.tabular.models import AbstractModel as AbstractTabularModel\nfrom autogluon.tabular.registry import ag_model_registry\nfrom autogluon.timeseries import TimeSeriesDataFrame\nfrom autogluon.timeseries.models.abstract import AbstractTimeSeriesModel\nfrom autogluon.timeseries.utils.datetime import get_lags_for_frequency, get_time_features_for_frequency\nfrom autogluon.timeseries.utils.warning_filters import set_loggers_level, warning_filter\n\nfrom .utils import MLF_ITEMID, MLF_TARGET, MLF_TIMESTAMP\n\nlogger = logging.getLogger(__name__)\n\n\nclass PerStepTabularModel(AbstractTimeSeriesModel):\n \"\"\"Fit a separate tabular regression model for each time step in the forecast horizon.\n\n Each model has access to the following features:\n\n - lag features (observed time series values) based on ``freq`` of the data\n - time features (e.g., day of the week) based on the timestamp of the measurement\n - known covariates (if available)\n - static features of each item (if available)\n\n This model is typically slower to fit compared to other tabular forecasting models.\n\n If ``eval_metric.needs_quantile``, the tabular regression models will be trained with ``\"quantile\"`` problem type.\n Otherwise, the models will be trained with ``\"regression\"`` problem type, and dummy quantiles will be\n obtained by assuming that the residuals follow zero-mean normal distribution.\n\n This model uses `mlforecast `_ under the hood for efficient preprocessing,\n but the implementation of the per-step forecasting strategy is different from the ``max_horizon`` in ``mlforecast``.\n\n\n Other Parameters\n ----------------\n trailing_lags : list[int], default = None\n Trailing window lags of the target that will be used as features for predictions.\n Trailing lags are shifted per forecast step: model for step ``h`` uses ``[lag+h for lag in trailing_lags]``.\n If None, defaults to ``[1, 2, ..., 12]``.\n seasonal_lags : list[int], default = None\n Seasonal lags of the target used as features. Unlike trailing lags, seasonal lags are not shifted\n but filtered by availability: model for step ``h`` uses ``[lag for lag in seasonal_lags if lag > h]``.\n If None, determined automatically based on data frequency.\n date_features : list[str | Callable], default = None\n Features computed from the dates. Can be pandas date attributes or functions that will take the dates as input.\n If None, will be determined automatically based on the frequency of the data.\n target_scaler : {\"standard\", \"mean_abs\", \"min_max\", \"robust\", None}, default = \"mean_abs\"\n Scaling applied to each time series.\n model_name : str, default = \"CAT\"\n Name of the tabular regression model. See ``autogluon.tabular.registry.ag_model_registry`` or\n `the documentation `_ for the list of available\n tabular models.\n model_hyperparameters : dict[str, Any], optional\n Hyperparameters passed to the tabular regression model.\n validation_fraction : float or None, default = 0.1\n Fraction of the training data to use for validation. If None or 0.0, no validation set is created.\n Validation set contains the most recent observations (chronologically). Must be between 0.0 and 1.0.\n max_num_items : int or None, default = 20_000\n If not None, the model will randomly select this many time series for training and validation.\n max_num_samples : int or None, default = 1_000_000\n If not None, training dataset passed to the tabular regression model will contain at most this many rows\n (starting from the end of each time series).\n n_jobs : int or None, default = None\n Number of parallel jobs for fitting models across forecast horizons.\n If None, automatically determined based on available memory to prevent OOM errors.\n \"\"\"\n\n ag_priority = 80\n _dummy_freq = \"D\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n # We save the relative paths to per-step models. Each worker process independently saves/loads the model.\n # This is much more efficient than passing around model objects that can get really large\n self._relative_paths_to_models: list[str]\n self._trailing_lags: list[int]\n self._seasonal_lags: list[int]\n self._date_features: list[Callable]\n self._model_cls: Type[AbstractTabularModel]\n self._n_jobs: int\n self._non_boolean_real_covariates: list[str] = []\n self._max_ts_length: int | None = None\n\n @property\n def allowed_hyperparameters(self) -> list[str]:\n # TODO: Differencing is currently not supported because it greatly complicates the preprocessing logic\n return super().allowed_hyperparameters + [\n \"trailing_lags\",\n \"seasonal_lags\",\n \"date_features\",\n # \"differences\",\n \"validation_fraction\",\n \"model_name\",\n \"model_hyperparameters\",\n \"max_num_items\",\n \"max_num_samples\",\n \"n_jobs\",\n ]\n\n @property\n def _ag_to_nixtla(self) -> dict:\n return {\n self.target: MLF_TARGET,\n TimeSeriesDataFrame.ITEMID: MLF_ITEMID,\n TimeSeriesDataFrame.TIMESTAMP: MLF_TIMESTAMP,\n }\n\n def _get_default_hyperparameters(self):\n return {\n \"model_name\": \"CAT\",\n \"model_hyperparameters\": {},\n \"target_scaler\": \"mean_abs\",\n \"validation_fraction\": 0.1,\n \"max_num_samples\": 1_000_000,\n \"max_num_items\": 20_000,\n }\n\n @classmethod\n def _fit_single_model(\n cls,\n train_df: pd.DataFrame,\n path_root: str,\n step: int,\n model_cls: Type[AbstractTabularModel],\n model_hyperparameters: dict,\n problem_type: Literal[\"quantile\", \"regression\"],\n eval_metric: str,\n validation_fraction: float | None,\n quantile_levels: list[float],\n lags: list[int],\n date_features: list[Callable],\n time_limit: float | None,\n num_cpus: int,\n verbosity: int,\n ) -> str:\n from mlforecast import MLForecast\n\n start_time = time.monotonic()\n\n mlf = MLForecast(models=[], freq=cls._dummy_freq, lags=lags, date_features=date_features)\n\n with warning_filter():\n features_df = mlf.preprocess(train_df, static_features=[], dropna=False)\n del train_df\n del mlf\n # Sort chronologically for efficient train/test split\n features_df = features_df.sort_values(by=MLF_TIMESTAMP)\n item_ids = features_df[MLF_ITEMID]\n X = features_df.drop(columns=[MLF_ITEMID, MLF_TIMESTAMP, MLF_TARGET])\n y = features_df[MLF_TARGET]\n del features_df\n\n y_is_valid = np.isfinite(y)\n X, y = X[y_is_valid], y[y_is_valid]\n X = X.replace(float(\"inf\"), float(\"nan\"))\n if validation_fraction is None or validation_fraction == 0.0:\n X_val = None\n y_val = None\n else:\n assert 0 < validation_fraction < 1, \"validation_fraction must be between 0.0 and 1.0\"\n num_val = math.ceil(len(X) * validation_fraction)\n X_val, y_val = X.iloc[-num_val:], y.iloc[-num_val:]\n X, y = X.iloc[:-num_val], y.iloc[:-num_val]\n if len(y) == 0:\n raise ValueError(\"Not enough valid target values to fit model\")\n\n elapsed = time.monotonic() - start_time\n time_left = time_limit - elapsed if time_limit is not None else None\n if problem_type == QUANTILE:\n model_hyperparameters = model_hyperparameters | {\"ag.quantile_levels\": quantile_levels}\n try:\n with set_loggers_level(regex=r\"^autogluon.tabular.*\", level=logging.ERROR):\n model = model_cls(\n path=os.path.join(path_root, f\"step_{step}\"),\n name=model_cls.__name__, # explicitly provide name to avoid warnings\n problem_type=problem_type,\n eval_metric=eval_metric,\n hyperparameters=model_hyperparameters,\n )\n model.fit(\n X=X,\n y=y,\n X_val=X_val,\n y_val=y_val,\n time_limit=time_left,\n num_cpus=num_cpus,\n num_gpus=0, # num_cpus is only used if num_gpus is set as well\n verbosity=verbosity,\n )\n except Exception as e:\n raise RuntimeError(f\"Failed when fitting model for {step=}\") from e\n model.save()\n if problem_type == REGRESSION:\n residuals_std = pd.Series((model.predict(X) - y) ** 2).groupby(item_ids).mean() ** 0.5\n save_pkl.save(cls._get_residuals_std_path(model.path), residuals_std)\n relative_path = os.path.relpath(path=model.path, start=path_root)\n return relative_path\n\n @staticmethod\n def _get_n_jobs(\n train_df: pd.DataFrame,\n num_extra_dynamic_features: int,\n model_cls: Type[AbstractTabularModel],\n model_hyperparameters: dict,\n overhead_factor: float = 2.0,\n ) -> int:\n \"\"\"Estimate the maximum number of jobs that can be run in parallel without encountering OOM errors.\"\"\"\n mem_usage_per_column = get_approximate_df_mem_usage(train_df)\n num_columns = len(train_df.columns)\n mem_usage_per_job = mem_usage_per_column.sum()\n try:\n mem_usage_per_job += model_cls.estimate_memory_usage_static(\n X=train_df, hyperparameters=model_hyperparameters, problem_type=\"regression\"\n )\n except NotImplementedError:\n mem_usage_per_job *= 2\n # Extra scaling factor because the preprocessed DF will have more columns for lags + date features\n mem_usage_per_job *= overhead_factor + (num_extra_dynamic_features + num_columns) / num_columns\n max_jobs_by_memory = int(ResourceManager.get_available_virtual_mem() / mem_usage_per_job)\n return max(1, max_jobs_by_memory)\n\n def preprocess(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n is_train: bool = False,\n **kwargs,\n ):\n # TODO: Make this toggleable with a hyperparameter\n # We add a scaled version of non-boolean known real covariates, same as in MLForecast models\n if is_train:\n for col in self.covariate_metadata.known_covariates_real:\n if not set(data[col].unique()) == set([0, 1]):\n self._non_boolean_real_covariates.append(col)\n\n if len(self._non_boolean_real_covariates) > 0:\n item_ids = data.index.get_level_values(level=TimeSeriesDataFrame.ITEMID)\n scale_per_column: dict[str, pd.Series] = {}\n columns_grouped = data[self._non_boolean_real_covariates].abs().groupby(item_ids)\n for col in self._non_boolean_real_covariates:\n scale_per_column[col] = columns_grouped[col].mean()\n data = data.assign(**{f\"__scaled_{col}\": data[col] / scale for col, scale in scale_per_column.items()})\n if known_covariates is not None:\n known_covariates = known_covariates.assign(\n **{f\"__scaled_{col}\": known_covariates[col] / scale for col, scale in scale_per_column.items()}\n )\n data = data.astype({self.target: \"float32\"})\n return data, known_covariates\n\n def _get_train_df(\n self, train_data: TimeSeriesDataFrame, max_num_items: int | None, max_num_samples: int | None\n ) -> pd.DataFrame:\n if max_num_items is not None and train_data.num_items > max_num_items:\n items_to_keep = train_data.item_ids.to_series().sample(n=int(max_num_items)) # noqa: F841\n train_data = train_data.query(\"item_id in @items_to_keep\")\n\n if max_num_samples is not None and len(train_data) > max_num_samples:\n max_samples_per_ts = max(200, math.ceil(max_num_samples / train_data.num_items))\n self._max_ts_length = max_samples_per_ts + self.prediction_length\n train_data = train_data.slice_by_timestep(-self._max_ts_length, None)\n\n if len(self.covariate_metadata.past_covariates) > 0:\n train_data = train_data.drop(columns=self.covariate_metadata.past_covariates)\n\n train_df = train_data.to_data_frame().reset_index()\n if train_data.static_features is not None:\n train_df = pd.merge(\n left=train_df,\n right=train_data.static_features,\n left_on=TimeSeriesDataFrame.ITEMID,\n right_index=True,\n how=\"left\",\n )\n train_df = train_df.rename(columns=self._ag_to_nixtla)\n train_df = train_df.assign(**{MLF_TARGET: train_df[MLF_TARGET].fillna(float(\"inf\"))})\n return train_df\n\n @staticmethod\n def _get_lags_for_step(\n trailing_lags: list[int],\n seasonal_lags: list[int],\n step: int,\n ) -> list[int]:\n \"\"\"Get the list of lags that can be used by the model for the given step.\"\"\"\n shifted_trailing_lags = [lag + step for lag in trailing_lags]\n # Only keep lags that are available for model predicting `step` values ahead at prediction time\n valid_lags = [lag for lag in shifted_trailing_lags + seasonal_lags if lag > step]\n return sorted(set(valid_lags))\n\n def _fit(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None = None,\n time_limit: float | None = None,\n num_cpus: int | None = None,\n num_gpus: int | None = None,\n verbosity: int = 2,\n **kwargs,\n ) -> None:\n self._check_fit_params()\n self._log_unused_hyperparameters()\n model_params = self.get_hyperparameters()\n\n train_df = self._get_train_df(\n train_data,\n max_num_items=model_params[\"max_num_items\"],\n max_num_samples=model_params[\"max_num_samples\"],\n )\n\n # Initialize MLForecast arguments\n assert self.freq is not None\n trailing_lags = model_params.get(\"trailing_lags\")\n if trailing_lags is None:\n trailing_lags = list(range(1, 13))\n # Ensure that lags have type list[int] and not, e.g., np.ndarray\n self._trailing_lags = [int(lag) for lag in trailing_lags]\n assert all(lag >= 1 for lag in self._trailing_lags), \"trailing_lags must be >= 1\"\n\n seasonal_lags = model_params.get(\"seasonal_lags\")\n if seasonal_lags is None:\n median_ts_length = int(train_df[MLF_ITEMID].value_counts(sort=False).median())\n seasonal_lags = get_lags_for_frequency(self.freq, num_default_lags=0, lag_ub=median_ts_length)\n self._seasonal_lags = [int(lag) for lag in seasonal_lags]\n assert all(lag >= 1 for lag in self._seasonal_lags), \"seasonal_lags must be >= 1\"\n\n date_features = model_params.get(\"date_features\")\n if date_features is None:\n date_features = get_time_features_for_frequency(self.freq)\n self._date_features = date_features\n\n model_name = model_params[\"model_name\"]\n self._model_cls = ag_model_registry.key_to_cls(model_name)\n model_hyperparameters = model_params[\"model_hyperparameters\"]\n # User-provided n_jobs takes priority over the automatic estimate\n if model_params.get(\"n_jobs\") is not None:\n self._n_jobs = model_params[\"n_jobs\"]\n else:\n self._n_jobs = self._get_n_jobs(\n train_df,\n num_extra_dynamic_features=len(set(self._seasonal_lags + self._trailing_lags))\n + len(self._date_features),\n model_cls=self._model_cls,\n model_hyperparameters=model_hyperparameters,\n )\n n_jobs = min(self._n_jobs, self.prediction_length, cpu_count(only_physical_cores=True))\n\n num_cpus_per_model = max(cpu_count(only_physical_cores=True) // n_jobs, 1)\n if time_limit is not None:\n time_limit_per_model = time_limit / math.ceil(self.prediction_length / n_jobs)\n else:\n time_limit_per_model = None\n\n if self.eval_metric.needs_quantile:\n problem_type = QUANTILE\n eval_metric = \"pinball_loss\"\n else:\n problem_type = REGRESSION\n eval_metric = self.eval_metric.equivalent_tabular_regression_metric or \"mean_absolute_error\"\n\n supported_problem_types = self._model_cls.supported_problem_types()\n if supported_problem_types is not None and problem_type not in supported_problem_types:\n raise ValueError(\n f\"Chosen model_name='{model_name}' cannot be used by {self.name} with eval_metric={self.eval_metric}\"\n f\"because {model_name} does not support problem_type={problem_type} ({supported_problem_types=})\"\n )\n model_fit_kwargs = dict(\n train_df=train_df,\n path_root=self.path,\n model_cls=self._model_cls,\n quantile_levels=self.quantile_levels,\n validation_fraction=model_params[\"validation_fraction\"],\n problem_type=problem_type,\n eval_metric=eval_metric,\n date_features=self._date_features,\n time_limit=time_limit_per_model,\n num_cpus=num_cpus_per_model,\n model_hyperparameters=model_hyperparameters.copy(),\n verbosity=verbosity - 1,\n )\n\n logger.debug(f\"Fitting models in parallel with {n_jobs=}, {num_cpus_per_model=}, {time_limit_per_model=}\")\n self._relative_paths_to_models = Parallel(n_jobs=n_jobs)( # type: ignore\n delayed(self._fit_single_model)(\n step=step,\n lags=self._get_lags_for_step(\n seasonal_lags=self._seasonal_lags, trailing_lags=self._trailing_lags, step=step\n ),\n **model_fit_kwargs,\n )\n for step in range(self.prediction_length)\n )\n\n @classmethod\n def _get_residuals_std_path(cls, model_path: str) -> str:\n \"\"\"Path to the pd.Series storing the standard deviation of residuals for each item_id.\"\"\"\n return os.path.join(model_path, \"residuals_std.pkl\")\n\n @classmethod\n def _predict_with_single_model(\n cls,\n full_df: pd.DataFrame,\n path_to_model: str,\n model_cls: Type[AbstractTabularModel],\n step: int,\n quantile_levels: list[float],\n prediction_length: int,\n lags: list[int],\n date_features: list[Callable],\n ) -> np.ndarray:\n \"\"\"Make predictions with the model for the given step.\n\n Returns\n -------\n predictions\n Predictions of the model for the given step. Shape: (num_items, len(quantile_levels)).\n \"\"\"\n from mlforecast import MLForecast\n\n mlf = MLForecast(models=[], freq=cls._dummy_freq, lags=lags, date_features=date_features)\n\n with warning_filter():\n features_df = mlf.preprocess(full_df, static_features=[], dropna=False)\n del mlf\n\n end_idx_per_item = np.cumsum(features_df[MLF_ITEMID].value_counts(sort=False).to_numpy(dtype=\"int32\"))\n features_for_step = features_df.iloc[end_idx_per_item - (prediction_length - step)]\n try:\n model: AbstractTabularModel = model_cls.load(path_to_model) # type: ignore\n except:\n logger.error(f\"Could not load model for {step=} from {path_to_model}\")\n raise\n predictions = model.predict(features_for_step)\n if model.problem_type == REGRESSION:\n predictions = np.tile(predictions[:, None], (1, len(quantile_levels)))\n residuals_std: pd.Series = load_pkl.load(cls._get_residuals_std_path(model.path))\n item_ids = features_for_step[MLF_ITEMID]\n residuals_repeated = residuals_std.reindex(item_ids).fillna(residuals_std.mean()).to_numpy()\n for i, q in enumerate(quantile_levels):\n predictions[:, i] += scipy.stats.norm.ppf(q) * residuals_repeated\n return predictions\n\n def _predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n if known_covariates is not None:\n X_df = known_covariates\n else:\n X_df = TimeSeriesDataFrame(\n pd.DataFrame(float(\"inf\"), index=self.get_forecast_horizon_index(data), columns=[self.target])\n )\n full_df = pd.concat([data, X_df])\n if self._max_ts_length is not None:\n full_df = full_df.slice_by_timestep(-(self._max_ts_length + self.prediction_length), None)\n full_df = full_df.to_data_frame().reset_index()\n if data.static_features is not None:\n full_df = pd.merge(\n full_df, data.static_features, left_on=TimeSeriesDataFrame.ITEMID, right_index=True, how=\"left\"\n )\n\n full_df = (\n full_df.rename(columns=self._ag_to_nixtla)\n .sort_values(by=[MLF_ITEMID, MLF_TIMESTAMP])\n .reset_index(drop=True)\n )\n full_df = full_df.assign(**{MLF_TARGET: full_df[MLF_TARGET].fillna(float(\"inf\"))})\n\n model_predict_kwargs = dict(\n full_df=full_df,\n quantile_levels=self.quantile_levels,\n prediction_length=self.prediction_length,\n model_cls=self._model_cls,\n date_features=self._date_features,\n )\n n_jobs = min(self._n_jobs, self.prediction_length, cpu_count(only_physical_cores=True))\n predictions_per_step = Parallel(n_jobs=n_jobs)(\n delayed(self._predict_with_single_model)(\n step=step,\n lags=self._get_lags_for_step(\n seasonal_lags=self._seasonal_lags, trailing_lags=self._trailing_lags, step=step\n ),\n path_to_model=os.path.join(self.path, suffix),\n **model_predict_kwargs,\n )\n for step, suffix in enumerate(self._relative_paths_to_models)\n )\n predictions = pd.DataFrame(\n np.stack(predictions_per_step, axis=1).reshape([-1, len(self.quantile_levels)]),\n columns=[str(q) for q in self.quantile_levels],\n index=self.get_forecast_horizon_index(data),\n )\n predictions[\"mean\"] = predictions[\"0.5\"]\n return TimeSeriesDataFrame(predictions)\n\n def _more_tags(self) -> dict[str, Any]:\n return {\"allow_nan\": True, \"can_refit_full\": True}\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/autogluon_tabular/transforms.py",
"content": "from typing import Literal\n\nimport numpy as np\nimport pandas as pd\nfrom mlforecast.target_transforms import (\n BaseTargetTransform,\n GroupedArray,\n _BaseGroupedArrayTargetTransform,\n)\n\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.transforms.target_scaler import TargetScaler, get_target_scaler\n\nfrom .utils import MLF_ITEMID, MLF_TIMESTAMP\n\n\nclass MLForecastScaler(BaseTargetTransform):\n def __init__(self, scaler_type: Literal[\"standard\", \"min_max\", \"mean_abs\", \"robust\"]):\n # For backward compatibility\n self.scaler_type: Literal[\"standard\", \"min_max\", \"mean_abs\", \"robust\"] = scaler_type\n self.ag_scaler: TargetScaler\n\n def _df_to_tsdf(self, df: pd.DataFrame) -> TimeSeriesDataFrame:\n return TimeSeriesDataFrame(\n df.rename(\n columns={self.id_col: TimeSeriesDataFrame.ITEMID, self.time_col: TimeSeriesDataFrame.TIMESTAMP}\n ).set_index([TimeSeriesDataFrame.ITEMID, TimeSeriesDataFrame.TIMESTAMP])\n )\n\n def _tsdf_to_df(self, ts_df: TimeSeriesDataFrame) -> pd.DataFrame:\n return (\n pd.DataFrame(ts_df)\n .reset_index()\n .rename(columns={TimeSeriesDataFrame.ITEMID: self.id_col, TimeSeriesDataFrame.TIMESTAMP: self.time_col})\n )\n\n def fit_transform(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore\n self.ag_scaler = get_target_scaler(name=self.scaler_type, target=self.target_col)\n transformed = self.ag_scaler.fit_transform(self._df_to_tsdf(df))\n return self._tsdf_to_df(transformed)\n\n def inverse_transform(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore\n assert self.ag_scaler is not None\n transformed = self.ag_scaler.inverse_transform(self._df_to_tsdf(df))\n return self._tsdf_to_df(transformed)\n\n\ndef apply_inverse_transform(\n df: pd.DataFrame,\n transform: _BaseGroupedArrayTargetTransform | BaseTargetTransform,\n) -> pd.DataFrame:\n \"\"\"Apply inverse transformation to a dataframe, converting to GroupedArray if necessary\"\"\"\n if isinstance(transform, BaseTargetTransform):\n inverse_transformed = transform.inverse_transform(df=df)\n assert isinstance(inverse_transformed, pd.DataFrame)\n return inverse_transformed\n elif isinstance(transform, _BaseGroupedArrayTargetTransform):\n indptr = np.concatenate([[0], df[MLF_ITEMID].value_counts().cumsum()])\n assignment = {}\n for col in df.columns.drop([MLF_ITEMID, MLF_TIMESTAMP]):\n ga = GroupedArray(data=df[col].to_numpy(), indptr=indptr)\n assignment[col] = transform.inverse_transform(ga).data\n return df.assign(**assignment)\n else:\n raise ValueError(\n f\"transform must be of type `_BaseGroupedArrayTargetTransform` or `BaseTargetTransform` (got {type(transform)})\"\n )\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/autogluon_tabular/utils.py",
"content": "MLF_TARGET = \"y\"\nMLF_ITEMID = \"unique_id\"\nMLF_TIMESTAMP = \"ds\"\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/chronos/__init__.py",
"content": "from .chronos2 import Chronos2Model\nfrom .model import ChronosModel\n\n__all__ = [\"ChronosModel\", \"Chronos2Model\"]\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/chronos/chronos2.py",
"content": "import logging\nimport os\nfrom typing import Any\n\nimport numpy as np\nimport pandas as pd\nfrom typing_extensions import Self\n\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.models.abstract import AbstractTimeSeriesModel\n\nlogger = logging.getLogger(__name__)\n\n\nclass Chronos2Model(AbstractTimeSeriesModel):\n \"\"\"Chronos-2 pretrained time series forecasting model [Ansari2025]_, which provides strong zero-shot forecasting\n capability natively taking advantage of covariates. The model can also be fine-tuned in a task specific manner.\n\n This implementation wraps the original implementation in the `chronos-forecasting`\n `library `_ .\n\n Chronos-2 can be used both on GPU and CPU. However, we recommend using a GPU for faster inference and fine-tuning.\n\n Chronos-2 variants can be fine-tuned by setting ``fine_tune=True`` and selecting appropriate fine-tuning parameters\n such as the learning rate (``fine_tune_lr``) and max steps (``fine_tune_steps``). By default, a low-rank adapter (LoRA)\n will be used for fine-tuning.\n\n References\n ----------\n .. [Ansari2025] Ansari, Abdul Fatir, Shchur, Oleksandr, Kuken, Jaris et al.\n \"Chronos-2: From Univariate to Universal Forecasting.\" (2025).\n https://arxiv.org/abs/2510.15821\n\n Other Parameters\n ----------------\n model_path : str, default = \"autogluon/chronos-2\"\n Model path used for the model, i.e., a Hugging Face transformers ``name_or_path``. Can be a\n compatible model name on Hugging Face Hub or a local path to a model directory.\n batch_size : int, default = 256\n Size of batches used during inference.\n device : str, default = None\n Device to use for inference (and fine-tuning, if enabled). If None, model will use the GPU if\n available.\n cross_learning : bool, default = True\n If True, the cross-learning mode of Chronos-2 is enabled. This means that the model will make joint\n predictions across time series in a batch, by default True\n Note: Enabling this mode makes the results sensitive to the ``batch_size`` used.\n context_length : int or None, default = None\n The context length to use for inference. If None, the model will use its default context length\n of 8192. Shorter context lengths may reduce accuracy, but result in faster inference.\n fine_tune : bool, default = False\n If True, the pretrained model will be fine-tuned.\n fine_tune_mode : str, default = \"lora\"\n Fine-tuning mode, either \"full\" for full fine-tuning or \"lora\" for Low Rank Adaptation (LoRA).\n LoRA is faster and uses less memory.\n fine_tune_lr : float, default = 1e-5\n The learning rate used for fine-tuning. When using full fine-tuning, a lower learning rate such as 1e-6\n is recommended.\n fine_tune_steps : int, default = 1000\n The number of gradient update steps to fine-tune for.\n fine_tune_batch_size : int, default = 32\n The batch size to use for fine-tuning.\n fine_tune_context_length : int, default = 2048\n The maximum context_length to use for fine-tuning\n eval_during_fine_tune : bool, default = False\n If True, validation will be performed during fine-tuning to select the best checkpoint. Setting this\n argument to True may result in slower fine-tuning. This parameter is ignored if ``skip_model_selection=True``\n in ``TimeSeriesPredictor.fit``.\n fine_tune_eval_max_items : int, default = 256\n The maximum number of randomly-sampled time series to use from the validation set for evaluation\n during fine-tuning. If None, the entire validation dataset will be used.\n fine_tune_lora_config : dict, optional\n Configuration for LoRA fine-tuning when ``fine_tune_mode=\"lora\"``. If None and LoRA is enabled,\n a default configuration will be used. Example: ``{\"r\": 8, \"lora_alpha\": 16}``.\n fine_tune_trainer_kwargs : dict, optional\n Extra keyword arguments passed to ``transformers.TrainingArguments``\n revision : str, default = None\n Model revision to use (branch name or commit hash). If None, the default branch (usually \"main\") is used.\n disable_known_covariates : bool, default = False\n If True, known covariates won't be used by the model even if they are present in the dataset.\n disable_past_covariates : bool, default = False\n If True, past covariates won't be used by the model even if they are present in the dataset.\n \"\"\"\n\n ag_model_aliases = [\"Chronos-2\"]\n ag_priority = 75\n fine_tuned_ckpt_name: str = \"fine-tuned-ckpt\"\n\n _supports_known_covariates = True\n _supports_past_covariates = True\n\n def __init__(\n self,\n freq: str | None = None,\n prediction_length: int = 1,\n path: str | None = None,\n name: str | None = None,\n eval_metric: str | None = None,\n hyperparameters: dict[str, Any] | None = None,\n **kwargs,\n ):\n super().__init__(\n path=path,\n freq=freq,\n prediction_length=prediction_length,\n name=name,\n eval_metric=eval_metric,\n hyperparameters=hyperparameters,\n **kwargs,\n )\n self._is_fine_tuned: bool = False\n self._model_pipeline = None\n\n @property\n def model_path(self) -> str:\n default_model_path = self.get_hyperparameter(\"model_path\")\n\n if self._is_fine_tuned:\n model_path = os.path.join(self.path, self.fine_tuned_ckpt_name)\n if not os.path.exists(model_path):\n raise ValueError(\"Cannot find finetuned checkpoint for Chronos-2.\")\n else:\n return model_path\n\n return default_model_path\n\n def save(self, path: str | None = None, verbose: bool = True) -> str:\n pipeline = self._model_pipeline\n self._model_pipeline = None\n path = super().save(path=path, verbose=verbose)\n self._model_pipeline = pipeline\n\n return str(path)\n\n def _fit(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None = None,\n time_limit: float | None = None,\n num_cpus: int | None = None,\n num_gpus: int | None = None,\n verbosity: int = 2,\n **kwargs,\n ) -> None:\n self._check_fit_params()\n self._log_unused_hyperparameters()\n self.load_model_pipeline()\n\n # NOTE: This must be placed after load_model_pipeline to ensure that the loggers are available in loggerDict\n self._update_transformers_loggers(logging.ERROR if verbosity <= 3 else logging.WARNING)\n\n if self.get_hyperparameter(\"fine_tune\"):\n self._fine_tune(train_data, val_data, time_limit=time_limit, verbosity=verbosity)\n\n def get_hyperparameters(self) -> dict:\n \"\"\"Gets params that are passed to the inner model.\"\"\"\n init_args = super().get_hyperparameters()\n\n fine_tune_trainer_kwargs = dict(disable_tqdm=True)\n user_fine_tune_trainer_kwargs = init_args.get(\"fine_tune_trainer_kwargs\", {})\n fine_tune_trainer_kwargs.update(user_fine_tune_trainer_kwargs)\n init_args[\"fine_tune_trainer_kwargs\"] = fine_tune_trainer_kwargs\n\n return init_args.copy()\n\n def _get_default_hyperparameters(self) -> dict:\n return {\n \"model_path\": \"autogluon/chronos-2\",\n \"batch_size\": 256,\n \"device\": None,\n \"cross_learning\": True,\n \"context_length\": None,\n \"fine_tune\": False,\n \"fine_tune_mode\": \"lora\",\n \"fine_tune_lr\": 1e-5,\n \"fine_tune_steps\": 1000,\n \"fine_tune_batch_size\": 32,\n \"fine_tune_context_length\": 2048,\n \"eval_during_fine_tune\": False,\n \"fine_tune_eval_max_items\": 256,\n \"fine_tune_lora_config\": None,\n \"revision\": None,\n \"disable_known_covariates\": False,\n \"disable_past_covariates\": False,\n }\n\n @property\n def allowed_hyperparameters(self) -> list[str]:\n return super().allowed_hyperparameters + [\n \"model_path\",\n \"batch_size\",\n \"device\",\n \"cross_learning\",\n \"context_length\",\n \"fine_tune\",\n \"fine_tune_mode\",\n \"fine_tune_lr\",\n \"fine_tune_steps\",\n \"fine_tune_batch_size\",\n \"fine_tune_context_length\",\n \"eval_during_fine_tune\",\n \"fine_tune_eval_max_items\",\n \"fine_tune_lora_config\",\n \"fine_tune_trainer_kwargs\",\n \"revision\",\n \"disable_known_covariates\",\n \"disable_past_covariates\",\n ]\n\n def _remove_disabled_covariates(\n self, past_df: pd.DataFrame, future_df: pd.DataFrame | None\n ) -> tuple[pd.DataFrame, pd.DataFrame | None]:\n \"\"\"Remove covariates from dataframes based on disable flags.\"\"\"\n cols_to_remove = []\n if self.get_hyperparameter(\"disable_past_covariates\"):\n cols_to_remove.extend(self.covariate_metadata.past_covariates)\n if self.get_hyperparameter(\"disable_known_covariates\"):\n cols_to_remove.extend(self.covariate_metadata.known_covariates)\n future_df = None\n\n if cols_to_remove:\n past_df = past_df.drop(columns=cols_to_remove)\n\n return past_df, future_df\n\n def _predict(\n self,\n data: TimeSeriesDataFrame,\n known_covariates: TimeSeriesDataFrame | None = None,\n **kwargs,\n ) -> TimeSeriesDataFrame:\n from .utils import timeout_callback\n\n if self._model_pipeline is None:\n self.load_model_pipeline()\n assert self._model_pipeline is not None\n\n if max(data.num_timesteps_per_item()) < 3:\n # If all time series have length 2 or less, we prepend 2 dummy timesteps to the first series\n first_item_id = data.index.get_level_values(0)[0]\n dummy_timestamps = pd.date_range(end=data.loc[first_item_id].index[0], periods=3, freq=self.freq)[:-1]\n full_time_index_first_item = data.loc[first_item_id].index.union(dummy_timestamps)\n new_index = (\n pd.MultiIndex.from_product([[first_item_id], full_time_index_first_item], names=data.index.names)\n ).union(data.index)\n context_df = data.reindex(new_index).reset_index()\n else:\n context_df = data.reset_index().to_data_frame()\n\n batch_size = self.get_hyperparameter(\"batch_size\")\n cross_learning = self.get_hyperparameter(\"cross_learning\")\n context_length = self.get_hyperparameter(\"context_length\")\n future_df = known_covariates.reset_index().to_data_frame() if known_covariates is not None else None\n time_limit = kwargs.get(\"time_limit\")\n\n context_df, future_df = self._remove_disabled_covariates(context_df, future_df)\n\n forecast_df = self._model_pipeline.predict_df(\n df=context_df,\n future_df=future_df,\n target=self.target,\n prediction_length=self.prediction_length,\n quantile_levels=self.quantile_levels,\n context_length=context_length,\n batch_size=batch_size,\n validate_inputs=False,\n cross_learning=cross_learning,\n after_batch=timeout_callback(time_limit),\n )\n\n forecast_df = forecast_df.rename(columns={\"predictions\": \"mean\"}).drop(columns=\"target_name\")\n\n return TimeSeriesDataFrame(forecast_df)\n\n def load_model_pipeline(self):\n from chronos.chronos2.pipeline import Chronos2Pipeline\n\n device = (self.get_hyperparameter(\"device\") or \"cuda\") if self._is_gpu_available() else \"cpu\"\n\n assert self.model_path is not None\n pipeline = Chronos2Pipeline.from_pretrained(\n self.model_path,\n device_map=device,\n revision=self.get_hyperparameter(\"revision\"),\n )\n\n self._model_pipeline = pipeline\n\n def persist(self) -> Self:\n self.load_model_pipeline()\n return self\n\n def _update_transformers_loggers(self, log_level: int):\n for logger_name in logging.root.manager.loggerDict:\n if \"transformers\" in logger_name:\n transformers_logger = logging.getLogger(logger_name)\n transformers_logger.setLevel(log_level)\n\n def _fine_tune(\n self,\n train_data: TimeSeriesDataFrame,\n val_data: TimeSeriesDataFrame | None,\n time_limit: float | None = None,\n verbosity: int = 2,\n ):\n from chronos.df_utils import convert_df_input_to_list_of_dicts_input\n\n from .utils import LoggerCallback, TimeLimitCallback\n\n def convert_data(df: TimeSeriesDataFrame):\n past_df = df.reset_index().to_data_frame()\n past_df, _ = self._remove_disabled_covariates(past_df, None)\n\n inputs, _, _ = convert_df_input_to_list_of_dicts_input(\n df=past_df,\n future_df=None,\n target_columns=[self.target],\n prediction_length=self.prediction_length,\n validate_inputs=False,\n )\n\n # The above utility will only split the dataframe into target and past_covariates, where past_covariates contains\n # past values of both past-only and known-future covariates. We need to add future_covariates to enable fine-tuning\n # with known covariates by indicating which covariates are known in the future.\n if not self.get_hyperparameter(\"disable_known_covariates\"):\n known_covariates = self.covariate_metadata.known_covariates\n if len(known_covariates) > 0:\n for input_dict in inputs:\n # NOTE: the covariates are empty because the actual values are not used\n # This only indicates which covariates are known in the future\n input_dict[\"future_covariates\"] = {name: np.array([]) for name in known_covariates}\n\n return inputs\n\n assert self._model_pipeline is not None\n hyperparameters = self.get_hyperparameters()\n\n callbacks = []\n if time_limit is not None:\n callbacks.append(TimeLimitCallback(time_limit=time_limit))\n\n val_inputs = None\n if val_data is not None and hyperparameters[\"eval_during_fine_tune\"]:\n # evaluate on a randomly-sampled subset\n fine_tune_eval_max_items = (\n min(val_data.num_items, hyperparameters[\"fine_tune_eval_max_items\"])\n if hyperparameters[\"fine_tune_eval_max_items\"] is not None\n else val_data.num_items\n )\n\n if fine_tune_eval_max_items < val_data.num_items:\n eval_items = np.random.choice(val_data.item_ids.values, size=fine_tune_eval_max_items, replace=False) # noqa: F841\n val_data = val_data.query(\"item_id in @eval_items\")\n\n assert isinstance(val_data, TimeSeriesDataFrame)\n val_inputs = convert_data(val_data)\n\n if verbosity >= 3:\n logger.warning(\n \"Transformers logging is turned on during fine-tuning. Note that losses reported by transformers \"\n \"do not correspond to those specified via `eval_metric`.\"\n )\n callbacks.append(LoggerCallback())\n\n self._model_pipeline = self._model_pipeline.fit(\n inputs=convert_data(train_data),\n prediction_length=self.prediction_length,\n validation_inputs=val_inputs,\n finetune_mode=hyperparameters[\"fine_tune_mode\"],\n lora_config=hyperparameters[\"fine_tune_lora_config\"],\n context_length=hyperparameters[\"fine_tune_context_length\"],\n learning_rate=hyperparameters[\"fine_tune_lr\"],\n num_steps=hyperparameters[\"fine_tune_steps\"],\n batch_size=hyperparameters[\"fine_tune_batch_size\"],\n output_dir=self.path,\n finetuned_ckpt_name=self.fine_tuned_ckpt_name,\n callbacks=callbacks,\n remove_printer_callback=True,\n min_past=1,\n **hyperparameters[\"fine_tune_trainer_kwargs\"],\n )\n self._is_fine_tuned = True\n\n def _more_tags(self) -> dict[str, Any]:\n do_fine_tune = self.get_hyperparameter(\"fine_tune\")\n return {\n \"allow_nan\": True,\n \"can_use_train_data\": do_fine_tune,\n \"can_use_val_data\": do_fine_tune,\n }\n\n def _is_gpu_available(self) -> bool:\n import torch.cuda\n\n return torch.cuda.is_available()\n"
},
{
"path": "timeseries/src/autogluon/timeseries/models/chronos/model.py",
"content": "import logging\nimport os\nimport shutil\nimport warnings\nfrom pathlib import Path\nfrom typing import Any\n\nimport numpy as np\nimport pandas as pd\nfrom typing_extensions import Self\n\nfrom autogluon.common.loaders import load_pkl\nfrom autogluon.common.space import Space\nfrom autogluon.timeseries.dataset import TimeSeriesDataFrame\nfrom autogluon.timeseries.models.abstract import AbstractTimeSeriesModel\nfrom autogluon.timeseries.utils.warning_filters import disable_duplicate_logs, warning_filter\n\nlogger = logging.getLogger(\"autogluon.timeseries.models.chronos\")\n\n# TODO: Replace `evaluation_strategy` with `eval_strategy` when upgrading to `transformers>=4.41` + remove warning filter\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, message=\"`evaluation_strategy` is deprecated\")\n# TODO: Remove warning filter when upgrading to `transformers>=4.40`\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, message=\"Passing the following arguments to \")\n\n\n# allowed HuggingFace model paths with custom parameter definitions\nMODEL_CONFIGS = {\n \"chronos-t5-tiny\": {\n \"num_gpus\": 0, # minimum number of required GPUs\n \"default_torch_dtype\": \"auto\",\n \"default_batch_size\": 16,\n },\n \"chronos-t5-mini\": {\n \"num_gpus\": 0,\n \"default_torch_dtype\": \"auto\",\n \"default_batch_size\": 16,\n },\n \"chronos-t5-small\": {\n \"num_gpus\": 1,\n \"default_torch_dtype\": \"bfloat16\",\n \"default_batch_size\": 16,\n },\n \"chronos-t5-base\": {\n \"num_gpus\": 1,\n \"default_torch_dtype\": \"bfloat16\",\n \"default_batch_size\": 16,\n },\n \"chronos-t5-large\": {\n \"num_gpus\": 1,\n \"default_torch_dtype\": \"bfloat16\",\n \"default_batch_size\": 8,\n },\n \"chronos-bolt-mini\": {\n \"num_gpus\": 0,\n \"default_torch_dtype\": \"auto\",\n \"default_batch_size\": 256,\n },\n \"chronos-bolt-small\": {\n \"num_gpus\": 0,\n \"default_torch_dtype\": \"auto\",\n \"default_batch_size\": 256,\n },\n \"chronos-bolt-base\": {\n \"num_gpus\": 0,\n \"default_torch_dtype\": \"auto\",\n \"default_batch_size\": 256,\n },\n}\n\n\nMODEL_ALIASES = {\n \"tiny\": \"autogluon/chronos-t5-tiny\",\n \"mini\": \"autogluon/chronos-t5-mini\",\n \"small\": \"autogluon/chronos-t5-small\",\n \"base\": \"autogluon/chronos-t5-base\",\n \"large\": \"autogluon/chronos-t5-large\",\n \"bolt_tiny\": \"autogluon/chronos-bolt-tiny\",\n \"bolt_mini\": \"autogluon/chronos-bolt-mini\",\n \"bolt_small\": \"autogluon/chronos-bolt-small\",\n \"bolt_base\": \"autogluon/chronos-bolt-base\",\n}\n\n\nclass ChronosModel(AbstractTimeSeriesModel):\n \"\"\"Chronos [Ansari2024]_ pretrained time series forecasting models which can be used for zero-shot\n forecasting or fine-tuned in a task-specific manner.\n\n Models can be based on the original\n `Chronos `_\n implementation, as well as a newer family of\n `Chronos-Bolt
\\n\",\n \"\\n\",\n \"\\n\",\n \" \\n\",\n \"
\\n\",\n \"
\"\n ]\n },\n \"execution_count\": 39,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"execution_count\": 39\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"ExecuteTime\": {\n \"end_time\": \"2026-01-07T01:37:13.879526Z\",\n \"start_time\": \"2026-01-07T01:37:13.878145Z\"\n }\n },\n \"source\": [],\n \"outputs\": [],\n \"execution_count\": null\n }\n ],\n \"metadata\": {\n \"anaconda-cloud\": {},\n \"kernelspec\": {\n \"display_name\": \"Python (agluon)\",\n \"language\": \"python\",\n \"name\": \"agluon\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.18\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n"
},
{
"path": "features/setup.py",
"content": "#!/usr/bin/env python\n###########################\n# This code block is a HACK (!), but is necessary to avoid code duplication. Do NOT alter these lines.\nimport importlib.util\nimport os\n\nfrom setuptools import setup\n\nfilepath = os.path.abspath(os.path.dirname(__file__))\nfilepath_import = os.path.join(filepath, \"..\", \"core\", \"src\", \"autogluon\", \"core\", \"_setup_utils.py\")\nif not os.path.exists(filepath_import):\n filepath_import = os.path.join(filepath, \"_setup_utils.py\")\n\nspec = importlib.util.spec_from_file_location(\"ag_min_dependencies\", filepath_import)\nag = importlib.util.module_from_spec(spec)\n# Identical to `from autogluon.core import _setup_utils as ag`, but works without `autogluon.core` being installed.\nspec.loader.exec_module(ag)\n###########################\n\nversion = ag.load_version_file()\nversion = ag.update_version(version)\n\nsubmodule = \"features\"\ninstall_requires = (\n [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"scikit-learn\", # version range defined in `core/_setup_utils.py`\n f\"autogluon.common=={version}\",\n ]\n if not ag.LITE_MODE\n else [\n # version ranges added in ag.get_dependency_version_ranges()\n \"numpy\", # version range defined in `core/_setup_utils.py`\n \"pandas\", # version range defined in `core/_setup_utils.py`\n \"scikit-learn\", # version range defined in `core/_setup_utils.py`\n f\"{ag.PACKAGE_NAME}.common=={version}\",\n ]\n)\n\ninstall_requires = ag.get_dependency_version_ranges(install_requires)\n\nif __name__ == \"__main__\":\n ag.create_version_file(version=version, submodule=submodule)\n setup_args = ag.default_setup_args(version=version, submodule=submodule)\n setup(\n install_requires=install_requires,\n **setup_args,\n )\n"
},
{
"path": "features/src/autogluon/features/__init__.py",
"content": "from .generators import * # noqa\nfrom .version import __version__\n"
},
{
"path": "features/src/autogluon/features/binning.py",
"content": "import logging\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, IntervalIndex, Series\n\nlogger = logging.getLogger(__name__)\n\n\ndef bin_column(series: Series, bins, dtype):\n return np.digitize(series, bins=bins, right=True).astype(dtype)\n\n\n# TODO: Rewrite with normalized value counts as binning technique, will be more performant and optimal\ndef generate_bins(X_features: DataFrame, features_to_bin: list, ideal_bins: int = 10):\n X_len = len(X_features)\n starting_cats = 1000\n bin_index_starting = [np.floor(X_len * (num + 1) / starting_cats) for num in range(starting_cats - 1)]\n bin_epsilon = 0.000000001\n bin_mapping = dict()\n max_iterations = 20\n for column in features_to_bin:\n num_cats_initial = starting_cats\n bins_value_counts = X_features[column].value_counts(ascending=False, normalize=True)\n max_bins = len(bins_value_counts)\n\n if max_bins <= ideal_bins:\n bins = pd.Series(data=sorted(X_features[column].unique()))\n num_cats_initial = max_bins\n cur_len = max_bins\n bin_index = list(range(num_cats_initial))\n interval_index = get_bins(bins=bins, bin_index=bin_index, bin_epsilon=bin_epsilon)\n else:\n cur_len = X_len\n bins = X_features[column].sort_values(ascending=True)\n interval_index = get_bins(bins=bins, bin_index=bin_index_starting, bin_epsilon=bin_epsilon)\n\n # TODO: max_desired_bins and min_desired_bins are currently equivalent, but in future they will be parameterized to allow for flexibility.\n max_desired_bins = min(ideal_bins, max_bins)\n min_desired_bins = min(ideal_bins, max_bins)\n\n is_satisfied = (len(interval_index) >= min_desired_bins) and (len(interval_index) <= max_desired_bins)\n\n num_cats_current = num_cats_initial\n cur_iteration = 0\n while not is_satisfied:\n ratio_reduction = max_desired_bins / len(interval_index)\n num_cats_current = int(np.floor(num_cats_current * ratio_reduction))\n bin_index = [np.floor(cur_len * (num + 1) / num_cats_current) for num in range(num_cats_current - 1)]\n interval_index = get_bins(bins=bins, bin_index=bin_index, bin_epsilon=bin_epsilon)\n\n if (len(interval_index) >= min_desired_bins) and (len(interval_index) <= max_desired_bins):\n is_satisfied = True\n # print('satisfied', column, len(interval_index))\n cur_iteration += 1\n if cur_iteration >= max_iterations:\n is_satisfied = True\n # print('max_iterations met, stopping prior to satisfaction!', column, len(interval_index))\n\n bins_final = interval_index.right.values\n bin_mapping[column] = bins_final\n return bin_mapping\n\n\n# TODO: Clean code\n# TODO: Consider reusing bins variable instead of making bins_2-7 variables\n# bins is a sorted int/float series, ascending=True\ndef get_bins(bins: Series, bin_index: list, bin_epsilon: float) -> IntervalIndex:\n max_val = bins.max()\n bins_2 = bins.iloc[bin_index]\n bins_3 = list(bins_2.values)\n bins_unique = sorted(list(set(bins_3)))\n bins_with_epsilon_max = set([i for i in bins_unique] + [i - bin_epsilon for i in bins_unique if i == max_val])\n removal_bins = set([bins_unique[index - 1] for index, i in enumerate(bins_unique[1:], start=1) if i == max_val])\n bins_4 = sorted(list(bins_with_epsilon_max - removal_bins))\n bins_5 = [np.inf if (x == max_val) else x for x in bins_4]\n bins_6 = sorted(list(set([-np.inf] + bins_5 + [np.inf])))\n bins_7 = [(bins_6[i], bins_6[i + 1]) for i in range(len(bins_6) - 1)]\n interval_index = IntervalIndex.from_tuples(bins_7)\n return interval_index\n"
},
{
"path": "features/src/autogluon/features/generators/__init__.py",
"content": "from .abstract import AbstractFeatureGenerator\nfrom .arithmetic.preprocessor import ArithmeticFeatureGenerator\nfrom .astype import AsTypeFeatureGenerator\nfrom .auto_ml_pipeline import AutoMLInterpretablePipelineFeatureGenerator, AutoMLPipelineFeatureGenerator\nfrom .binned import BinnedFeatureGenerator\nfrom .bulk import BulkFeatureGenerator\nfrom .cat_as_num import CatAsNumFeatureGenerator\nfrom .cat_int import CategoricalInteractionFeatureGenerator\nfrom .category import CategoryFeatureGenerator\nfrom .datetime import DatetimeFeatureGenerator\nfrom .drop_duplicates import DropDuplicatesFeatureGenerator\nfrom .drop_unique import DropUniqueFeatureGenerator\nfrom .dummy import DummyFeatureGenerator\nfrom .fillna import FillNaFeatureGenerator\nfrom .frequency import FrequencyFeatureGenerator\nfrom .groupby import GroupByFeatureGenerator\nfrom .identity import IdentityFeatureGenerator\nfrom .isnan import IsNanFeatureGenerator\nfrom .label_encoder import LabelEncoderFeatureGenerator\nfrom .memory_minimize import CategoryMemoryMinimizeFeatureGenerator, NumericMemoryMinimizeFeatureGenerator\nfrom .one_hot_encoder import OneHotEncoderFeatureGenerator\nfrom .oof_target_encoder import OOFTargetEncodingFeatureGenerator\nfrom .pipeline import PipelineFeatureGenerator\nfrom .rename import RenameFeatureGenerator\nfrom .rsfc import RandomSubsetFeatureCompressionGenerator\nfrom .selection import SpearmanFeatureSelector\nfrom .text_ngram import TextNgramFeatureGenerator\nfrom .text_special import TextSpecialFeatureGenerator\n\nREGISTERED_FE_CLS_LST = [\n AbstractFeatureGenerator,\n ArithmeticFeatureGenerator,\n AsTypeFeatureGenerator,\n AutoMLInterpretablePipelineFeatureGenerator,\n AutoMLPipelineFeatureGenerator,\n BinnedFeatureGenerator,\n BulkFeatureGenerator,\n CatAsNumFeatureGenerator,\n CategoricalInteractionFeatureGenerator,\n CategoryFeatureGenerator,\n DatetimeFeatureGenerator,\n DropDuplicatesFeatureGenerator,\n DropUniqueFeatureGenerator,\n DummyFeatureGenerator,\n FillNaFeatureGenerator,\n FrequencyFeatureGenerator,\n GroupByFeatureGenerator,\n IdentityFeatureGenerator,\n IsNanFeatureGenerator,\n LabelEncoderFeatureGenerator,\n CategoryMemoryMinimizeFeatureGenerator,\n NumericMemoryMinimizeFeatureGenerator,\n OneHotEncoderFeatureGenerator,\n OOFTargetEncodingFeatureGenerator,\n PipelineFeatureGenerator,\n RandomSubsetFeatureCompressionGenerator,\n RenameFeatureGenerator,\n SpearmanFeatureSelector,\n TextNgramFeatureGenerator,\n TextSpecialFeatureGenerator,\n]\n"
},
{
"path": "features/src/autogluon/features/generators/abstract.py",
"content": "from __future__ import annotations\n\nimport copy\nimport inspect\nimport logging\nimport time\nfrom collections import defaultdict\nfrom typing import Dict, List, Literal\n\nimport pandas as pd\nfrom pandas import DataFrame, Series\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.infer_types import get_type_group_map_special, get_type_map_raw, get_type_map_real\nfrom autogluon.common.savers import save_pkl\n\nfrom ..utils import is_useless_feature\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add option to minimize memory usage of feature names by making them integers / strings of integers\n# TODO: Add ability to track which input features created which output features.\n# TODO: Add log of # of observation counts to high cardinality categorical features\nclass AbstractFeatureGenerator:\n \"\"\"\n Abstract feature generator implementation from which all AutoGluon feature generators inherit.\n The purpose of a feature generator is to transform data from one form to another in a stateful manner.\n First, the generator is initialized with various arguments that dictate the way features are generated.\n Then, the generator is fit through either the `.fit()` or `.fit_transform()` methods using training data typically in pandas DataFrame format.\n Finally, the generator can transform new data with the same initial format as the training data through the `.transform()` method.\n\n Parameters\n ----------\n features_in : list, default None\n List of feature names the generator will expect and use in the fit and transform methods.\n Any feature in an incoming DataFrame that is not present in features_in is dropped and will not influence the transformation logic.\n If None, infer during fit from the _infer_features_in method.\n Equivalent to feature_metadata_in.get_features() post-fit.\n feature_metadata_in : :class:`autogluon.common.features.feature_metadata.FeatureMetadata`, default None\n :class:`FeatureMetadata` object corresponding to the training data input features.\n If None, infer during fit from the _infer_feature_metadata_in method.\n Any features not present in features_in (if provided) will be removed from feature_metadata_in.\n post_generators : list of FeatureGenerators, default None\n FeatureGenerators which will fit and transform sequentially after this object's transformation logic,\n feeding their output into the next generator's input.\n The output of the final FeatureGenerator will be the used as the transformed output.\n pre_enforce_types : bool, default False\n If True, the exact raw types (int64, float32, etc.) of the training data will be enforced on future data,\n either converting the types to the training types or raising an exception if unable.\n This is important to set to True on the outer feature generator in a feature generation pipeline to ensure\n incorrect dtypes are not passed downstream, but is often redundant when used on inner feature generators inside a pipeline.\n pre_drop_useless : bool, default False\n If True, features_in will be pruned at fit time of features containing only a single unique value across all rows.\n post_drop_duplicates : bool, default False\n If True, a :class:`DropDuplicatesFeatureGenerator` will be appended to post_generators.\n This feature generator will drop any duplicate features found in the data, keeping only one feature within any duplicate feature sets.\n Warning: For large datasets with many features, this may be very computationally expensive or even computationally infeasible.\n reset_index : bool, default False\n If True, for the duration of fit and transform, the input data's index is reset to be monotonically increasing from 0 to N-1 for a dataset of N rows.\n At the end of fit and transform, the original index is re-applied to the output data.\n This is important to set to True on the outer feature generator in a feature generation pipeline to ensure that a non-default\n index does not cause corruption of the inner feature generation if any inner feature generator does not properly handle non-default indices.\n This index reset is also applied to the y label data if provided during fit.\n column_names_as_str : bool, default True\n If True, the column names of the input data are converted to string if they were not already.\n This solves any issues related to downstream FeatureGenerators and models which cannot handle integer column names, and allows\n column name prefix and suffix operations to avoid errors.\n Note that for performance purposes, column names are only converted at transform time if they were not strings at fit time.\n Ensure consistent column names as input to avoid errors.\n name_prefix : str, default None\n Name prefix to add to all output feature names.\n name_suffix : str, default None\n Name suffix to add to all output feature names.\n infer_features_in_args : dict, default None\n Used as the kwargs input to FeatureMetadata.get_features(**kwargs) when inferring self.features_in.\n This is merged with the output dictionary of self.get_default_infer_features_in_args() depending on the value of infer_features_in_args_strategy.\n Only used when features_in is None.\n If None, then self.get_default_infer_features_in_args() is used directly.\n Refer to FeatureMetadata.get_features documentation for a full description of valid keys.\n Note: This is advanced functionality that is not necessary for most situations.\n infer_features_in_args_strategy : str, default 'overwrite'\n Determines how infer_features_in_args and self.get_default_infer_features_in_args() are combined to result in self._infer_features_in_args\n which dictates the features_in inference logic.\n If 'overwrite': infer_features_in_args is used exclusively and self.get_default_infer_features_in_args() is ignored.\n If 'update': self.get_default_infer_features_in_args() is dictionary updated by infer_features_in_args.\n If infer_features_in_args is None, this is ignored.\n banned_feature_special_types : List[str], default None\n List of feature special types to additionally exclude from input. Will update self.get_default_infer_features_in_args().\n target_type: str | None, default None\n The problem type of the target variable, such as \"binary\", \"multiclass\", \"regression\".\n If None and the preprocessor requires target_type, an exception will be raised.\n random_state: int | None, default 0\n The random state to use when fitting this generator.\n log_prefix : str, default ''\n Prefix string added to all logging statements made by the generator.\n verbosity : int, default 2\n Controls the verbosity of logging.\n 0 will silence logs, 1 will only log warnings, 2 will log info level information, and 3 will log info level information and provide detailed\n feature type input and output information.\n Logging is still controlled by the global logger configuration, and therefore a verbosity of 3 does not guarantee that logs will be output.\n\n Attributes\n ----------\n features_in : list of str\n List of feature names the generator will expect and use in the fit and transform methods.\n Equivalent to feature_metadata_in.get_features() post-fit.\n features_out : list of str\n List of feature names present in the output of fit_transform and transform methods.\n Equivalent to feature_metadata.get_features() post-fit.\n feature_metadata_in : FeatureMetadata\n The FeatureMetadata of data pre-transformation (data used as input to fit and transform methods).\n feature_metadata : FeatureMetadata\n The FeatureMetadata of data post-transformation (data outputted by fit_transform and transform methods).\n feature_metadata_real : FeatureMetadata\n The FeatureMetadata of data post-transformation consisting of the exact dtypes as opposed to the grouped raw dtypes found in feature_metadata_in,\n with grouped raw dtypes substituting for the special dtypes.\n This is only used in the print_feature_metadata_info method and is intended for introspection. It can be safely set to None to reduce memory and\n disk usage post-fit.\n \"\"\"\n\n def __init__(\n self,\n features_in: list = None,\n feature_metadata_in: FeatureMetadata = None,\n post_generators: list = None,\n passthrough: bool = False,\n passthrough_stage: Literal[\"first\", \"last\"] = \"first\", # FIXME: bug: \"last\" crashes if X_out is empty\n passthrough_types: dict = None,\n pre_enforce_types=False,\n pre_drop_useless=False,\n post_drop_duplicates=False,\n reset_index=False,\n column_names_as_str=True,\n name_prefix: str = None,\n name_suffix: str = None,\n infer_features_in_args: dict = None,\n infer_features_in_args_strategy=\"overwrite\",\n banned_feature_special_types: List[str] = None,\n target_type: Literal[\"regression\", \"multiclass\", \"binary\", None] = None,\n random_state: int | None = 0,\n log_prefix=\"\",\n verbosity=2,\n ):\n self._is_fit = False # Whether the feature generator has been fit\n self.features_in = features_in # Original features to use as input to feature generation\n self.features_out = None # Final list of features after transformation\n self.feature_metadata_in: FeatureMetadata = (\n feature_metadata_in # FeatureMetadata object based on the original input features.\n )\n\n # FeatureMetadata object based on the processed features. Pass to models to enable advanced functionality.\n self.feature_metadata: FeatureMetadata = None\n\n self.passthrough = passthrough\n self.passthrough_features = None\n assert passthrough_stage in [\"first\", \"last\"]\n self.passthrough_stage = passthrough_stage\n self.passthrough_types = passthrough_types\n\n # TODO: Consider merging feature_metadata and feature_metadata_real, have FeatureMetadata contain exact dtypes, grouped raw dtypes,\n # and special dtypes all at once.\n # FeatureMetadata object based on the processed features, containing the true raw dtype information (such as int32, float64, etc.).\n # Pass to models to enable advanced functionality.\n self.feature_metadata_real: FeatureMetadata = None\n self._feature_metadata_before_post = None # FeatureMetadata directly prior to applying self._post_generators.\n self._infer_features_in_args = self.get_default_infer_features_in_args()\n if infer_features_in_args is not None:\n if infer_features_in_args_strategy == \"overwrite\":\n self._infer_features_in_args = copy.deepcopy(infer_features_in_args)\n elif infer_features_in_args_strategy == \"update\":\n self._infer_features_in_args.update(infer_features_in_args)\n else:\n raise ValueError(\n f\"infer_features_in_args_strategy must be one of: {['overwrite', 'update']}, but was: '{infer_features_in_args_strategy}'\"\n )\n if banned_feature_special_types:\n if \"invalid_special_types\" not in self._infer_features_in_args:\n self._infer_features_in_args[\"invalid_special_types\"] = banned_feature_special_types\n else:\n for f in banned_feature_special_types:\n if f not in self._infer_features_in_args[\"invalid_special_types\"]:\n self._infer_features_in_args[\"invalid_special_types\"].append(f)\n\n if post_generators is None:\n post_generators = []\n elif not isinstance(post_generators, list):\n post_generators = [post_generators]\n self._post_generators: list = post_generators\n if post_drop_duplicates:\n from .drop_duplicates import DropDuplicatesFeatureGenerator\n\n self._post_generators.append(DropDuplicatesFeatureGenerator(post_drop_duplicates=False))\n if name_prefix or name_suffix:\n from .rename import RenameFeatureGenerator\n\n # inplace=False required to avoid altering outer context: refer to https://github.com/autogluon/autogluon/issues/2688\n self._post_generators.append(\n RenameFeatureGenerator(name_prefix=name_prefix, name_suffix=name_suffix, inplace=False)\n )\n\n if self._post_generators:\n if not self.get_tags().get(\"allow_post_generators\", True):\n raise AssertionError(\n f\"{self.__class__.__name__} is not allowed to have post_generators, \"\n f\"but found: {[generator.__class__.__name__ for generator in self._post_generators]}\"\n )\n\n self.pre_enforce_types = pre_enforce_types\n self._pre_astype_generator = None\n self.pre_drop_useless = pre_drop_useless\n self.reset_index = reset_index\n self.column_names_as_str = column_names_as_str\n self._useless_features_in: list = None\n\n self._is_updated_name = False # If feature names have been altered by name_prefix or name_suffix\n\n self.target_type = target_type\n self.random_state = random_state\n self.log_prefix = log_prefix\n self.verbosity = verbosity\n\n self.fit_time = None\n\n def fit(self, X: DataFrame, **kwargs):\n \"\"\"\n Fit generator to the provided data.\n Because of how the generators track output features and types, it is generally required that the data be transformed during fit, so the fit\n function is rarely useful to implement beyond a simple call to fit_transform.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n **kwargs\n Any additional arguments that a particular generator implementation could use.\n See fit_transform method for common kwargs values.\n \"\"\"\n self.fit_transform(X, **kwargs)\n\n def fit_transform(\n self, X: DataFrame, y: Series = None, feature_metadata_in: FeatureMetadata = None, **kwargs\n ) -> DataFrame:\n \"\"\"\n Fit generator to the provided data and return the transformed version of the data as if fit and transform were called sequentially with the same data.\n This is generally more efficient than calling fit and transform separately and can be up to twice as fast if the fit process requires transformation\n of the data.\n This cannot be called after the generator has been fit, and will result in an AssertionError.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n y : Series, optional\n Input data's labels used to fit the generator. Most generators do not utilize labels.\n y.index must be equal to X.index to avoid misalignment.\n feature_metadata_in : FeatureMetadata, optional\n Identical to providing feature_metadata_in during generator initialization. Ignored if self.feature_metadata_in is already specified.\n If neither are set, feature_metadata_in will be inferred from the _infer_feature_metadata_in method.\n **kwargs\n Any additional arguments that a particular generator implementation could use. Passed to _fit_transform and _fit_generators methods.\n\n Returns\n -------\n X_out : DataFrame object which is the transformed version of the input data X.\n\n \"\"\"\n start_time = time.time()\n self._log(20, f\"Fitting {self.__class__.__name__}...\")\n if self._is_fit:\n raise AssertionError(f\"{self.__class__.__name__} is already fit.\")\n self._pre_fit_validate(X=X, y=y, feature_metadata_in=feature_metadata_in, **kwargs)\n\n if self.reset_index:\n X_index = copy.deepcopy(X.index)\n # TODO: Theoretically inplace=True avoids data copy, but can lead to altering of original DataFrame outside of method context.\n X = X.reset_index(drop=True)\n if y is not None and isinstance(y, Series):\n y = y.reset_index(drop=True) # TODO: this assumes y and X had matching indices prior\n else:\n X_index = None\n if self.column_names_as_str:\n columns_orig = list(X.columns)\n X.columns = X.columns.astype(str) # Ensure all column names are strings\n columns_new = list(X.columns)\n if columns_orig != columns_new:\n rename_map = {orig: new for orig, new in zip(columns_orig, columns_new)}\n if feature_metadata_in is not None:\n feature_metadata_in.rename_features(rename_map=rename_map)\n self._rename_features_in(rename_map)\n else:\n self.column_names_as_str = False # Columns were already string, so don't do conversion. Better to error if they change types at inference.\n self._ensure_no_duplicate_column_names(X=X)\n self._infer_features_in_full(X=X, feature_metadata_in=feature_metadata_in)\n if self.pre_drop_useless:\n self._useless_features_in = self._get_useless_features(X, columns_to_check=self.features_in)\n if self._useless_features_in:\n self._remove_features_in(self._useless_features_in)\n if self.pre_enforce_types:\n from .astype import AsTypeFeatureGenerator\n\n self._pre_astype_generator = AsTypeFeatureGenerator(\n features_in=self.features_in,\n feature_metadata_in=self.feature_metadata_in,\n log_prefix=self.log_prefix + \"\\t\",\n )\n self._pre_astype_generator.fit(X)\n\n # TODO: Add option to return feature_metadata instead to avoid data copy\n # If so, consider adding validation step to check that X_out matches the feature metadata, error/warning if not\n X_out, type_family_groups_special = self._fit_transform(X[self.features_in], y=y, **kwargs)\n\n type_map_raw = get_type_map_raw(X_out)\n self.feature_metadata = FeatureMetadata(\n type_map_raw=type_map_raw, type_group_map_special=type_family_groups_special\n )\n\n if self.passthrough and self.passthrough_stage == \"first\" and self.features_in:\n self.feature_metadata, self.passthrough_features = self._fit_passthrough()\n if self.passthrough_features:\n X_out = self._transform_passthrough(X=X, X_out=X_out)\n\n self._feature_metadata_before_post = self.feature_metadata\n\n if self._post_generators:\n X_out, self.feature_metadata, self._post_generators = self._fit_generators(\n X=X_out,\n y=y,\n feature_metadata=self.feature_metadata,\n generators=self._post_generators,\n **kwargs,\n )\n\n # FIXME: This is bugged if `self.feature_metadata` is empty, crashes at transform\n if self.passthrough and self.passthrough_stage == \"last\" and self.features_in:\n self.feature_metadata, self.passthrough_features = self._fit_passthrough()\n if self.passthrough_features:\n X_out = self._transform_passthrough(X=X, X_out=X_out)\n\n type_map_real = get_type_map_real(X_out)\n self.features_out = list(X_out.columns)\n self.feature_metadata_real = FeatureMetadata(\n type_map_raw=type_map_real, type_group_map_special=self.feature_metadata.get_type_group_map_raw()\n )\n\n self._post_fit_cleanup()\n if self.reset_index:\n X_out.index = X_index\n self._is_fit = True\n end_time = time.time()\n self.fit_time = end_time - start_time\n if self.verbosity >= 3:\n self.print_feature_metadata_info(log_level=20)\n self.print_generator_info(log_level=20)\n elif self.verbosity == 2:\n self.print_feature_metadata_info(log_level=15)\n self.print_generator_info(log_level=15)\n return X_out\n\n def _fit_passthrough(self) -> tuple[FeatureMetadata, list[str]]:\n if self.passthrough_types:\n get_features_kwargs = self.passthrough_types\n else:\n get_features_kwargs = dict()\n features_out_set = set(self.feature_metadata.get_features())\n passthrough_features_unsorted = set(self.feature_metadata_in.get_features(**get_features_kwargs))\n passthrough_features = [f for f in self.features_in if f in passthrough_features_unsorted]\n passthrough_features = [f for f in passthrough_features if f not in features_out_set]\n if passthrough_features:\n passthrough_metadata = self.feature_metadata_in.keep_features(features=passthrough_features)\n feature_metadata = self._merge_feature_metadata(\n feature_metadata_lst=[\n passthrough_metadata,\n self.feature_metadata,\n ],\n )\n else:\n feature_metadata = self.feature_metadata\n return feature_metadata, passthrough_features\n\n def _transform_passthrough(self, X: DataFrame, X_out: DataFrame) -> DataFrame:\n return self._concat_features(feature_df_list=[X[self.passthrough_features], X_out], index=X.index)\n\n def transform(self, X: DataFrame) -> DataFrame:\n \"\"\"\n Transforms input data into the output data format.\n Will raise an AssertionError if called before the generator has been fit using fit or fit_transform methods.\n\n Parameters\n ----------\n X : DataFrame\n Input data to be transformed by the generator.\n Input data must contain all features in features_in, and should have the same dtypes as in the data provided to fit.\n Extra columns present in X that are not in features_in will be ignored and not affect the output.\n\n Returns\n -------\n X_out : DataFrame object which is the transformed version of the input data X.\n \"\"\"\n if not self._is_fit:\n raise AssertionError(f\"{self.__class__.__name__} is not fit.\")\n if self.reset_index:\n X_index = copy.deepcopy(X.index)\n # TODO: Theoretically inplace=True avoids data copy, but can lead to altering of original DataFrame outside of method context.\n X = X.reset_index(drop=True)\n else:\n X_index = None\n if self.column_names_as_str:\n X.columns = X.columns.astype(str) # Ensure all column names are strings\n try:\n if list(X.columns) != self.features_in:\n # It comes at a cost when making a copy of the DataFrame,\n # therefore, try avoid copying by checking the expected features first.\n X = X[self.features_in]\n except KeyError:\n missing_cols = []\n for col in self.features_in:\n if col not in X.columns:\n missing_cols.append(col)\n raise KeyError(\n f\"{len(missing_cols)} required columns are missing from the provided dataset to transform using {self.__class__.__name__}. \"\n f\"{len(missing_cols)} missing columns: {missing_cols} | \"\n f\"{len(list(X.columns))} available columns: {list(X.columns)}\"\n )\n if self._pre_astype_generator:\n X = self._pre_astype_generator.transform(X)\n X_out = self._transform(X)\n if self.passthrough and self.passthrough_stage == \"first\" and self.passthrough_features:\n X_out = self._transform_passthrough(X=X, X_out=X_out)\n if self._post_generators:\n X_out = self._transform_generators(X=X_out, generators=self._post_generators)\n if self.passthrough and self.passthrough_stage == \"last\" and self.passthrough_features:\n X_out = self._transform_passthrough(X=X, X_out=X_out)\n if self.reset_index:\n X_out.index = X_index\n return X_out\n\n def _fit_transform(self, X: DataFrame, y: Series, **kwargs) -> (DataFrame, dict):\n \"\"\"\n Performs the inner fit_transform logic that is non-generic (specific to the generator implementation).\n When creating a new generator class, this should be implemented.\n At the point this method is called, self.features_in and self.features_metadata_in will be set, and can be accessed and altered freely.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n This data will have already been limited to only the columns present in self.features_in.\n This data may have been altered by the fit_transform method prior to entering _fit_transform in a variety of ways, but self.features_in and\n self.features_metadata_in will correctly correspond to X at this point in the generator's fit process.\n y : Series, optional\n Input data's labels used to fit the generator. Most generators do not utilize labels.\n y.index is always equal to X.index.\n **kwargs\n Any additional arguments that a particular generator implementation could use. Received from the fit_transform method.\n\n Returns\n -------\n (X_out : DataFrame, type_group_map_special : dict)\n X_out is the transformed version of the input data X\n type_group_map_special is the type_group_map_special value of X_out's intended FeatureMetadata object.\n If special types are not relevant to the generator, this can simply be dict()\n If the input and output features are identical in name and type, it may be valid to return self.feature_metadata_in.type_group_map_special\n to maintain any pre-existing special type information.\n Refer to existing generator implementations for guidance on setting the dict output of _fit_transform.\n\n \"\"\"\n raise NotImplementedError\n\n def _transform(self, X: DataFrame) -> DataFrame:\n \"\"\"\n Performs the inner transform logic that is non-generic (specific to the generator implementation).\n When creating a new generator class, this should be implemented.\n At the point this method is called, self.features_in and self.features_metadata_in will be set, and can be accessed freely.\n\n Parameters\n ----------\n X : DataFrame\n Input data to be transformed by the generator.\n This data will have already been limited to only the columns present in self.features_in.\n This data may have been altered by the transform method prior to entering _transform in a variety of ways, but self.features_in and\n self.features_metadata_in will correctly correspond to X at this point in the generator's transform process.\n\n Returns\n -------\n X_out : DataFrame object which is the transformed version of the input data X.\n \"\"\"\n raise NotImplementedError\n\n def _infer_features_in_full(self, X: DataFrame, feature_metadata_in: FeatureMetadata = None):\n \"\"\"\n Infers all input related feature information of X.\n This can be extended when additional input information is desired beyond feature_metadata_in and features_in.\n For example, AsTypeFeatureGenerator extends this method to also compute the exact raw feature types of the input for later use.\n After this method returns, self.features_in and self.feature_metadata_in will be set to proper values.\n This method is called by fit_transform prior to calling _fit_transform.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n feature_metadata_in : FeatureMetadata, optional\n If passed, then self.feature_metadata_in will be set to feature_metadata_in assuming self.feature_metadata_in was None prior.\n If both are None, then self.feature_metadata_in is inferred through _infer_feature_metadata_in(X)\n \"\"\"\n if self.feature_metadata_in is None:\n self.feature_metadata_in = feature_metadata_in\n elif feature_metadata_in is not None:\n self._log(\n 30,\n \"\\tWarning: feature_metadata_in passed as input to fit_transform, but self.feature_metadata_in was already set. \"\n \"Ignoring feature_metadata_in.\",\n )\n if self.feature_metadata_in is None:\n self._log(\n 20,\n \"\\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special \"\n \"dtypes of the features.\",\n )\n self.feature_metadata_in = self._infer_feature_metadata_in(X=X)\n if self.features_in is None:\n self.features_in = self._infer_features_in(X=X)\n self.features_in = [feature for feature in self.features_in if feature in X.columns]\n self.feature_metadata_in = self.feature_metadata_in.keep_features(features=self.features_in)\n\n # TODO: Find way to increase flexibility here, possibly through init args\n def _infer_features_in(self, X: DataFrame) -> list:\n \"\"\"\n Infers the features_in of X.\n This is used if features_in was not provided by the user prior to fit.\n This can be overwritten in a new generator to use new infer logic.\n self.feature_metadata_in is available at the time this method is called.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n\n Returns\n -------\n feature_in : list of str feature names inferred from X.\n \"\"\"\n return self.feature_metadata_in.get_features(**self._infer_features_in_args)\n\n # TODO: Use code from problem type detection for column types. Ints/Floats could be Categorical through this method. Maybe try both?\n @staticmethod\n def _infer_feature_metadata_in(X: DataFrame) -> FeatureMetadata:\n \"\"\"\n Infers the feature_metadata_in of X.\n This is used if feature_metadata_in was not provided by the user prior to fit.\n This can be overwritten in a new generator to use new infer logic, but it is preferred to keep the default logic for consistency with other generators.\n\n Parameters\n ----------\n X : DataFrame\n Input data used to fit the generator.\n\n Returns\n -------\n feature_metadata_in : FeatureMetadata object inferred from X.\n \"\"\"\n type_map_raw = get_type_map_raw(X)\n type_group_map_special = get_type_group_map_special(X)\n return FeatureMetadata(type_map_raw=type_map_raw, type_group_map_special=type_group_map_special)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n raise NotImplementedError\n\n @staticmethod\n def get_infer_features_in_args_to_drop() -> dict:\n \"\"\"Return a dict of kwargs for FeatureMetadata.get_features().\n\n This allows to specify which features should be dropped after running this\n feature generator in a feature generator group.\n\n For example, assume you are using a feature generator to apply PCA to all\n features of special type S_TEXT_EMBEDDING, then this function could return:\n {\n \"invalid_special_types\": [S_TEXT_EMBEDDING]\n }\n to inform the user that all S_TEXT_EMBEDDING features that are captured by PCA\n should be dropped from the output of the feature generator group.\n \"\"\"\n return {}\n\n def estimate_output_feature_metadata(self, feature_metadata_in: FeatureMetadata, **kwargs) -> FeatureMetadata:\n \"\"\"Return an estimated representation of the feature metadata after fit_transform.\"\"\"\n raise NotImplementedError(\"This method is not implemented for this generator.\")\n\n def _fit_generators(\n self, X, y, feature_metadata, generators: list[\"AbstractFeatureGenerator\"], **kwargs\n ) -> (DataFrame, FeatureMetadata, list):\n \"\"\"\n Fit a list of AbstractFeatureGenerator objects in sequence, with the output of generators[i] fed as the input to generators[i+1]\n This is called to sequentially fit self._post_generators generators on the output of _fit_transform to obtain the final output of the generator.\n This should not be overwritten by implementations of AbstractFeatureGenerator.\n \"\"\"\n for generator in generators:\n generator.verbosity = min(self.verbosity, generator.verbosity)\n generator.set_log_prefix(log_prefix=self.log_prefix + \"\\t\", prepend=True)\n X = generator.fit_transform(X=X, y=y, feature_metadata_in=feature_metadata, **kwargs)\n feature_metadata = generator.feature_metadata\n return X, feature_metadata, generators\n\n @staticmethod\n def _transform_generators(X, generators: list[\"AbstractFeatureGenerator\"]) -> DataFrame:\n \"\"\"\n Transforms X through a list of AbstractFeatureGenerator objects in sequence, with the output of generators[i] fed as the input to generators[i+1]\n This is called to sequentially transform self._post_generators generators on the output of _transform to obtain the final output of the generator.\n This should not be overwritten by implementations of AbstractFeatureGenerator.\n \"\"\"\n for generator in generators:\n X = generator.transform(X=X)\n return X\n\n @classmethod\n def _merge_feature_metadata(\n cls,\n feature_metadata_lst: list[FeatureMetadata],\n shared_raw_features: str = \"error\",\n ) -> FeatureMetadata:\n if not feature_metadata_lst:\n return FeatureMetadata(type_map_raw=dict())\n feature_metadata = FeatureMetadata.join_metadatas(\n feature_metadata_lst,\n shared_raw_features=shared_raw_features,\n )\n return feature_metadata\n\n @classmethod\n def _concat_features(cls, feature_df_list: list[DataFrame], index: pd.Index) -> DataFrame:\n if not feature_df_list:\n X = DataFrame(index=index)\n elif len(feature_df_list) == 1:\n X = feature_df_list[0]\n else:\n X = pd.concat(feature_df_list, axis=1, ignore_index=False, copy=False)\n return X\n\n def _keep_features_in(self, features: list):\n features = set(features)\n features_to_remove = [f for f in self.features_in if f not in features]\n return self._remove_features_in(features=features_to_remove)\n\n def _remove_features_in(self, features: list):\n \"\"\"\n Removes features from all relevant objects which represent the content of the input data or how the input features are used.\n For example, DropDuplicatesFeatureGenerator calls this method during _fit_transform with the list of duplicate features.\n This allows DropDuplicatesFeatureGenerator's _transform method to simply return X, as the duplicate features are already dropped in the transform\n method due to not being in self.features_in.\n\n Parameters\n ----------\n features : list of str\n List of feature names to remove from the expected input.\n \"\"\"\n if features:\n if self._feature_metadata_before_post:\n feature_links_chain = self.get_feature_links_chain()\n for feature in features:\n feature_links_chain[0].pop(feature)\n features_to_keep = set()\n for features_out in feature_links_chain[0].values():\n features_to_keep = features_to_keep.union(features_out)\n self._feature_metadata_before_post = self._feature_metadata_before_post.keep_features(features_to_keep)\n\n self.feature_metadata_in = self.feature_metadata_in.remove_features(features=features)\n features_in_new = set(self.feature_metadata_in.get_features())\n self.features_in = [f for f in self.features_in if f in features_in_new]\n if self._pre_astype_generator:\n self._pre_astype_generator._remove_features_out(features)\n\n # TODO: Ensure arbitrary feature removal does not result in inconsistencies (add unit test)\n def _remove_features_out(self, features: list):\n \"\"\"\n Removes features from the output data.\n This is used for cleaning complex pipelines of unnecessary operations after fitting a sequence of generators.\n Implementations of AbstractFeatureGenerator should not need to alter this method.\n\n Parameters\n ----------\n features : list of str\n List of feature names to remove from the output of self.transform().\n \"\"\"\n feature_links_chain = self.get_feature_links_chain()\n if features:\n self.feature_metadata = self.feature_metadata.remove_features(features=features)\n self.feature_metadata_real = self.feature_metadata_real.remove_features(features=features)\n self.features_out = self.feature_metadata.get_features()\n feature_links_chain[-1] = {\n feature_in: [feature_out for feature_out in features_out if feature_out not in features]\n for feature_in, features_out in feature_links_chain[-1].items()\n }\n self._remove_unused_features(feature_links_chain=feature_links_chain)\n\n def _remove_unused_features(self, feature_links_chain):\n unused_features = self._get_unused_features(feature_links_chain=feature_links_chain)\n self._remove_features_in(features=unused_features[0])\n for i, generator in enumerate(self._post_generators):\n for feature in unused_features[i + 1]:\n if feature in feature_links_chain[i + 1]:\n feature_links_chain[i + 1].pop(feature)\n generated_features = set()\n for feature_in in feature_links_chain[i + 1]:\n generated_features = generated_features.union(feature_links_chain[i + 1][feature_in])\n features_out_to_remove = [\n feature for feature in generator.features_out if feature not in generated_features\n ]\n generator._remove_features_out(features_out_to_remove)\n\n def _rename_features_in(self, column_rename_map: dict):\n if self.feature_metadata_in is not None:\n self.feature_metadata_in = self.feature_metadata_in.rename_features(column_rename_map)\n if self.features_in is not None:\n self.features_in = [column_rename_map.get(col, col) for col in self.features_in]\n\n def _pre_fit_validate(self, X: DataFrame, y: Series, **kwargs):\n \"\"\"\n Any data validation checks prior to fitting the data should be done here.\n \"\"\"\n if y is not None and isinstance(y, Series):\n if list(y.index) != list(X.index):\n raise AssertionError(\n f\"y.index and X.index must be equal when fitting {self.__class__.__name__}, but they differ.\"\n )\n\n def _post_fit_cleanup(self):\n \"\"\"\n Any cleanup operations after all metadata objects have been constructed, but prior to feature renaming, should be done here.\n This includes removing keys from internal lists and dictionaries of features which have been removed, and deletion of any temp variables.\n \"\"\"\n pass\n\n def _ensure_no_duplicate_column_names(self, X: DataFrame):\n if len(X.columns) != len(set(X.columns)):\n count_dict = defaultdict(int)\n invalid_columns = []\n for column in list(X.columns):\n count_dict[column] += 1\n for column in count_dict:\n if count_dict[column] > 1:\n invalid_columns.append(column)\n raise AssertionError(\n f\"Columns appear multiple times in X. Columns must be unique. Invalid columns: {invalid_columns}\"\n )\n\n # TODO: Move to a generator\n @staticmethod\n def _get_useless_features(X: DataFrame, columns_to_check: List[str] = None) -> list:\n useless_features = []\n if columns_to_check is None:\n columns_to_check = list(X.columns)\n for column in columns_to_check:\n if is_useless_feature(X[column]):\n useless_features.append(column)\n return useless_features\n\n # TODO: Consider adding _log and verbosity methods to mixin\n def set_log_prefix(self, log_prefix, prepend=False):\n if prepend:\n self.log_prefix = log_prefix + self.log_prefix\n else:\n self.log_prefix = log_prefix\n\n def set_verbosity(self, verbosity: int):\n self.verbosity = verbosity\n\n def _log(self, level, msg, log_prefix=None, verb_min=None):\n if self.verbosity == 0:\n return\n if verb_min is None or self.verbosity >= verb_min:\n if log_prefix is None:\n log_prefix = self.log_prefix\n logger.log(level, f\"{log_prefix}{msg}\")\n\n def is_fit(self):\n return self._is_fit\n\n # TODO: Handle cases where self.features_in or self.feature_metadata_in was already set at init.\n def is_valid_metadata_in(self, feature_metadata_in: FeatureMetadata):\n \"\"\"\n True if input data with feature metadata of feature_metadata_in could result in non-empty output.\n This is dictated by `feature_metadata_in.get_features(**self._infer_features_in_args)` not being empty.\n False if the features represented in feature_metadata_in do not contain any usable types for the generator.\n For example, if only numeric features are passed as input to TextSpecialFeatureGenerator which requires text input features, this will return False.\n However, if both numeric and text features are passed, this will return True since the text features would be valid input (the numeric features\n would simply be dropped).\n \"\"\"\n features_in = feature_metadata_in.get_features(**self._infer_features_in_args)\n if features_in:\n return True\n else:\n return False\n\n def get_feature_links(self) -> Dict[str, List[str]]:\n \"\"\"Returns feature links including all pre and post generators.\"\"\"\n return self._get_feature_links_from_chain(self.get_feature_links_chain())\n\n def _get_feature_links(self, features_in: List[str], features_out: List[str]) -> Dict[str, List[str]]:\n \"\"\"Returns feature links ignoring all pre and post generators.\"\"\"\n feature_links = {}\n if self.get_tags().get(\"feature_interactions\", True):\n for feature_in in features_in:\n feature_links[feature_in] = features_out\n else:\n for feat_old, feat_new in zip(features_in, features_out):\n feature_links[feat_old] = feature_links.get(feat_old, []) + [feat_new]\n return feature_links\n\n def get_feature_links_chain(self) -> List[Dict[str, List[str]]]:\n \"\"\"Get the feature dependence chain between this generator and all of its post generators.\"\"\"\n features_out_internal = self._feature_metadata_before_post.get_features()\n\n generators = [self] + self._post_generators\n features_in_list = [self.features_in] + [generator.features_in for generator in self._post_generators]\n features_out_list = [features_out_internal] + [generator.features_out for generator in self._post_generators]\n\n feature_links_chain = []\n for i in range(len(features_in_list)):\n generator = generators[i]\n features_in = features_in_list[i]\n features_out = features_out_list[i]\n feature_chain = generator._get_feature_links(features_in=features_in, features_out=features_out)\n feature_links_chain.append(feature_chain)\n return feature_links_chain\n\n @staticmethod\n def _get_feature_links_from_chain(feature_links_chain: List[Dict[str, List[str]]]) -> Dict[str, List[str]]:\n \"\"\"Get the final input and output feature links by travelling the feature link chain\"\"\"\n features_out = []\n for val in feature_links_chain[-1].values():\n if val not in features_out:\n features_out.append(val)\n features_in = list(feature_links_chain[0].keys())\n feature_links = feature_links_chain[0]\n for i in range(1, len(feature_links_chain)):\n feature_links_new = {}\n for feature in features_in:\n feature_links_new[feature] = set()\n for feature_out in feature_links[feature]:\n feature_links_new[feature] = feature_links_new[feature].union(\n feature_links_chain[i].get(feature_out, [])\n )\n feature_links_new[feature] = list(feature_links_new[feature])\n feature_links = feature_links_new\n return feature_links\n\n def _get_unused_features(self, feature_links_chain: List[Dict[str, List[str]]]):\n features_in_list = [self.features_in]\n if self._post_generators:\n for i in range(len(self._post_generators)):\n if i == 0:\n features_in = self._feature_metadata_before_post.get_features()\n else:\n features_in = self._post_generators[i - 1].features_out\n features_in_list.append(features_in)\n return self._get_unused_features_generic(\n feature_links_chain=feature_links_chain, features_in_list=features_in_list\n )\n\n # TODO: Unit test this\n @staticmethod\n def _get_unused_features_generic(\n feature_links_chain: List[Dict[str, List[str]]], features_in_list: List[List[str]]\n ) -> List[List[str]]:\n unused_features = []\n unused_features_by_stage = []\n for i, chain in enumerate(reversed(feature_links_chain)):\n stage = len(feature_links_chain) - i\n used_features = set()\n for key in chain.keys():\n new_val = [val for val in chain[key] if val not in unused_features]\n if new_val:\n used_features.add(key)\n features_in = features_in_list[stage - 1]\n unused_features = []\n for feature in features_in:\n if feature not in used_features:\n unused_features.append(feature)\n unused_features_by_stage.append(unused_features)\n unused_features_by_stage = list(reversed(unused_features_by_stage))\n return unused_features_by_stage\n\n def print_generator_info(self, log_level: int = 20):\n \"\"\"\n Outputs detailed logs of the generator, such as the fit runtime.\n\n Parameters\n ----------\n log_level : int, default 20\n Log level of the logging statements.\n \"\"\"\n if self.fit_time:\n self._log(log_level, f\"\\t{round(self.fit_time, 1)}s = Fit runtime\")\n self._log(\n log_level,\n f\"\\t{len(self.features_in)} features in original data used to generate {len(self.features_out)} features in processed data.\",\n )\n\n def print_feature_metadata_info(self, log_level: int = 20):\n \"\"\"\n Outputs detailed logs of a fit feature generator including the input and output FeatureMetadata objects' feature types.\n\n Parameters\n ----------\n log_level : int, default 20\n Log level of the logging statements.\n \"\"\"\n self._log(log_level, \"\\tTypes of features in original data (raw dtype, special dtypes):\")\n self.feature_metadata_in.print_feature_metadata_full(self.log_prefix + \"\\t\\t\", log_level=log_level)\n if self.feature_metadata_real:\n self._log(log_level - 5, \"\\tTypes of features in processed data (exact raw dtype, raw dtype):\")\n self.feature_metadata_real.print_feature_metadata_full(\n self.log_prefix + \"\\t\\t\", print_only_one_special=True, log_level=log_level - 5\n )\n self._log(log_level, \"\\tTypes of features in processed data (raw dtype, special dtypes):\")\n self.feature_metadata.print_feature_metadata_full(self.log_prefix + \"\\t\\t\", log_level=log_level)\n\n def save(self, path: str):\n save_pkl.save(path=path, object=self)\n\n def _more_tags(self) -> dict:\n \"\"\"\n Special values to enable advanced functionality.\n\n Tags\n ----\n feature_interactions : bool, default True\n If True, then treat all features_out as if they depend on all features_in.\n If False, then treat each features_out as if it was generated by a 1:1 mapping (no feature interactions).\n This enables advanced functionality regarding automated feature pruning, but is only valid for generators which only transform each feature\n and do not perform interactions.\n allow_post_generators : bool, default True\n If False, will raise an AssertionError if post_generators is specified during init.\n This is reserved for very simple generators where including post_generators would not be sensible, such as in RenameFeatureGenerator.\n \"\"\"\n return {}\n\n def get_tags(self) -> dict:\n \"\"\"Gets the tags for this generator.\"\"\"\n collected_tags = {}\n for base_class in reversed(inspect.getmro(self.__class__)):\n if hasattr(base_class, \"_more_tags\"):\n # need the if because mixins might not have _more_tags\n # but might do redundant work in estimators\n # (i.e. calling more tags on BaseEstimator multiple times)\n more_tags = base_class._more_tags(self)\n collected_tags.update(more_tags)\n return collected_tags\n\n\n# FIXME: this logic still needs more work to become general purpose.\n# - Needs to make it work for multiple feature generator groups\n# - Need to support for all possible feature generators\ndef estimate_feature_metadata_after_generators(\n *, feature_generators: list[list[AbstractFeatureGenerator]] | None, feature_metadata_in: FeatureMetadata, **kwargs\n) -> FeatureMetadata:\n \"\"\"Estimate the feature metadata after applying a set of feature generators.\"\"\"\n feature_metadata = copy.deepcopy(feature_metadata_in)\n if feature_generators is not None:\n for fg_group in feature_generators:\n feature_metadatas = [\n fg.estimate_output_feature_metadata(feature_metadata_in=feature_metadata, **kwargs) for fg in fg_group\n ]\n feature_metadata = FeatureMetadata.join_metadatas(\n feature_metadatas,\n shared_raw_features=\"error\",\n )\n return feature_metadata\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/__init__.py",
"content": "from .preprocessor import ArithmeticFeatureGenerator\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/_numba_opt.py",
"content": "import numpy as np\nfrom numba import njit, prange\n\n\n@njit(parallel=True, fastmath=True)\ndef eval_order_fused(X_base: np.ndarray, idx_mat: np.ndarray, op_mat: np.ndarray) -> np.ndarray:\n n_rows, n_base = X_base.shape\n n_feats, order = idx_mat.shape\n\n out = np.empty((n_rows, n_feats), dtype=X_base.dtype)\n\n for i in prange(n_rows):\n for f in range(n_feats):\n idx_row = idx_mat[f] # 1D view: length = order\n ops_row = op_mat[f] # 1D view: length = order-1\n\n v = X_base[i, idx_row[0]]\n\n for k in range(1, order):\n b = X_base[i, idx_row[k]]\n op = ops_row[k - 1]\n\n if op == 0: # +\n v += b\n elif op == 1: # -\n v -= b\n elif op == 2: # *\n v *= b\n else: # /\n if b == 0.0:\n v = np.nan\n else:\n v /= b\n\n out[i, f] = v\n\n return out\n\n\n@njit(parallel=True, fastmath=False)\ndef _pearson_pairwise_nan(A: np.ndarray) -> np.ndarray:\n \"\"\"Compute pairwise Pearson correlation with NaN handling.\"\"\"\n n, p = A.shape\n out = np.empty((p, p), dtype=np.float64)\n\n # diagonals first\n for i in prange(p):\n out[i, i] = 1.0\n\n # upper triangle\n for i in prange(p):\n xi = A[:, i]\n for j in range(i + 1, p):\n x = xi\n y = A[:, j]\n\n # pairwise mask (ignore NaNs)\n m = (~np.isnan(x)) & (~np.isnan(y))\n cnt = np.sum(m)\n if cnt < 2:\n out[i, j] = np.nan\n continue\n\n xm = np.mean(x[m])\n ym = np.mean(y[m])\n dx = x[m] - xm\n dy = y[m] - ym\n num = np.sum(dx * dy)\n den = np.sqrt(np.sum(dx * dx) * np.sum(dy * dy))\n out[i, j] = num / den if den > 0 else np.nan\n\n # mirror to lower triangle\n for i in prange(p):\n for j in range(i):\n out[i, j] = out[j, i]\n\n return out\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/canonical_key.py",
"content": "from __future__ import annotations\n\nfrom typing import Dict, Hashable, List, Tuple, Union\n\nimport numpy as np\n\nfrom .operation import Operation\n\n# ----------------------------\n# Internal polynomial utilities\n# ----------------------------\n\n# A monomial is represented by a frozenset of (var, exponent) pairs.\nMonom = frozenset[tuple[str, int]]\nPoly = Dict[Monom, int] # monom -> integer coefficient\n\n\ndef _monom_from_var(var: str, exp: int = 1) -> Monom:\n if exp == 0:\n return frozenset()\n return frozenset(((var, exp),))\n\n\ndef _monom_add(m: Monom, var: str, delta: int) -> Monom:\n d = dict(m)\n d[var] = d.get(var, 0) + delta\n if d[var] == 0:\n del d[var]\n return frozenset(d.items())\n\n\ndef _poly_add(a: Poly, b: Poly, sign: int = 1) -> Poly:\n out = dict(a)\n for m, c in b.items():\n out[m] = out.get(m, 0) + sign * c\n if out[m] == 0:\n del out[m]\n return out\n\n\ndef _poly_mul(a: Poly, b: Poly) -> Poly:\n out: Poly = {}\n for m1, c1 in a.items():\n for m2, c2 in b.items():\n d = dict(m1)\n for v, e in m2:\n d[v] = d.get(v, 0) + e\n if d[v] == 0:\n del d[v]\n m = frozenset(d.items())\n out[m] = out.get(m, 0) + c1 * c2\n if out[m] == 0:\n del out[m]\n return out\n\n\ndef _is_single_monomial(poly: Poly) -> Tuple[bool, Union[None, Monom], Union[None, int]]:\n \"\"\"Return (True, monom, coeff) if poly is exactly coeff * monom with one term.\"\"\"\n if len(poly) != 1:\n return False, None, None\n ((m, c),) = poly.items()\n return True, m, c\n\n\ndef _poly_div_by_monomial(num: Poly, denom_m: Monom, denom_c: int) -> Union[Poly, None]:\n \"\"\"\n Divide polynomial num by (denom_c * denom_m) if denom_c divides coefficients.\n We only allow division by a single monomial term.\n \"\"\"\n if denom_c == 0:\n return None\n\n out: Poly = {}\n denom_exp = dict(denom_m)\n for m, c in num.items():\n if c % denom_c != 0:\n return None\n d = dict(m)\n # subtract exponents\n for v, e in denom_exp.items():\n d[v] = d.get(v, 0) - e\n if d[v] == 0:\n del d[v]\n nm = frozenset(d.items())\n nc = c // denom_c\n out[nm] = out.get(nm, 0) + nc\n if out[nm] == 0:\n del out[nm]\n return out\n\n\n# ----------------------------\n# Canonicalization\n# ----------------------------\n\n\ndef _canonical_items_from_poly(poly: Poly) -> Tuple[Tuple[int, Tuple[Tuple[str, int], ...]], ...]:\n \"\"\"Canonical hashable representation used for equality/dedup.\"\"\"\n items: List[Tuple[int, Tuple[Tuple[str, int], ...]]] = []\n for m, c in poly.items():\n if c == 0:\n continue\n vars_exps = tuple(sorted(m)) # (var, exponent)\n items.append((c, vars_exps))\n items.sort()\n return tuple(items)\n\n\ndef _structural_key(expr: Union[str, Operation]) -> Tuple:\n \"\"\"\n Fallback structural key (no algebra), still commutative for + and *.\n Useful if we hit unsupported constructs (e.g., division by a sum).\n \"\"\"\n if isinstance(expr, Operation):\n op = expr.op\n lk = _structural_key(expr.left)\n rk = _structural_key(expr.right)\n if op in (\"+\", \"*\"):\n a, b = sorted((lk, rk))\n return (op, a, b)\n return (op, lk, rk)\n return (\"var\", str(expr))\n\n\ndef _expr_to_canonical_key(expr: Union[str, Operation]) -> Tuple:\n \"\"\"\n Convert an Operation tree into a hashable canonical key that is invariant to:\n - Commutativity/associativity of * and +\n - Interaction of * and / via exponent bookkeeping\n \"\"\"\n\n def to_poly(node: Union[str, Operation]) -> Union[Poly, None]:\n if isinstance(node, Operation):\n op = node.op\n left = to_poly(node.left)\n right = to_poly(node.right)\n if left is None or right is None:\n return None\n\n if op == \"+\":\n return _poly_add(left, right, sign=1)\n if op == \"-\":\n return _poly_add(left, right, sign=-1)\n if op == \"*\":\n return _poly_mul(left, right)\n if op == \"/\":\n ok, denom_m, denom_c = _is_single_monomial(right)\n if not ok:\n return None\n # divide by denom_c * denom_m\n out = _poly_div_by_monomial(left, denom_m, denom_c)\n return out\n raise ValueError(f\"Unknown operator {op!r}\")\n else:\n # atomic variable name\n var = str(node)\n return {_monom_from_var(var, 1): 1}\n\n poly = to_poly(expr)\n if poly is None:\n # Non-monomial division or other unsupported form\n return (\"struct\", _structural_key(expr))\n\n return (\"poly\",) + (_canonical_items_from_poly(poly),)\n\n\ndef filter_canonical_expressions(exprs: list[Operation]) -> np.ndarray:\n \"\"\"\n Return indices of the *first* occurrence of each canonical expression.\n keep='first' semantics.\n \"\"\"\n seen: Dict[Hashable, int] = {}\n keep_indices: List[int] = []\n\n for i, expr in enumerate(exprs):\n key = _expr_to_canonical_key(expr)\n if key not in seen:\n seen[key] = i\n keep_indices.append(i)\n\n return np.asarray(keep_indices, dtype=int)\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/combinations.py",
"content": "from __future__ import annotations\n\nfrom itertools import combinations, product\nfrom typing import Dict, Hashable, Iterable, List, Tuple\n\nimport numpy as np\nimport pandas as pd\n\n# Precomputed translation table to strip parentheses quickly\n_PAREN_TRANS = str.maketrans(\"\", \"\", \"()\")\n\n\ndef estimate_no_higher_interaction_features(num_base_feats, num_new_feats):\n n_combinations = (num_base_feats - 2) * num_new_feats\n unique_div_add_sub = n_combinations * 0.8 * 3\n unique_prod = n_combinations * 0.4666666666666667\n return int(unique_div_add_sub + unique_prod)\n\n\ndef _expr_to_canonical_key(expr: str) -> Tuple:\n \"\"\"\n Convert an interaction expression like 'A_/_B_*_C' or '(A_*_B)_-_C'\n into a hashable canonical key that is invariant to:\n\n - Commutativity of * and +\n - Associativity of * and +\n - Interaction of * and / (e.g. (A/B)*C == (C*A)/B)\n - Superfluous parentheses introduced by the generator\n\n The expression format is assumed to be:\n var op var op var ...\n with '_' separating tokens, and optional parentheses.\n \"\"\"\n # Remove parentheses and split by the '_' tokens\n tokens = expr.translate(_PAREN_TRANS).split(\"_\")\n if not tokens:\n return ()\n\n # Each monomial is represented by:\n # frozenset({var: exponent, ...}.items()) -> coefficient (int)\n # The overall expression is a sum of such monomials.\n monoms: Dict[frozenset, int] = {frozenset({tokens[0]: 1}.items()): 1}\n\n # Process operator / variable pairs: (op, var), (op, var), ...\n # tokens = [v0, op1, v1, op2, v2, ...]\n it = iter(tokens[1:])\n for op, var in zip(it, it):\n if op in (\"*\", \"/\"):\n # Scale EVERY monomial by *var or /var\n delta = 1 if op == \"*\" else -1\n new_monoms: Dict[frozenset, int] = {}\n for exp_fs, coeff in monoms.items():\n exp_dict = dict(exp_fs)\n exp_dict[var] = exp_dict.get(var, 0) + delta\n if exp_dict[var] == 0:\n del exp_dict[var]\n\n key = frozenset(exp_dict.items())\n new_monoms[key] = new_monoms.get(key, 0) + coeff\n monoms = new_monoms\n\n elif op in (\"+\", \"-\"):\n # Add a new monomial Âą var (does NOT touch previous monoms)\n sign = 1 if op == \"+\" else -1\n key = frozenset({var: 1}.items())\n monoms[key] = monoms.get(key, 0) + sign\n\n else:\n raise ValueError(f\"Unknown operator {op!r} in expression {expr!r}\")\n\n # Build a canonical, hashable representation:\n # key = tuple(sorted( (coeff, tuple(sorted((var, exp), ...))) ))\n canonical_items: List[Tuple[int, Tuple[Tuple[str, int], ...]]] = []\n for exp_fs, coeff in monoms.items():\n if coeff == 0:\n continue\n vars_exps = tuple(sorted(exp_fs)) # (var, exponent)\n canonical_items.append((coeff, vars_exps))\n\n canonical_items.sort()\n return tuple(canonical_items)\n\n\ndef filter_canonical_expressions(exprs: Iterable[str]) -> np.ndarray:\n \"\"\"\n Given an iterable of expression strings, return indices of the\n *first* occurrence of each canonical expression.\n\n This is a drop-in replacement for the original Sympy-based\n implementation, but much faster and with no Sympy dependency.\n\n Parameters\n ----------\n exprs : Iterable[str]\n Expressions like 'A_*_B_-_C', '(A_/_B)_*_C', etc.\n\n Returns\n -------\n np.ndarray\n Indices of non-duplicate expressions (keep='first' semantics).\n \"\"\"\n seen: Dict[Hashable, int] = {}\n keep_indices: List[int] = []\n\n for i, expr in enumerate(exprs):\n key = _expr_to_canonical_key(str(expr))\n if key not in seen:\n seen[key] = i\n keep_indices.append(i)\n\n out = np.asarray(keep_indices, dtype=int)\n return out\n\n\ndef get_all_bivariate_interactions(\n X_num,\n max_feats=2000,\n interaction_types=[\"/\", \"*\", \"-\", \"+\"], # TODO: make inverse_div an own op\n random_state=None,\n):\n \"\"\"\n Generate all bivariate interactions of numeric features up to a maximum number of features.\n Parameters\n ----------\n X_num : pd.DataFrame\n Input numeric DataFrame.\n max_feats : int, default=2000\n Maximum number of interaction features to generate.\n interaction_types : list of str, default=['/', '*', '-', '+']\n Types of interactions to generate.\n random_state : int or None, default=None\n Random state for reproducibility.\n Returns\n -------\n pd.DataFrame\n DataFrame containing the generated interaction features.\n \"\"\"\n if random_state is None or isinstance(random_state, int):\n rng = np.random.default_rng(random_state)\n else:\n rng = random_state\n\n cols = X_num.columns.to_numpy()\n combs = np.array(list(combinations(cols, 2)))\n\n # Sample combinations directly instead of shuffling entire array\n combs = combs[rng.choice(len(combs), np.min([len(combs), max_feats]), replace=False)]\n\n feat0, feat1 = combs.T\n\n # Pre-cache arrays for speed\n arr = X_num.to_numpy()\n col_idx = {c: i for i, c in enumerate(cols)}\n\n # Helper to quickly extract columns\n def get_pair_arrays(f0, f1):\n return arr[:, [col_idx[c] for c in f0]], arr[:, [col_idx[c] for c in f1]]\n\n new_data = {}\n\n A, B = get_pair_arrays(feat0, feat1)\n # Avoid division by zero with masking instead of replace()\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n if \"/\" in interaction_types:\n div1 = A / np.where(B == 0, np.nan, B)\n names = [f\"{a}_/_{b}\" for a, b in zip(feat0, feat1)]\n new_data.update(dict(zip(names, div1.T)))\n if \"*\" in interaction_types:\n names = [f\"{a}_*_{b}\" for a, b in zip(feat0, feat1)]\n new_data.update(dict(zip(names, (A * B).T)))\n if \"-\" in interaction_types:\n names = [f\"{a}_-_{b}\" for a, b in zip(feat0, feat1)]\n new_data.update(dict(zip(names, (A - B).T)))\n if \"+\" in interaction_types:\n names = [f\"{a}_+_{b}\" for a, b in zip(feat0, feat1)]\n new_data.update(dict(zip(names, (A + B).T)))\n if \"/\" in interaction_types:\n div2 = B / np.where(A == 0, np.nan, A)\n names = [f\"{b}_/_{a}\" for a, b in zip(feat0, feat1)]\n new_data.update(dict(zip(names, div2.T)))\n\n return pd.DataFrame(new_data, index=X_num.index)\n\n\ndef add_higher_interaction(\n X_base,\n X_interact,\n max_feats=2000,\n interaction_types=[\n \"/\",\n \"*\",\n \"-\",\n \"+\",\n ], # FIXME: Might need to fix bug if one of these operators occurs in feature names\n random_state=None,\n):\n \"\"\"\n Generate higher-order interaction features between two sets of numeric features.\n Parameters\n ----------\n X_base : pd.DataFrame\n Base numeric features DataFrame.\n X_interact : pd.DataFrame\n Numeric features DataFrame to interact with base features.\n max_feats : int, default=2000\n Maximum number of interaction features to generate.\n interaction_types : list of str, default=['/', '*', '-', '+']\n Types of interactions to generate.\n random_state : int or None, default=None\n Random state for reproducibility.\n Returns\n -------\n pd.DataFrame\n DataFrame containing the generated higher-order interaction features.\n \"\"\"\n if random_state is None or isinstance(random_state, int):\n rng = np.random.default_rng(random_state)\n else:\n rng = random_state\n\n # Generate valid column pairs (avoid j inside i for safety)\n all_pairs = [(i, j) for i, j in product(X_interact.columns, X_base.columns) if j not in i]\n all_pairs = np.array(all_pairs)\n all_pairs = all_pairs[rng.choice(len(all_pairs), np.min([len(all_pairs), max_feats]), replace=False)]\n feat0, feat1 = all_pairs.T\n\n # Convert to numpy arrays once for speed\n X_interact_vals = X_interact[feat0].to_numpy(dtype=float)\n X_base_vals = X_base[feat1].to_numpy(dtype=float)\n\n new_data = {}\n\n # --- Division (/)\n if \"/\" in interaction_types:\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n # Forward A/B\n res_arr = X_interact_vals / np.where(X_base_vals == 0, np.nan, X_base_vals)\n name_arr = np.array([f\"{a}_/_{b}\" for a, b in zip(feat0, feat1)])\n canonical_expr_deduplicated = filter_canonical_expressions(name_arr)\n new_data.update(dict(zip(name_arr[canonical_expr_deduplicated], res_arr.T[canonical_expr_deduplicated])))\n\n # --- Multiplication (*), commutative: remove duplicates\n if \"*\" in interaction_types:\n # Identify unique sorted pairs\n unique_pairs = {}\n for a, b in zip(feat0, feat1):\n key = tuple(sorted((a, b)))\n if key not in unique_pairs:\n unique_pairs[key] = (a, b)\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n # Compute all multiplications at once\n res_list = [(X_interact[a].values * X_base[b].values).astype(float) for (a, b) in unique_pairs.values()]\n name_arr = np.array([f\"{a}_*_{b}\" for (a, b) in unique_pairs.values()])\n\n if res_list:\n res_arr = np.column_stack(res_list)\n canonical_expr_deduplicated = filter_canonical_expressions(name_arr)\n new_data.update(dict(zip(name_arr[canonical_expr_deduplicated], res_arr.T[canonical_expr_deduplicated])))\n\n # --- Addition (+), commutative: remove duplicates\n if \"+\" in interaction_types:\n unique_pairs = {}\n for a, b in zip(feat0, feat1):\n key = tuple(sorted((a, b)))\n if key not in unique_pairs:\n unique_pairs[key] = (a, b)\n\n res_list = [(X_interact[a].values + X_base[b].values).astype(float) for (a, b) in unique_pairs.values()]\n name_arr = np.array([f\"{a}_+_{b}\" for (a, b) in unique_pairs.values()])\n\n if res_list:\n res_arr = np.column_stack(res_list)\n canonical_expr_deduplicated = filter_canonical_expressions(name_arr)\n new_data.update(dict(zip(name_arr[canonical_expr_deduplicated], res_arr.T[canonical_expr_deduplicated])))\n\n # --- Subtraction (â), non-commutative\n if \"-\" in interaction_types:\n res_arr = X_interact_vals - X_base_vals\n name_arr = np.array([f\"{a}_-_{b}\" for a, b in zip(feat0, feat1)])\n canonical_expr_deduplicated = filter_canonical_expressions(name_arr)\n new_data.update(dict(zip(name_arr[canonical_expr_deduplicated], res_arr.T[canonical_expr_deduplicated])))\n\n # Build the new DataFrame once (fast)\n X_int_new = pd.DataFrame(new_data, index=X_interact.index)\n\n # Filter canonical expressions to remove duplicates\n canonical_expr_deduplicated = filter_canonical_expressions(X_int_new.columns.tolist())\n X_int_new = X_int_new.iloc[:, canonical_expr_deduplicated]\n\n # Return combined features\n return X_int_new # pd.concat([X_interact, X_int_new], axis=1)\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/combinations_lite.py",
"content": "from __future__ import annotations\n\nfrom itertools import combinations, product\n\nimport numpy as np\n\nfrom .canonical_key import filter_canonical_expressions\nfrom .operation import Operation\n\n\ndef get_all_bivariate_interactions(\n base_feats: list[str],\n max_feats=2000,\n interaction_types=[\"/\", \"*\", \"-\", \"+\"],\n random_state=None,\n) -> list[Operation]:\n \"\"\"\n Generate all bivariate interactions of numeric features up to a maximum number of features.\n Parameters\n ----------\n base_feats : list[str]\n Input feature names.\n max_feats : int, default=2000\n Maximum number of interaction features to generate.\n interaction_types : list of str, default=['/', '*', '-', '+']\n Types of interactions to generate.\n random_state : int or None, default=None\n Random state for reproducibility.\n Returns\n -------\n list[str]\n List containing the generated interaction feature names.\n \"\"\"\n if random_state is None or isinstance(random_state, int):\n rng = np.random.default_rng(random_state)\n else:\n rng = random_state\n\n cols = base_feats\n combs = np.array(list(combinations(cols, 2)))\n\n # Sample combinations directly instead of shuffling entire array\n combs = combs[rng.choice(len(combs), np.min([len(combs), max_feats]), replace=False)]\n\n feat0, feat1 = combs.T\n\n new_data = []\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n if \"/\" in interaction_types:\n names = [Operation(a, b, \"/\") for a, b in zip(feat0, feat1)]\n new_data += names\n if \"*\" in interaction_types:\n names = [Operation(a, b, \"*\") for a, b in zip(feat0, feat1)]\n new_data += names\n if \"-\" in interaction_types:\n names = [Operation(a, b, \"-\") for a, b in zip(feat0, feat1)]\n new_data += names\n if \"+\" in interaction_types:\n names = [Operation(a, b, \"+\") for a, b in zip(feat0, feat1)]\n new_data += names\n if \"/\" in interaction_types:\n names = [Operation(b, a, \"/\") for a, b in zip(feat0, feat1)]\n new_data += names\n\n return new_data\n\n\ndef add_higher_interaction(\n base_feats: list[str],\n interact_feats: list[Operation],\n max_feats=2000,\n interaction_types=(\"/\", \"*\", \"-\", \"+\"),\n random_state=None,\n) -> list[Operation]:\n \"\"\"\n Generate higher-order interaction features between two sets of numeric features.\n \"\"\"\n if random_state is None or isinstance(random_state, int):\n rng = np.random.default_rng(random_state)\n else:\n rng = random_state\n\n # Generate valid column pairs (FIXME: flawed logic retained)\n all_pairs = [(i, j) for i, j in product(interact_feats, base_feats) if j not in i.name()]\n\n if not all_pairs:\n return []\n\n all_pairs = np.array(all_pairs, dtype=object)\n all_pairs = all_pairs[rng.choice(len(all_pairs), min(len(all_pairs), max_feats), replace=False)]\n\n feat0, feat1 = all_pairs.T\n\n new_ops: list[Operation] = []\n\n # --- Division (/), non-commutative\n if \"/\" in interaction_types:\n new_ops.extend(Operation(a, b, \"/\") for a, b in zip(feat0, feat1))\n\n # --- Subtraction (-), non-commutative\n if \"-\" in interaction_types:\n new_ops.extend(Operation(a, b, \"-\") for a, b in zip(feat0, feat1))\n\n # --- Multiplication (*), commutative\n if \"*\" in interaction_types:\n seen = {}\n for a, b in zip(feat0, feat1):\n key = tuple(sorted((a.name(), str(b))))\n if key not in seen:\n seen[key] = Operation(a, b, \"*\")\n new_ops.extend(seen.values())\n\n # --- Addition (+), commutative\n if \"+\" in interaction_types:\n seen = {}\n for a, b in zip(feat0, feat1):\n key = tuple(sorted((a.name(), str(b))))\n if key not in seen:\n seen[key] = Operation(a, b, \"+\")\n new_ops.extend(seen.values())\n\n # --- Canonical deduplication (string-based, unchanged)\n mask = filter_canonical_expressions(new_ops)\n\n return [op for op, keep in zip(new_ops, mask) if keep]\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/filtering.py",
"content": "import re\n\nimport numpy as np\nimport pandas as pd\n\n\"\"\"\nFurther filtering ideas:\n- based on target_corr - if its the same, the feature is likely to contain the same info\n\"\"\"\n\n\ndef remove_mostlynan_features(X: pd.DataFrame, nan_threshold: float = 0.99) -> pd.DataFrame:\n return X.loc[:, X.isna().mean() < nan_threshold]\n\n\ndef remove_imbalanced(X: pd.DataFrame, mode_imbalance_threshold=0.99) -> pd.DataFrame:\n feature_imbalance = X.apply(mode_freq_fast)\n return X.loc[:, feature_imbalance < mode_imbalance_threshold]\n\n\ndef mode_freq_fast(s: pd.Series) -> float:\n vals, counts = np.unique(s.to_numpy(), return_counts=True)\n return counts.max() / len(s)\n\n\ndef remove_same_range_features(X: pd.DataFrame, x: pd.Series) -> float:\n # TODO: Change function name to be more descriptive of what actually happens\n col = x.name\n feature_names = [f for f in re.split(r\"_(\\*|/|\\+|\\-)_\", col) if f not in {\"*\", \"/\", \"+\", \"-\"}]\n\n return X[feature_names].corrwith(x, method=\"spearman\").max()\n\n\ndef basic_filter(\n X: pd.DataFrame,\n # y: pd.Series,\n min_cardinality: int = 3,\n candidate_cols: list = None,\n use_polars: bool = False,\n data_cleaning: bool = True,\n nan_threshold: float = 0.99,\n mode_imbalance_threshold: float = 0.99,\n) -> pd.DataFrame:\n \"\"\"\n Basic filtering of base and generated features:\n - Remove features with cardinality < min_cardinality\n - Optionally remove features that are constant or mostly NaN\n Parameters\n ----------\n X : pd.DataFrame\n Input features to filter\n min_cardinality : int, default=3\n Minimum cardinality required to keep a feature\n candidate_cols : list, default=None\n If provided, only these columns will be considered for filtering\n use_polars : bool, default=False\n Whether to use Polars for filtering (faster for large datasets)\n remove_constant_mostlynan : bool, default=True\n Whether to remove features that are nearly constant or mostly NaN\n Returns\n -------\n pd.DataFrame\n Filtered features\n \"\"\"\n # Filter by minimum cardinality\n X = X.loc[:, X.nunique() >= min_cardinality]\n\n if X.empty:\n return X\n\n # if predetermined candidate columns are given, use them\n if candidate_cols is not None:\n X = X[candidate_cols]\n\n if data_cleaning:\n X = remove_mostlynan_features(X, nan_threshold=nan_threshold)\n X = remove_imbalanced(X, mode_imbalance_threshold=mode_imbalance_threshold)\n\n return X\n\n\ndef fast_spearman(X: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Compute Spearman correlation matrix using rank transformation and pairwise Pearson correlation.\"\"\"\n # 1) Rank in pandas to match tie handling exactly\n R = X.rank(method=\"average\", na_option=\"keep\")\n A = R.to_numpy(float)\n # A = R.to_numpy(dtype=np.float32) # Could be float32 for less memory, but float64 is more accurate\n from ._numba_opt import _pearson_pairwise_nan\n\n C = _pearson_pairwise_nan(A) # numba-accelerated pairwise corr\n return pd.DataFrame(C, index=X.columns, columns=X.columns)\n\n\ndef drop_high_corr(corr: pd.DataFrame, corr_threshold: float = 0.9) -> list:\n \"\"\"Identify columns to drop based on a correlation threshold.\"\"\"\n ac = corr.abs().copy()\n np.fill_diagonal(ac.values, 0.0)\n upper = ac.where(np.triu(np.ones(ac.shape), k=1).astype(bool))\n return upper.gt(corr_threshold).any(axis=0)[lambda s: s].index.tolist()\n\n\ndef filter_by_spearman(X: pd.DataFrame, corr_threshold: float = 0.95) -> list:\n \"\"\"Filtering of a base feature set based on correlation.\"\"\"\n spearman_corr = fast_spearman(X)\n np.fill_diagonal(spearman_corr.values, 0)\n drop_cols = drop_high_corr(spearman_corr, corr_threshold=corr_threshold)\n return [col for col in X.columns if col not in drop_cols]\n\n\ndef cross_spearman(df1, df2):\n \"\"\"\n Compute Spearman correlations between all features in df1 vs df2.\n Missing values allowed. Pairwise-complete observations used.\n \"\"\"\n # ---- 1) Rank-transform dataframes ----\n r1 = df1.rank(axis=0, na_option=\"keep\")\n r2 = df2.rank(axis=0, na_option=\"keep\")\n\n r1 = r1.to_numpy(float)\n r2 = r2.to_numpy(float)\n\n # ---- 2) Masks for non-missing ranks ----\n mask1 = ~np.isnan(r1)\n mask2 = ~np.isnan(r2)\n\n # ---- 3) Center ranks but keep NaN in place ----\n # Need nanmean to avoid biasing mean when NaNs present\n r1_centered = r1 - np.nanmean(r1, axis=0)\n r2_centered = r2 - np.nanmean(r2, axis=0)\n\n # Replace NaNs with zero so dot products ignore them\n r1c = np.where(mask1, r1_centered, 0.0)\n r2c = np.where(mask2, r2_centered, 0.0)\n\n # ---- 4) Compute pairwise sample sizes (valid rows per pair) ----\n # Broadcast masks: (samples à n1 à n2)\n n_valid = mask1[:, :, None] & mask2[:, None, :]\n n_valid = n_valid.sum(axis=0) # shape (n1, n2)\n\n # ---- 5) Compute covariance using dot product but divide by valid count-1 ----\n cov = (r1c.T @ r2c) / (n_valid - 1)\n\n # ---- 6) Compute std dev for each feature using valid data ----\n # std = sqrt( sum((x-mean)^2) / (n_valid-1) )\n ss1 = (r1c**2).sum(axis=0) # sum of squares for each column\n ss2 = (r2c**2).sum(axis=0)\n\n std1 = np.sqrt(ss1 / (mask1.sum(axis=0) - 1)) # shape (n1,)\n std2 = np.sqrt(ss2 / (mask2.sum(axis=0) - 1))\n\n # Outer-normalize to get Spearman correlations\n corr = cov / np.outer(std1, std2)\n\n # Where n_valid < 2 â correlation is undefined\n corr[n_valid < 2] = np.nan\n\n return pd.DataFrame(corr, index=df1.columns, columns=df2.columns)\n\n\ndef filter_by_cross_correlation(X_base: pd.DataFrame, X_new: pd.DataFrame, corr_threshold: float = 0.95) -> list:\n \"\"\"Filter new features by cross-correlation with base features.\"\"\"\n cross_corr = cross_spearman(X_base, X_new)\n to_drop = set()\n for new_col in X_new.columns:\n if (cross_corr[new_col].abs() > corr_threshold).any():\n to_drop.add(new_col)\n to_keep = [col for col in X_new.columns if col not in to_drop]\n novelty_scores = 1 - cross_corr[to_keep].abs().max() # Higher --> more novel\n return to_keep, novelty_scores\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/memory.py",
"content": "from typing import Literal\n\nimport numpy as np\nimport pandas as pd\n\n\ndef dataset_magnitude(X_in: pd.DataFrame, method: Literal[\"rms\", \"max\", \"median\"] = \"rms\") -> float:\n \"\"\"\n Compute a single magnitude statistic over all numeric values in the dataset,\n ignoring NaNs.\n \"\"\"\n X = X_in.copy()\n Xv = X.select_dtypes(include=[np.number]).to_numpy(dtype=np.float64)\n if Xv.size == 0:\n return 1.0\n if method == \"rms\":\n return float(np.sqrt(np.nanmean(Xv**2)))\n elif method == \"max\":\n return float(np.nanmax(np.abs(Xv)))\n elif method == \"median\":\n return float(np.nanmedian(np.abs(Xv)))\n else:\n raise ValueError(\"method must be 'rms', 'max', or 'median'\")\n\n\ndef global_scale_preserve_ops(\n X_in: pd.DataFrame,\n target: Literal[\"rms\", \"max\", \"median\"] = \"rms\",\n target_value: float = 1.0,\n out_dtype: np.dtype = np.float64,\n) -> tuple[pd.DataFrame, float, float]:\n \"\"\"\n Scale the entire dataset by a single positive constant a, preserving\n +, -, *, / correlations exactly (NaNs preserved). Then cast to out_dtype.\n \"\"\"\n X = X_in.copy()\n M = dataset_magnitude(X, method=target) # NaN-aware\n a = 1.0 if not np.isfinite(M) or M == 0.0 else (target_value / M)\n # Apply to all numeric columns only; NaNs remain NaN\n Xs = X.copy()\n num_cols = Xs.select_dtypes(include=[np.number]).columns\n Xs[num_cols] = (Xs[num_cols].to_numpy(dtype=np.float64) * a).astype(out_dtype)\n return Xs, a, M\n\n\ndef minimize_numeric_dtypes(X):\n \"\"\"Safely downcast all columns in X to minimal possible dtypes without overflow.\"\"\"\n X_opt = X.copy()\n\n int_types = [np.int8, np.int16, np.int32, np.int64]\n float_types = [np.float16, np.float32, np.float64]\n\n for col in X_opt.columns:\n s = X_opt[col]\n if pd.api.types.is_integer_dtype(s):\n cmin, cmax = s.min(), s.max()\n for t in int_types:\n if np.iinfo(t).min <= cmin and cmax <= np.iinfo(t).max:\n X_opt[col] = s.astype(t)\n break\n\n elif pd.api.types.is_float_dtype(s):\n cmin, cmax = s.min(skipna=True), s.max(skipna=True)\n for t in float_types:\n finfo = np.finfo(t)\n if np.isfinite(cmin) and np.isfinite(cmax):\n if finfo.min <= cmin and cmax <= finfo.max:\n X_opt[col] = s.astype(t)\n break\n else:\n X_opt[col] = s.astype(np.float32)\n break\n\n return X_opt\n\n\ndef reduce_memory_usage(X_in: pd.DataFrame, verbose: bool = True, rescale: bool = True) -> pd.DataFrame:\n \"\"\"Reduce memory usage of DataFrame by downcasting numeric types.\n Parameters\n ----------\n X_in : pd.DataFrame\n Input DataFrame.\n verbose : bool, default=True\n Whether to print memory usage before and after optimization.\n rescale : bool, default=True\n Whether to rescale numeric values before downcasting to preserve numeric stability.\n Returns\n -------\n pd.DataFrame\n Optimized DataFrame with reduced memory usage.\n \"\"\"\n X_out = X_in.copy()\n if verbose:\n print(f\"Memory before: {X_in.memory_usage(deep=True).sum() / 1e6:.2f} MB\")\n if rescale:\n X_out, a, M = global_scale_preserve_ops(X_out, target=\"rms\", target_value=1000, out_dtype=np.float64)\n X_out = minimize_numeric_dtypes(X_out)\n if verbose:\n print(f\"Memory after: {X_out.memory_usage(deep=True).sum() / 1e6:.2f} MB\")\n return X_out\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/operation.py",
"content": "from __future__ import annotations\n\n\nclass Operation:\n def __init__(self, left, right, op: str):\n self.left = left\n self.right = right\n self.op = op\n\n def _render(self, x):\n if isinstance(x, Operation):\n return f\"({x.name()})\"\n return str(x)\n\n def name(self) -> str:\n left = self._render(self.left)\n right = self._render(self.right)\n return f\"{left}{self.op}{right}\"\n\n def __str__(self):\n return self.name()\n"
},
{
"path": "features/src/autogluon/features/generators/arithmetic/preprocessor.py",
"content": "from __future__ import annotations\n\nimport operator\nimport warnings\nfrom contextlib import contextmanager\nfrom math import comb\nfrom time import perf_counter\nfrom typing import Literal, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame, Series\nfrom pandas.api.types import is_numeric_dtype\n\nfrom ..abstract import AbstractFeatureGenerator\nfrom ..cat_as_num import CatAsNumFeatureGenerator\nfrom .combinations import (\n add_higher_interaction,\n estimate_no_higher_interaction_features,\n get_all_bivariate_interactions,\n)\nfrom .combinations_lite import (\n add_higher_interaction as add_higher_interaction_lite,\n)\nfrom .combinations_lite import (\n get_all_bivariate_interactions as get_all_bivariate_interactions_lite,\n)\nfrom .filtering import basic_filter, filter_by_cross_correlation, filter_by_spearman\nfrom .memory import reduce_memory_usage\nfrom .operation import Operation\n\n\nclass TimerLog:\n # TODO: Mainly used for debugging and tracking runtimes during development. Not needed for preprocessing logic. Better remove?\n def __init__(self):\n self.times = {}\n\n @contextmanager\n def block(self, name: str):\n t0 = perf_counter()\n try:\n yield\n finally:\n dt = perf_counter() - t0\n self.times[name] = self.times.get(name, 0) + dt\n\n def summary(self, verbose: bool = False) -> dict:\n if verbose:\n print(\"\\n--- Timing Summary (in order) ---\")\n for name, total in self.times.items():\n print(f\"{name:<20} {total:.3f}s\")\n return dict(self.times)\n\n\n# Compact op codes for numba\nOP_CODE = {\n \"+\": 0,\n \"-\": 1,\n \"*\": 2,\n \"/\": 3,\n}\n\n\nclass ArithmeticFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Converts category features to one-hot boolean features by mapping to the category codes.\n\n Parameters\n ----------\n selection_method : str, default = 'random'\n Method to select features for interaction generation. Options are 'spearman' or 'random'.\n max_order : int, default = 3\n The maximum number of features considered to use for arithmetic interaction feature generation.\n max_base_feats : int, default = 150\n The maximum number of (randomly selected) base features to consider for arithmetic interaction generation.\n max_new_feats : int, default = 2000\n The maximum number of new arithmetic interaction features to generate.\n cat_as_num: bool, default = False\n Whether to transform categorical features to numeric and use them as base features in arithmetic interaction generation.\n min_cardinality : int, default = 3\n Minimum cardinality for a feature to be considered as a base feature in arithmetic interaction generation.\n interaction_types : list of str, default = ['/', '*', '-', '+']\n List of arithmetic interaction types to consider.\n data_cleaning : bool, default = True\n Whether to remove features that are nearly constant or mostly NaN after generation.\n nan_threshold : float, default = 0.95\n Threshold for removing features with too many NaN values.\n mode_imbalance_threshold : float, default = 0.95\n Threshold for mode imbalance to remove features.\n subsample : int, default = 100000\n Number of samples to use when selecting new features, used to improve efficiency.\n reduce_memory : bool, default = True\n Whether to reduce memory usage during feature generation. Might affect precision.\n rescale_avoid_overflow : bool, default = True\n Whether to normalize all features to the same constant to deal with large integer values, mainly to avoid overflow during arithmetic operations.\n corr_threshold : float, default = 0.95\n Correlation threshold for feature selection with selection_method=='spearman'.\n use_cross_corr : bool, default = False\n Whether to use cross-correlation with the base features for feature selection when selection_method=='spearman'.\n cross_corr_n_block_size : int, default = 5000\n Block size for cross-correlation computation. Used to limit memory usage.\n max_accept_for_pairwise : int, default = 10000\n Maximum number of accepted features for generating pairwise interactions when selection_method=='spearman'. At most max_accept_for_pairwise feature are generated and then filtered to max_new_feats using spearman correlation.\n verbose : bool, default = False\n Whether to print logs.\n\n \"\"\"\n\n def __init__(\n self,\n selection_method: Literal[\"spearman\", \"random\"] = \"random\",\n max_order: int = 3,\n max_base_feats: int = 150, # TODO: Need to implement a better heuristic than choosing randomly\n max_new_feats: int = 2000, # FIXME: 2000 originally\n cat_as_num: bool = False,\n min_cardinality: int = 3,\n interaction_types: list[str] = [\"/\", \"*\", \"-\", \"+\"],\n data_cleaning: bool = True,\n nan_threshold: float = 0.95,\n mode_imbalance_threshold: float = 0.9,\n subsample: int = 100000, # TODO: Need to implement\n reduce_memory: bool = False,\n rescale_avoid_overflow: bool = True,\n corr_threshold: float = 0.95,\n use_cross_corr: bool = False,\n cross_corr_n_block_size: int = 5000,\n max_accept_for_pairwise: int = 10000,\n inference_mode: Literal[\"dag\"] = \"dag\",\n inner_dtype=np.float32,\n out_dtype=np.float32,\n verbose: bool = False,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self.max_order = max_order\n self.cat_as_num = cat_as_num\n self.min_cardinality = min_cardinality\n self.max_base_feats = max_base_feats\n self.max_new_feats = max_new_feats\n self.selection_method = selection_method\n self.subsample = subsample\n self.reduce_memory = reduce_memory\n self.rescale_avoid_overflow = rescale_avoid_overflow\n self.corr_threshold = corr_threshold\n self.use_cross_corr = use_cross_corr\n self.cross_corr_n_block_size = cross_corr_n_block_size\n self.max_accept_for_pairwise = max_accept_for_pairwise\n self.inference_mode = inference_mode\n self.verbose = verbose\n self.inner_dtype = inner_dtype\n self.out_dtype = out_dtype\n self.interaction_types = interaction_types\n self.data_cleaning = data_cleaning\n self.nan_threshold = nan_threshold\n self.mode_imbalance_threshold = mode_imbalance_threshold\n\n for i in self.interaction_types:\n if i not in OP_CODE:\n raise ValueError(f\"Unsupported interaction type: {i}\")\n\n self.rng = np.random.default_rng(self.random_state)\n\n self.used_base_cols = []\n\n self.timelog = TimerLog()\n self.new_feats = []\n\n def estimate_new_dtypes(self, n_numeric, n_categorical, n_binary, **kwargs) -> int:\n num_base_feats = n_numeric\n if self.min_cardinality < 2:\n num_base_feats += n_binary\n if self.cat_as_num:\n num_base_feats += n_categorical\n\n # 2. Estimate the no. of new arithmetic features per order\n no_interaction_types = len(self.interaction_types)\n num_new_feats = 0\n for order in range(2, self.max_order + 1):\n if order > num_base_feats:\n break\n if order == 2:\n if \"/\" in self.interaction_types:\n no_interaction_types += 1\n num_new_feats = (\n comb(num_base_feats, 2) * no_interaction_types\n ) # num_base_feats*(num_base_feats-1)/2*no_interaction_types\n if \"/\" in self.interaction_types:\n no_interaction_types -= 1\n else:\n # num_new_feats += ((num_base_feats - 2) * (num_new_feats)) * no_interaction_types\n num_new_feats += estimate_no_higher_interaction_features(num_base_feats, num_new_feats)\n if num_new_feats > self.max_new_feats:\n num_new_feats = self.max_new_feats\n break\n return n_numeric + int(num_new_feats), n_categorical, n_binary\n\n def estimate_no_of_new_features(self, X: pd.DataFrame, **kwargs) -> int:\n if self.selection_method != \"random\":\n warnings.warn(\n \"Estimation of new features is only implemented for selection_method='random'. Returning max_new_feats.\",\n UserWarning,\n )\n return self.max_new_feats\n\n # 1. Determine the no. of base features\n pass_cardinality_filter = X.nunique().values >= self.min_cardinality\n pass_cat_filter = X.apply(is_numeric_dtype).values if not self.cat_as_num else np.array([True] * X.shape[1])\n base_feat_mask = pass_cardinality_filter & pass_cat_filter\n num_base_feats = min(np.sum(base_feat_mask), self.max_base_feats)\n\n # 2. Estimate the no. of new arithmetic features per order\n no_interaction_types = len(self.interaction_types)\n num_new_feats = 0\n for order in range(2, self.max_order + 1):\n if order > num_base_feats:\n break\n if order == 2:\n if \"/\" in self.interaction_types:\n no_interaction_types += 1\n num_new_feats = (\n comb(num_base_feats, 2) * no_interaction_types\n ) # num_base_feats*(num_base_feats-1)/2*no_interaction_types\n if \"/\" in self.interaction_types:\n no_interaction_types -= 1\n else:\n # num_new_feats += ((num_base_feats - 2) * (num_new_feats)) * no_interaction_types\n num_new_feats += estimate_no_higher_interaction_features(num_base_feats, num_new_feats)\n if num_new_feats > self.max_new_feats:\n num_new_feats = self.max_new_feats\n break\n return int(num_new_feats), X.columns[base_feat_mask].tolist()\n\n def spearman_selection(self, X: pd.DataFrame, y: pd.Series):\n # FIXME: Currently heavily relies on polars for performance, make sure a numpy and a polars version exists for each function\n ### Apply advanced filtering steps (spearman correlation thresholding)\n # TODO: Might skip that and instead add the corr based filter to basic + use a max_base_features parameter\n with self.timelog.block(\"advanced_filter_base\"):\n use_cols = filter_by_spearman(X, corr_threshold=self.corr_threshold)\n\n if self.verbose:\n print(f\"Using {len(use_cols)}/{X.shape[1]} features after advanced filtering\")\n X = X[use_cols]\n\n if X.shape[1] == 0:\n if self.verbose:\n print(\"No features left after filtering. Exiting.\")\n return self\n\n X_dict = {1: X}\n for order in range(2, self.max_order + 1):\n if order > X.shape[1]:\n break\n if self.verbose:\n print(\"---\" * 20)\n print(f\"Generating order {order} interaction features\")\n\n # 6. Generate higher-order interaction features\n with self.timelog.block(f\"get_interactions_{order}-order\"):\n if order == 2:\n X_dict[2] = get_all_bivariate_interactions(\n X,\n max_feats=int(self.max_accept_for_pairwise / 5),\n random_state=self.rng,\n interaction_types=self.interaction_types,\n )\n else:\n X_dict[order] = add_higher_interaction(\n X,\n X_dict[order - 1],\n max_feats=int(self.max_accept_for_pairwise / 5),\n random_state=self.rng,\n interaction_types=self.interaction_types,\n )\n\n if self.reduce_memory:\n with self.timelog.block(f\"reduce_memory_{order}-order\"):\n X_dict[order] = reduce_memory_usage(X_dict[order], rescale=False, verbose=self.verbose)\n if self.verbose:\n print(f\"Generated {X_dict[order].shape[1]} {order}-order interaction features\")\n\n # 7. Filter higher-order interaction features\n n_feats_start = X_dict[order].shape[1]\n # basic\n with self.timelog.block(f\"basic_filter_{order}-order\"):\n X_dict[order] = basic_filter(\n X_dict[order],\n use_polars=False,\n min_cardinality=self.min_cardinality,\n data_cleaning=self.data_cleaning,\n nan_threshold=self.nan_threshold,\n mode_imbalance_threshold=self.mode_imbalance_threshold,\n )\n if self.verbose:\n print(f\"Using {len(X_dict[order].columns)}/{n_feats_start} features after basic filtering\")\n\n # based on correlations among interaction features\n if X_dict[order].shape[1] > self.max_accept_for_pairwise:\n if self.verbose:\n print(\n f\"Limiting interaction features to {self.max_accept_for_pairwise} (from {X_dict[order].shape[1]})\"\n )\n X_dict[order] = X_dict[order].sample(n=self.max_accept_for_pairwise, random_state=42, axis=1)\n\n # TODO: Implement filtering in chunks. Currently it is too slow and too memory intensive\n n_feats_start = X_dict[order].shape[1]\n with self.timelog.block(f\"spearman_int_filter_{order}-order\"):\n use_cols = filter_by_spearman(X_dict[order], corr_threshold=self.corr_threshold)\n X_dict[order] = X_dict[order][use_cols]\n if self.verbose:\n print(f\"Using {len(use_cols)}/{n_feats_start} features after spearman filtering\")\n\n # based on cross-correlation with base features\n if self.use_cross_corr:\n n_feats_start = X_dict[order].shape[1]\n with self.timelog.block(f\"cross_correlation_{order}-order\"):\n use_cols, novelty_scores = filter_by_cross_correlation(\n X_dict[order - 1], X_dict[order], corr_threshold=self.corr_threshold\n )\n X_dict[order] = X_dict[order][use_cols]\n if self.verbose:\n print(f\"Using {len(use_cols)}/{n_feats_start} features after cross-correlation filtering\")\n\n if len(self.new_feats) + X_dict[order].shape[1] >= self.max_new_feats:\n if self.use_cross_corr:\n max_new = self.max_new_feats - len(self.new_feats)\n self.new_feats.extend(novelty_scores.sort_values(ascending=False).index[:max_new].tolist())\n else:\n self.new_feats.extend(X_dict[order].columns.tolist()[: self.max_new_feats - len(self.new_feats)])\n if self.verbose:\n print(f\"Reached max new features limit of {self.max_new_feats}. Stopping.\")\n break\n else:\n self.new_feats.extend(X_dict[order].columns.tolist())\n\n def random_selection(self, X: pd.DataFrame, y: pd.Series):\n # TODO: Improve memory efficiency for max_order > 3 by deleting unneeded intermediate results\n X_columns = list(X.columns)\n X_dict = {1: X_columns}\n\n for order in range(2, self.max_order + 1):\n if order > X.shape[1]:\n break\n if self.verbose:\n print(\"---\" * 20)\n print(f\"Generating order {order} interaction features\")\n remaining_new_feats = self.max_new_feats - len(self.new_feats)\n\n if remaining_new_feats <= 0:\n self.new_feats = self.new_feats[: self.max_new_feats]\n break\n\n # 6. Generate higher-order interaction features\n with self.timelog.block(f\"get_interactions_{order}-order\"):\n if order == 2:\n X_dict[2] = get_all_bivariate_interactions_lite(\n X_columns,\n max_feats=remaining_new_feats,\n random_state=self.rng,\n interaction_types=self.interaction_types,\n )\n else:\n X_dict[order] = add_higher_interaction_lite(\n X_columns,\n X_dict[order - 1],\n max_feats=remaining_new_feats,\n random_state=self.rng,\n interaction_types=self.interaction_types,\n )\n\n if self.verbose:\n print(f\"Generated {len(X_dict[order])} {order}-order interaction features\")\n\n self.new_feats.extend(X_dict[order])\n\n if len(self.new_feats) >= self.max_new_feats:\n self.new_feats = self.new_feats[: self.max_new_feats]\n if self.verbose:\n print(f\"Reached max new features limit of {self.max_new_feats}. Stopping.\")\n break\n\n def _fit(self, X: pd.DataFrame, y: pd.Series | None, **kwargs):\n # TODO: Add a check that the original features names don't contain arithmetic operators to avoid issues in transform\n use_y = (self.selection_method != \"random\") and (y is not None)\n\n # ------------------------------------------------------\n # 0) Optional row subsampling\n # ------------------------------------------------------\n with self.timelog.block(\"row_subsample\"):\n if self.subsample and self.subsample < X.shape[0]:\n X = X.sample(n=self.subsample, random_state=self.rng, axis=0)\n sample_index = X.index\n\n # ------------------------------------------------------\n # 1) Prepare y only if needed\n # ------------------------------------------------------\n with self.timelog.block(\"prepare_y\"):\n if use_y:\n y = y.loc[sample_index]\n else:\n y = None\n\n # ------------------------------------------------------\n # 2) Detect numeric columns early (non-cat_as_num)\n # ------------------------------------------------------\n with self.timelog.block(\"detect_numeric_cols\"):\n if not self.cat_as_num:\n num_cols = X.select_dtypes(include=[np.number]).columns\n if self.verbose:\n print(f\"Using {len(num_cols)}/{X.shape[1]} base features (filtering to numeric)\")\n if len(num_cols) == 0:\n if self.verbose:\n print(\"No numeric features available. Exiting.\")\n self.new_feats = []\n self.time_logs = {}\n return self\n X = X[num_cols]\n\n # ------------------------------------------------------\n # 3) Basic filtering\n # ------------------------------------------------------\n with self.timelog.block(\"basic_filter_base\"):\n n_start = X.shape[1]\n X = basic_filter(\n X,\n use_polars=False,\n min_cardinality=self.min_cardinality,\n data_cleaning=self.data_cleaning,\n nan_threshold=self.nan_threshold,\n mode_imbalance_threshold=self.mode_imbalance_threshold,\n )\n\n if self.verbose:\n print(f\"Using {X.shape[1]}/{n_start} base features after basic filtering\")\n\n if X.shape[1] == 0:\n if self.verbose:\n print(\"All base features removed after basic filtering.\")\n self.new_feats = []\n self.time_logs = {}\n return self\n\n # ------------------------------------------------------\n # 4) Column subsampling for base features and clean column names\n # ------------------------------------------------------\n with self.timelog.block(\"column_subsample\"):\n if X.shape[1] > self.max_base_feats:\n if self.verbose:\n print(f\"Limiting base features to {self.max_base_feats} (from {X.shape[1]})\")\n sampled_cols = set(\n self.rng.choice(\n X.columns,\n size=self.max_base_feats,\n replace=False,\n )\n )\n\n X = X[[c for c in X.columns if c in sampled_cols]]\n\n self.used_base_cols = X.columns.tolist()\n self._keep_features_in(features=X.columns.tolist())\n\n # ------------------------------------------------------\n # 5) Cat-as-num or numeric selection\n # ------------------------------------------------------\n with self.timelog.block(\"apply_cat_as_num_or_select_numeric\"):\n if self.cat_as_num:\n self.cat_as_num_preprocessor = CatAsNumFeatureGenerator(\n keep_original=False,\n )\n X = self.cat_as_num_preprocessor.fit_transform(X)\n if X.shape[1] == 0:\n if self.verbose:\n print(\"No features left after CatAsNum conversion.\")\n self.new_feats = []\n self.time_logs = {}\n return self\n\n # ------------------------------------------------------\n # 6) Memory reduction\n # ------------------------------------------------------\n if self.reduce_memory and X.shape[1] > 0:\n with self.timelog.block(\"reduce_memory_usage_base\"):\n X = reduce_memory_usage(X, rescale=self.rescale_avoid_overflow, verbose=self.verbose)\n\n # ------------------------------------------------------\n # 7) Interaction generation\n # ------------------------------------------------------\n if self.selection_method == \"random\":\n with self.timelog.block(\"random_selection\"):\n self.random_selection(X, y)\n else:\n with self.timelog.block(\"spearman_selection\"):\n self.spearman_selection(X, y)\n\n # ------------------------------------------------------\n # 9) Collect timing logs\n # ------------------------------------------------------\n with self.timelog.block(\"collect_time_logs\"):\n self.time_logs = self.timelog.summary(verbose=False)\n\n # ------------------------------------------------------\n # 11) Print summary at end\n # ------------------------------------------------------\n if self.verbose:\n print(\"\\n=== ArithmeticFeatureGenerator Timing Summary ===\")\n for k, v in self.time_logs.items():\n print(f\"{k:<32} {v:.4f} sec\")\n print(\"=\" * 52)\n self.time_logs = {}\n\n return self\n\n def _fit_transform(self, X: DataFrame, y: Series, **kwargs) -> Tuple[DataFrame, dict]:\n self._fit(X, y, **kwargs)\n X_out = self._transform(X[self.features_in])\n\n # features_out = list(X_out.columns)\n # type_group_map_special = {R_FLOAT: features_out}\n return X_out, dict() # TODO: Unsure whether we need to return anything special here\n\n def _add_arithmetic_dag(\n self,\n X: pd.DataFrame,\n ) -> pd.DataFrame:\n \"\"\"\n Fast evaluator with DAG optimization.\n Computes common sub-expressions only once.\n expressions: list[Operation]\n \"\"\"\n\n # ---------------------------------------------------------------------\n # 1) Prepare input array\n # ---------------------------------------------------------------------\n if self.inner_dtype != np.float64:\n arr = X.to_numpy(dtype=self.inner_dtype)\n else:\n arr = X.to_numpy()\n if not np.issubdtype(arr.dtype, np.floating):\n arr = arr.astype(float)\n col_idx = {c: i for i, c in enumerate(self.used_base_cols)}\n opmap = {\n \"+\": operator.add,\n \"-\": operator.sub,\n \"*\": operator.mul,\n \"/\": operator.truediv,\n }\n\n # ---------------------------------------------------------------------\n # 2) Compile DAG (only if expressions changed)\n # ---------------------------------------------------------------------\n compile_needed = not hasattr(self, \"_compiled_dag\")\n\n # FIXME: Add support for recompiling in case we pruned features\n if compile_needed:\n expressions: list[Operation] = self.new_feats\n\n nodes = {} # key -> None (placeholder)\n expr_roots = {} # Operation -> node key or int\n\n def lower(node):\n \"\"\"Recursively lower Operation â DAG node or base index\"\"\"\n if isinstance(node, Operation):\n left = lower(node.left)\n right = lower(node.right)\n key = (left, node.op, right)\n if key not in nodes:\n nodes[key] = None\n return key\n else:\n # base feature name â column index\n return col_idx[node]\n\n for expr in expressions:\n expr_roots[expr] = lower(expr)\n\n self._compiled_dag = {\n \"nodes\": list(nodes.keys()),\n \"expr_roots\": expr_roots,\n \"exprs\": tuple(expressions),\n \"names\": tuple([expression.name() for expression in expressions]),\n }\n\n # ---------------------------------------------------------------------\n # 3) Evaluate DAG\n # ---------------------------------------------------------------------\n nodes = self._compiled_dag[\"nodes\"]\n expr_roots = self._compiled_dag[\"expr_roots\"]\n exprs = self._compiled_dag[\"exprs\"]\n names = self._compiled_dag[\"names\"]\n\n base_values = {i: arr[:, i] for i in range(arr.shape[1])}\n results = {}\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n for key in nodes:\n left, op, right = key\n\n left_arr = base_values[left] if isinstance(left, int) else results[left]\n right_arr = base_values[right] if isinstance(right, int) else results[right]\n\n results[key] = opmap[op](left_arr, right_arr)\n\n # ---------------------------------------------------------------------\n # 4) Build output DataFrame\n # ---------------------------------------------------------------------\n output = {}\n for expr, name in zip(exprs, names):\n root = expr_roots[expr]\n if isinstance(root, int):\n out = base_values[root]\n else:\n out = results[root]\n\n out[np.isinf(out)] = np.nan\n output[name] = out\n\n return pd.DataFrame(output, index=X.index)\n\n def _transform(self, X: DataFrame) -> DataFrame:\n # Note: It is important that X have the same order as `self.features_in` when entering this method\n if not self.new_feats:\n return pd.DataFrame(index=X.index)\n\n X = X[self.used_base_cols] # TODO: Remove this for a slight speedup, but need to be careful\n\n if self.cat_as_num:\n X = self.cat_as_num_preprocessor.transform(X)\n\n if self.inference_mode == \"dag\":\n X_new = self._add_arithmetic_dag(X)\n\n # All columns in X_new are numeric floats created from numpy, so\n # cast the whole block at once instead of per-column astype.\n if self.inner_dtype != self.out_dtype:\n if np.issubdtype(self.out_dtype, np.floating):\n with np.errstate(divide=\"ignore\", invalid=\"ignore\", over=\"ignore\"):\n arr = X_new.to_numpy(dtype=self.out_dtype, copy=False)\n X_new = pd.DataFrame(arr, index=X_new.index, columns=X_new.columns)\n\n # Combine original and interaction features\n return X_new\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict() # TODO: Unsure what to include here\n"
},
{
"path": "features/src/autogluon/features/generators/astype.py",
"content": "import logging\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.infer_types import get_bool_true_val, get_type_map_raw, get_type_map_real\nfrom autogluon.common.features.types import R_INT, S_BOOL\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add int fillna input value options: 0, set value, mean, mode, median\nclass AsTypeFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Enforces type conversion on the data to match the types seen during fitting.\n If a feature cannot be converted to the correct type, an exception will be raised.\n\n Parameters\n ----------\n convert_bool : bool, default True\n Whether to automatically convert features with only two unique values to boolean.\n convert_bool_method : str, default \"auto\"\n [Advanced] The processing method to convert boolean features. Recommended to keep as \"auto\".\n If \"auto\": Will attempt to automatically select the best method based on the data.\n If \"v1\": Will use a simple method that was the default prior to v0.7 (`_convert_to_bool_simple`)\n If \"v2\": Will use an optimized method that was introduced in v0.7 (`_convert_to_bool_fast`)\n Note that \"v2\" is not always faster than \"v1\", and is often slower when there are few boolean columns.\n All options produce identical results, except in extreme synthetic edge-cases.\n convert_bool_method_v2_threshold : int, default 15\n [Advanced] If `convert_bool_method=\"auto\"`, this value determines which method is used.\n If the number of boolean features is >= this value, then \"v2\" is used. Otherwise, \"v1\" is used.\n 15 is roughly the optimal value on average.\n convert_bool_method_v2_row_threshold : int, default 128\n [Advanced] If using \"v2\" bool method, this is the row count in which when >=, the batch method is used instead of the realtime method.\n 128 is roughly the optimal value on average.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(\n self,\n convert_bool: bool = True,\n convert_bool_method: str = \"auto\",\n convert_bool_method_v2_threshold: int = 15,\n convert_bool_method_v2_row_threshold: int = 128,\n **kwargs,\n ):\n super().__init__(**kwargs)\n # FeatureMetadata object based on the original input features real dtypes\n # (will contain dtypes such as 'int16' and 'float32' instead of 'int' and 'float').\n self._feature_metadata_in_real: FeatureMetadata = None\n self._type_map_real_opt: dict = None # Optimized representation of data types, saves a few milliseconds during comparisons in online inference\n # self.inplace = inplace # TODO, also add check if dtypes are same as expected and skip .astype\n self._int_features = None\n self._bool_features = None\n self._convert_bool = convert_bool\n self._convert_bool_method_v2_threshold = convert_bool_method_v2_threshold\n self._convert_bool_method_v2_row_threshold = convert_bool_method_v2_row_threshold\n if convert_bool_method == \"v1\":\n self._use_fast_bool_method = False\n elif convert_bool_method == \"v2\":\n self._use_fast_bool_method = True\n elif convert_bool_method == \"auto\":\n self._use_fast_bool_method = \"auto\"\n else:\n raise ValueError(\n f'Unknown `convert_bool_method` value: {convert_bool_method}. Valid values: [\"v1\", \"v2\", \"auto\"]'\n )\n self._bool_features_list = None\n self._non_bool_features_list = None\n self._bool_features_val = None\n self._bool_features_val_np = None\n\n # TODO: consider returning self._transform(X) if we allow users to specify real dtypes as input\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n feature_type_raw_cur_dict = get_type_map_raw(X)\n feature_map_to_update = dict()\n type_map_special = self.feature_metadata_in.get_type_map_special()\n for feature in self.features_in:\n feature_type_raw = self.feature_metadata_in.get_feature_type_raw(feature)\n feature_type_raw_cur = feature_type_raw_cur_dict[feature]\n if feature_type_raw != feature_type_raw_cur:\n self._log(\n 30,\n f'\\tWARNING: Actual dtype differs from dtype in FeatureMetadata for feature \"{feature}\". '\n f\"Actual dtype: {feature_type_raw_cur} | Expected dtype: {feature_type_raw}\",\n )\n feature_map_to_update[feature] = feature_type_raw\n if feature_map_to_update:\n self._log(\n 30,\n \"\\tWARNING: Forcefully converting features to expected dtypes. \"\n \"Please manually align the input data with the expected dtypes if issues occur.\",\n )\n X = X.astype(feature_map_to_update)\n\n self._bool_features = dict()\n if self._convert_bool:\n num_rows = len(X)\n if num_rows > 1000:\n # Sample and filter out features that already have >2 unique values\n # in the first 500 rows from bool consideration\n X_nunique_sample = X[self.features_in].head(500).nunique(dropna=False)\n X_nunique_sample = X_nunique_sample[X_nunique_sample <= 2]\n bool_candidates = list(X_nunique_sample.index)\n else:\n bool_candidates = self.features_in\n for feature in bool_candidates:\n if S_BOOL not in type_map_special[feature]:\n uniques = X[feature].unique()\n if len(uniques) == 2:\n feature_bool_val = get_bool_true_val(uniques=uniques)\n self._bool_features[feature] = feature_bool_val\n\n if self._bool_features:\n self._log(\n 20,\n f\"\\tNote: Converting {len(self._bool_features)} features to boolean dtype \"\n f\"as they only contain 2 unique values.\",\n )\n self._set_bool_features_val()\n if self._use_fast_bool_method == \"auto\":\n self._use_fast_bool_method = len(self._bool_features) >= self._convert_bool_method_v2_threshold\n X = self._convert_to_bool(X)\n for feature in self._bool_features:\n type_map_special[feature] = [S_BOOL]\n self._type_map_real_opt[feature] = np.int8\n type_group_map_special = FeatureMetadata.get_type_group_map_special_from_type_map_special(type_map_special)\n else:\n type_group_map_special = self.feature_metadata_in.type_group_map_special\n self._int_features = np.array(self.feature_metadata_in.get_features(valid_raw_types=[R_INT]))\n return X, type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n if self._bool_features:\n X = self._convert_to_bool(X)\n # check if not same\n if self._type_map_real_opt != X.dtypes.to_dict():\n if self._int_features.size:\n null_count = X[self._int_features].isnull().any()\n # If int feature contains null during inference but not during fit.\n if null_count.any():\n # TODO: Consider imputing to mode? This is tricky because training data had no missing values.\n # TODO: Add unit test for this situation, to confirm it is handled properly.\n with_null = null_count[null_count]\n with_null_features = list(with_null.index)\n logger.warning(\n \"WARNING: Int features without null values \"\n \"at train time contain null values at inference time! \"\n \"Imputing nulls to 0. To avoid this, pass the features as floats during fit!\"\n )\n logger.warning(f\"WARNING: Int features with nulls: {with_null_features}\")\n X[with_null_features] = X[with_null_features].fillna(0)\n\n if self._type_map_real_opt:\n # TODO: Confirm this works with sparse and other feature types!\n # FIXME: Address situation where test-time invalid type values cause crash:\n # https://stackoverflow.com/questions/49256211/how-to-set-unexpected-data-type-to-na?noredirect=1&lq=1\n try:\n X = X.astype(self._type_map_real_opt)\n except Exception as e:\n self._log_invalid_dtypes(X=X)\n raise e\n return X\n\n def _log_invalid_dtypes(self, X: pd.DataFrame):\n \"\"\"\n Logs detailed information on all feature transformations, including exceptions that occur.\n \"\"\"\n pd_cols = [\"feature\", \"status\", \"dtype_input\", \"dtype_to_convert_to\", \"exception\"]\n rows = []\n\n logger.log(\n 40,\n f\"Exception encountered in {self.__class__.__name__} ... \"\n f\"Please check if feature data types differ between train and test (via df.dtypes).\\nException breakdown by feature:\",\n )\n for f in self._type_map_real_opt.keys():\n f_type_out = self._type_map_real_opt[f]\n f_type_in = X[f].dtype\n try:\n X[f].astype(f_type_out)\n except Exception as e:\n status = f\"{e.__class__.__name__}\"\n exception = e\n else:\n status = \"Success\"\n exception = None\n row = [f, status, f_type_in, f_type_out, exception]\n rows.append(row)\n df_debug = pd.DataFrame(rows, columns=pd_cols)\n with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None, \"display.width\", 1000):\n logger.log(40, df_debug)\n\n def _convert_to_bool(self, X: DataFrame) -> DataFrame:\n if self._use_fast_bool_method:\n return self._convert_to_bool_fast(X)\n else:\n return self._convert_to_bool_simple(X)\n\n def _convert_to_bool_simple(self, X: DataFrame) -> DataFrame:\n \"\"\"Generic method to convert feature types to booleans. Efficient with small amounts of features.\"\"\"\n for feature in self._bool_features_list:\n # Note, this edits inplace, altering outer context.\n # This is ok when used in PipelineFeatureGenerator, as the data is already deep copied.\n # We avoid deep copying here to speed up processing.\n X[feature] = (X[feature] == self._bool_features[feature]).astype(np.int8)\n return X\n\n def _convert_to_bool_fast(self, X: DataFrame) -> DataFrame:\n \"\"\"\n Faster method to convert feature types to boolean when many features must be converted at once.\n Can be >10x faster than the simple version, particularly when len(X) < 100\n\n Note that the fast method alters the column order with boolean features being last.\n \"\"\"\n if len(X) >= self._convert_bool_method_v2_row_threshold:\n return self._convert_to_bool_fast_batch(X)\n else:\n return self._convert_to_bool_fast_realtime(X)\n\n def _convert_to_bool_fast_batch(self, X: DataFrame) -> DataFrame:\n \"\"\"Optimized for when X is > 100 rows\"\"\"\n X_bool_list = []\n for feature in self._bool_features_list:\n X_bool_list.append((X[feature] == self._bool_features[feature]).astype(np.int8))\n X_bool = pd.concat(X_bool_list, axis=1)\n\n # TODO: re-order columns to features_in required because `feature_interactions=False` to avoid error when feature prune.\n # Note that this is slower than avoiding the re-order, but avoiding the re-order is very complicated to do correctly.\n return pd.concat([X[self._non_bool_features_list], X_bool], axis=1)[self.features_in]\n\n def _convert_to_bool_fast_realtime(self, X: DataFrame) -> DataFrame:\n \"\"\"Optimized for when X is <= 100 rows\"\"\"\n X_bool_features_np = X[self._bool_features_list].to_numpy(dtype=\"object\")\n X_bool_numpy = X_bool_features_np == self._bool_features_val_np\n X_bool = pd.DataFrame(X_bool_numpy, columns=self._bool_features_list, dtype=np.int8, index=X.index)\n\n # TODO: re-order columns to features_in required because `feature_interactions=False` to avoid error when feature prune.\n # Note that this is slower than avoiding the re-order, but avoiding the re-order is very complicated to do correctly.\n return pd.concat([X[self._non_bool_features_list], X_bool], axis=1)[self.features_in]\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n def _infer_features_in_full(self, X: DataFrame, feature_metadata_in: FeatureMetadata = None):\n super()._infer_features_in_full(X=X, feature_metadata_in=feature_metadata_in)\n type_map_real = get_type_map_real(X[self.feature_metadata_in.get_features()])\n self._type_map_real_opt = X[self.feature_metadata_in.get_features()].dtypes.to_dict()\n self._feature_metadata_in_real = FeatureMetadata(\n type_map_raw=type_map_real, type_group_map_special=self.feature_metadata_in.get_type_group_map_raw()\n )\n\n def _remove_features_in(self, features):\n super()._remove_features_in(features)\n if features:\n self._feature_metadata_in_real = self._feature_metadata_in_real.remove_features(features=features)\n for feature in features:\n self._type_map_real_opt.pop(feature, None)\n self._bool_features.pop(feature, None)\n self._set_bool_features_val()\n self._int_features = np.array(self.feature_metadata_in.get_features(valid_raw_types=[R_INT]))\n\n def _set_bool_features_val(self):\n self._bool_features_val = [self._bool_features[f] for f in self._bool_features]\n self._bool_features_val_np = np.array(self._bool_features_val, dtype=\"object\")\n self._bool_features_list = list(self._bool_features.keys())\n self._non_bool_features_list = [f for f in self.features_in if f not in self._bool_features]\n\n def print_feature_metadata_info(self, log_level=20):\n self._log(log_level, \"\\tOriginal Features (exact raw dtype, raw dtype):\")\n self._feature_metadata_in_real.print_feature_metadata_full(\n self.log_prefix + \"\\t\\t\", print_only_one_special=True, log_level=log_level\n )\n super().print_feature_metadata_info(log_level=log_level)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/auto_ml_pipeline.py",
"content": "import logging\n\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom autogluon.common.features.types import (\n R_FLOAT,\n R_INT,\n R_OBJECT,\n S_IMAGE_BYTEARRAY,\n S_IMAGE_PATH,\n S_TEXT,\n)\nfrom autogluon.features.generators.abstract import AbstractFeatureGenerator\n\nfrom .category import CategoryFeatureGenerator\nfrom .datetime import DatetimeFeatureGenerator\nfrom .identity import IdentityFeatureGenerator\nfrom .isnan import IsNanFeatureGenerator\nfrom .one_hot_encoder import OneHotEncoderFeatureGenerator\nfrom .pipeline import PipelineFeatureGenerator\nfrom .text_ngram import TextNgramFeatureGenerator\nfrom .text_special import TextSpecialFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: write out in English the full set of transformations that are applied (and eventually host page on website).\n# Also explicitly write out all of the feature-generator \"hyperparameters\" that might affect the results from the AutoML FeatureGenerator\nclass AutoMLPipelineFeatureGenerator(PipelineFeatureGenerator):\n \"\"\"Pipeline feature generator with simplified arguments to handle most Tabular data including text and dates adequately.\n\n This is the default feature generation pipeline used by AutoGluon when unspecified.\n For more customization options, refer to :class:`PipelineFeatureGenerator` and :class:`BulkFeatureGenerator`.\n\n Parameters\n ----------\n enable_numeric_features : bool, default True\n Whether to keep features of 'int' and 'float' raw types.\n These features are passed without alteration to the models.\n Appends IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=['int', 'float'])) to the generator group.\n enable_categorical_features : bool, default True\n Whether to keep features of 'object' and 'category' raw types.\n These features are processed into memory optimized 'category' features.\n Appends CategoryFeatureGenerator() to the generator group.\n enable_datetime_features : bool, default True\n Whether to keep features of 'datetime' raw type and 'object' features identified as 'datetime_as_object' features.\n These features will be converted to 'int' features representing milliseconds since epoch.\n Appends DatetimeFeatureGenerator() to the generator group.\n enable_text_special_features : bool, default True\n Whether to use 'object' features identified as 'text' features to generate 'text_special' features such as word count,\n capital letter ratio, and symbol counts.\n Appends TextSpecialFeatureGenerator() to the generator group.\n enable_text_ngram_features : bool, default True\n Whether to use 'object' features identified as 'text' features to generate 'text_ngram' features.\n Appends TextNgramFeatureGenerator(vectorizer=vectorizer, text_ngram_params) to the generator group. See text_ngram.py for valid parameters.\n enable_raw_text_features : bool, default False\n Whether to use the raw text features. The generated raw text features will end up with '_raw_text' suffix.\n For example, 'sentence' --> 'sentence_raw_text'\n enable_vision_features : bool, default True\n [Experimental]\n Whether to keep 'object' features identified as 'image_path' special type.\n Features of this form should have a string path to an image file as their value.\n Only vision models can leverage these features, and these features will not be treated as categorical.\n Note: 'image_path' features will not be automatically inferred. These features must be explicitly specified as such in a custom FeatureMetadata object.\n Note: It is recommended that the string paths use absolute paths rather than relative, as they will likely be more stable.\n vectorizer : :class:`sklearn.feature_extraction.text.CountVectorizer`, default CountVectorizer(min_df=30, ngram_range=(1, 3), max_features=10000, dtype=np.uint8) # noqa\n sklearn CountVectorizer object to use in :class:`TextNgramFeatureGenerator`.\n Only used if `enable_text_ngram_features=True`.\n text_ngram_params : dict, default None\n Parameters besides vectorizer passed to the :class:`TextNgramFeatureGenerator`.\n custom_feature_generators : list of :class:`AbstractFeatureGenerator`, default None\n Lists of custom feature generators. This list is inserted in the first generator\n step that is getting the original X data (i.e. before any other feature\n generators are applied) as input.\n Note, there might be an overlap of custom feature generators with the default\n feature generators used in AutoMLPipelineFeatureGenerator. It is the user's\n responsibility to avoid unwanted overlap and disable default generators if\n needed.\n If None, no custom feature generators are added.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularDataset\n >>> from autogluon.features.generators import AutoMLPipelineFeatureGenerator\n >>>\n >>> feature_generator = AutoMLPipelineFeatureGenerator()\n >>>\n >>> label = 'class'\n >>> train_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv')\n >>> X_train = train_data.drop(labels=[label], axis=1)\n >>> y_train = train_data[label]\n >>>\n >>> X_train_transformed = feature_generator.fit_transform(X=X_train, y=y_train)\n >>>\n >>> test_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv')\n >>>\n >>> X_test_transformed = feature_generator.transform(test_data)\n \"\"\"\n\n def __init__(\n self,\n enable_numeric_features: bool = True,\n enable_categorical_features: bool = True,\n enable_datetime_features: bool = True,\n enable_text_special_features: bool = True,\n enable_text_ngram_features: bool = True,\n enable_raw_text_features: bool = False,\n enable_vision_features: bool = True,\n vectorizer: CountVectorizer | None = None,\n text_ngram_params: dict | None = None,\n custom_feature_generators: list[AbstractFeatureGenerator] | None = None,\n **kwargs,\n ):\n if \"generators\" in kwargs:\n raise KeyError(\n f\"generators is not a valid parameter to {self.__class__.__name__}. \"\n f\"Use {PipelineFeatureGenerator.__name__} to specify custom generators.\"\n )\n if \"enable_raw_features\" in kwargs:\n enable_numeric_features = kwargs.pop(\"enable_raw_features\")\n logger.warning(\n \"'enable_raw_features' is a deprecated parameter, use 'enable_numeric_features' instead. \"\n \"Specifying 'enable_raw_features' will raise an exception in a future release\"\n )\n\n self.enable_numeric_features = enable_numeric_features\n self.enable_categorical_features = enable_categorical_features\n self.enable_datetime_features = enable_datetime_features\n self.enable_text_special_features = enable_text_special_features\n self.enable_text_ngram_features = enable_text_ngram_features\n self.enable_raw_text_features = enable_raw_text_features\n self.enable_vision_features = enable_vision_features\n self.text_ngram_params = text_ngram_params if text_ngram_params else {}\n self.custom_feature_generators = custom_feature_generators\n\n generators = self._get_default_generators(vectorizer=vectorizer)\n super().__init__(generators=generators, **kwargs)\n\n # TODO: switch to / add skrub's String or Text encoders\n def _get_default_generators(self, vectorizer=None):\n generator_group = []\n\n if self.enable_numeric_features:\n generator_group.append(\n IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT]))\n )\n if self.enable_raw_text_features:\n generator_group.append(\n IdentityFeatureGenerator(\n infer_features_in_args=dict(\n required_special_types=[S_TEXT],\n invalid_special_types=[S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n ),\n name_suffix=\"_raw_text\",\n )\n )\n if self.enable_categorical_features:\n generator_group.append(self._get_category_feature_generator())\n if self.enable_datetime_features:\n generator_group.append(DatetimeFeatureGenerator())\n if self.enable_text_special_features:\n generator_group.append(TextSpecialFeatureGenerator())\n if self.enable_text_ngram_features:\n generator_group.append(TextNgramFeatureGenerator(vectorizer=vectorizer, **self.text_ngram_params))\n if self.enable_vision_features:\n generator_group.append(\n IdentityFeatureGenerator(\n infer_features_in_args=dict(\n valid_raw_types=[R_OBJECT],\n valid_special_types=[S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n required_at_least_one_special=True,\n )\n )\n )\n generator_group.append(\n IsNanFeatureGenerator(\n infer_features_in_args=dict(\n valid_raw_types=[R_OBJECT],\n valid_special_types=[S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n required_at_least_one_special=True,\n )\n )\n )\n\n if self.custom_feature_generators is not None:\n generator_group = [\n *generator_group,\n *self.custom_feature_generators,\n ]\n\n generators = [generator_group]\n return generators\n\n def _get_category_feature_generator(self):\n return CategoryFeatureGenerator()\n\n\nclass AutoMLInterpretablePipelineFeatureGenerator(AutoMLPipelineFeatureGenerator):\n def _get_category_feature_generator(self):\n return CategoryFeatureGenerator(\n minimize_memory=False,\n maximum_num_cat=10,\n post_generators=[OneHotEncoderFeatureGenerator()],\n )\n"
},
{
"path": "features/src/autogluon/features/generators/binned.py",
"content": "import copy\nimport logging\n\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_FLOAT, R_INT, S_BINNED\n\nfrom .. import binning\nfrom ..utils import get_smallest_valid_dtype_int\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add more parameters (possibly pass in binning function as an argument for full control)\nclass BinnedFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"BinnedFeatureGenerator bins incoming int and float features to num_bins unique int values, maintaining relative rank order.\"\"\"\n\n def __init__(self, num_bins=10, **kwargs):\n super().__init__(**kwargs)\n self.num_bins = num_bins\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n self._bin_map = self._get_bin_map(X=X)\n self._astype_map = {\n feature: get_smallest_valid_dtype_int(min_val=0, max_val=len(bin_index))\n for feature, bin_index in self._bin_map.items()\n }\n X_out = self._transform(X)\n type_group_map_special = copy.deepcopy(self.feature_metadata_in.type_group_map_special)\n type_group_map_special[S_BINNED] += list(X_out.columns)\n return X_out, type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self._transform_bin(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_INT, R_FLOAT])\n\n def _get_bin_map(self, X: DataFrame) -> dict:\n return binning.generate_bins(X, list(X.columns), ideal_bins=self.num_bins)\n\n def _transform_bin(self, X: DataFrame):\n X_out = dict()\n for column in self._bin_map:\n X_out[column] = binning.bin_column(\n series=X[column], bins=self._bin_map[column], dtype=self._astype_map[column]\n )\n X_out = pd.DataFrame(X_out, index=X.index)\n return X_out\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self._bin_map:\n for feature in features:\n if feature in self._bin_map:\n self._bin_map.pop(feature)\n if self._astype_map:\n for feature in features:\n if feature in self._astype_map:\n self._astype_map.pop(feature)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/bulk.py",
"content": "from __future__ import annotations\n\nimport logging\nfrom typing import List\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add parameter to add prefix to each generator to guarantee no name collisions: 'G1_', 'G2_', etc.\n# TODO: Add argument keep_unused, which creates an identity feature generator at each stage to pipe unused\n# input features into the next stage instead of dropping them.\nclass BulkFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n BulkFeatureGenerator is used for complex feature generation pipelines where multiple generators are required,\n with some generators requiring the output of other generators as input (multi-stage generation).\n For ML problems, it is expected that the user uses a feature generator that is an instance of or inherits from BulkFeatureGenerator,\n as single feature generators typically will not satisfy the feature generation needs of all input data types.\n Unless you are an expert user, we recommend you create custom FeatureGenerators based off of PipelineFeatureGenerator instead of BulkFeatureGenerator.\n\n Parameters\n ----------\n generators : List[List[:class:`AbstractFeatureGenerator`]]\n generators is a list of generator groups, where a generator group is a list of generators.\n Feature generators within generators[i] (generator group) are all fit on the same data,\n and their outputs are then concatenated to form the output of generators[i].\n generators[i+1] are then fit on the output of generators[i].\n The last generator group's output is the output of _fit_transform and _transform methods.\n Due to the flexibility of generators, at the time of initialization, generators will prepend pre_generators and append post_generators\n if they are not None.\n If pre/post generators are specified, the supplied generators will be extended like this:\n pre_generators = [[pre_generator] for pre_generator in pre_generators]\n post_generators = [[post_generator] for post_generator in self._post_generators]\n self.generators: List[List[AbstractFeatureGenerator]] = pre_generators + generators + post_generators\n self._post_generators = []\n This means that self._post_generators will be empty as post_generators will be incorporated into self.generators instead.\n Note that if generators within a generator group produce a feature with the same name, an AssertionError will be raised as features\n with the same name cannot be present within a valid DataFrame output.\n If both features are desired, specify a name_prefix parameter in one of the generators to prevent name collisions.\n If experimenting with different generator groups, it is encouraged to try fitting your experimental\n feature-generators to the data without any ML model training to ensure validity and avoid name collisions.\n pre_generators: List[AbstractFeatureGenerator], optional\n pre_generators are generators which are sequentially fit prior to generators.\n Functions identically to post_generators argument, but pre_generators are called before generators, while post_generators are called after generators.\n Provided for convenience to classes inheriting from BulkFeatureGenerator.\n Common pre_generators include :class:`AsTypeFeatureGenerator` and :class:`FillNaFeatureGenerator`, which act to prune and clean the data instead\n of generating entirely new features.\n remove_unused_features: {True, False, \"false_recursive\"}, default True\n If True, will try to remove unused input features based on the output features post-fit.\n This is done to optimize inference speed.\n If False, will not perform this operation.\n If \"false_recursive\", will also disable this operation in all inner generators.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n\n Examples\n --------\n >>> from autogluon.tabular import TabularDataset\n >>> from autogluon.features.generators import AsTypeFeatureGenerator, BulkFeatureGenerator, CategoryFeatureGenerator, DropDuplicatesFeatureGenerator, FillNaFeatureGenerator, IdentityFeatureGenerator # noqa\n >>> from autogluon.common.features.types import R_INT, R_FLOAT\n >>>\n >>> generators = [\n >>> [AsTypeFeatureGenerator()], # Convert all input features to the exact same types as they were during fit.\n >>> [FillNaFeatureGenerator()], # Fill all NA values in the data\n >>> [\n >>> CategoryFeatureGenerator(), # Convert object types to category types and minimize their memory usage\n >>> # Carry over all features that are not objects and categories (without this, the int features would be dropped).\n >>> IdentityFeatureGenerator(infer_features_in_args=dict(valid_raw_types=[R_INT, R_FLOAT])),\n >>> ],\n >>> # CategoryFeatureGenerator and IdentityFeatureGenerator will have their outputs concatenated together\n >>> # before being fed into DropDuplicatesFeatureGenerator\n >>> [DropDuplicatesFeatureGenerator()] # Drops any features which are duplicates of each-other\n >>> ]\n >>> feature_generator = BulkFeatureGenerator(generators=generators, verbosity=3)\n >>>\n >>> label = 'class'\n >>> train_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv')\n >>> X_train = train_data.drop(labels=[label], axis=1)\n >>> y_train = train_data[label]\n >>>\n >>> X_train_transformed = feature_generator.fit_transform(X=X_train, y=y_train)\n >>>\n >>> test_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv')\n >>>\n >>> X_test_transformed = feature_generator.transform(test_data)\n \"\"\"\n\n def __init__(\n self,\n generators: list[list[AbstractFeatureGenerator | list]],\n pre_generators: list[AbstractFeatureGenerator | List[AbstractFeatureGenerator]] = None,\n remove_unused_features: bool | str = True,\n **kwargs,\n ):\n super().__init__(**kwargs)\n if isinstance(remove_unused_features, str):\n assert remove_unused_features == \"false_recursive\", (\n \"remove_unused_features only accepts bool or 'false_recursive'\"\n )\n self._remove_unused_features_flag = False\n else:\n assert isinstance(remove_unused_features, bool)\n self._remove_unused_features_flag = remove_unused_features\n if not isinstance(generators, list):\n generators = [[generators]]\n elif len(generators) == 0:\n raise AssertionError(\"generators must contain at least one AbstractFeatureGenerator.\")\n _generators_init = [\n generator_group if isinstance(generator_group, list) else [generator_group]\n for generator_group in generators\n ]\n generators = []\n for generator_group in _generators_init:\n _generators_group = []\n for generator_group_inner in generator_group:\n if isinstance(generator_group_inner, list):\n inner_kwargs = {}\n if isinstance(remove_unused_features, str) and remove_unused_features == \"false_recursive\":\n inner_kwargs[\"remove_unused_features\"] = remove_unused_features\n _generators_group.append(\n BulkFeatureGenerator(\n generators=generator_group_inner,\n verbosity=self.verbosity,\n **inner_kwargs,\n )\n )\n else:\n _generators_group.append(generator_group_inner)\n generators.append(_generators_group)\n _generators_init = None\n\n if pre_generators is None:\n pre_generators = []\n elif not isinstance(pre_generators, list):\n pre_generators = [pre_generators]\n if self.pre_enforce_types:\n from .astype import AsTypeFeatureGenerator\n\n pre_generators = [AsTypeFeatureGenerator()] + pre_generators\n self.pre_enforce_types = False\n pre_generators = [\n pre_generator if isinstance(pre_generator, list) else [pre_generator] for pre_generator in pre_generators\n ]\n\n if self._post_generators is not None:\n post_generators = [\n post_generator if isinstance(post_generator, list) else [post_generator]\n for post_generator in self._post_generators\n ]\n self._post_generators = []\n else:\n post_generators = []\n self.generators: List[List[AbstractFeatureGenerator]] = pre_generators + generators + post_generators\n\n for generator_group in self.generators:\n for generator in generator_group:\n if not isinstance(generator, AbstractFeatureGenerator):\n raise AssertionError(\n f\"generators contains an object which is not an instance of AbstractFeatureGenerator. Invalid generator: {generator}\"\n )\n\n # FeatureMetadata object based on the original input features that were unused by any feature generator.\n self._feature_metadata_in_unused: FeatureMetadata = None\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> tuple[DataFrame, dict]:\n feature_metadata = self.feature_metadata_in\n for i in range(len(self.generators)):\n self._log(20, f\"\\tStage {i + 1} Generators:\")\n X, self.generators[i], feature_metadata = self._fit_transform_stage(\n X=X,\n generators=self.generators[i],\n feature_metadata_in=feature_metadata,\n **kwargs,\n )\n\n if self._remove_unused_features_flag:\n self._remove_features_out(features=[])\n # Remove useless generators\n # TODO: consider moving to self._remove_features_out\n for i in range(len(self.generators)):\n generator_group_valid = []\n for j in range(len(self.generators[i])):\n if self.generators[i][j].features_out:\n generator_group_valid.append(self.generators[i][j])\n self.generators[i] = generator_group_valid\n\n return X, feature_metadata.type_group_map_special\n\n def _fit_transform_stage(\n self,\n X: DataFrame,\n generators: list[\"AbstractFeatureGenerator\"],\n feature_metadata_in: FeatureMetadata,\n **kwargs,\n ) -> tuple[DataFrame, list[\"AbstractFeatureGenerator\"], FeatureMetadata]:\n feature_df_list = []\n generator_group_valid = []\n for generator in generators:\n if generator.is_valid_metadata_in(feature_metadata_in):\n if generator.verbosity > self.verbosity:\n generator.verbosity = self.verbosity\n generator.set_log_prefix(log_prefix=self.log_prefix + \"\\t\\t\", prepend=True)\n feature_df_list.append(generator.fit_transform(X, feature_metadata_in=feature_metadata_in, **kwargs))\n generator_group_valid.append(generator)\n else:\n self._log(15, f\"\\t\\tSkipping {generator.__class__.__name__}: No input feature with required dtypes.\")\n\n generators = generator_group_valid\n\n generators = [\n generator\n for j, generator in enumerate(generators)\n if feature_df_list[j] is not None and len(feature_df_list[j].columns) > 0\n ]\n feature_df_list = [\n feature_df for feature_df in feature_df_list if feature_df is not None and len(feature_df.columns) > 0\n ]\n\n if generators:\n # Raise an exception if generators expect different raw input types for the same feature.\n FeatureMetadata.join_metadatas(\n [generator.feature_metadata_in for generator in generators],\n shared_raw_features=\"error_if_diff\",\n )\n\n feature_metadata = self._merge_feature_metadata(\n feature_metadata_lst=[generator.feature_metadata for generator in generators],\n shared_raw_features=\"error\",\n )\n\n X = self._concat_features(\n feature_df_list=feature_df_list,\n index=X.index,\n )\n return X, generators, feature_metadata\n\n def _transform(self, X: DataFrame) -> DataFrame:\n for generator_group in self.generators:\n feature_df_list = []\n for generator in generator_group:\n feature_df_list.append(generator.transform(X))\n\n X = self._concat_features(\n feature_df_list=feature_df_list,\n index=X.index,\n )\n return X\n\n def _transform_stage(\n self,\n X: DataFrame,\n generators: list[\"AbstractFeatureGenerator\"],\n ) -> DataFrame:\n feature_df_list = []\n for generator in generators:\n feature_df_list.append(generator.transform(X))\n\n X = self._concat_features(\n feature_df_list=feature_df_list,\n index=X.index,\n )\n return X\n\n def get_feature_links_chain(self):\n feature_links_chain = []\n for i in range(len(self.generators)):\n feature_links_group = {}\n for generator in self.generators[i]:\n feature_links = generator.get_feature_links()\n for feature_in, features_out in feature_links.items():\n if feature_in in feature_links_group:\n feature_links_group[feature_in] += features_out\n else:\n feature_links_group[feature_in] = features_out\n feature_links_chain.append(feature_links_group)\n return feature_links_chain\n\n def _remove_unused_features(self, feature_links_chain):\n unused_features_by_stage = self._get_unused_features(feature_links_chain)\n if unused_features_by_stage:\n unused_features_in = [\n feature\n for feature in self.feature_metadata_in.get_features()\n if feature in unused_features_by_stage[0]\n ]\n feature_metadata_in_unused = self.feature_metadata_in.keep_features(features=unused_features_in)\n if self._feature_metadata_in_unused:\n self._feature_metadata_in_unused = self._feature_metadata_in_unused.join_metadata(\n feature_metadata_in_unused\n )\n else:\n self._feature_metadata_in_unused = feature_metadata_in_unused\n self._remove_features_in(features=unused_features_in)\n\n for i, generator_group in enumerate(self.generators):\n unused_features_in_stage = unused_features_by_stage[i]\n unused_features_out_stage = [\n feature_links_chain[i][feature_in]\n for feature_in in unused_features_in_stage\n if feature_in in feature_links_chain[i]\n ]\n unused_features_out_stage = list(\n set([feature for sublist in unused_features_out_stage for feature in sublist])\n )\n for generator in generator_group:\n unused_features_out_generator = [\n feature for feature in generator.features_out if feature in unused_features_out_stage\n ]\n generator._remove_features_out(features=unused_features_out_generator)\n\n def _get_unused_features(self, feature_links_chain):\n features_in_list = []\n for i in range(len(self.generators)):\n stage = i + 1\n if stage > 1:\n if self.generators[stage - 2]:\n features_in = FeatureMetadata.join_metadatas(\n [generator.feature_metadata for generator in self.generators[stage - 2]],\n shared_raw_features=\"error\",\n ).get_features()\n else:\n features_in = []\n else:\n features_in = self.features_in\n features_in_list.append(features_in)\n return self._get_unused_features_generic(\n feature_links_chain=feature_links_chain, features_in_list=features_in_list\n )\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n"
},
{
"path": "features/src/autogluon/features/generators/cat_as_num.py",
"content": "from typing import Dict, List, Literal, Tuple\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import OrdinalEncoder\n\nfrom autogluon.common.features.types import (\n R_CATEGORY,\n R_OBJECT,\n)\n\nfrom .abstract import AbstractFeatureGenerator\n\n\nclass CatAsNumFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Convert object/category columns:\n - If a column's non-null values are all numeric-like, cast it with pd.to_numeric.\n - Otherwise, apply an OrdinalEncoder (per-column) to map categories to integers.\n\n Parameters\n ----------\n keep_original : bool, default=False\n Whether to keep the original columns alongside the converted ones.\n handle_unknown : {\"use_encoded_value\", \"error\"}, default=\"use_encoded_value\"\n Passed to each internal OrdinalEncoder.\n unknown_value : int or float, default=-1\n Value to use for unknown categories when handle_unknown=\"use_encoded_value\".\n dtype : numpy dtype, default=np.float64\n Output dtype for numeric-like conversions and encoded columns.\n \"\"\"\n\n def __init__(\n self,\n keep_original=False,\n handle_unknown: str = \"use_encoded_value\",\n unknown_value: float | int = -1,\n dtype=np.float64,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self.keep_original = keep_original\n self.handle_unknown = handle_unknown\n self.unknown_value = unknown_value\n self.dtype = dtype\n\n def estimate_no_of_new_features(self, X: pd.DataFrame, **kwargs) -> int:\n # TODO: Add dtype of new features to estimation for all preprocessors\n if self.keep_original:\n return X.select_dtypes(include=[\"object\", \"category\"]).shape[1]\n else: # TODO: Implement proper handling\n return X.select_dtypes(include=[\"object\", \"category\"]).shape[1]\n\n def _fit(self, X_in: pd.DataFrame, y_in=None):\n X = X_in.copy()\n self.input_columns_: List[str] = list(X.columns)\n\n obj_cat_cols = X.select_dtypes(include=[R_OBJECT, R_CATEGORY]).columns.tolist()\n self.numeric_like_cols_: List[str] = []\n self.ordinal_cols_: List[str] = []\n self.passthrough_cols_: List[str] = [col for col in X.columns if col not in obj_cat_cols]\n self.encoders_: Dict[str, OrdinalEncoder] = {}\n\n # Decide column-by-column and fit encoders where needed\n for col in obj_cat_cols:\n s = X[col]\n # Determine if all non-null values can be parsed as numbers\n non_null = s.dropna()\n num_convertible = pd.to_numeric(non_null, errors=\"coerce\").notna().all()\n\n if num_convertible:\n self.numeric_like_cols_.append(col)\n else:\n self.ordinal_cols_.append(col)\n enc = OrdinalEncoder(\n handle_unknown=self.handle_unknown,\n unknown_value=self.unknown_value,\n dtype=self.dtype,\n )\n # Fit on a 2D array\n enc.fit(s.astype(\"category\").to_frame())\n self.encoders_[col] = enc\n\n return self\n\n def _fit_transform(self, X: pd.DataFrame, y: pd.Series, **kwargs) -> Tuple[pd.DataFrame, dict]:\n self._fit(X, y, **kwargs)\n X_out = self._transform(X)\n return X_out, dict()\n\n def _transform(self, X_in: pd.DataFrame, **kwargs) -> pd.DataFrame:\n X_out = pd.DataFrame(index=X_in.index)\n\n # 1) Cast numeric-like string/category columns\n for col in self.numeric_like_cols_:\n if col in X_in.columns:\n X_out[col] = pd.to_numeric(X_in[col], errors=\"coerce\").astype(self.dtype)\n\n # 2) Ordinal-encode the remaining categorical columns (per-column encoders)\n for col in self.ordinal_cols_:\n if col in X_in.columns:\n enc = self.encoders_[col]\n # transform expects 2D\n X_out[col] = enc.transform(X_in[[col]]).astype(self.dtype).ravel()\n\n X_out.columns = [f\"{col}_num\" for col in X_out.columns]\n if self.keep_original:\n X_out = pd.concat([X_in, X_out], axis=1)\n else:\n X_out = pd.concat([X_in[self.passthrough_cols_], X_out], axis=1)\n return X_out\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n"
},
{
"path": "features/src/autogluon/features/generators/cat_int.py",
"content": "from collections import defaultdict\nfrom itertools import combinations\nfrom math import comb\nfrom typing import List, Literal, Tuple\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import (\n R_CATEGORY,\n R_OBJECT,\n S_DATETIME_AS_OBJECT,\n S_IMAGE_BYTEARRAY,\n S_IMAGE_PATH,\n)\nfrom autogluon.features.generators.category import CategoryFeatureGenerator\n\nfrom .abstract import AbstractFeatureGenerator\nfrom .frequency import FrequencyFeatureGenerator\n\n\nclass CategoricalInteractionFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Generate new categorical features by combining existing categorical features.\n Parameters\n ----------\n max_order : int, default=2\n The maximum order of interactions to generate.\n max_new_feats : int, default=100\n Maximum number of new features to generate.\n candidate_cols : list of str or None, default=None\n List of candidate columns to consider for interaction generation. If None, all categorical columns are used.\n add_freq : bool, default=False\n Whether to add frequency encoding for the new features.\n only_freq : bool, default=False\n Whether to only keep frequency encoded features.\n min_cardinality : int, default=6\n Minimum cardinality of categorical features to be considered for interaction generation.\n min_count : int, default=2\n Minimum count of the most frequent category in a new feature to be retained.\n fillna : int, default=0\n Value to fill NaNs in frequency encoding.\n log : bool, default=False\n Whether to apply log transformation in frequency encoding.\n random_state : int, default=42\n Random state for reproducibility.\n **kwargs\n Additional keyword arguments.\n Returns\n -------\n self : CategoricalInteractionFeatureGenerator\n Fitted CategoricalInteractionFeatureGenerator instance.\n \"\"\"\n\n def __init__(\n self,\n max_order: int = 3,\n max_new_feats: int = 100,\n candidate_cols: List[str] = None,\n add_freq: bool = False,\n only_freq: bool = False,\n min_cardinality: int = 2,\n min_count: int = 2,\n fillna: int = 0,\n log: bool = False,\n random_state: int = 42,\n inference_mode: Literal[\"string\", \"category\"] = \"category\",\n make_categoricals: bool = True,\n **kwargs,\n ):\n super().__init__(**kwargs)\n\n self.__name__ = \"CatIntAdder\"\n self.max_order = max_order\n self.candidate_cols = candidate_cols\n self.add_freq = add_freq\n self.only_freq = only_freq\n self.min_cardinality = min_cardinality\n self.min_count = min_count\n self.fillna = fillna\n self.log = log\n self.max_new_feats = max_new_feats\n self.inference_mode = inference_mode\n\n self.rng = np.random.default_rng(random_state)\n\n self.new_dtypes = {}\n\n self.make_categoricals = make_categoricals\n self.fitted_ = False\n self.cat_sizes_ = {} # base categorical cardinalities\n self.reverse_mapping_ = {} # interaction_col â sorted unique int combos\n\n # FIXME: Implement passthrough a bit better -> If feature in output that is also in input, drop the input feature?\n # TODO: This is the correct one, but due to the `passthrough` logic, need to keep all.\n # Need to refactor passthrough logic to be more nuanced in order to enable this\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(\n valid_raw_types=[R_OBJECT, R_CATEGORY],\n invalid_special_types=[S_DATETIME_AS_OBJECT, S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n # required_raw_special_pairs=[\n # (R_BOOL, None),\n # (R_OBJECT, None),\n # (R_CATEGORY, None),\n # # (R_INT, S_BOOL),\n # # (R_FLOAT, S_BOOL),\n # ],\n )\n\n # @staticmethod\n # def get_default_infer_features_in_args() -> dict:\n # return dict()\n\n def estimate_new_dtypes(self, n_numeric, n_categorical, n_binary, **kwargs) -> int:\n num_new_feats = 0\n for order in range(2, self.max_order + 1):\n if n_categorical < order:\n continue\n num_new_feats += comb(n_categorical, order)\n\n if num_new_feats >= self.max_new_feats:\n num_new_feats = self.max_new_feats\n break\n\n if self.only_freq:\n return n_numeric + num_new_feats, n_categorical, n_binary\n elif self.add_freq:\n return n_numeric + num_new_feats, n_categorical + num_new_feats, n_binary\n else:\n return n_numeric, n_categorical + num_new_feats, n_binary\n\n def estimate_no_of_new_features(self, X: pd.DataFrame, **kwargs) -> int:\n \"\"\"\n Estimate the number of new features that will be generated.\n Parameters\n ----------\n X : pd.DataFrame\n Input DataFrame.\n Returns\n -------\n int\n Estimated number of new features.\n Notes\n -----\n This is a rough estimate based on the number of categorical features\n that pass the cardinality filter and the specified maximum order.\n Currently, doesn't consider the min_count filter. Therefore, the actual\n number of generated features may be lower than this estimate.\n \"\"\"\n X_cat = X.select_dtypes(include=[\"object\", \"category\"])\n pass_cardinality_filter = X_cat.nunique().values >= self.min_cardinality\n num_base_feats = np.sum(pass_cardinality_filter)\n affected_features = X_cat.columns[pass_cardinality_filter].tolist()\n\n num_new_feats = 0\n for order in range(2, self.max_order + 1):\n if num_base_feats < order:\n continue\n num_new_feats += comb(num_base_feats, order)\n\n return np.min([self.max_new_feats, num_new_feats]), affected_features\n\n def group_feat_combs_by_order(self, new_col_set):\n feat_combs = [s.split(\"_&_\") for s in new_col_set]\n by_order = defaultdict(list)\n for cols in feat_combs:\n by_order[len(cols)].append(cols)\n return dict(by_order)\n\n def get_interaction_names_dict(self, X: pd.DataFrame, max_order: int = 2, max_feats: int = 100):\n self.new_col_set = []\n for o in range(2, max_order + 1):\n feat_combs_use = list(combinations(np.unique(X.columns), o))\n add_cols = [\"_&_\".join(f) for f in feat_combs_use]\n max_feats_order = max_feats - len(self.new_col_set)\n if len(add_cols) > max_feats_order:\n add_cols = self.rng.choice(add_cols, max_feats_order, replace=False).tolist()\n self.new_col_set.extend(add_cols)\n if len(self.new_col_set) >= max_feats:\n self.new_col_set = self.new_col_set[:max_feats]\n break\n self.new_cols_by_order = self.group_feat_combs_by_order(self.new_col_set)\n\n self.used_cols = []\n for col in self.new_col_set:\n self.used_cols.extend(col.split(\"_&_\"))\n self.used_cols = list(set(self.used_cols))\n\n def combine_predefined(self, X_in: pd.DataFrame, comb_lst: List[str], fit_mode=True, **kwargs) -> pd.DataFrame:\n \"\"\"Generate interaction features based on predefined combinations.\n Parameters\n ----------\n X_in : pd.DataFrame\n Input categorical features DataFrame.\n comb_lst : List[str]\n List of predefined combinations as strings. Can be higher order combinations, but one call to this function should only handle one order at a time.\n Returns\n -------\n pd.DataFrame\n DataFrame containing the generated interaction features.\n \"\"\"\n X = X_in.copy()\n X = X_in.astype(\"U\")\n feat_combs_use = [i.split(\"_&_\") for i in comb_lst]\n feat_combs_use_arr = np.array(feat_combs_use).transpose()\n\n features = X[feat_combs_use_arr[0]].values\n for num, arr in enumerate(feat_combs_use_arr[1:]):\n features += \"_&_\" + X[arr].values\n\n X_out = pd.DataFrame(features, columns=comb_lst, index=X.index)\n\n return X_out\n\n # ----------------------------------------------------------------------\n # INTERNAL: build interactions (faster version)\n # ----------------------------------------------------------------------\n def _build_interactions(self, X, fit_mode):\n # Precompute base codes (faster than repeatedly calling .cat.codes)\n base_codes = {col: X[col].cat.codes.to_numpy(np.int32) for col in X.columns}\n\n outputs = {} # column_name â numpy array\n\n # --------------------------\n # ORDER 2 interactions\n # --------------------------\n if 2 in self.new_cols_by_order:\n for cols in self.new_cols_by_order[2]:\n name = \"_&_\".join(cols)\n col1, col2 = cols\n k = self.cat_sizes_[col2]\n\n combined = base_codes[col1] * k + base_codes[col2]\n outputs[name] = self._fit_or_map(name, combined, fit_mode)\n\n # Expose newly created interaction codes for higher-order use\n base_codes[name] = outputs[name]\n\n # --------------------------\n # ORDER >= 3 interactions\n # --------------------------\n for order in sorted(k for k in self.new_cols_by_order if k >= 3):\n for cols in self.new_cols_by_order[order]:\n name = \"_&_\".join(cols)\n\n prefix = \"_&_\".join(cols[:-1])\n last = cols[-1]\n\n k = self.cat_sizes_[last]\n combined = base_codes[prefix] * k + base_codes[last]\n\n outputs[name] = self._fit_or_map(name, combined, fit_mode)\n base_codes[name] = outputs[name]\n\n # ------------------------------------------------------------\n # Build final DataFrame in one fast pass\n # ------------------------------------------------------------\n if not outputs:\n return pd.DataFrame(index=X.index)\n\n df_out = pd.DataFrame(outputs, index=X.index)\n\n # Convert to categoricals with consistent ordering\n if self.make_categoricals:\n for col in df_out.columns:\n uniques = self.reverse_mapping_[col]\n df_out[col] = pd.Categorical.from_codes(\n df_out[col],\n categories=range(len(uniques)), # ensures 0..n-1\n ordered=False,\n )\n\n return df_out\n\n # ----------------------------------------------------------------------\n # INTERNAL: fit or transform mapping (FAST version)\n # ----------------------------------------------------------------------\n def _fit_or_map(self, name, combined, fit_mode):\n \"\"\"\n combined: int64 array of encoded interaction IDs\n \"\"\"\n\n if fit_mode:\n valid = combined[combined >= 0]\n uniques = np.unique(valid)\n self.reverse_mapping_[name] = uniques\n return self._vectorized_map(combined, uniques)\n else:\n uniques = self.reverse_mapping_[name]\n return self._vectorized_map(combined, uniques)\n\n # ----------------------------------------------------------------------\n # INTERNAL: vectorized mapping using np.searchsorted (FAST)\n # ----------------------------------------------------------------------\n @staticmethod\n def _vectorized_map(combined, uniques):\n \"\"\"\n Map combined IDs to 0..(n-1) using vectorized searchsorted.\n Unseen combos â -1.\n \"\"\"\n mapped = np.full(combined.shape, -1, dtype=np.int64)\n\n valid_mask = combined >= 0\n vals = combined[valid_mask]\n\n # vectorized position lookup\n idx = np.searchsorted(uniques, vals)\n\n # Step 1: out-of-range positions (unseen)\n oob = idx == len(uniques)\n\n # Step 2: in-range but not equal â unseen mismatch\n in_range = ~oob\n mismatch = np.zeros_like(oob)\n mismatch[in_range] = uniques[idx[in_range]] != vals[in_range]\n\n miss = oob | mismatch\n\n # Assign -1 to misses, otherwise idx\n out = np.full(vals.shape, -1, dtype=np.int64)\n out[~miss] = idx[~miss]\n\n mapped[valid_mask] = out\n return mapped.astype(np.int32)\n\n def frequency_encode_new_features(self, X: pd.DataFrame, y: pd.Series = None):\n if self.add_freq or self.only_freq:\n if not self.fitted_:\n # TODO: Unclear whether there is a more efficient way to do this\n candidate_cols = FrequencyFeatureGenerator.filter_candidates_by_distinctiveness(X[self.new_col_set])\n if len(candidate_cols) > 0:\n keep_original = not self.only_freq\n self.cat_freq = FrequencyFeatureGenerator(\n candidate_cols=candidate_cols, fillna=self.fillna, log=self.log, keep_original=keep_original\n )\n return self.cat_freq.fit_transform(X[candidate_cols], y)\n else:\n return X\n else:\n return self.cat_freq.transform(X)\n else:\n return X\n\n def _fit(self, X: pd.DataFrame, y: pd.Series, **kwargs):\n if self.candidate_cols is None:\n self.candidate_cols = X.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n self.candidate_cols = [\n i for i in self.candidate_cols if X[i].nunique() >= self.min_cardinality\n ] # TODO: Make this a parameter\n\n # FIXME: why `self.max_order` and not 2?\n if len(self.candidate_cols) < self.max_order:\n self.new_col_set = []\n return self\n\n X = X[self.candidate_cols].copy()\n # Ensure categorical only when needed\n for col in self.candidate_cols:\n if not isinstance(X[col].dtype, pd.CategoricalDtype):\n X[col] = X[col].astype(\"category\")\n\n self.get_interaction_names_dict(X[self.candidate_cols], max_order=self.max_order, max_feats=self.max_new_feats)\n\n # Cache category sizes (+1 for missing index -1)\n self.cat_sizes_ = {col: len(X[col].cat.categories) + 1 for col in self.used_cols}\n\n self.fitted_ = True\n\n return self\n\n # FIXME: Filter if too unique?\n def _fit_transform(self, X: pd.DataFrame, y: pd.Series, **kwargs) -> Tuple[pd.DataFrame, dict]:\n self._fit(X, y, **kwargs)\n X_out = self._transform(X, fit_mode=True)\n\n # features_out = list(X_out.columns)\n\n # type_group_map_special = {R_CATEGORY: features_out} # TODO: Find out whether that is needed\n return X_out, dict()\n\n def _transform(self, X: pd.DataFrame, y: pd.Series = None, fit_mode: bool = False, **kwargs) -> pd.DataFrame:\n if len(self.new_col_set) == 0:\n return pd.DataFrame(index=X.index)\n\n if self.inference_mode == \"string\":\n X_out = pd.DataFrame(index=X.index)\n for degree in range(2, self.max_order + 1):\n col_set_use = [col for col in self.new_col_set if col.count(\"_&_\") + 1 == degree]\n if len(col_set_use) > 0:\n X_degree = self.combine_predefined(X[self.used_cols], col_set_use, fit_mode=fit_mode)\n X_out = pd.concat([X_out, X_degree], axis=1)\n if fit_mode:\n self.cat_transformer = CategoryFeatureGenerator(minimum_cat_count=1)\n X_out = self.cat_transformer.fit_transform(X_out)\n else:\n X_out = self.cat_transformer.transform(X_out)\n\n else: # new inference mode\n for col in self.used_cols:\n if not isinstance(X[col].dtype, pd.CategoricalDtype):\n X[col] = X[col].astype(\"category\")\n X_out = self._build_interactions(X[self.used_cols], fit_mode=fit_mode)\n\n X_out = self.frequency_encode_new_features(X_out)\n\n return X_out\n"
},
{
"path": "features/src/autogluon/features/generators/category.py",
"content": "import copy\nimport logging\n\nimport pandas as pd\nfrom pandas import DataFrame\nfrom pandas.api.types import CategoricalDtype\n\nfrom autogluon.common.features.types import (\n R_BOOL,\n R_CATEGORY,\n R_OBJECT,\n S_DATETIME_AS_OBJECT,\n S_IMAGE_BYTEARRAY,\n S_IMAGE_PATH,\n S_TEXT,\n S_TEXT_AS_CATEGORY,\n)\n\nfrom .abstract import AbstractFeatureGenerator\nfrom .memory_minimize import CategoryMemoryMinimizeFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add hashing trick if minimize_memory=True to avoid storing full original mapping\n# TODO: fill_nan add additional options: group_rares, possibly percentile based\nclass CategoryFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n CategoryFeatureGenerator is used to convert object types to category types, as well as remove rare categories and optimize memory usage.\n After fitting, previously unseen categories during transform are treated as missing values.\n\n Parameters\n ----------\n stateful_categories : bool, default True\n If True, categories from training are applied to transformed data, and any unknown categories from input data will be treated as missing values.\n It is recommended to keep this value as True to avoid strange downstream behavior.\n minimize_memory : bool, default True\n If True, minimizes category memory usage by converting all category values to sequential integers.\n This replaces any string data present in the categories but does not alter the behavior of models when using the category as a feature so long\n as the original string values are not required downstream.\n It is recommended to keep this value as True to dramatically reduce memory usage with no cost to accuracy.\n cat_order : str, default 'original'\n Determines the order in which categories are stored.\n This is important when minimize_memory is True, as the order will determine which categories are converted to which integer values.\n Valid values:\n 'original' : Keep the original order. If the feature was originally an object, this is equivalent to 'alphanumeric'.\n 'alphanumeric' : Sort the categories alphanumerically.\n 'count' : Sort the categories by frequency (Least frequent in front with code of 0)\n minimum_cat_count : int, default None\n The minimum number of occurrences a category must have in the training data to avoid being considered a rare category.\n Rare categories are removed and treated as missing values.\n If None, no minimum count is required. This includes categories that never occur in the data but are present in the category object\n as possible categories.\n maximum_num_cat : int, default None\n The maximum amount of categories that can be considered non-rare.\n Sorted by occurrence count, up to the N highest count categories will be kept if maximum_num_cat=N. All others will be considered rare categories.\n fillna : str, default None\n The method used to handle missing values. Only valid if stateful_categories=True.\n Missing values include the values that were originally NaN and values converted to NaN from other parameters such as minimum_cat_count.\n Valid values:\n None : Keep missing values as is. They will appear as NaN and have no category assigned to them.\n 'mode' : Set missing values to the most frequent category in their feature.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(\n self,\n stateful_categories=True,\n minimize_memory=True,\n cat_order=\"original\",\n minimum_cat_count: int = 2,\n maximum_num_cat: int = None,\n fillna: str = None,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self._stateful_categories = stateful_categories\n if minimum_cat_count is not None and minimum_cat_count < 1:\n minimum_cat_count = None\n if cat_order not in [\"original\", \"alphanumeric\", \"count\"]:\n raise ValueError(f\"cat_order must be one of {['original', 'alphanumeric', 'count']}, but was: {cat_order}\")\n self.cat_order = cat_order\n self._minimum_cat_count = minimum_cat_count\n self._maximum_num_cat = maximum_num_cat\n self.category_map = None\n if fillna is not None:\n if fillna not in [\"mode\"]:\n raise ValueError(f\"fillna={fillna} is not a valid value. Valid values: {[None, 'mode']}\")\n self._fillna = fillna\n self._fillna_flag = self._fillna is not None\n self._fillna_map = None\n\n if minimize_memory:\n self._post_generators = [CategoryMemoryMinimizeFeatureGenerator()] + self._post_generators\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n if self._stateful_categories:\n X_out, self.category_map, self._fillna_map = self._generate_category_map(X=X)\n if self._fillna_map is not None:\n for column in self._fillna_map:\n X_out[column] = X_out[column].fillna(self._fillna_map[column])\n else:\n X_out = self._transform(X)\n feature_metadata_out_type_group_map_special = copy.deepcopy(self.feature_metadata_in.type_group_map_special)\n if S_TEXT in feature_metadata_out_type_group_map_special:\n text_features = feature_metadata_out_type_group_map_special.pop(S_TEXT)\n feature_metadata_out_type_group_map_special[S_TEXT_AS_CATEGORY] += [\n feature\n for feature in text_features\n if feature not in feature_metadata_out_type_group_map_special[S_TEXT_AS_CATEGORY]\n ]\n return X_out, feature_metadata_out_type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self._generate_features_category(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(\n valid_raw_types=[R_OBJECT, R_CATEGORY, R_BOOL],\n invalid_special_types=[S_DATETIME_AS_OBJECT, S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n )\n\n def _generate_features_category(self, X: DataFrame) -> DataFrame:\n if self.features_in:\n X_category = dict()\n if self.category_map is not None:\n for column, column_map in self.category_map.items():\n X_category[column] = pd.Categorical(X[column], categories=column_map)\n X_category = DataFrame(X_category, index=X.index)\n if self._fillna_map is not None:\n for column, column_map in self._fillna_map.items():\n X_category[column] = X_category[column].fillna(column_map)\n else:\n X_category = DataFrame(index=X.index)\n return X_category\n\n def _generate_category_map(self, X: DataFrame) -> (DataFrame, dict):\n if self.features_in:\n fill_nan_map = dict()\n category_map = dict()\n X_category = X.astype(\"category\")\n for column in X_category:\n rank = X_category[column].value_counts().sort_values(ascending=True)\n if self._minimum_cat_count is not None:\n rank = rank[rank >= self._minimum_cat_count]\n if self._maximum_num_cat is not None:\n rank = rank[-self._maximum_num_cat :]\n if (\n self.cat_order == \"count\"\n or self._minimum_cat_count is not None\n or self._maximum_num_cat is not None\n ):\n category_list = list(rank.index) # category_list in 'count' order\n if len(category_list) > 1:\n if self.cat_order == \"original\":\n original_cat_order = list(X_category[column].cat.categories)\n set_category_list = set(category_list)\n category_list = [cat for cat in original_cat_order if cat in set_category_list]\n elif self.cat_order == \"alphanumeric\":\n category_list.sort()\n X_category[column] = X_category[column].astype(\n CategoricalDtype(categories=category_list)\n ) # TODO: Remove columns if all NaN after this?\n X_category[column] = X_category[column].cat.reorder_categories(category_list)\n elif self.cat_order == \"alphanumeric\":\n category_list = list(X_category[column].cat.categories)\n category_list.sort()\n X_category[column] = X_category[column].astype(CategoricalDtype(categories=category_list))\n X_category[column] = X_category[column].cat.reorder_categories(category_list)\n category_map[column] = copy.deepcopy(X_category[column].cat.categories)\n if self._fillna_flag:\n if self._fillna == \"mode\":\n if len(rank) > 0:\n fill_nan_map[column] = list(rank.index)[-1]\n if not self._fillna_flag:\n fill_nan_map = None\n return X_category, category_map, fill_nan_map\n else:\n return DataFrame(index=X.index), None, None\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self.category_map:\n for feature in features:\n if feature in self.category_map:\n self.category_map.pop(feature)\n if self._fillna_map:\n for feature in features:\n if feature in self._fillna_map:\n self._fillna_map.pop(feature)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/datetime.py",
"content": "import logging\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_DATETIME, S_DATETIME_AS_OBJECT\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass DatetimeFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"Transforms datetime features into numeric features.\n\n Parameters\n ----------\n features : list, optional\n A list of datetime features to parse out of dates.\n For a full list of options see the methods inside pandas.Series.dt at https://pandas.pydata.org/docs/reference/api/pandas.Series.html\n \"\"\"\n\n def __init__(self, features: list = None, **kwargs):\n super().__init__(**kwargs)\n if features is None:\n features = [\"year\", \"month\", \"day\", \"dayofweek\"]\n self.features = features\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n self._fillna_map = dict()\n X_out = self._transform(X, is_fit=True)\n type_family_groups_special = dict(datetime_as_int=list(X_out.columns))\n return X_out, type_family_groups_special\n\n def _transform(self, X: DataFrame, is_fit=False) -> DataFrame:\n return self._generate_features_datetime(X, is_fit=is_fit)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(required_raw_special_pairs=[(R_DATETIME, None), (None, [S_DATETIME_AS_OBJECT])])\n\n def normalize_timeseries(self, X: pd.DataFrame, feature: str, is_fit: bool) -> pd.Series:\n # TODO: Be aware: When converted to float32 by downstream models, the seconds value will be up to 3 seconds off the true time due to rounding error.\n # If seconds matter, find a separate way to generate (Possibly subtract smallest datetime from all values).\n # TODO: could also return an extra boolean column is_nan which could provide predictive signal.\n # Note: The .replace call is required to handle the obnoxious edge-case of:\n # NaN, empty string, datetime without timezone, and datetime with timezone, all as an object type, all being present in the same column.\n # I don't know why, but in this specific situation (and not otherwise), NaN will be filled by .fillna, but empty string will be converted to NaT\n # and refuses to be filled by .fillna, requiring a dedicated replace call. (NaT is filled by .fillna in every other situation...)\n # Note: The below `pd.to_datetime` line can take a long time if the format is mixed.\n # For an example of this problem, refer to https://www.kaggle.com/code/kuldeepnpatel/to-datetime-is-too-slow-on-large-dataset\n # There does not appear to be a straightforward way to avoid this,\n # and the trick that was used in the above notebook was removed in Pandas 2.0 due to being unsafe.\n # Alternatives like Polars do not offer the same datetime conversion logic, and thus aren't valid to use.\n # The runtime is approximately 0.08 seconds per 1000 rows in worst case.\n series = pd.to_datetime(X[feature].copy(), utc=True, errors=\"coerce\", format=\"mixed\")\n broken_idx = series[(series == \"NaT\") | series.isna() | series.isnull()].index\n bad_rows = series.iloc[broken_idx]\n if is_fit:\n good_rows = series[~series.isin(bad_rows)].astype(np.int64)\n self._fillna_map[feature] = pd.to_datetime(int(good_rows.mean()), utc=True, format=\"mixed\")\n series[broken_idx] = self._fillna_map[feature]\n return series\n\n # TODO: Improve handling of missing datetimes\n def _generate_features_datetime(self, X: DataFrame, is_fit: bool) -> DataFrame:\n X_datetime = DataFrame(index=X.index)\n for datetime_feature in self.features_in:\n X_datetime[datetime_feature] = self.normalize_timeseries(X, datetime_feature, is_fit=is_fit)\n for feature in self.features:\n X_datetime[datetime_feature + \".\" + feature] = getattr(\n X_datetime[datetime_feature].dt, feature\n ).astype(np.int64)\n X_datetime[datetime_feature] = pd.to_numeric(X_datetime[datetime_feature])\n return X_datetime\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self._fillna_map:\n for feature in features:\n if feature in self._fillna_map:\n self._fillna_map.pop(feature)\n"
},
{
"path": "features/src/autogluon/features/generators/drop_duplicates.py",
"content": "from __future__ import annotations\n\nimport hashlib\nimport logging\nfrom collections import defaultdict\nfrom typing import Union\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_BOOL, R_CATEGORY, R_FLOAT, R_INT\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Not necessary to exist after fitting, can just update outer context feature_out/feature_in and then delete this\nclass DropDuplicatesFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Drops features which are exact duplicates of other features, leaving only one instance of the data.\n\n Parameters\n ----------\n sample_size_init : int, default 1000\n The number of rows to sample when doing an initial filter of duplicate feature candidates.\n Usually, the majority of features can be filtered out using this smaller amount of rows which greatly speeds up the computation of the final check.\n If None or greater than the number of rows, no initial filter will occur. This may increase the time to fit immensely for large datasets.\n sample_size_final : int, default 5000\n The number of rows to sample when doing the final filter to determine duplicate features.\n This theoretically can lead to features that are very nearly duplicates but not exact duplicates being removed,\n but should be near impossible in practice.\n If None or greater than the number of rows, will perform exact duplicate detection (most expensive).\n It is recommended to keep this value below 100000 to maintain reasonable fit times.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(self, sample_size_init=1000, sample_size_final=5000, **kwargs):\n super().__init__(**kwargs)\n self.sample_size_init = sample_size_init\n self.sample_size_final = sample_size_final\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n if self.sample_size_init is not None and len(X) > self.sample_size_init:\n features_to_check = self._drop_duplicate_features(\n X, self.feature_metadata_in, keep=False, sample_size=self.sample_size_init\n )\n X_candidates = X[features_to_check]\n else:\n X_candidates = X\n features_to_drop = self._drop_duplicate_features(\n X_candidates, self.feature_metadata_in, sample_size=self.sample_size_final\n )\n self._remove_features_in(features_to_drop)\n if features_to_drop:\n self._log(15, f\"\\t{len(features_to_drop)} duplicate columns removed: {features_to_drop}\")\n # Avoid creating an unnecessary copy with X[self.features_in], if possible\n if self.features_in != X.columns.to_list():\n X = X[self.features_in]\n return X, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return X\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n @classmethod\n def _drop_duplicate_features(\n cls, X: DataFrame, feature_metadata_in, keep: Union[str, bool] = \"first\", sample_size=None\n ) -> list:\n if sample_size is not None and len(X) > sample_size:\n # Sampling with replacement is much faster than without replacement\n X = X.sample(sample_size, random_state=0, replace=True)\n features_to_remove = []\n\n X_columns = set(X.columns)\n features_to_check_numeric = feature_metadata_in.get_features(valid_raw_types=[R_INT, R_FLOAT])\n features_to_check_numeric = [feature for feature in features_to_check_numeric if feature in X_columns]\n if features_to_check_numeric:\n features_to_remove += cls._drop_duplicate_features_numeric(X=X[features_to_check_numeric], keep=keep)\n X = X.drop(columns=features_to_check_numeric)\n\n X_columns = set(X.columns)\n features_to_check_categorical = feature_metadata_in.get_features(valid_raw_types=[R_CATEGORY, R_BOOL])\n features_to_check_categorical = [feature for feature in features_to_check_categorical if feature in X_columns]\n if features_to_check_categorical:\n features_to_remove += cls._drop_duplicate_features_categorical(\n X=X[features_to_check_categorical], keep=keep\n )\n X = X.drop(columns=features_to_check_categorical)\n\n if len(X.columns) > 0:\n features_to_remove += cls._drop_duplicate_features_generic(X=X, keep=keep)\n\n return features_to_remove\n\n @classmethod\n def _drop_duplicate_features_generic(cls, X: DataFrame, keep: Union[str, bool] = \"first\"):\n \"\"\"Generic duplication dropping method. Much slower than optimized variants, but can handle all data types.\"\"\"\n X_columns = list(X.columns)\n features_to_keep = set(X.T.drop_duplicates(keep=keep).T.columns)\n features_to_remove = [column for column in X_columns if column not in features_to_keep]\n return features_to_remove\n\n @staticmethod\n def _fingerprint_numeric_series_full(s: pd.Series) -> bytes:\n \"\"\"\n Deterministic full-column fingerprint for numeric-like Series.\n\n - Includes an isna mask, so NaN/NA positions matter.\n - Normalizes missing values by zeroing them in the value buffer (mask carries the info).\n - Normalizes -0.0 to +0.0 for float types.\n - Uses blake2b (fast, stable) to produce a 16-byte digest.\n \"\"\"\n # Convert to numpy array (may include float w/ NaN or pandas nullable types)\n arr = s.to_numpy(copy=False)\n\n # Build missing mask robustly (works for float NaN, pandas NA, etc.)\n # Note: pd.isna on ndarray returns ndarray[bool] of same shape.\n mask = pd.isna(arr)\n\n # Normalize to a stable numeric dtype + stable value buffer\n if np.issubdtype(arr.dtype, np.floating):\n vals = np.asarray(arr, dtype=np.float64).copy()\n # Normalize -0.0 -> +0.0 (0.0 == -0.0 but different bit pattern)\n vals[vals == 0.0] = 0.0\n # Zero-out missing to avoid NaN payload differences; mask preserves NA pattern\n if mask.any():\n vals[mask] = 0.0\n else:\n # For ints / bools / nullable integer arrays that became object,\n # coerce to float64 so NA can be represented; mask preserves NA pattern.\n # (If you prefer strict dtype separation, change this to int64 and handle NA separately.)\n vals = np.asarray(arr, dtype=np.float64).copy()\n if mask.any():\n vals[mask] = 0.0\n\n # Hash both: value bytes + mask bytes\n h = hashlib.blake2b(digest_size=16)\n h.update(np.ascontiguousarray(vals).view(np.uint8))\n h.update(np.ascontiguousarray(mask).view(np.uint8))\n return h.digest()\n\n @classmethod\n def _drop_duplicate_features_numeric(\n cls,\n X: DataFrame,\n keep: Union[str, bool] = \"first\",\n ) -> list[str]:\n \"\"\"\n >100x faster than pandas drop_duplicates\n\n Numeric duplicate detection using:\n 1) summary-stat bucketing (sum, std, min, max)\n 2) full-column fingerprint hash (values + missing mask)\n\n Never calls pandas drop_duplicates.\n\n keep semantics:\n - \"first\": keep earliest column in X.columns order\n - \"last\": keep latest column in X.columns order\n - False: drop all columns in duplicate groups\n - True: treated like \"first\" (for compatibility)\n \"\"\"\n if X.empty or X.shape[1] <= 1:\n return []\n\n # Normalize keep\n if keep is True:\n keep_mode = \"first\"\n elif keep in (\"first\", \"last\") or keep is False:\n keep_mode = keep\n else:\n raise ValueError(f\"Invalid keep={keep!r}. Expected 'first', 'last', False (or True).\")\n\n cols = list(X.columns)\n\n # ---- Vectorized stats pass (cheap) ----\n # Note: pandas reductions skipna by default, consistent across these stats.\n stats = pd.DataFrame(\n {\n \"sum\": X.sum(axis=0),\n \"std\": X.std(axis=0, ddof=0),\n \"min\": X.min(axis=0),\n \"max\": X.max(axis=0),\n }\n ).round(6)\n\n # ---- Bucket by stats ----\n bucket_map: dict[tuple[float, float, float, float], list[str]] = defaultdict(list)\n for c in cols:\n key = (stats.at[c, \"sum\"], stats.at[c, \"std\"], stats.at[c, \"min\"], stats.at[c, \"max\"])\n bucket_map[key].append(c)\n\n # ---- Within each stats bucket, bucket by full fingerprint ----\n to_remove: list[str] = []\n\n for bucket_cols in bucket_map.values():\n if len(bucket_cols) <= 1:\n continue\n\n fp_map: dict[bytes, list[str]] = defaultdict(list)\n for c in bucket_cols:\n fp = cls._fingerprint_numeric_series_full(X[c])\n fp_map[fp].append(c)\n\n for dup_group in fp_map.values():\n if len(dup_group) <= 1:\n continue\n\n # Deterministic ordering: preserve original column order\n # (dup_group already follows bucket_cols order which follows cols order,\n # but we enforce explicitly to be safe.)\n dup_group_sorted = sorted(dup_group, key=lambda k: cols.index(k))\n\n if keep_mode == \"first\":\n to_remove.extend(dup_group_sorted[1:])\n elif keep_mode == \"last\":\n to_remove.extend(dup_group_sorted[:-1])\n else: # keep_mode is False\n to_remove.extend(dup_group_sorted)\n\n return to_remove\n\n @classmethod\n def _drop_duplicate_features_categorical(cls, X: DataFrame, keep: Union[str, bool] = \"first\"):\n \"\"\"\n Drops duplicate features if they contain the same information, ignoring the actual values in the features.\n For example, ['a', 'b', 'b'] is considered a duplicate of ['b', 'a', 'a'], but not ['a', 'b', 'a'].\n \"\"\"\n X_columns = list(X.columns)\n mapping_features_val_dict = {}\n features_unique_count_dict = defaultdict(list)\n features_to_remove = []\n for feature in X_columns:\n feature_unique_vals = X[feature].unique()\n mapping_features_val_dict[feature] = dict(zip(feature_unique_vals, range(len(feature_unique_vals))))\n features_unique_count_dict[len(feature_unique_vals)].append(feature)\n\n for feature_unique_count in features_unique_count_dict:\n # Only need to check features that have same amount of unique values.\n features_to_check = features_unique_count_dict[feature_unique_count]\n if len(features_to_check) <= 1:\n continue\n mapping_features_val_dict_cur = {\n feature: mapping_features_val_dict[feature] for feature in features_to_check\n }\n # Converts ['a', 'd', 'f', 'a'] to [0, 1, 2, 0]\n # Converts [5, 'a', np.nan, 5] to [0, 1, 2, 0], these would be considered duplicates since they carry the same information.\n\n # Have to convert to object dtype because category dtype for unknown reasons will refuse to replace NaNs.\n try:\n # verify that the option exists (pandas >2.1)\n pd.get_option(\"future.no_silent_downcasting\")\n except pd.errors.OptionError:\n X_cur = X[features_to_check].astype(\"object\").replace(mapping_features_val_dict_cur).astype(np.int64)\n else:\n # refer to https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#deprecated-automatic-downcasting\n with pd.option_context(\"future.no_silent_downcasting\", True):\n X_cur = (\n X[features_to_check].astype(\"object\").replace(mapping_features_val_dict_cur).astype(np.int64)\n )\n features_to_remove += cls._drop_duplicate_features_numeric(X=X_cur, keep=keep)\n\n return features_to_remove\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/drop_unique.py",
"content": "import logging\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.types import R_CATEGORY, R_OBJECT, S_IMAGE_BYTEARRAY, S_IMAGE_PATH, S_TEXT\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Not necessary to exist after fitting, can just update outer context feature_out/feature_in and then delete this\nclass DropUniqueFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"Drops features which only have 1 unique value or which have nearly no repeated values (based on max_unique_ratio) and are of category or object type.\"\"\"\n\n def __init__(self, max_unique_ratio=0.99, **kwargs):\n super().__init__(**kwargs)\n self.max_unique_ratio = max_unique_ratio\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n features_to_drop = self._drop_unique_features(\n X, self.feature_metadata_in, max_unique_ratio=self.max_unique_ratio\n )\n self._remove_features_in(features_to_drop)\n X_out = X[self.features_in]\n return X_out, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return X\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n # TODO: Consider NaN?\n @staticmethod\n def _drop_unique_features(X: DataFrame, feature_metadata: FeatureMetadata, max_unique_ratio) -> list:\n features_to_drop = []\n X_len = len(X)\n max_unique_value_count = X_len * max_unique_ratio\n for column in X:\n unique_value_count = len(X[column].unique())\n # Drop features that are always the same\n if unique_value_count == 1:\n features_to_drop.append(column)\n elif feature_metadata.get_feature_type_raw(column) in [R_CATEGORY, R_OBJECT] and (\n unique_value_count > max_unique_value_count\n ):\n special_types = feature_metadata.get_feature_types_special(column)\n if S_TEXT in special_types:\n # We should not drop a text column\n continue\n elif S_IMAGE_PATH in special_types:\n # We should not drop an image path column\n continue\n elif S_IMAGE_BYTEARRAY in special_types:\n # We should not drop an image bytearray column\n continue\n else:\n features_to_drop.append(column)\n return features_to_drop\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/dummy.py",
"content": "import logging\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass DummyFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Ignores all input features and returns a single int feature with all 0 values.\n Useful for testing purposes or to avoid crashes if no features were given.\n \"\"\"\n\n def __init__(self, features_in=\"empty\", feature_metadata_in=\"empty\", **kwargs):\n if features_in == \"empty\":\n features_in = []\n if feature_metadata_in == \"empty\":\n feature_metadata_in = FeatureMetadata(type_map_raw={})\n super().__init__(features_in=features_in, feature_metadata_in=feature_metadata_in, **kwargs)\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n X_out = self._transform(X)\n return X_out, dict()\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self._generate_features_dummy(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[])\n\n @staticmethod\n def _generate_features_dummy(X: DataFrame):\n X_out = DataFrame(index=X.index)\n X_out[\"__dummy__\"] = 0\n return X_out\n\n def is_valid_metadata_in(self, feature_metadata_in: FeatureMetadata):\n return True\n"
},
{
"path": "features/src/autogluon/features/generators/fillna.py",
"content": "import logging\nimport warnings\n\nimport numpy as np\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_OBJECT\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Add fillna_special_map, fillna_combined_map to increase options\n# TODO: Add options to specify mean/median/mode for int/float\n# TODO: Add fillna_features for feature specific fill values\nclass FillNaFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Fills missing values in the data.\n\n Parameters\n ----------\n fillna_map : dict, default {'object': ''}\n Map which dictates the fill values of NaNs.\n Keys are the raw types of the features as in self.feature_metadata_in.type_map_raw.\n If a feature's raw type is not present in fillna_map, its NaN values are filled to fillna_default.\n fillna_default, default np.nan\n The default fillna value if the feature's raw type is not present in fillna_map.\n Be careful about setting this to anything other than np.nan, as not all raw types can handle int, float, or string values.\n inplace : bool, default False\n If True, then the NaN values are filled inplace without copying the input data.\n This will alter the input data outside of the scope of this function.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(self, fillna_map=None, fillna_default=np.nan, inplace=False, **kwargs):\n super().__init__(**kwargs)\n if fillna_map is None:\n fillna_map = {R_OBJECT: \"\"}\n self.fillna_map = fillna_map\n self.fillna_default = fillna_default\n self._fillna_feature_map = None\n self.inplace = inplace\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n features = self.feature_metadata_in.get_features()\n self._fillna_feature_map = dict()\n for feature in features:\n feature_raw_type = self.feature_metadata_in.get_feature_type_raw(feature)\n feature_fillna_val = self.fillna_map.get(feature_raw_type, self.fillna_default)\n if feature_fillna_val is not np.nan:\n self._fillna_feature_map[feature] = feature_fillna_val\n return self._transform(X), self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n if self._fillna_feature_map:\n with warnings.catch_warnings():\n warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n # FIXME: v1.1 Remove this warning filter and resolve.\n # In Pandas 2.1, the `downcast` argument was deprecated,\n # but we need it to avoid incorrect type conversion.\n # Pandas authors may have not considered our edge-case.\n # We specifically want to have an object dtype not be converted to a numeric dtype,\n # even if all of the values can be converted to numeric.\n # However, without specifying `downcast=False`, it will be converted to numeric, which we don't want.\n # Note: Non-trivial to keep current functionality without specifying `downcast=False`...\n # Doing so may end up slowing down the code noticeably.\n if self.inplace:\n X.fillna(self._fillna_feature_map, inplace=True, downcast=False)\n else:\n X = X.fillna(self._fillna_feature_map, inplace=False, downcast=False)\n return X\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n def _remove_features_in(self, features):\n super()._remove_features_in(features)\n if features:\n for feature in features:\n self._fillna_feature_map.pop(feature, None)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/frequency.py",
"content": "from typing import Literal, Tuple\n\nimport numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import (\n R_BOOL,\n R_CATEGORY,\n R_FLOAT,\n R_INT,\n R_OBJECT,\n S_DATETIME_AS_OBJECT,\n S_IMAGE_BYTEARRAY,\n S_IMAGE_PATH,\n)\n\nfrom .abstract import AbstractFeatureGenerator\n\n\nclass FrequencyFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Generate frequency encoded features for categorical variables.\n Parameters\n ----------\n keep_original : bool, default=True\n Whether to keep the original features.\n only_categorical : bool, default=True\n Whether to only apply frequency encoding to categorical features.\n candidate_cols : list of str or None, default=None\n List of candidate columns to consider for frequency encoding. If None, all columns are used.\n use_filters : bool, default=True\n Whether to filter candidate columns based on distinctiveness.\n fillna : int or None, default=0\n Value to fill NaNs in frequency encoding. If None, NaNs are kept as NaN.\n log : bool, default=False\n Whether to apply log transformation to frequency encoded values.\n **kwargs\n Additional keyword arguments.\n Returns\n -------\n self : FrequencyFeatureGenerator\n Fitted FrequencyFeatureGenerator instance.\n \"\"\"\n\n def __init__(\n self,\n keep_original: bool = True,\n only_categorical: bool = True,\n candidate_cols: list = None,\n use_filters: bool = True,\n fillna: int | None = 0,\n log: bool = False,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self.keep_original = keep_original # TODO: Clarify if and how something similar to keep_original logic is already in AG preprocessors\n self.only_categorical = only_categorical\n self.candidate_cols = candidate_cols\n self.use_filters = use_filters\n self.log = log\n\n if fillna is None:\n self.fillna = np.nan\n else:\n self.fillna = fillna\n\n self.freq_maps = {}\n self.passthrough_cols = []\n\n def estimate_no_of_new_features(self, X: pd.DataFrame, **kwargs) -> int:\n # TODO: Improve estimation using other hyperparameters\n # TODO: Account for the fact that some columns may be removed if keep_original is False\n if self.only_categorical:\n return X.select_dtypes(include=[\"object\", \"category\"]).shape[1]\n else:\n return X.shape[1]\n\n @classmethod\n def filter_candidates_by_distinctiveness(cls, X: pd.DataFrame) -> list:\n candidate_cols = []\n for col in X.columns:\n x_new = X[col].map(X[col].value_counts().to_dict())\n if all((pd.crosstab(X[col], x_new) > 0).sum() == 1):\n continue\n else:\n candidate_cols.append(col)\n\n return candidate_cols\n\n def _fit(self, X_in: pd.DataFrame, y_in: pd.Series = None):\n X = X_in.copy()\n\n if self.candidate_cols is not None:\n self.passthrough_cols.extend([col for col in X.columns if col not in self.candidate_cols])\n X = X[self.candidate_cols]\n if self.only_categorical:\n self.passthrough_cols.extend(\n [col for col in X.columns if col not in X.select_dtypes(include=[\"object\", \"category\"]).columns]\n )\n X = X.select_dtypes(include=[\"object\", \"category\"])\n\n for col in X.columns:\n x = X[col]\n self.freq_maps[x.name] = x.value_counts().to_dict()\n\n return self\n\n def _fit_transform(self, X: pd.DataFrame, y: pd.Series, **kwargs) -> Tuple[pd.DataFrame, dict]:\n self._fit(X, y, **kwargs)\n X_out = self._transform(X)\n\n if self.keep_original:\n X_out = pd.concat([X, X_out], axis=1)\n\n # if self.log:\n # type_group_map_special = {R_FLOAT: list(self.freq_maps.keys())}\n # else:\n # type_group_map_special = {R_INT: list(self.freq_maps.keys())}\n return X_out, dict() # TODO: Unsure whether we need to return anything special here\n\n def _transform(self, X_in, **kwargs):\n X = X_in.copy()\n\n new_cols = []\n for col in X.columns:\n x = X[col]\n if x.name in self.freq_maps:\n new_col = x.map(self.freq_maps[x.name]).astype(float).fillna(self.fillna)\n new_col.name = x.name + \"_freq\"\n if self.log:\n new_col = np.log1p(new_col)\n new_col.name += \"_log\"\n new_cols.append(new_col)\n else:\n continue\n if self.keep_original:\n return pd.concat([X] + new_cols, axis=1)\n else:\n return pd.concat([X[self.passthrough_cols]] + new_cols, axis=1)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(\n valid_raw_types=[R_OBJECT, R_CATEGORY, R_BOOL, R_INT, R_FLOAT],\n invalid_special_types=[S_DATETIME_AS_OBJECT, S_IMAGE_PATH, S_IMAGE_BYTEARRAY],\n )\n"
},
{
"path": "features/src/autogluon/features/generators/groupby.py",
"content": "import numpy as np\nimport pandas as pd\n\nfrom autogluon.common.features.types import R_CATEGORY, R_OBJECT\n\nfrom .abstract import AbstractFeatureGenerator\n\n\n# ----------------------------\n# Aggregations\n# ----------------------------\ndef q25(series):\n return series.quantile(0.25)\n\n\ndef q75(series):\n return series.quantile(0.75)\n\n\ndef q10(series):\n return series.quantile(0.10)\n\n\ndef q90(series):\n return series.quantile(0.90)\n\n\nAGGREGATION_REGISTRY = {\n \"mean\": {\"kind\": \"group\", \"agg\": \"mean\"},\n \"std\": {\"kind\": \"group\", \"agg\": \"std\"},\n \"median\": {\"kind\": \"group\", \"agg\": \"median\"},\n \"count\": {\"kind\": \"group\", \"agg\": \"count\"},\n \"nunique\": {\"kind\": \"group\", \"agg\": pd.Series.nunique},\n \"min\": {\"kind\": \"group\", \"agg\": \"min\"},\n \"max\": {\"kind\": \"group\", \"agg\": \"max\"},\n \"q10\": {\"kind\": \"group\", \"agg\": q10},\n \"q25\": {\"kind\": \"group\", \"agg\": q25},\n \"q75\": {\"kind\": \"group\", \"agg\": q75},\n \"q90\": {\"kind\": \"group\", \"agg\": q90},\n \"pct_rank\": {\"kind\": \"rowwise\"},\n}\n\n\ndef rank_categoricals_by_small_counts(\n X: pd.DataFrame,\n categorical_cols,\n min_count: int = 1,\n top_k_smallest: int = 10,\n require_at_least_levels: int = 2,\n observed: bool = True,\n):\n \"\"\"\n Returns categorical_cols sorted best->worst by lexicographic comparison of the\n smallest group sizes (min, 2nd-min, ...).\n\n Score vector per cat:\n v = sorted(counts[counts >= min_count])[:top_k_smallest]\n Pad with +inf to fixed length so fewer levels doesn't get penalized.\n Sort by v descending lexicographically.\n \"\"\"\n scores = {}\n for cat in categorical_cols:\n counts = X[cat].value_counts(dropna=True)\n counts = counts[counts >= min_count].sort_values() # ascending\n\n if len(counts) < require_at_least_levels:\n v = np.full(top_k_smallest, -np.inf, dtype=float)\n else:\n v = counts.to_numpy(dtype=float)[:top_k_smallest]\n if v.size < top_k_smallest:\n v = np.pad(v, (0, top_k_smallest - v.size), constant_values=np.inf)\n\n scores[cat] = v\n\n ranked = sorted(scores.keys(), key=lambda c: tuple(scores[c]), reverse=True)\n return ranked, scores\n\n\nclass GroupByFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Output-identical, faster transform:\n - Preallocates output 2D array + builds DataFrame once\n - Caches numeric arrays once\n - Keeps EXACT category-to-code mapping semantics (Index.get_indexer on raw arrays)\n - Keeps EXACT feature insertion order as the original dict-based transform\n - Keeps pct_rank semantics identical (pct_rank is output whenever requested, even if drop_basic=True)\n \"\"\"\n\n def __init__(\n self,\n aggregations=(\n \"mean\",\n \"pct_rank\",\n ),\n relative_to_aggs=(\"mean\",),\n relative_ops=(\"ratio\",),\n drop_basic_groupby_when_relative=True,\n fill_value=\"nan\",\n eps=1e-8,\n return_dataframe=True,\n num_as_cat_cardinality_thresh=2,\n min_num_cardinality_thresh=10,\n max_features=500,\n random_state=42,\n **kwargs,\n ):\n super().__init__(random_state=random_state, **kwargs)\n self.relative_to_aggs = relative_to_aggs\n self.relative_ops = relative_ops\n self.aggregations = aggregations\n self.max_features = max_features\n\n self.drop_basic_groupby_when_relative = drop_basic_groupby_when_relative\n\n self.fill_value = np.nan if fill_value == \"nan\" else fill_value\n self.eps = eps\n self.return_dataframe = return_dataframe\n\n self.num_as_cat_cardinality_thresh = num_as_cat_cardinality_thresh\n self.min_num_cardinality_thresh = min_num_cardinality_thresh\n\n unknown = set(self.aggregations) - set(AGGREGATION_REGISTRY)\n if unknown:\n raise ValueError(f\"Unknown aggregations: {unknown}\")\n\n def _to_dataframe(self, X):\n if isinstance(X, pd.DataFrame):\n return X\n raise ValueError(\"Input must be a pandas DataFrame\")\n\n def _split_aggs(self):\n group_aggs, rowwise_aggs = [], []\n for name in self.aggregations:\n entry = AGGREGATION_REGISTRY[name]\n if entry[\"kind\"] == \"group\":\n group_aggs.append(name)\n elif entry[\"kind\"] == \"rowwise\":\n rowwise_aggs.append(name)\n else:\n raise ValueError(f\"Unknown agg kind for {name}: {entry}\")\n return group_aggs, rowwise_aggs\n\n def _relative_enabled(self) -> bool:\n return (\"diff\" in self.relative_ops) or (\"ratio\" in self.relative_ops)\n\n def _drop_basic(self) -> bool:\n # Same logic as your original\n return bool(self.drop_basic_groupby_when_relative and self._relative_enabled() and self.relative_to_aggs)\n\n def _features_per_pair(self):\n \"\"\"\n IMPORTANT: kept identical to your original implementation for pair selection / budgeting.\n (Yes, this means pct_rank is NOT counted when drop_basic=True, matching your current behavior.)\n \"\"\"\n group_aggs, rowwise_aggs = self._split_aggs()\n\n base = 0\n if not self._drop_basic():\n base += len(group_aggs)\n base += int(\"pct_rank\" in rowwise_aggs)\n\n rel = 0\n if self._relative_enabled():\n rel += len(self.relative_to_aggs) * (int(\"diff\" in self.relative_ops) + int(\"ratio\" in self.relative_ops))\n return int(base + rel)\n\n # ----------------------------\n # FIT\n # ----------------------------\n def _fit(self, X, y=None):\n X = self._to_dataframe(X)\n\n # infer types\n self.categorical_features = X.columns[X.nunique() < self.num_as_cat_cardinality_thresh].tolist()\n self.categorical_features += X.select_dtypes(include=[R_CATEGORY, R_OBJECT]).columns.tolist()\n self.categorical_features = np.unique(self.categorical_features).tolist()\n\n self.numeric_features = [\n col\n for col in X.columns\n if col not in self.categorical_features\n and X[col].dtype not in [\"category\", \"object\"]\n and X[col].nunique() >= self.min_num_cardinality_thresh\n ]\n\n if len(self.categorical_features) == 0 or len(self.numeric_features) == 0:\n self.group_stats_ = {}\n self.group_index_ = {}\n self.group_values_ = {}\n self.pct_rank_keys_ = {}\n self.pct_rank_vals_ = {}\n self.pct_rank_by_code_ = {}\n self.global_stats_ = {}\n self.output_columns_ = []\n self.pairs_ = []\n return self\n\n group_aggs, rowwise_aggs = self._split_aggs()\n drop_basic = self._drop_basic()\n\n ranked_cats, _ = rank_categoricals_by_small_counts(\n X,\n categorical_cols=self.categorical_features,\n min_count=20,\n top_k_smallest=10,\n )\n ranked_nums = X[self.numeric_features].nunique().sort_values(ascending=False).index.to_list()\n\n self.group_stats_ = {}\n self.group_index_ = {}\n self.group_values_ = {}\n self.pct_rank_keys_ = {}\n self.pct_rank_vals_ = {}\n self.pct_rank_by_code_ = {} # keep same lazy cache behavior, but initialize dict\n self.global_stats_ = {num: float(X[num].mean()) for num in self.numeric_features}\n\n # NEW: store exact transform iteration order + exact output column order\n self.pairs_ = []\n self.output_columns_ = []\n\n features_per_pair = self._features_per_pair()\n budget = self.max_features if self.max_features is not None else float(\"inf\")\n used_features = 0\n\n for cat in ranked_cats:\n if cat not in self.categorical_features:\n continue\n\n for num in ranked_nums:\n if num not in self.numeric_features:\n continue\n\n if used_features + features_per_pair > budget:\n return self\n\n # group-level stats\n if group_aggs:\n named_aggs = {\n name: pd.NamedAgg(column=num, aggfunc=AGGREGATION_REGISTRY[name][\"agg\"]) for name in group_aggs\n }\n stats = X.groupby(cat, observed=True).agg(**named_aggs).astype(float)\n else:\n stats = pd.DataFrame(index=X[cat].dropna().unique())\n\n self.group_stats_[(cat, num)] = stats\n\n idx = stats.index\n self.group_index_[(cat, num)] = idx\n self.group_values_[(cat, num)] = {\n agg: stats[agg].to_numpy(dtype=float, copy=False) for agg in stats.columns\n }\n\n if \"pct_rank\" in rowwise_aggs:\n g = X[[cat, num]].dropna()\n s = g.groupby(cat, observed=True)[num].apply(\n lambda t: np.sort(t.to_numpy(dtype=float, copy=False))\n )\n self.pct_rank_keys_[(cat, num)] = s.index.to_numpy()\n self.pct_rank_vals_[(cat, num)] = s.to_numpy() # dtype=object\n\n # record order (matches original dict insertion order)\n self.pairs_.append((cat, num))\n\n # EXACT output layout:\n # - group aggs only when not drop_basic\n if not drop_basic:\n for agg in group_aggs:\n self.output_columns_.append(f\"{num}__by__{cat}__{agg}\")\n\n # - pct_rank ALWAYS when requested (matches your original transform)\n if \"pct_rank\" in rowwise_aggs:\n self.output_columns_.append(f\"{num}__by__{cat}__pct_rank\")\n\n # - relatives in the same nested loop order as original\n if self._relative_enabled():\n for agg in self.relative_to_aggs:\n if \"diff\" in self.relative_ops:\n self.output_columns_.append(f\"{num}__minus__by__{cat}_{agg}\")\n if \"ratio\" in self.relative_ops:\n self.output_columns_.append(f\"{num}__ratio__by__{cat}_{agg}\")\n\n used_features += features_per_pair\n\n return self\n\n # ----------------------------\n # TRANSFORM\n # ----------------------------\n def _transform(self, X):\n X = self._to_dataframe(X)\n\n if len(getattr(self, \"pairs_\", [])) == 0:\n empty = pd.DataFrame(index=X.index)\n return empty if self.return_dataframe else empty.values\n\n group_aggs, rowwise_aggs = self._split_aggs()\n drop_basic = self._drop_basic()\n rel_enabled = self._relative_enabled()\n\n n = len(X)\n m = len(self.output_columns_)\n out = np.empty((n, m), dtype=float)\n col_i = 0\n\n # cache numeric arrays once\n used_nums = {num for _, num in self.pairs_}\n num_cache = {num: X[num].astype(float).to_numpy(copy=False) for num in used_nums}\n\n # cache category codes once per cat (EXACT original semantics)\n codes_cache = {} # cat -> np.ndarray[int]\n missing_cache = {} # cat -> np.ndarray[bool]\n safe_cache = {} # cat -> np.ndarray[int] with -1 replaced by 0 for take()\n\n for cat, num in self.pairs_:\n x = num_cache[num]\n\n idx = self.group_index_.get((cat, num), None)\n vals_dict = self.group_values_.get((cat, num), None)\n\n mapped = {}\n\n # compute codes once per cat, using the FIRST idx encountered for that cat\n if cat not in codes_cache:\n if idx is None:\n codes = np.full(n, -1, dtype=int)\n else:\n cat_arr = X[cat].to_numpy()\n codes = idx.get_indexer(cat_arr) # exact same as your original\n codes_cache[cat] = codes\n missing = codes == -1\n missing_cache[cat] = missing\n\n safe_codes = codes.copy()\n safe_codes[missing] = 0\n safe_cache[cat] = safe_codes\n\n codes = codes_cache[cat]\n missing = missing_cache[cat]\n safe_codes = safe_cache[cat]\n\n # ---- group aggs mapping ----\n if idx is not None and vals_dict is not None and len(vals_dict) > 0:\n for agg, vals in vals_dict.items():\n col = vals.take(safe_codes).astype(float, copy=False)\n if missing.any():\n col = col.copy()\n col[missing] = np.nan\n\n mapped[agg] = col\n\n if not drop_basic:\n if self.fill_value is np.nan:\n out[:, col_i] = col\n else:\n out[:, col_i] = np.where(np.isnan(col), self.fill_value, col)\n col_i += 1\n else:\n # fallback identical to your original\n stats = self.group_stats_.get((cat, num), None)\n if stats is not None and len(stats.columns) > 0:\n cat_series = X[cat]\n for agg in stats.columns:\n col = cat_series.map(stats[agg]).astype(float).to_numpy(copy=False)\n mapped[agg] = col\n if not drop_basic:\n if self.fill_value is np.nan:\n out[:, col_i] = col\n else:\n out[:, col_i] = np.where(np.isnan(col), self.fill_value, col)\n col_i += 1\n\n # ---- pct_rank: ALWAYS output when requested (matches your original) ----\n if \"pct_rank\" in rowwise_aggs:\n pr = np.full(n, 0.5, dtype=float)\n vals = x\n\n valid = (codes != -1) & (~np.isnan(vals))\n if valid.any():\n keys = self.pct_rank_keys_.get((cat, num), None)\n dists = self.pct_rank_vals_.get((cat, num), None)\n\n if idx is not None and keys is not None and dists is not None and len(keys) > 0:\n dist_by_code = self.pct_rank_by_code_.get((cat, num), None)\n if dist_by_code is None:\n dist_by_code = np.empty(len(idx), dtype=object)\n dist_by_code[:] = None\n pos = idx.get_indexer(keys)\n ok = pos != -1\n dist_by_code[pos[ok]] = dists[ok]\n self.pct_rank_by_code_[(cat, num)] = dist_by_code\n\n vidx = np.flatnonzero(valid)\n vcode = codes[vidx]\n order = np.argsort(vcode, kind=\"mergesort\")\n vidx = vidx[order]\n vcode = vcode[order]\n\n breaks = np.r_[0, 1 + np.flatnonzero(vcode[1:] != vcode[:-1]), vcode.size]\n for b0, b1 in zip(breaks[:-1], breaks[1:]):\n c = vcode[b0]\n arr = dist_by_code[c]\n if arr is None or arr.size == 0:\n continue\n rows = vidx[b0:b1]\n ranks = np.searchsorted(arr, vals[rows], side=\"right\")\n pr[rows] = ranks / arr.size\n\n out[:, col_i] = pr\n col_i += 1\n\n # ---- relatives ----\n if rel_enabled:\n missing_aggs = set(self.relative_to_aggs) - set(mapped)\n if missing_aggs:\n raise ValueError(\n f\"Requested relative_to_aggs {missing_aggs} not present in computed \"\n f\"group aggregations {list(mapped)} for pair ({cat}, {num}). \"\n f\"Make sure those aggs are included in `aggregations=`.\"\n )\n\n for agg in self.relative_to_aggs:\n ref = mapped[agg]\n if \"diff\" in self.relative_ops:\n out[:, col_i] = x - ref\n col_i += 1\n if \"ratio\" in self.relative_ops:\n out[:, col_i] = x / (ref + self.eps)\n col_i += 1\n\n if col_i != m:\n raise RuntimeError(\n f\"Internal error: wrote {col_i} columns but expected {m}. \"\n f\"This indicates a mismatch between output_columns_ and transform writing order.\"\n )\n\n result = pd.DataFrame(out, columns=self.output_columns_, index=X.index)\n return result if self.return_dataframe else result.values\n\n def _fit_transform(self, X, y, **kwargs):\n self._fit(X, y)\n return self._transform(X), dict()\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n"
},
{
"path": "features/src/autogluon/features/generators/identity.py",
"content": "import logging\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass IdentityFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"IdentityFeatureGenerator simply passes the data along without alterations.\"\"\"\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n X_out = self._transform(X)\n return X_out, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return X\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n\n def estimate_output_feature_metadata(self, feature_metadata_in: FeatureMetadata, **kwargs) -> FeatureMetadata:\n features_to_remove = feature_metadata_in.get_features(**self._infer_features_in_args)\n return feature_metadata_in.keep_features(features_to_remove, inplace=False)\n"
},
{
"path": "features/src/autogluon/features/generators/isnan.py",
"content": "import logging\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_OBJECT, S_BOOL\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass IsNanFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Transforms features into isnull flags.\n\n Parameters\n ----------\n null_map : dict, default {'object': ''}\n Map which dictates the values to consider as NaN.\n Keys are the raw types of the features as in self.feature_metadata_in.type_map_raw.\n If a feature's raw type is not present in null_map, np.nan is treated as NaN.\n If a value other than np.nan is specified, np.nan is not considered NaN.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(self, null_map=None, **kwargs):\n super().__init__(**kwargs)\n if null_map is None:\n null_map = {R_OBJECT: \"\"}\n self.null_map = null_map\n self._null_feature_map = None\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n features = self.feature_metadata_in.get_features()\n self._null_feature_map = dict()\n for feature in features:\n feature_raw_type = self.feature_metadata_in.get_feature_type_raw(feature)\n if feature_raw_type in self.null_map:\n self._null_feature_map[feature] = self.null_map[feature_raw_type]\n X_out = self._transform(X)\n type_family_groups_special = {S_BOOL: list(X_out.columns)}\n return X_out, type_family_groups_special\n\n # TODO: Try returning bool type instead of uint8\n def _transform(self, X: DataFrame) -> DataFrame:\n is_nan_features = dict()\n for feature in self.features_in:\n if feature in self._null_feature_map:\n null_val = self._null_feature_map[feature]\n is_nan_features[\"__nan__.\" + feature] = (X[feature] == null_val).astype(np.uint8)\n else:\n is_nan_features[\"__nan__.\" + feature] = X[feature].isnull().astype(np.uint8)\n return pd.DataFrame(is_nan_features, index=X.index)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self._null_feature_map:\n for feature in features:\n if feature in self._null_feature_map:\n self._null_feature_map.pop(feature)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/label_encoder.py",
"content": "import copy\nimport logging\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.types import R_CATEGORY, S_TEXT_AS_CATEGORY\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: LabelEncoderTransformer\nclass LabelEncoderFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"Converts category features to int features by mapping to the category codes.\"\"\"\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n X_out = self._transform(X)\n feature_metadata_out_type_group_map_special = copy.deepcopy(self.feature_metadata_in.type_group_map_special)\n if S_TEXT_AS_CATEGORY in feature_metadata_out_type_group_map_special:\n feature_metadata_out_type_group_map_special.pop(S_TEXT_AS_CATEGORY)\n return X_out, feature_metadata_out_type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self.convert_category_to_int(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_CATEGORY])\n\n @staticmethod\n def convert_category_to_int(X: DataFrame) -> DataFrame:\n # TODO: add inplace option?\n X = X.apply(lambda x: x.cat.codes)\n return X\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/memory_minimize.py",
"content": "import logging\n\nimport numpy as np\nfrom pandas import DataFrame, RangeIndex\n\nfrom autogluon.common.features.types import R_CATEGORY, R_INT\n\nfrom ..utils import clip_and_astype\nfrom . import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass CategoryMemoryMinimizeFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Minimizes memory usage of category features by converting the category values to monotonically increasing int values.\n This is important for category features with string values which can take up significant memory despite the string information not being used downstream.\n \"\"\"\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n self._category_maps = self._get_category_map(X=X)\n\n X_out = self._transform(X)\n return X_out, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n return self._minimize_categorical_memory_usage(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_CATEGORY])\n\n def _get_category_map(self, X: DataFrame) -> dict:\n category_maps = {}\n for column in X:\n old_categories = list(X[column].cat.categories.values)\n new_categories = RangeIndex(len(old_categories)) # Memory optimal categories\n category_maps[column] = new_categories\n return category_maps\n\n def _minimize_categorical_memory_usage(self, X: DataFrame):\n if self._category_maps:\n X_renamed = dict()\n for column in self._category_maps:\n # rename_categories(inplace=True) is faster but it is deprecated as of pandas 1.3.0\n X_renamed[column] = X[column].cat.rename_categories(self._category_maps[column])\n X = DataFrame(X_renamed)\n return X\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if self._category_maps:\n for feature in features:\n if feature in self._category_maps:\n self._category_maps.pop(feature)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n\n\n# TODO: What about nulls / unknowns?\nclass NumericMemoryMinimizeFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Clips and converts dtype of int features to minimize memory usage.\n\n dtype_out : np.dtype, default np.uint8\n dtype to clip and convert features to.\n Clipping will automatically use the correct min and max values for the dtype provided.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(self, dtype_out=np.uint8, **kwargs):\n super().__init__(**kwargs)\n self.dtype_out, self._clip_min, self._clip_max = self._get_dtype_clip_args(dtype_out)\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n X_out = self._transform(X)\n return X_out, self.feature_metadata_in.type_group_map_special\n\n def _transform(self, X):\n return self._minimize_numeric_memory_usage(X)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_INT])\n\n @staticmethod\n def _get_dtype_clip_args(dtype) -> (np.dtype, int, int):\n try:\n dtype_info = np.iinfo(dtype)\n except ValueError:\n dtype_info = np.finfo(dtype)\n return dtype_info.dtype, dtype_info.min, dtype_info.max\n\n def _minimize_numeric_memory_usage(self, X: DataFrame):\n return clip_and_astype(df=X, clip_min=self._clip_min, clip_max=self._clip_max, dtype=self.dtype_out)\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/one_hot_encoder.py",
"content": "import logging\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nfrom pandas import DataFrame\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom autogluon.common.features.types import R_CATEGORY, R_INT, S_BOOL, S_SPARSE\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass CatToInt:\n \"\"\"\n Converts pandas categoricals to numpy int in preparation for OHE.\n\n Parameters\n ----------\n max_levels : int\n The maximum number of unique values for OHE. Selected categories are based on frequency.\n fillna_val : int, default = None\n The default value to fill NaN.\n If None, automatically inferred as a new value not present in existing categories.\n infrequent_val : int or {'na', 'na+1'}, default = 'na'\n The value to group all infrequent categories to (those that aren't within the max_levels most frequent categories).\n If 'na', uses `fillna_val`.\n If 'na+1', uses `fillna_val+1`. This guarantees a new category for infrequent values separate from missing values if `fillna_val=None`.\n \"\"\"\n\n def __init__(self, max_levels, fillna_val=None, infrequent_val=\"na\"):\n self.max_levels = max_levels\n self.fillna_val = fillna_val\n self.infrequent_val = infrequent_val\n self.cats = dict()\n self.num_cols = None\n self._dtype = None\n\n def fit(self, X: DataFrame):\n # dtype_buffer=2 is required to avoid edge case errors with invalid self.infrequent_val in 'na+1' mode.\n self._dtype, fillna_val = self._get_dtype_and_fillna(X, dtype_buffer=2)\n if self.fillna_val is None:\n self.fillna_val = fillna_val\n if self.infrequent_val == \"na\":\n self.infrequent_val = self.fillna_val\n elif self.infrequent_val == \"na+1\":\n self.infrequent_val = self.fillna_val + 1\n\n X = self.pd_to_np(X)\n self.num_cols = X.shape[1]\n for col in range(self.num_cols):\n data = X[:, col]\n uniques, counts = np.unique(data, return_counts=True)\n self.cats[col] = uniques[np.argsort(counts)[-self.max_levels :]]\n with warnings.catch_warnings():\n # Refer to https://stackoverflow.com/questions/40659212/futurewarning-elementwise-comparison-failed-returning-scalar-but-in-the-futur\n warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n if self.infrequent_val in self.cats[col] or str(self.infrequent_val) in self.cats[col]:\n # Add one extra level since NaN values shouldn't be counted towards max levels\n self.cats[col] = uniques[np.argsort(counts)[-(self.max_levels + 1) :]]\n\n def transform(self, X: DataFrame):\n X = self.pd_to_np(X)\n mask = np.zeros(shape=X.shape, dtype=bool)\n for col in range(self.num_cols):\n mask[:, col] = np.isin(X[:, col], self.cats[col], invert=True)\n X[mask] = self.infrequent_val\n return X\n\n def pd_to_np(self, X: DataFrame) -> np.ndarray:\n \"\"\"\n Converts pandas categoricals to a numpy ndarray of the codes of the categories.\n \"\"\"\n with warnings.catch_warnings():\n if np.issubdtype(self._dtype, np.integer):\n # Filter incorrect pandas RuntimeWarning message\n # For more details, refer to https://github.com/autogluon/autogluon/pull/4224#issuecomment-2156423410\n warnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n return X.to_numpy(dtype=self._dtype, na_value=self.fillna_val, copy=True)\n\n def _get_dtype_and_fillna(self, X: DataFrame, dtype_buffer=2):\n assert dtype_buffer >= 1, \"dtype_buffer must be >= 1 or else fillna_val could be invalid.\"\n dtype = None\n max_val_all = None\n for col in X.columns:\n try:\n max_val = X[col].dtype.categories.max()\n min_val = X[col].dtype.categories.min()\n except:\n max_val = X[col].max()\n min_val = X[col].min()\n if isinstance(max_val, str):\n max_dtype = np.min_scalar_type(max_val)\n else:\n if max_val_all is None:\n max_val_all = max_val\n else:\n max_val_all = max(max_val_all, max_val)\n max_val_with_buffer = max_val + dtype_buffer\n max_dtype = np.min_scalar_type(max_val_with_buffer)\n min_dtype = np.min_scalar_type(min_val)\n cur_dtype = np.promote_types(min_dtype, max_dtype)\n\n if dtype is None:\n dtype = cur_dtype\n else:\n dtype = np.promote_types(dtype, cur_dtype)\n if max_val_all is None:\n fillna_val = 0\n else:\n fillna_val = max_val_all + 1\n return dtype, fillna_val\n\n\n# TODO: Replace XGBoost, NN, and Linear Model OHE logic with this\nclass OneHotEncoderFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n Converts category features to one-hot boolean features by mapping to the category codes.\n\n Parameters\n ----------\n max_levels : int\n The maximum number of categories to use for OHE per feature. Selected categories are based on frequency.\n dtype : number type, default = np.uint8\n Desired dtype of output.\n sparse : bool, default = True\n Will return sparse matrix if set to True else will return an array.\n drop : str, default = None\n Refer to OneHotEncoder documentation for details.\n \"\"\"\n\n def __init__(self, max_levels=None, dtype=np.uint8, sparse=True, drop=None, **kwargs):\n super().__init__(**kwargs)\n self.max_levels = max_levels\n self.sparse = sparse\n self._ohe = OneHotEncoder(dtype=dtype, sparse_output=self.sparse, handle_unknown=\"ignore\", drop=drop)\n self._ohe_columns = None\n self._cat_feat_gen = None\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n if self.max_levels is not None:\n self._cat_feat_gen = CatToInt(max_levels=self.max_levels)\n self._cat_feat_gen.fit(X)\n X_out = self._cat_feat_gen.transform(X)\n else:\n X_out = X\n\n self._ohe.fit(X_out)\n self._ohe_columns = self._ohe.get_feature_names_out()\n self._ohe_columns = [\"_ohe_\" + str(i) for i in range(len(self._ohe_columns))]\n X_out = self._transform(X)\n\n features_out = list(X_out.columns)\n\n feature_metadata_out_type_group_map_special = {S_BOOL: features_out}\n if self.sparse:\n feature_metadata_out_type_group_map_special[S_SPARSE] = features_out\n return X_out, feature_metadata_out_type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n X_out = self.transform_ohe(X)\n if self.sparse:\n X_out = pd.DataFrame.sparse.from_spmatrix(X_out, columns=self._ohe_columns, index=X.index)\n else:\n X_out = pd.DataFrame(X_out, columns=self._ohe_columns, index=X.index)\n return X_out\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict(valid_raw_types=[R_CATEGORY, R_INT])\n\n def transform_ohe(self, X: DataFrame):\n \"\"\"\n Call this method directly to get numpy output.\n Skips pandas conversion (much faster if only the numpy output is required).\n \"\"\"\n if self._cat_feat_gen is not None:\n X = self._cat_feat_gen.transform(X)\n X = self._ohe.transform(X)\n return X\n\n def _more_tags(self):\n return {\"feature_interactions\": False}\n"
},
{
"path": "features/src/autogluon/features/generators/oof_target_encoder.py",
"content": "import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nfrom autogluon.common.utils.cv_splitter import CVSplitter\n\nfrom .abstract import AbstractFeatureGenerator\n\n\nclass OOFTargetEncodingFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n KFold out-of-fold target encoding (regression / binary / multiclass)\n Parameters\n ----------\n n_splits : int, default=5\n Number of folds for KFold or StratifiedKFold.\n alpha : float, default=10.0\n Smoothing parameter for target encoding.\n random_state : int, default=42\n Random state for reproducibility.\n keep_original : bool, default=False\n Whether to keep the original features.\n **kwargs\n Additional keyword arguments.\n Returns\n -------\n self : OOFTargetEncoderFeatureGenerator\n Fitted OOFTargetEncoderFeatureGenerator instance.\n Notes\n -----\n The target encoding is performed using K-Fold cross-validation to prevent data leakage.\n For regression and binary classification, the target mean is used for encoding.\n For multiclass classification, separate encodings are created for each class.\n During transformation on unseen data, the full training statistics are used to encode new data.\n - fit(...) computes + stores OOF TRAIN encodings and full-train stats\n - transform(..., is_train=True) returns stored OOF TRAIN encodings\n - transform(..., is_train=False) encodes new data using full stats\n \"\"\"\n\n def __init__(\n self,\n keep_original: bool = False,\n n_splits: int = 5,\n alpha: float = 10.0,\n # TODO: Consider adding max_classes to select only the most frequent classes for multi-class to avoid feature explosion for many-class problems\n random_state: int = 42,\n **kwargs,\n ):\n super().__init__(**kwargs)\n assert self.target_type in {\"regression\", \"binary\", \"multiclass\", \"quantile\"}\n if self.target_type == \"quantile\":\n self.target_type = \"regression\" # FIXME: this is a hack\n self.keep_original = keep_original\n self.n_splits = n_splits\n self.alpha = alpha\n self.random_state = random_state\n\n def estimate_new_dtypes(self, n_numeric, n_categorical, n_binary, num_classes=None, **kwargs) -> int:\n num_new_feats = n_categorical\n\n if self.target_type == \"multiclass\":\n num_new_feats *= num_classes\n if self.keep_original:\n return n_numeric + num_new_feats, n_categorical, n_binary\n else:\n return n_numeric + num_new_feats, 0, n_binary\n\n def estimate_no_of_new_features(self, X: pd.DataFrame, num_classes: int, **kwargs) -> int:\n X_cat = X.select_dtypes(include=[\"object\", \"category\"])\n num_cat_cols = X_cat.shape[1]\n if self.target_type == \"multiclass\":\n return num_classes * num_cat_cols, X_cat.columns.tolist()\n else:\n return num_cat_cols, X_cat.columns.tolist()\n\n def _fit(self, X: pd.DataFrame, y: pd.Series, **kwargs):\n if y is None:\n raise AssertionError(f\"y must be present during fit\")\n original_index = X.index\n\n # Identify categorical vs passthrough cols\n self.cols_ = X.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n self.passthrough_cols_ = [col for col in X.columns if col not in self.cols_]\n\n # Early exit: nothing to encode\n if len(self.cols_) == 0:\n self.encodings_ = {}\n self.train_encoded_ = pd.DataFrame(index=original_index)\n return self\n\n # Work only on the categorical part (for encoding), keep object dtype\n X_cat = X[self.cols_].reset_index(drop=True).astype(\"object\")\n y = pd.Series(y).reset_index(drop=True)\n\n n = len(X_cat)\n\n if self.target_type in [\"binary\", \"multiclass\"]:\n splitter_cls = StratifiedKFold\n stratify = True\n else:\n splitter_cls = KFold\n stratify = False\n\n kf = CVSplitter(\n splitter_cls=splitter_cls,\n n_splits=self.n_splits,\n random_state=self.random_state,\n stratify=stratify,\n shuffle=True,\n # bin=True if self.target_type == 'regression' else False,\n # n_bins=50 if self.target_type == 'regression' else None,\n )\n\n # ------------------------\n # Build target matrix Y\n # ------------------------\n if self.target_type == \"regression\":\n Y = y.to_numpy().reshape(-1, 1).astype(float)\n\n elif self.target_type == \"binary\":\n if y.dtype.name == \"category\":\n y = y.cat.codes\n classes = np.unique(y)\n assert len(classes) == 2, \"binary target_type but >2 classes\"\n self.classes_ = classes\n Y = (y.to_numpy() == classes[-1]).astype(float).reshape(-1, 1)\n\n else: # multiclass\n if y.dtype.name == \"category\":\n y = y.cat.codes\n classes = np.unique(y)\n self.classes_ = classes\n arr = y.to_numpy()\n Y = (arr[:, None] == classes[None, :]).astype(float) # [n_samples, n_classes]\n\n self.n_targets = Y.shape[1]\n\n # Precompute splits once\n kf_splits = list(kf.split(np.zeros(n), y))\n\n alpha = self.alpha\n\n store = []\n self.encodings_ = {}\n\n # =====================================================\n # Per-column encoding using factorize + np.bincount\n # =====================================================\n for col in self.cols_:\n col_values = X_cat[col]\n\n # Factorize categories once; sorted to mimic groupby index order\n codes, uniques = pd.factorize(col_values, sort=True)\n # codes == -1 corresponds to NaN / missing category\n mask_valid = codes >= 0\n codes_valid = codes[mask_valid]\n Y_valid = Y[mask_valid]\n n_cat = len(uniques)\n\n # ---------------------------------------\n # Global (all-data) sums & counts\n # ---------------------------------------\n count_all = np.bincount(codes_valid, minlength=n_cat).astype(float) # (n_cat,)\n\n sum_all = np.vstack(\n [np.bincount(codes_valid, weights=Y_valid[:, j], minlength=n_cat) for j in range(self.n_targets)]\n ).T # (n_cat, n_targets)\n\n with np.errstate(invalid=\"ignore\", divide=\"ignore\"):\n mean_all = sum_all / count_all[:, None] # (n_cat, n_targets)\n\n # global_mean as in original _transform:\n # mean of per-category means\n global_mean = np.nanmean(mean_all, axis=0) # (n_targets,)\n\n # Smoothed per-category encodings used at inference\n denom_all = count_all[:, None] + alpha\n num_all = mean_all * count_all[:, None] + alpha * global_mean[None, :]\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\n enc_all = num_all / denom_all # (n_cat, n_targets)\n\n # Names for encoded features\n if self.n_targets == 1:\n names_out = [f\"{col}__te\"]\n else:\n names_out = [f\"{col}__te_class{j}\" for j in range(self.n_targets)]\n\n # enc_df = pd.DataFrame(enc_all, index=uniques, columns=names_out)\n # global_mean_series = pd.Series(global_mean, index=names_out)\n #\n # # Store lightweight stats for inference\n # self.encodings_[col] = dict(\n # enc=enc_df,\n # global_mean=global_mean_series,\n # )\n\n # Store lightweight, numeric-only info for fast transform\n self.encodings_[col] = dict(\n categories=uniques.to_numpy(copy=False), # np.array of categories, length n_cat\n enc_matrix=enc_all.astype(float, copy=False), # shape (n_cat, n_targets)\n global_mean=global_mean.astype(float, copy=False), # shape (n_targets,)\n names=names_out, # list of output column names for this feature\n )\n\n # ------------------------\n # OOF encodings (fast)\n # ------------------------\n oof = np.zeros((n, self.n_targets), dtype=float)\n\n for tr_idx, val_idx in kf_splits:\n val_mask = np.zeros(n, dtype=bool)\n val_mask[val_idx] = True\n tr_mask_valid = mask_valid & ~val_mask\n\n codes_tr = codes[tr_mask_valid]\n Y_tr = Y[tr_mask_valid]\n\n if codes_tr.size == 0:\n # Degenerate case: no training rows for this fold\n oof[val_idx, :] = global_mean[None, :]\n continue\n\n # Per-category sums & counts on the *training* portion\n count_tr = np.bincount(codes_tr, minlength=n_cat).astype(float) # (n_cat,)\n sum_tr = np.vstack(\n [np.bincount(codes_tr, weights=Y_tr[:, j], minlength=n_cat) for j in range(self.n_targets)]\n ).T # (n_cat, n_targets)\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\n mean_tr = sum_tr / count_tr[:, None] # (n_cat, n_targets)\n\n valid_cats = count_tr > 0\n\n # m_mean: mean of per-category means over categories that appear in training\n m_mean = np.where(valid_cats[:, None], mean_tr, np.nan)\n m_mean = np.nanmean(m_mean, axis=0) # (n_targets,)\n\n denom = count_tr[:, None] + alpha\n num = mean_tr * count_tr[:, None] + alpha * m_mean[None, :]\n\n with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\n enc_tr = num / denom # (n_cat, n_targets)\n\n enc_tr[~valid_cats, :] = m_mean\n\n # Assign encodings to OOF for this fold\n enc_val = np.zeros((len(val_idx), self.n_targets), dtype=float)\n enc_val[:] = m_mean[None, :]\n\n val_codes = codes[val_idx]\n non_nan_mask = val_codes >= 0\n if np.any(non_nan_mask):\n enc_val[non_nan_mask, :] = enc_tr[val_codes[non_nan_mask]]\n\n oof[val_idx, :] = enc_val\n\n # Build DataFrame for this column, aligned to original index\n if self.n_targets == 1:\n df_oof = pd.DataFrame(oof, columns=names_out, index=original_index)\n else:\n df_oof = pd.DataFrame(oof, columns=names_out, index=original_index)\n store.append(df_oof)\n\n self.train_encoded_ = pd.concat(store, axis=1)\n\n return self\n\n def _fit_transform(self, X: pd.DataFrame, y: pd.Series, **kwargs):\n self._fit(X, y)\n\n if len(self.cols_) == 0:\n return X.copy(), dict()\n if self.keep_original:\n X_out = pd.concat([X, self.train_encoded_], axis=1)\n else:\n X_out = pd.concat([X[self.passthrough_cols_], self.train_encoded_], axis=1)\n self.train_encoded_ = None\n return X_out, dict()\n\n def _transform(self, X, **kwargs):\n if len(self.cols_) == 0:\n return X.copy()\n\n n_rows = len(X)\n\n # Precompute how many encoded features we will create\n total_new_feats = 0\n for col in self.cols_:\n total_new_feats += len(self.encodings_[col][\"names\"])\n\n # Single big output array for all encoded columns\n encoded_all = np.empty((n_rows, total_new_feats), dtype=float)\n encoded_colnames: list[str] = []\n\n offset = 0\n for col in self.cols_:\n info = self.encodings_[col]\n categories = info[\"categories\"] # np.array, shape (n_cat,)\n enc_matrix = info[\"enc_matrix\"] # np.array, shape (n_cat, n_targets)\n global_mean = info[\"global_mean\"] # np.array, shape (n_targets,)\n names = info[\"names\"] # list[str], length n_targets\n\n n_targets = enc_matrix.shape[1]\n\n # Extract raw values once, no copy if possible\n col_vals = X[col].to_numpy(dtype=object, copy=False)\n\n # Use a Categorical with fixed categories from fit to get stable codes\n cat = pd.Categorical(col_vals, categories=categories, ordered=False)\n codes = cat.codes # int array; -1 for NaN/unseen categories\n\n # Prepare block for this column\n block = np.empty((n_rows, n_targets), dtype=float)\n # Start with global_mean everywhere (handles NaN/unseen)\n block[:] = global_mean[None, :]\n\n # For valid codes, index into enc_matrix\n valid_mask = codes >= 0\n if np.any(valid_mask):\n block[valid_mask, :] = enc_matrix[codes[valid_mask]]\n\n # Place block into the big encoded_all array\n encoded_all[:, offset : offset + n_targets] = block\n encoded_colnames.extend(names)\n offset += n_targets\n\n # Wrap all encoded columns into a single DataFrame\n encoded_df = pd.DataFrame(encoded_all, index=X.index, columns=encoded_colnames)\n\n if self.keep_original:\n # Preserve original columns + new encodings\n return pd.concat([X, encoded_df], axis=1)\n else:\n # Only passthrough + new encodings\n return pd.concat([X[self.passthrough_cols_], encoded_df], axis=1)\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n"
},
{
"path": "features/src/autogluon/features/generators/pipeline.py",
"content": "import copy\nimport logging\n\nfrom pandas import DataFrame\n\nfrom autogluon.common.features.feature_metadata import FeatureMetadata\nfrom autogluon.common.features.infer_types import get_type_map_real\nfrom autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage\nfrom autogluon.common.utils.resource_utils import ResourceManager\n\nfrom .bulk import BulkFeatureGenerator\nfrom .drop_unique import DropUniqueFeatureGenerator\nfrom .dummy import DummyFeatureGenerator\nfrom .fillna import FillNaFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\n# TODO: Documentation\nclass PipelineFeatureGenerator(BulkFeatureGenerator):\n \"\"\"\n PipelineFeatureGenerator is an implementation of BulkFeatureGenerator with various smart defaults and edge case handling functionality to enable\n robust data handling.\n It is recommended that users base any custom feature generators meant for end-to-end data transformation from PipelineFeatureGenerator.\n Reference AutoMLPipelineFeatureGenerator for an example of extending PipelineFeatureGenerator.\n It is not recommended that PipelineFeatureGenerator be used as a generator within any other generator's pre or post generators.\n \"\"\"\n\n def __init__(\n self,\n pre_generators=None,\n post_generators=None,\n pre_drop_useless=True,\n pre_enforce_types=True,\n reset_index=True,\n post_drop_duplicates=True,\n verbosity=3,\n **kwargs,\n ):\n if pre_generators is None:\n pre_generators = [FillNaFeatureGenerator(inplace=True)]\n if post_generators is None:\n post_generators = [DropUniqueFeatureGenerator()]\n\n super().__init__(\n pre_generators=pre_generators,\n post_generators=post_generators,\n post_drop_duplicates=post_drop_duplicates,\n pre_drop_useless=pre_drop_useless,\n pre_enforce_types=pre_enforce_types,\n reset_index=reset_index,\n verbosity=verbosity,\n **kwargs,\n )\n\n # FeatureMetadata object based on the original input features real dtypes\n # (will contain dtypes such as 'int16' and 'float32' instead of 'int' and 'float').\n self._feature_metadata_in_real: FeatureMetadata = None\n\n self._is_dummy = (\n False # If True, returns a single dummy feature as output. Occurs if fit with no useful features.\n )\n\n self.pre_memory_usage = None\n self.pre_memory_usage_per_row = None\n self.post_memory_usage = None\n self.post_memory_usage_per_row = None\n\n def fit_transform(self, X: DataFrame, y=None, feature_metadata_in: FeatureMetadata = None, **kwargs) -> DataFrame:\n X_out = super().fit_transform(X=X, y=y, feature_metadata_in=feature_metadata_in, **kwargs)\n self._compute_post_memory_usage(X_out)\n # TODO: Consider adding final check of validity/that features are reasonable.\n\n return X_out\n\n def _fit_transform(self, X: DataFrame, y=None, **kwargs):\n X_out, type_group_map_special = super()._fit_transform(X=X, y=y, **kwargs)\n X_out, type_group_map_special = self._fit_transform_custom(\n X_out=X_out, type_group_map_special=type_group_map_special, y=y\n )\n return X_out, type_group_map_special\n\n def _fit_transform_custom(self, X_out: DataFrame, type_group_map_special: dict, y=None) -> (DataFrame, dict):\n if len(list(X_out.columns)) == 0:\n self._is_dummy = True\n self._log(\n 30,\n \"\\tWARNING: No useful features were detected in the data! AutoGluon will train using 0 features, \"\n \"and will always predict the same value. Ensure that you are passing the correct data to AutoGluon!\",\n )\n dummy_generator = DummyFeatureGenerator()\n X_out = dummy_generator.fit_transform(X=X_out)\n type_group_map_special = copy.deepcopy(dummy_generator.feature_metadata.type_group_map_special)\n self.generators = [[dummy_generator]]\n self._remove_features_in(features=self.features_in)\n return X_out, type_group_map_special\n\n def _infer_features_in_full(self, X: DataFrame, feature_metadata_in: FeatureMetadata = None):\n super()._infer_features_in_full(X=X, feature_metadata_in=feature_metadata_in)\n type_map_real = get_type_map_real(X[self.feature_metadata_in.get_features()])\n self._feature_metadata_in_real = FeatureMetadata(\n type_map_raw=type_map_real, type_group_map_special=self.feature_metadata_in.get_type_group_map_raw()\n )\n\n def _remove_features_in(self, features: list):\n super()._remove_features_in(features)\n if features:\n self._feature_metadata_in_real = self._feature_metadata_in_real.remove_features(features=features)\n\n def _pre_fit_validate(self, X: DataFrame, **kwargs):\n super()._pre_fit_validate(X=X, **kwargs)\n self._ensure_no_duplicate_column_names(\n X=X\n ) # TODO: Remove this, move pre_memory_usage and post_memory_usage into super().\n self._compute_pre_memory_usage(X)\n\n def _compute_pre_memory_usage(self, X: DataFrame):\n X_len = len(X)\n self.pre_memory_usage = get_approximate_df_mem_usage(X, sample_ratio=0.2).sum()\n pre_memory_usage_mb = ResourceManager.bytes_converter(\n value=self.pre_memory_usage, format_in=\"B\", format_out=\"MB\"\n )\n self.pre_memory_usage_per_row = self.pre_memory_usage / X_len\n available_mem_mb = ResourceManager.get_available_virtual_mem(format=\"MB\")\n pre_memory_usage_percent = pre_memory_usage_mb / (available_mem_mb + pre_memory_usage_mb)\n self._log(20, f\"\\tAvailable Memory: {(pre_memory_usage_mb + available_mem_mb):.2f} MB\")\n self._log(\n 20,\n f\"\\tTrain Data (Original) Memory Usage: {pre_memory_usage_mb:.2f} MB \"\n f\"({(pre_memory_usage_percent * 100):.1f}% of available memory)\",\n )\n if pre_memory_usage_percent > 0.05:\n self._log(\n 30,\n f\"\\tWarning: Data size prior to feature transformation consumes {(pre_memory_usage_percent * 100):.1f}% of available memory. \"\n f\"Consider increasing memory or subsampling the data to avoid instability.\",\n )\n\n def _compute_post_memory_usage(self, X: DataFrame):\n X_len = len(X)\n self.post_memory_usage = get_approximate_df_mem_usage(X, sample_ratio=0.2).sum()\n self.post_memory_usage_per_row = self.post_memory_usage / X_len\n post_memory_usage_mb = ResourceManager.bytes_converter(\n value=self.post_memory_usage, format_in=\"B\", format_out=\"MB\"\n )\n pre_memory_usage_mb = ResourceManager.bytes_converter(\n value=self.pre_memory_usage, format_in=\"B\", format_out=\"MB\"\n )\n available_mem_mb = ResourceManager.get_available_virtual_mem(format=\"MB\")\n post_memory_usage_percent = post_memory_usage_mb / (\n available_mem_mb + post_memory_usage_mb + pre_memory_usage_mb\n )\n self._log(\n 20,\n f\"\\tTrain Data (Processed) Memory Usage: {post_memory_usage_mb:.2f} MB \"\n f\"({(post_memory_usage_percent * 100):.1f}% of available memory)\",\n )\n if post_memory_usage_percent > 0.15:\n self._log(\n 30,\n f\"\\tWarning: Data size post feature transformation consumes {(post_memory_usage_percent * 100):.1f}% of available memory. \"\n f\"Consider increasing memory or subsampling the data to avoid instability.\",\n )\n\n def print_feature_metadata_info(self, log_level=20):\n if self._useless_features_in:\n self._log(\n log_level,\n f\"\\tUseless Original Features (Count: {len(self._useless_features_in)}): {list(self._useless_features_in)}\",\n )\n # TODO: What about features with 1 unique value but also np.nan?\n self._log(log_level, \"\\t\\tThese features carry no predictive signal and should be manually investigated.\")\n self._log(log_level, \"\\t\\tThis is typically a feature which has the same value for all rows.\")\n self._log(log_level, \"\\t\\tThese features do not need to be present at inference time.\")\n if self._feature_metadata_in_unused is not None and self._feature_metadata_in_unused.get_features():\n # TODO: Consider highlighting why a feature was unused\n # (complex to implement, can check if was valid input to any generator in a generator group through feature chaining)\n self._log(\n log_level,\n f\"\\tUnused Original Features (Count: {len(self._feature_metadata_in_unused.get_features())}): \"\n f\"{self._feature_metadata_in_unused.get_features()}\",\n )\n self._log(\n log_level,\n \"\\t\\tThese features were not used to generate any of the output features. \"\n \"Add a feature generator compatible with these features to utilize them.\",\n )\n self._log(\n log_level,\n \"\\t\\tFeatures can also be unused if they carry very little information, \"\n \"such as being categorical but having almost entirely unique values or being duplicates of other features.\",\n )\n self._log(log_level, \"\\t\\tThese features do not need to be present at inference time.\")\n self._feature_metadata_in_unused.print_feature_metadata_full(self.log_prefix + \"\\t\\t\", log_level=log_level)\n self._log(log_level - 5, \"\\tTypes of features in original data (exact raw dtype, raw dtype):\")\n self._feature_metadata_in_real.print_feature_metadata_full(\n self.log_prefix + \"\\t\\t\", print_only_one_special=True, log_level=log_level - 5\n )\n super().print_feature_metadata_info(log_level=log_level)\n"
},
{
"path": "features/src/autogluon/features/generators/rename.py",
"content": "import copy\nimport logging\n\nfrom pandas import DataFrame\n\nfrom .abstract import AbstractFeatureGenerator\n\nlogger = logging.getLogger(__name__)\n\n\nclass RenameFeatureGenerator(AbstractFeatureGenerator):\n \"\"\"\n RenameFeatureGenerator renames the columns without altering their values.\n This can be used to avoid column name collisions when transforming the same feature in multiple ways,\n or to highlight that a feature was derived from a particular pipeline.\n\n Parameters\n ----------\n name_prefix : str, default None\n Name prefix to add to all output feature names.\n name_suffix : str, default None\n Name suffix to add to all output feature names.\n inplace : bool, default False\n If True, then the column names are renamed inplace without copying the input data.\n This will alter the input data outside of the scope of this function.\n **kwargs :\n Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.\n \"\"\"\n\n def __init__(self, name_prefix=None, name_suffix=None, inplace=False, **kwargs):\n super().__init__(**kwargs)\n self._name_prefix = name_prefix\n self._name_suffix = name_suffix\n self.inplace = inplace\n self._is_updated_name = None\n\n def _fit_transform(self, X: DataFrame, **kwargs) -> (DataFrame, dict):\n column_rename_map, self._is_updated_name = self._get_renamed_features(X)\n if not self.inplace:\n X = copy.deepcopy(X)\n X.columns = [column_rename_map.get(col, col) for col in X.columns]\n\n feature_metadata_out = self.feature_metadata_in.rename_features(column_rename_map)\n return X, feature_metadata_out.type_group_map_special\n\n def _transform(self, X: DataFrame) -> DataFrame:\n if self._is_updated_name:\n if not self.inplace:\n X = copy.deepcopy(X)\n X.columns = self.features_out\n return X\n\n def _get_renamed_features(self, X: DataFrame) -> (DataFrame, dict):\n X_columns_orig = list(X.columns)\n X_columns_new = list(X.columns)\n if self._name_prefix:\n X_columns_new = [self._name_prefix + column for column in X_columns_new]\n if self._name_suffix:\n X_columns_new = [column + self._name_suffix for column in X_columns_new]\n if X_columns_orig != X_columns_new:\n is_updated_name = True\n else:\n is_updated_name = False\n column_rename_map = {orig: new for orig, new in zip(X_columns_orig, X_columns_new)}\n return column_rename_map, is_updated_name\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n\n def _more_tags(self):\n return {\n \"feature_interactions\": False,\n \"allow_post_generators\": False, # TODO: This might not be necessary anymore\n }\n"
},
{
"path": "features/src/autogluon/features/generators/rsfc.py",
"content": "from __future__ import annotations\n\nfrom contextlib import contextmanager\nfrom time import perf_counter\nfrom typing import Literal, Optional, Sequence\n\nimport numpy as np\nimport pandas as pd\n\nfrom .abstract import AbstractFeatureGenerator\nfrom .oof_target_encoder import OOFTargetEncodingFeatureGenerator\n\n\nclass TimerLog:\n # TODO: Mainly used for debugging and tracking runtimes during development. Not needed for preprocessing logic. Better remove?\n # TODO: Copied from arithmetic preprocessors, might move it somewhere else if we want to reuse it.\n def __init__(self):\n self.times = {}\n\n @contextmanager\n def block(self, name: str):\n t0 = perf_counter()\n try:\n yield\n finally:\n dt = perf_counter() - t0\n self.times[name] = self.times.get(name, 0) + dt\n\n def summary(self, verbose: bool = False) -> dict:\n if verbose:\n print(\"\\n--- Timing Summary (in order) ---\")\n for name, total in self.times.items():\n print(f\"{name:<20} {total:.3f}s\")\n return dict(self.times)\n\n\nclass RandomSubsetFeatureCompressionGenerator(AbstractFeatureGenerator):\n \"\"\"\n Random Subset Feature Compression (RSFC) with target-awareness via OOF-TE.\n\n Runs target-aware random subset compression on:\n - full feature set (always attempted)\n - additional random subsets of features\n\n Output columns:\n RSFC_0_* -> full-feature set compression\n RSFC_1_* -> first random subset\n ...\n \"\"\"\n\n def __init__(\n self,\n only_cat: bool = False,\n binary_as_cat: bool = True,\n max_cardinality: Optional[int] = None,\n round_numerical: Optional[int] = 2,\n n_subsets: int = 50,\n subset_size: Optional[int] = None,\n min_subset_size: int = 2,\n max_subset_size: Optional[int] = None,\n max_base_feats_to_consider: Optional[int] = 150,\n select_for_multiclass: bool = False,\n random_state: int = 42,\n **kwargs,\n ):\n super().__init__(random_state=random_state, **kwargs)\n\n assert self.target_type is not None, f\"Must specify target_type for {self.__class__.__name__}\"\n\n self.n_subsets = int(n_subsets)\n self.subset_size = subset_size\n self.min_subset_size = int(min_subset_size)\n self.max_subset_size = max_subset_size\n self.only_cat = bool(only_cat)\n self.binary_as_cat = bool(binary_as_cat)\n self.max_cardinality = int(max_cardinality) if max_cardinality is not None else None\n self.round_numerical = int(round_numerical) if round_numerical is not None else None\n self.select_for_multiclass = bool(select_for_multiclass)\n\n self.max_base_feats_to_consider = (\n int(max_base_feats_to_consider) if max_base_feats_to_consider is not None else None\n )\n\n # fitted state\n self.base_features_: Optional[tuple[str, ...]] = None\n\n self.timelog = TimerLog()\n\n @staticmethod\n def _select_top_mode_features(X: pd.DataFrame, k: Optional[int]) -> list[str]:\n \"\"\"\n Pick top-k columns by mode strength:\n max(value_count(col)) / n_rows (dropna=False)\n \"\"\"\n if k is None or k <= 0 or X.shape[1] == 0:\n return list(X.columns)\n\n k = min(k, X.shape[1])\n n = len(X)\n if n == 0:\n return list(X.columns)[:k]\n\n scores = {}\n for c in X.columns:\n vc = X[c].value_counts(dropna=False)\n top = int(vc.iloc[0]) if len(vc) else 0\n scores[c] = top / n\n\n return sorted(scores.keys(), key=lambda c: (-scores[c], str(c)))[:k]\n\n @staticmethod\n def _select_features_by_dtype_and_cardinality(X: pd.DataFrame, k: Optional[int]) -> list[str]:\n if k is not None and k <= 0:\n return []\n\n ordered = (\n list(X.select_dtypes(include=\"category\").nunique().sort_values().index)\n + list(X.select_dtypes(include=\"object\").nunique().sort_values().index)\n + list(X.columns[X.nunique() == 2])\n + list(X.select_dtypes(include=\"integer\").nunique().sort_values().index) # Note: Just int not captured\n + list(X.select_dtypes(include=\"float\").nunique().sort_values().index)\n )\n\n # remove duplicates, keep order\n ordered = list(dict.fromkeys(ordered))\n\n return ordered if k is None else ordered[:k]\n\n @staticmethod\n def _sample_unique_subsets(\n features: Sequence[str],\n rng: np.random.Generator,\n n: int,\n subset_size: Optional[int],\n min_k: int,\n max_k: Optional[int],\n max_tries_multiplier: int = 50,\n ) -> list[tuple[str, ...]]:\n feats = np.asarray(list(features), dtype=object)\n p = len(feats)\n if p <= 1 or n <= 0:\n return []\n\n max_k_eff = min(max_k if max_k is not None else (p - 1), p - 1)\n min_k_eff = max(min_k, 1)\n\n if subset_size is not None:\n k_min = k_max = int(subset_size)\n if not (1 <= k_min <= p - 1):\n return []\n else:\n k_min, k_max = min_k_eff, max_k_eff\n if k_min > k_max:\n return []\n\n selected: list[tuple[str, ...]] = []\n seen: set[tuple[str, ...]] = set()\n max_tries = max_tries_multiplier * n\n\n for _ in range(max_tries):\n if len(selected) >= n:\n break\n\n k = k_min if subset_size is not None else int(rng.integers(k_min, k_max + 1))\n subset = tuple(sorted(rng.choice(feats, size=k, replace=False).tolist()))\n if subset in seen:\n continue\n seen.add(subset)\n selected.append(subset)\n\n return selected\n\n def _prepare_X(self, X: pd.DataFrame) -> pd.DataFrame:\n if X.shape[1] == 0:\n # Degenerate: no columns; hash empty rows\n return pd.Series(pd.util.hash_pandas_object(X, index=False).astype(\"uint64\").astype(str), index=X.index)\n\n # Select columns\n if self.only_cat:\n cols = X.select_dtypes(include=[\"object\", \"category\"]).columns\n numeric_cols = X.select_dtypes(include=[\"number\"]).columns\n if self.binary_as_cat:\n binary_cols = X.select_dtypes(include=[\"number\"]).columns[\n X[numeric_cols].nunique() <= 2\n ] # NOTE: uniform may occur at test, hence <=, should generally make train/test prepare versions\n cols = cols.union(binary_cols)\n else:\n cols = X.columns\n\n if self.max_cardinality is not None and len(cols) > 0:\n nunique = X[cols].nunique(dropna=False)\n cols = nunique[nunique < self.max_cardinality].index\n\n if len(cols) == 0:\n cols = X.columns\n\n X_candidates = X.loc[:, cols]\n\n # Round numeric columns (copy only when needed)\n num_cols = X_candidates.select_dtypes(include=[\"number\"]).columns\n if len(num_cols) > 0 and self.round_numerical is not None:\n X_candidates = X_candidates.copy()\n X_candidates.loc[:, num_cols] = X_candidates.loc[:, num_cols].round(self.round_numerical)\n\n return X_candidates\n\n def _make_key(self, X: pd.DataFrame) -> pd.Series:\n \"\"\"\n Build a deterministic key per row by hashing selected columns.\n \"\"\"\n # hash rows -> uint64 -> str (OOF-TE expects single column input)\n return pd.util.hash_pandas_object(X, index=False).astype(\"uint64\").astype(str)\n\n @staticmethod\n def collapse_singletons(s, threshold=1, label=\"__single__\"):\n vc = s.value_counts()\n return s.where(s.map(vc) > threshold, label)\n\n def _fit_transform(self, X: pd.DataFrame, y: pd.Series, **kwargs):\n # Prepare X\n with self.timelog.block(\"prepare_input\"):\n X_local = self._prepare_X(X)\n\n # Restrict base feature space if requested\n with self.timelog.block(\"select_base_features\"):\n if self.max_base_feats_to_consider is not None and X.shape[1] > 0:\n selected = self._select_features_by_dtype_and_cardinality(X_local, self.max_base_feats_to_consider)\n X_local = X_local[selected]\n self.base_features_ = list(selected)\n else:\n self.base_features_ = list(X.columns)\n\n features = list(X_local.columns)\n rng = np.random.default_rng(self.random_state)\n\n # ---- 1..n) Random subsets ----\n with self.timelog.block(\"select_random_subsets\"):\n self.selected_subsets = [tuple(features)] # always include full set\n n_random = max(self.n_subsets - 1, 0)\n self.selected_subsets += self._sample_unique_subsets(\n features=features,\n rng=rng,\n n=n_random,\n subset_size=self.subset_size,\n min_k=self.min_subset_size,\n max_k=self.max_subset_size,\n )\n\n with self.timelog.block(\"make_key\"):\n X_str = pd.concat([self._make_key(X_local[list(i)]) for i in self.selected_subsets], axis=1)\n\n # with self.timelog.block(\"filter_uninformative_keys\"): # Improves efficiency but hurts performance\n # X_str = X_str.apply(self.collapse_singletons)\n\n with self.timelog.block(\"oof-te\"):\n self.subset_oof = OOFTargetEncodingFeatureGenerator(\n target_type=self.target_type, verbosity=0, alpha=0, random_state=self.random_state\n )\n X_oof = self.subset_oof.fit_transform(X_str, y)\n\n self.col_names = [f\"RSFC_{i}\" for i in range(X_oof.shape[1])]\n X_oof.columns = self.col_names\n\n if self.select_for_multiclass and self.target_type == \"multiclass\":\n y_corrs = pd.get_dummies(y).apply(lambda y_: X_oof.corrwith(y_))\n best_corr_rank = y_corrs.abs().rank().min(axis=1)\n self.selected_cols = best_corr_rank.index[best_corr_rank < self.n_subsets]\n X_oof = X_oof[self.selected_cols]\n\n return X_oof, {}\n\n def _transform(self, X: pd.DataFrame) -> pd.DataFrame:\n # Prepare X\n with self.timelog.block(\"transform_prepare_input\"):\n X_local = self._prepare_X(X[self.base_features_])\n with self.timelog.block(\"transform_make_key\"):\n X_str = pd.concat([self._make_key(X_local[list(i)]) for i in self.selected_subsets], axis=1)\n with self.timelog.block(\"transform_oof-transform\"):\n out = self.subset_oof.transform(X_str)\n out.columns = self.col_names\n if self.select_for_multiclass and self.target_type == \"multiclass\":\n out = out[self.selected_cols]\n return out\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return {}\n"
},
{
"path": "features/src/autogluon/features/generators/selection.py",
"content": "from .abstract import AbstractFeatureGenerator\n\n\nclass SpearmanFeatureSelector(AbstractFeatureGenerator):\n \"\"\"Select features based on Spearman correlation with the target.\n\n Parameters\n ----------\n threshold : float, default=None\n Features with absolute Spearman correlation below this threshold will be removed.\n Ignored if None.\n \"\"\"\n\n def __init__(self, threshold: float = None, max_features: int = 2000, **kwargs):\n super().__init__(**kwargs)\n if threshold is None and max_features is None:\n raise ValueError(\"Either 'threshold' or 'max_features' must be provided.\")\n self.threshold = threshold\n self.max_features = max_features\n self.selected_features_ = []\n\n def _fit(self, X, y):\n # Compute Spearman correlation\n # TODO: Add nan filtering\n # TODO: Add option for AUC for binary\n # TODO: Properly handle multiclass targets\n # X.columns[X.isna().mean()<0.99]\n corr = X.corrwith(y, method=\"spearman\")\n abs_corr = corr.abs().sort_values(ascending=False).dropna()\n\n if self.threshold is not None:\n self.selected_features_ = abs_corr[abs_corr >= self.threshold].index.tolist()\n elif self.max_features is not None:\n self.selected_features_ = abs_corr.head(self.max_features).index.tolist()\n\n def _transform(self, X):\n return X[self.selected_features_]\n\n def _fit_transform(self, X, y, **kwargs):\n self._fit(X, y)\n return self._transform(X), dict()\n\n @staticmethod\n def get_default_infer_features_in_args() -> dict:\n return dict()\n"
},
{
"path": "features/src/autogluon/features/generators/skrub/__init__.py",
"content": ""
},
{
"path": "features/src/autogluon/features/generators/skrub/_sklearn_compat.py",
"content": "\"\"\"\nCode adapted from skrub==0.6.2\n\"\"\"\n\nfrom __future__ import annotations\n\nimport sklearn\nfrom sklearn.utils.fixes import parse_version\n\nsklearn_version = parse_version(parse_version(sklearn.__version__).base_version)\n\n\nif sklearn_version < parse_version(\"1.6\"):\n\n def validate_data(_estimator, /, **kwargs):\n if \"ensure_all_finite\" in kwargs:\n force_all_finite = kwargs.pop(\"ensure_all_finite\")\n else:\n force_all_finite = True\n return _estimator._validate_data(**kwargs, force_all_finite=force_all_finite)\nelse:\n from sklearn.utils.validation import validate_data # noqa: F401\n"
},
{
"path": "features/src/autogluon/features/generators/skrub/_squashing_scaler.py",
"content": "\"\"\"\nCode adapted from skrub==0.6.2\n\"\"\"\n\nfrom __future__ import annotations\n\nimport numbers\n\nimport numpy as np\nfrom sklearn.base import BaseEstimator, OneToOneFeatureMixin, TransformerMixin\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.utils.validation import FLOAT_DTYPES, check_is_fitted\n\nfrom ._sklearn_compat import validate_data\n\n\ndef _mask_inf(X):\n \"\"\"Replace infinite values with NaN and return their sign.\"\"\"\n if (mask_inf := np.isinf(X)).any():\n sign = np.sign(X)\n X = np.where(mask_inf, np.nan, X)\n # 0 when X is finite, 1 when X is +inf, -1 when X is -inf\n mask_inf = mask_inf.astype(X.dtype) * sign\n\n return X, mask_inf\n\n\ndef _set_zeros(X, zero_cols):\n \"\"\"Set the finite values of the specified columns to zero.\"\"\"\n mask = np.isfinite(X)\n mask[:, ~zero_cols] = False\n X[mask] = 0.0\n return X\n\n\ndef _soft_clip(X, max_absolute_value, mask_inf):\n \"\"\"Apply a soft clipping to the data.\n\n Parameters\n ----------\n X : array-like, shape (n_samples, n_features)\n The data to be clipped.\n max_absolute_value : float, default=3.0\n Maximum absolute value that the transformed data can take.\n mask_inf : array-like, shape (n_samples, n_features)\n A mask indicating the positions of infinite values in the input data and their\n signs.\n\n Returns\n -------\n X_clipped : array-like, shape (n_samples, n_features)\n The clipped version of the input.\n \"\"\"\n X = X / np.sqrt(1 + (X / max_absolute_value) ** 2)\n X = np.where(mask_inf == 1, max_absolute_value, X)\n X = np.where(mask_inf == -1, -max_absolute_value, X)\n return X\n\n\nclass _MinMaxScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):\n \"\"\"A variation of scikit-learn MinMaxScaler.\n\n A simple min-max scaler that centers the median to zero and scales\n the data to the range [-2, 2].\n\n scikit-learn MinMaxScaler computes the following::\n\n X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))\n X_scaled = X_std * (max - min) + min\n\n This scaler computes the following::\n\n X_std = (X - median) / (X.max(axis=0) - X.min(axis=0) + eps)\n X_scaled = X_std * (max - min) + min\n\n where we set min = 0 and max = 2.\n \"\"\"\n\n def fit(self, X, y=None):\n del y\n eps = np.finfo(\"float32\").tiny\n self.median_ = np.nanmedian(X, axis=0)\n self.scale_ = 2 / (np.nanmax(X, axis=0) - np.nanmin(X, axis=0) + eps)\n return self\n\n def transform(self, X):\n check_is_fitted(self, [\"median_\", \"scale_\"])\n return self.scale_ * (X - self.median_)\n\n\nclass SquashingScaler(OneToOneFeatureMixin, TransformerMixin, BaseEstimator):\n r\"\"\"Perform robust centering and scaling followed by soft clipping.\n\n When features have large outliers, smooth clipping prevents the outliers from\n affecting the result too strongly, while robust scaling prevents the outliers from\n affecting the inlier scaling. Infinite values are mapped to the corresponding\n boundaries of the interval. NaN values are preserved.\n\n Parameters\n ----------\n max_absolute_value : float, default=3.0\n Maximum absolute value that the transformed data can take.\n\n quantile_range : tuple of float, default=(0.25, 0.75)\n The quantiles used to compute the scaling factor. The first value is the lower\n quantile and the second value is the upper quantile. The default values are the\n 25th and 75th percentiles, respectively. The quantiles are used to compute the\n scaling factor for the robust scaling step. The quantiles are computed from the\n finite values in the input column. If the two quantiles are equal, the scaling\n factor is computed from the 0th and 100th percentiles (i.e., the minimum and\n maximum values of the finite values in the input column).\n\n Notes\n -----\n This transformer is applied to each column independently. It uses two stages:\n\n 1. The first stage centers the median of the data to zero and multiplies the data by a\n scaling factor determined from quantiles of the distribution, using\n scikit-learn's :class:`~sklearn.preprocessing.RobustScaler`. It also handles\n edge-cases in which the two quantiles are equal by following-up with a\n :class:`~sklearn.preprocessing.MinMaxScaler`.\n 2. The second stage applies a soft clipping to the transformed data to limit the\n data to the interval ``[-max_absolute_value, max_absolute_value]`` in an\n injective way.\n\n Infinite values will be mapped to the corresponding boundaries of the interval. NaN\n values will be preserved.\n\n The formula for the transform is:\n\n .. math::\n\n \\begin{align*}\n a &:= \\begin{cases}\n 1/(q_{\\beta} - q_{\\alpha}) &\\text{if} \\quad q_{\\beta} \\neq q_{\\alpha} \\\\\n 2/(q_1 - q_0) &\\text{if}\\quad q_{\\beta} = q_{\\alpha} \\text{ and } q_1\n \\neq q_0 \\\\ 0 & \\text{otherwise}\n \\end{cases} \\\\ z &:= a.(x - q_{1/2}), \\\\ x_{\\text{out}} &:= \\frac{z}{\\sqrt{1\n + (z/B)^2}},\n \\end{align*}\n\n where:\n\n - :math:`x` is a value in the input column.\n - :math:`q_{\\gamma}` is the :math:`\\gamma`-quantile of the finite values in X,\n - :math:`B` is max_abs_value\n - :math:`\\alpha` is the lower quantile\n - :math:`\\beta` is the upper quantile.\n\n References\n ----------\n This method has been introduced as the robust scaling and smooth clipping transform\n in `Better by Default: Strong Pre-Tuned MLPs and Boosted Trees on Tabular Data\n (HolzmÃŧller et al., 2024) | \\n\",\n \" | importance | \\n\",\n \"stddev | \\n\",\n \"p_value | \\n\",\n \"n | \\n\",\n \"p99_high | \\n\",\n \"p99_low | \\n\",\n \"
|---|---|---|---|---|---|---|
| marital-status | \\n\",\n \"0.368 | \\n\",\n \"0.030332 | \\n\",\n \"0.000005 | \\n\",\n \"5 | \\n\",\n \"0.430453 | \\n\",\n \"0.305547 | \\n\",\n \"
| sex | \\n\",\n \"0.208 | \\n\",\n \"0.057619 | \\n\",\n \"0.000640 | \\n\",\n \"5 | \\n\",\n \"0.326639 | \\n\",\n \"0.089361 | \\n\",\n \"
| occupation | \\n\",\n \"0.032 | \\n\",\n \"0.022804 | \\n\",\n \"0.017460 | \\n\",\n \"5 | \\n\",\n \"0.078953 | \\n\",\n \"-0.014953 | \\n\",\n \"
| class | \\n\",\n \"0.028 | \\n\",\n \"0.010954 | \\n\",\n \"0.002318 | \\n\",\n \"5 | \\n\",\n \"0.050555 | \\n\",\n \"0.005445 | \\n\",\n \"
| age | \\n\",\n \"0.020 | \\n\",\n \"0.024495 | \\n\",\n \"0.070964 | \\n\",\n \"5 | \\n\",\n \"0.070435 | \\n\",\n \"-0.030435 | \\n\",\n \"
| education-num | \\n\",\n \"0.016 | \\n\",\n \"0.016733 | \\n\",\n \"0.049650 | \\n\",\n \"5 | \\n\",\n \"0.050454 | \\n\",\n \"-0.018454 | \\n\",\n \"
| education | \\n\",\n \"0.004 | \\n\",\n \"0.008944 | \\n\",\n \"0.186950 | \\n\",\n \"5 | \\n\",\n \"0.022416 | \\n\",\n \"-0.014416 | \\n\",\n \"
| hours-per-week | \\n\",\n \"0.004 | \\n\",\n \"0.016733 | \\n\",\n \"0.310654 | \\n\",\n \"5 | \\n\",\n \"0.038454 | \\n\",\n \"-0.030454 | \\n\",\n \"
| capital-gain | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| workclass | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| capital-loss | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| race | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| native-country | \\n\",\n \"0.000 | \\n\",\n \"0.000000 | \\n\",\n \"0.500000 | \\n\",\n \"5 | \\n\",\n \"0.000000 | \\n\",\n \"0.000000 | \\n\",\n \"
| fnlwgt | \\n\",\n \"-0.012 | \\n\",\n \"0.017889 | \\n\",\n \"0.896000 | \\n\",\n \"5 | \\n\",\n \"0.024833 | \\n\",\n \"-0.048833 | \\n\",\n \"